import { deliveredCodexEvidence, evaluateKnownAnswers, type FixtureClaimExpectation, type KnownAnswerAssessment, } from "./KnownAnswerEvaluation.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; /** One REA MCP invocation observed in a Codex JSONL transcript. */ interface CodexMcpCall { readonly id: string | null; readonly server: string | null; readonly tool: string; readonly arguments: unknown; readonly evidenceIds: readonly string[]; readonly error: boolean; readonly errorCode: string | null; } /** Transcript measurements, text heuristics, and optional closed fixture-claim assessment. */ export interface CodexAgentMetrics { readonly naturalUse: boolean; readonly correctFirstTool: boolean; readonly firstTool: string | null; readonly reaCalls: readonly CodexMcpCall[]; readonly repeatedCallCount: number; readonly inputValidationFailureCount: number; readonly requiredToolSubsequenceMet: boolean; readonly inputTokens: number; readonly cachedInputTokens: number; readonly outputTokens: number; readonly finalMessage: string; readonly evidenceIds: readonly string[]; /** Whether the answer contains a produced Evidence ID; claim support is not checked. */ readonly finalCitesEvidence: boolean; /** Case-insensitive substring coverage, including terms in negated claims. */ readonly answerTermCoverageMet: boolean; /** Text length, term/cue coverage, and configured transcript checks only. */ readonly answerHeuristicsMet: boolean; /** Whether an epistemic keyword occurs; this does not establish honest authority use. */ readonly epistemicCuePresent: boolean; /** Limited to configured fixture claims; free-form factual correctness is not assessed. */ readonly factualCorrectness: KnownAnswerAssessment["status"]; readonly factualAssessment: KnownAnswerAssessment; } const isRecord = (value: unknown): value is Record => typeof value === "object" && value !== null && !Array.isArray(value); const record = (value: unknown): Record | undefined => isRecord(value) ? value : undefined; const textValue = (value: unknown): string | undefined => typeof value === "string" && value.length > 0 ? value : undefined; const numberValue = (value: unknown): number => typeof value === "number" && Number.isFinite(value) ? value : 0; const canonicalValue = (value: unknown): unknown => { if (Array.isArray(value)) return value.map(canonicalValue); const object = record(value); if (object === undefined) return value; return Object.fromEntries( Object.entries(object) .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) .map(([key, child]) => [key, canonicalValue(child)]), ); }; const toolResultError = ( item: Record, ): Record | undefined => { const result = record(item.result ?? item.output); const structured = record( result?.structured_content ?? result?.structuredContent, ); const direct = record(structured?.error ?? result?.error ?? item.error); if (direct !== undefined || structured !== undefined) return direct; // Codex preserves some REA failures only as one complete JSON text block. // A client-truncated preview or SDK prose is not a structured error record. const content = result?.content; if (!Array.isArray(content) || content.length !== 1) return undefined; const block = record(content[0]); if (block?.type !== "text" || typeof block.text !== "string") return undefined; try { return record(record(JSON.parse(block.text))?.error); } catch (cause) { if (cause instanceof SyntaxError || cause instanceof RangeError) return undefined; throw cause; } }; const toolResultFailed = ( item: Record, error: Record | undefined, ): boolean => { if ( item.status === "failed" || (item.error !== undefined && item.error !== null) ) return true; const result = record(item.result ?? item.output); const structured = record( result?.structured_content ?? result?.structuredContent, ); return ( result?.isError === true || structured?.error !== undefined || error !== undefined ); }; const callFromEvent = ( event: Record, ): CodexMcpCall | undefined => { const item = record( event.type === "item.completed" ? event.item : event.type === "mcp_tool_call" ? event : undefined, ); if (item?.type !== "mcp_tool_call") return undefined; const tool = textValue(item.tool) ?? textValue(item.name); if (tool === undefined) return undefined; const error = toolResultError(item); return { id: textValue(item.id) ?? null, server: textValue(item.server) ?? textValue(item.server_name) ?? null, tool, arguments: item.arguments ?? item.input ?? {}, evidenceIds: deliveredCodexEvidence(event).map( ({ evidence_id }) => evidence_id, ), error: toolResultFailed(item, error), errorCode: textValue(error?.code) ?? null, }; }; const agentTextFromItem = (value: unknown): string | undefined => { const item = record(value); if (item?.type !== "agent_message") return undefined; const direct = textValue(item.text) ?? textValue(item.message); if (direct !== undefined) return direct; if (!Array.isArray(item.content)) return undefined; const parts = item.content.flatMap((part) => { const object = record(part); const text = object === undefined ? undefined : textValue(object.text); return text === undefined ? [] : [text]; }); return parts.length === 0 ? undefined : parts.join("\n"); }; interface EvaluationState { readonly calls: CodexMcpCall[]; readonly seenCallIds: Set; readonly finalMessages: string[]; inputTokens: number; cachedInputTokens: number; outputTokens: number; } const consumeEvent = (state: EvaluationState, value: unknown): void => { const event = record(value); if (event === undefined) return; const completedItem = event.type === "item.completed" ? event.item : undefined; const call = callFromEvent(event); if (call !== undefined) { const identity = call.id ?? `${call.server ?? ""}:${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`; if (!state.seenCallIds.has(identity)) { state.seenCallIds.add(identity); state.calls.push(call); } } const message = agentTextFromItem(completedItem); if (message !== undefined) state.finalMessages.push(message); if (event.type !== "turn.completed") return; const usage = record(event.usage); state.inputTokens = Math.max( state.inputTokens, numberValue(usage?.input_tokens), ); state.cachedInputTokens = Math.max( state.cachedInputTokens, numberValue(usage?.cached_input_tokens), ); state.outputTokens = Math.max( state.outputTokens, numberValue(usage?.output_tokens), ); }; /** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */ export const evaluateCodexEvents = ( events: readonly unknown[], expectedFirstTool: string | null, options: { readonly requireEvidence?: boolean; readonly requiredAnswerTermGroups?: readonly (readonly string[])[]; readonly requiredToolSubsequence?: readonly string[]; readonly forbidInputValidationFailures?: boolean; readonly fixtureClaims?: readonly FixtureClaimExpectation[]; } = {}, ): CodexAgentMetrics => { const state: EvaluationState = { calls: [], seenCallIds: new Set(), finalMessages: [], inputTokens: 0, cachedInputTokens: 0, outputTokens: 0, }; for (const event of events) consumeEvent(state, event); const reaCalls = state.calls.filter( ({ server }) => server === null || server.toLowerCase() === "rea", ); const signatures = new Map(); let targetRevision = 0; for (const call of reaCalls) { // A status read before and after a successful target lifecycle change is // useful verification, not a duplicate analysis or unchanged failed retry. const context = call.tool === "binary_session" ? `${String(targetRevision)}:` : ""; const signature = `${context}${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`; signatures.set(signature, (signatures.get(signature) ?? 0) + 1); if ( !call.error && (call.tool === "open_binary" || call.tool === "close_binary") ) targetRevision += 1; } const repeatedCallCount = [...signatures.values()].reduce( (total, count) => total + Math.max(0, count - 1), 0, ); const inputValidationFailureCount = reaCalls.filter( (call) => call.errorCode === "invalid_request" || (call.error && TOOL_CONTRACTS.find( ({ name }) => name === call.tool, )?.inputSchema.safeParse(call.arguments).success === false), ).length; const requiredToolSubsequenceMet = containsOrderedSubsequence( reaCalls.filter(({ error }) => !error).map(({ tool }) => tool), options.requiredToolSubsequence ?? [], ); const finalMessage = state.finalMessages.at(-1) ?? ""; const evidenceIds = [ ...new Set(reaCalls.flatMap(({ evidenceIds: ids }) => ids)), ]; const finalCitesEvidence = evidenceIds.some((evidenceId) => finalMessage.includes(evidenceId), ); const normalizedFinalMessage = finalMessage.toLocaleLowerCase("en-US"); const answerTermCoverageMet = (options.requiredAnswerTermGroups ?? []).every( (terms) => terms.some((term) => normalizedFinalMessage.includes(term.toLocaleLowerCase("en-US")), ), ); const epistemicCuePresent = /\b(evidence|observed|inferred|unknown|unavailable|limitation|authority|not configured|could not|requires approval)\b/iu.test( finalMessage, ); const factualAssessment = evaluateKnownAnswers( events, finalMessage, options.fixtureClaims, ); return { naturalUse: reaCalls.length > 0, correctFirstTool: (reaCalls[0]?.tool ?? null) === expectedFirstTool, firstTool: reaCalls[0]?.tool ?? null, reaCalls, repeatedCallCount, inputValidationFailureCount, requiredToolSubsequenceMet, inputTokens: state.inputTokens, cachedInputTokens: state.cachedInputTokens, outputTokens: state.outputTokens, finalMessage, evidenceIds, finalCitesEvidence, answerTermCoverageMet, answerHeuristicsMet: finalMessage.trim().length >= 80 && epistemicCuePresent && answerTermCoverageMet && requiredToolSubsequenceMet && (options.forbidInputValidationFailures !== true || inputValidationFailureCount === 0) && (options.requireEvidence !== true || finalCitesEvidence), epistemicCuePresent, factualCorrectness: factualAssessment.status, factualAssessment, }; }; const containsOrderedSubsequence = ( values: readonly string[], required: readonly string[], ): boolean => { let index = 0; for (const value of values) { if (value === required[index]) index += 1; if (index === required.length) return true; } return required.length === 0; };