Guide task-centered recovery from observed client, registration, PATH and provider blockers. Install the recovery reference, document exact JavaScript module selectors, and require clarification before searching for an unnamed app. Validate public setup in isolated Codex accounts and preserve failed or unrecognized tool outcomes in the existing evaluator. Refs: #1587 BREAKING CHANGE: Optional verify:agent reports now use schemaVersion 4. Consumers must accept expectedFirstTool: null and targetClarificationPassed for missing-target scenarios. Production CLI/MCP contracts are unchanged.
315 lines
11 KiB
TypeScript
315 lines
11 KiB
TypeScript
import {
|
|
evaluateKnownAnswers,
|
|
type FixtureClaimExpectation,
|
|
type KnownAnswerAssessment,
|
|
} from "./KnownAnswerEvaluation.js";
|
|
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
|
|
import { TOOL_CONTRACTS } from "../contracts/toolContracts.js";
|
|
|
|
/** One REA MCP invocation observed in a Codex JSONL transcript. */
|
|
interface CodexMcpCall {
|
|
readonly id: string | null;
|
|
readonly server: string | null;
|
|
readonly tool: string;
|
|
readonly arguments: unknown;
|
|
readonly evidenceIds: readonly string[];
|
|
readonly error: boolean;
|
|
readonly errorCode: string | null;
|
|
}
|
|
|
|
/** Transcript measurements, text heuristics, and optional closed fixture-claim assessment. */
|
|
export interface CodexAgentMetrics {
|
|
readonly naturalUse: boolean;
|
|
readonly correctFirstTool: boolean;
|
|
readonly firstTool: string | null;
|
|
readonly reaCalls: readonly CodexMcpCall[];
|
|
readonly repeatedCallCount: number;
|
|
readonly inputValidationFailureCount: number;
|
|
readonly requiredToolSubsequenceMet: boolean;
|
|
readonly inputTokens: number;
|
|
readonly cachedInputTokens: number;
|
|
readonly outputTokens: number;
|
|
readonly finalMessage: string;
|
|
readonly evidenceIds: readonly string[];
|
|
/** Whether the answer contains a produced Evidence ID; claim support is not checked. */
|
|
readonly finalCitesEvidence: boolean;
|
|
/** Case-insensitive substring coverage, including terms in negated claims. */
|
|
readonly answerTermCoverageMet: boolean;
|
|
/** Text length, term/cue coverage, and configured transcript checks only. */
|
|
readonly answerHeuristicsMet: boolean;
|
|
/** Whether an epistemic keyword occurs; this does not establish honest authority use. */
|
|
readonly epistemicCuePresent: boolean;
|
|
/** Limited to configured fixture claims; free-form factual correctness is not assessed. */
|
|
readonly factualCorrectness: KnownAnswerAssessment["status"];
|
|
readonly factualAssessment: KnownAnswerAssessment;
|
|
}
|
|
|
|
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
typeof value === "object" && value !== null && !Array.isArray(value);
|
|
|
|
const record = (value: unknown): Record<string, unknown> | undefined =>
|
|
isRecord(value) ? value : undefined;
|
|
|
|
const textValue = (value: unknown): string | undefined =>
|
|
typeof value === "string" && value.length > 0 ? value : undefined;
|
|
|
|
const numberValue = (value: unknown): number =>
|
|
typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
|
|
const canonicalValue = (value: unknown): unknown => {
|
|
if (Array.isArray(value)) return value.map(canonicalValue);
|
|
const object = record(value);
|
|
if (object === undefined) return value;
|
|
return Object.fromEntries(
|
|
Object.entries(object)
|
|
.sort(([left], [right]) => compareUnicodeCodePoints(left, right))
|
|
.map(([key, child]) => [key, canonicalValue(child)]),
|
|
);
|
|
};
|
|
|
|
const evidenceIdsFrom = (value: unknown): readonly string[] => {
|
|
const encoded = JSON.stringify(value);
|
|
if (encoded === undefined) return [];
|
|
return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])];
|
|
};
|
|
|
|
const toolResultError = (
|
|
item: Record<string, unknown>,
|
|
): Record<string, unknown> | undefined => {
|
|
const result = record(item.result ?? item.output);
|
|
const structured = record(
|
|
result?.structured_content ?? result?.structuredContent,
|
|
);
|
|
const direct = record(structured?.error ?? result?.error ?? item.error);
|
|
if (direct !== undefined || structured !== undefined) return direct;
|
|
// Codex preserves some REA failures only as one complete JSON text block.
|
|
// A client-truncated preview or SDK prose is not a structured error record.
|
|
const content = result?.content;
|
|
if (!Array.isArray(content) || content.length !== 1) return undefined;
|
|
const block = record(content[0]);
|
|
if (block?.type !== "text" || typeof block.text !== "string")
|
|
return undefined;
|
|
try {
|
|
return record(record(JSON.parse(block.text))?.error);
|
|
} catch (cause) {
|
|
if (cause instanceof SyntaxError || cause instanceof RangeError)
|
|
return undefined;
|
|
throw cause;
|
|
}
|
|
};
|
|
|
|
const toolResultFailed = (
|
|
item: Record<string, unknown>,
|
|
error: Record<string, unknown> | undefined,
|
|
): boolean => {
|
|
if (
|
|
item.status === "failed" ||
|
|
(item.error !== undefined && item.error !== null)
|
|
)
|
|
return true;
|
|
const result = record(item.result ?? item.output);
|
|
const structured = record(
|
|
result?.structured_content ?? result?.structuredContent,
|
|
);
|
|
return (
|
|
result?.isError === true ||
|
|
structured?.error !== undefined ||
|
|
error !== undefined
|
|
);
|
|
};
|
|
|
|
const callFromItem = (value: unknown): CodexMcpCall | undefined => {
|
|
const item = record(value);
|
|
if (item?.type !== "mcp_tool_call") return undefined;
|
|
const tool = textValue(item.tool) ?? textValue(item.name);
|
|
if (tool === undefined) return undefined;
|
|
const error = toolResultError(item);
|
|
return {
|
|
id: textValue(item.id) ?? null,
|
|
server: textValue(item.server) ?? textValue(item.server_name) ?? null,
|
|
tool,
|
|
arguments: item.arguments ?? item.input ?? {},
|
|
evidenceIds: evidenceIdsFrom(item.result ?? item.output),
|
|
error: toolResultFailed(item, error),
|
|
errorCode: textValue(error?.code) ?? null,
|
|
};
|
|
};
|
|
|
|
const agentTextFromItem = (value: unknown): string | undefined => {
|
|
const item = record(value);
|
|
if (item?.type !== "agent_message") return undefined;
|
|
const direct = textValue(item.text) ?? textValue(item.message);
|
|
if (direct !== undefined) return direct;
|
|
if (!Array.isArray(item.content)) return undefined;
|
|
const parts = item.content.flatMap((part) => {
|
|
const object = record(part);
|
|
const text = object === undefined ? undefined : textValue(object.text);
|
|
return text === undefined ? [] : [text];
|
|
});
|
|
return parts.length === 0 ? undefined : parts.join("\n");
|
|
};
|
|
|
|
interface EvaluationState {
|
|
readonly calls: CodexMcpCall[];
|
|
readonly seenCallIds: Set<string>;
|
|
readonly finalMessages: string[];
|
|
inputTokens: number;
|
|
cachedInputTokens: number;
|
|
outputTokens: number;
|
|
}
|
|
|
|
const consumeEvent = (state: EvaluationState, value: unknown): void => {
|
|
const event = record(value);
|
|
if (event === undefined) return;
|
|
const completedItem =
|
|
event.type === "item.completed" ? event.item : undefined;
|
|
const call =
|
|
callFromItem(completedItem) ??
|
|
(event.type === "mcp_tool_call" ? callFromItem(event) : undefined);
|
|
if (call !== undefined) {
|
|
const identity =
|
|
call.id ??
|
|
`${call.server ?? ""}:${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`;
|
|
if (!state.seenCallIds.has(identity)) {
|
|
state.seenCallIds.add(identity);
|
|
state.calls.push(call);
|
|
}
|
|
}
|
|
const message = agentTextFromItem(completedItem);
|
|
if (message !== undefined) state.finalMessages.push(message);
|
|
if (event.type !== "turn.completed") return;
|
|
const usage = record(event.usage);
|
|
state.inputTokens = Math.max(
|
|
state.inputTokens,
|
|
numberValue(usage?.input_tokens),
|
|
);
|
|
state.cachedInputTokens = Math.max(
|
|
state.cachedInputTokens,
|
|
numberValue(usage?.cached_input_tokens),
|
|
);
|
|
state.outputTokens = Math.max(
|
|
state.outputTokens,
|
|
numberValue(usage?.output_tokens),
|
|
);
|
|
};
|
|
|
|
/** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */
|
|
export const evaluateCodexEvents = (
|
|
events: readonly unknown[],
|
|
expectedFirstTool: string | null,
|
|
options: {
|
|
readonly requireEvidence?: boolean;
|
|
readonly requiredAnswerTermGroups?: readonly (readonly string[])[];
|
|
readonly requiredToolSubsequence?: readonly string[];
|
|
readonly forbidInputValidationFailures?: boolean;
|
|
readonly fixtureClaims?: readonly FixtureClaimExpectation[];
|
|
} = {},
|
|
): CodexAgentMetrics => {
|
|
const state: EvaluationState = {
|
|
calls: [],
|
|
seenCallIds: new Set(),
|
|
finalMessages: [],
|
|
inputTokens: 0,
|
|
cachedInputTokens: 0,
|
|
outputTokens: 0,
|
|
};
|
|
for (const event of events) consumeEvent(state, event);
|
|
|
|
const reaCalls = state.calls.filter(
|
|
({ server }) => server === null || server.toLowerCase() === "rea",
|
|
);
|
|
const signatures = new Map<string, number>();
|
|
let targetRevision = 0;
|
|
for (const call of reaCalls) {
|
|
// A status read before and after a successful target lifecycle change is
|
|
// useful verification, not a duplicate analysis or unchanged failed retry.
|
|
const context =
|
|
call.tool === "binary_session" ? `${String(targetRevision)}:` : "";
|
|
const signature = `${context}${call.tool}:${JSON.stringify(canonicalValue(call.arguments))}`;
|
|
signatures.set(signature, (signatures.get(signature) ?? 0) + 1);
|
|
if (
|
|
!call.error &&
|
|
(call.tool === "open_binary" || call.tool === "close_binary")
|
|
)
|
|
targetRevision += 1;
|
|
}
|
|
const repeatedCallCount = [...signatures.values()].reduce(
|
|
(total, count) => total + Math.max(0, count - 1),
|
|
0,
|
|
);
|
|
const inputValidationFailureCount = reaCalls.filter(
|
|
(call) =>
|
|
call.errorCode === "invalid_request" ||
|
|
(call.error &&
|
|
TOOL_CONTRACTS.find(
|
|
({ name }) => name === call.tool,
|
|
)?.inputSchema.safeParse(call.arguments).success === false),
|
|
).length;
|
|
const requiredToolSubsequenceMet = containsOrderedSubsequence(
|
|
reaCalls.filter(({ error }) => !error).map(({ tool }) => tool),
|
|
options.requiredToolSubsequence ?? [],
|
|
);
|
|
const finalMessage = state.finalMessages.at(-1) ?? "";
|
|
const evidenceIds = [
|
|
...new Set(reaCalls.flatMap(({ evidenceIds: ids }) => ids)),
|
|
];
|
|
const finalCitesEvidence = evidenceIds.some((evidenceId) =>
|
|
finalMessage.includes(evidenceId),
|
|
);
|
|
const normalizedFinalMessage = finalMessage.toLocaleLowerCase("en-US");
|
|
const answerTermCoverageMet = (options.requiredAnswerTermGroups ?? []).every(
|
|
(terms) =>
|
|
terms.some((term) =>
|
|
normalizedFinalMessage.includes(term.toLocaleLowerCase("en-US")),
|
|
),
|
|
);
|
|
const epistemicCuePresent =
|
|
/\b(evidence|observed|inferred|unknown|unavailable|limitation|authority|not configured|could not|requires approval)\b/iu.test(
|
|
finalMessage,
|
|
);
|
|
const factualAssessment = evaluateKnownAnswers(
|
|
events,
|
|
finalMessage,
|
|
options.fixtureClaims,
|
|
);
|
|
return {
|
|
naturalUse: reaCalls.length > 0,
|
|
correctFirstTool: (reaCalls[0]?.tool ?? null) === expectedFirstTool,
|
|
firstTool: reaCalls[0]?.tool ?? null,
|
|
reaCalls,
|
|
repeatedCallCount,
|
|
inputValidationFailureCount,
|
|
requiredToolSubsequenceMet,
|
|
inputTokens: state.inputTokens,
|
|
cachedInputTokens: state.cachedInputTokens,
|
|
outputTokens: state.outputTokens,
|
|
finalMessage,
|
|
evidenceIds,
|
|
finalCitesEvidence,
|
|
answerTermCoverageMet,
|
|
answerHeuristicsMet:
|
|
finalMessage.trim().length >= 80 &&
|
|
epistemicCuePresent &&
|
|
answerTermCoverageMet &&
|
|
requiredToolSubsequenceMet &&
|
|
(options.forbidInputValidationFailures !== true ||
|
|
inputValidationFailureCount === 0) &&
|
|
(options.requireEvidence !== true || finalCitesEvidence),
|
|
epistemicCuePresent,
|
|
factualCorrectness: factualAssessment.status,
|
|
factualAssessment,
|
|
};
|
|
};
|
|
|
|
const containsOrderedSubsequence = (
|
|
values: readonly string[],
|
|
required: readonly string[],
|
|
): boolean => {
|
|
let index = 0;
|
|
for (const value of values) {
|
|
if (value === required[index]) index += 1;
|
|
if (index === required.length) return true;
|
|
}
|
|
return required.length === 0;
|
|
};
|