From 27a9ceade7232c1c9a24dc7d454ae060948183a1 Mon Sep 17 00:00:00 2001 From: YouWish <29498139+Y0UW1SH@users.noreply.github.com> Date: Fri, 9 Oct 2026 02:49:09 -0400 Subject: [PATCH] fix!: restore caching, preserve failures, and verify fixture claims (#1211) Restore profile-bound Ghidra snapshot reuse, preserve direct-analysis results through cleanup failures, and grade configured agent fixture claims against returned evidence. Register the compiled agent-evaluation fixture entry for static analysis. Co-authored-by: YouWish <29498139+Y0UW1SH@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- docs/testing.md | 44 +- knip.json | 1 + scripts/verify-agent-experience.mjs | 113 ++--- src/application/AnalysisProvider.ts | 7 +- src/application/DirectAnalysis.ts | 231 ++++----- src/application/DirectAnalysisCleanup.ts | 97 ++++ .../binary/AnalysisSnapshotCache.test.ts | 68 +++ .../binary/AnalysisSnapshotCache.ts | 3 +- src/evaluation/AgentEvaluationFixtures.ts | 315 ++++++++++++ src/evaluation/AgentEvaluationReport.ts | 56 +++ src/evaluation/CodexAgentEval.ts | 44 +- src/evaluation/KnownAnswerEvaluation.ts | 421 ++++++++++++++++ src/ghidra/GhidraProviderCapabilities.ts | 6 + src/ghidra/GhidraSnapshotCache.test.ts | 236 +++++++++ .../boundary/mcp/agentKnownAnswerMcp.test.ts | 302 ++++++++++++ .../composition/directAnalysisCleanup.test.ts | 242 ++++++++++ .../directAnalysisSnapshotReplay.test.ts | 59 +++ .../model-evals/agentEvaluationReport.test.ts | 163 +++++++ .../model-evals/codexAgentEval.test.ts | 67 ++- .../model-evals/knownAnswerEvaluation.test.ts | 449 ++++++++++++++++++ 20 files changed, 2714 insertions(+), 210 deletions(-) create mode 100644 src/application/DirectAnalysisCleanup.ts create mode 100644 src/evaluation/AgentEvaluationFixtures.ts create mode 100644 src/evaluation/AgentEvaluationReport.ts create mode 100644 src/evaluation/KnownAnswerEvaluation.ts create mode 100644 src/ghidra/GhidraSnapshotCache.test.ts create mode 100644 tests/boundary/mcp/agentKnownAnswerMcp.test.ts create mode 100644 tests/composition/directAnalysisCleanup.test.ts create mode 100644 tests/evaluation/model-evals/agentEvaluationReport.test.ts create mode 100644 tests/evaluation/model-evals/knownAnswerEvaluation.test.ts diff --git a/docs/testing.md b/docs/testing.md index 4f4633bcf..4b77bd650 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -885,8 +885,48 @@ real local Codex CLI with: npm run verify:agent ``` -Its report records tool selection, repeated calls, token use, completion quality, -and handling of permissions and unknowns. +The packaged desktop and parser-comparison scenarios now assess a closed set of +known fixture claims. Each final answer must be a strict JSON manifest containing +exactly the requested claim IDs, values, Evidence IDs, and Evidence authority and +confidence. Golden values are not included in the prompt. The evaluator checks +exact values against both the fixture oracle and authenticated successful REA +results; an Evidence ID by itself is insufficient. Missing, duplicate, extra, +contradictory, incorrectly sourced, or unsupported claims fail the scenario. + +The desktop rubric covers the exposed bridge API and members, renderer and main +IPC operations, and the resolved preload. It binds its Evidence to the exact +packaged artifact path and SHA-256. The parser rubric checks the precise heading +depth addition, discriminant, complete comparison counts, and the explicit limit +on runtime semantics. Its comparison must link to the two delivered source +analyses and use the requested module and export selectors. Boundary tests package +the same source-owned desktop fixture and compare the same source-owned parser +files through actual MCP tool results, then verify that fabricated answers fail. + +Per-scenario `factualCorrectness` is `passed`, `failed`, or `not_assessed` within +the `configured_fixture_claims` scope. A pass establishes only the configured +claims, not unrestricted factual correctness. Native, managed, browser, +navigation-context, and address-context scenarios have no factual rubric and +remain `not_assessed`; the managed workflow requires both artifact and member +inspection to answer its type and entry-point question. + +All scenarios retain routing, workflow, validation, repetition, process-exit, +and token-use gates. Configured factual scenarios additionally require a factual +pass. Their answer-text heuristics are diagnostic and do not affect the gate. +Scenarios without a rubric retain the legacy text-heuristic gate. +`answerTermCoverageMet` checks case-insensitive substrings, +`epistemicCuePresent` checks keywords, and `finalCitesEvidence` checks only an ID's +presence. These metrics can still accept fabricated prose and must not be read +as factual assessments. Set `REA_AGENT_EVAL_TRANSCRIPT_DIR` to retain complete +tool results and final answers for review. + +Report schema version 3 changes the top-level `factualCorrectness` from a constant +string to an assessment summary with status, scope, and assessed/passed/failed/ +not-assessed scenario counts. Scenario records include the factual assessment and +configured claim IDs. The evaluation scope is +`routing_workflow_and_configured_fixture_claims`. Update report consumers for +these changes. The narrower heuristic field names introduced in version 2 remain: +`answerHeuristicsMet`, `epistemicCuePresent`, `answerTermCoverageMet`, and +`requiredAnswerTermGroups`. Regenerate the managed conformance manifest and Evidence completion ledger from live verification results, or check them for drift: diff --git a/knip.json b/knip.json index ae8742b2c..032a6cfb2 100644 --- a/knip.json +++ b/knip.json @@ -4,6 +4,7 @@ "src/main.ts", "src/cli.ts", "src/contracts/errorSchemas.ts", + "src/evaluation/AgentEvaluationFixtures.ts", "tests/processOwnershipRunner.ts" ], "project": ["src/**/*.ts", "tests/**/*.ts", "scripts/**/*.mjs"], diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 62ee3ffeb..513835860 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -5,9 +5,16 @@ import { dirname, join, resolve } from "node:path"; import { createInterface } from "node:readline"; import { fileURLToPath } from "node:url"; -import { createPackage } from "@electron/asar"; - import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js"; +import { + createAgentEvaluationFixtures, + agentFixtureClaims, + agentFixtureAnswerInstructions, +} from "../dist/evaluation/AgentEvaluationFixtures.js"; +import { + agentEvaluationPassed, + summarizeFactualCorrectness, +} from "../dist/evaluation/AgentEvaluationReport.js"; import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; @@ -91,6 +98,10 @@ try { expectedFirstTool: "inspect_managed_artifact", requiresEvidence: true, requiredAnswerTermGroups: [["profile"], ["main", "entry point"]], + requiredToolSubsequence: [ + "inspect_managed_artifact", + "inspect_managed_members", + ], prompt: `Explain the public managed types and entry point in ${targets.managed}. Use shipped-artifact evidence and distinguish unavailable behavior from observed metadata.`, }, { @@ -134,7 +145,13 @@ try { const results = []; for (const scenario of scenarios) { process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`); - const execution = await runCodex(scenario.prompt); + const fixtureClaims = agentFixtureClaims(scenario.id, targets); + const answerInstructions = agentFixtureAnswerInstructions(scenario.id); + const execution = await runCodex( + answerInstructions.length === 0 + ? scenario.prompt + : `${scenario.prompt}\n\n${answerInstructions}`, + ); const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR; if (transcriptDirectory !== undefined) { const transcriptPath = resolve( @@ -155,14 +172,16 @@ try { requiredAnswerTermGroups: scenario.requiredAnswerTermGroups, requiredToolSubsequence: scenario.requiredToolSubsequence, forbidInputValidationFailures: true, + ...(fixtureClaims === undefined ? {} : { fixtureClaims }), }, ); const result = { id: scenario.id, expectedFirstTool: scenario.expectedFirstTool, requiresEvidence: scenario.requiresEvidence, - qualityCriteria: scenario.requiredAnswerTermGroups, + requiredAnswerTermGroups: scenario.requiredAnswerTermGroups, requiredToolSubsequence: scenario.requiredToolSubsequence ?? [], + configuredClaimIds: fixtureClaims?.map(({ id }) => id) ?? [], exitCode: execution.exitCode, stderr: execution.stderr, ...metrics, @@ -173,12 +192,16 @@ try { ); } + const factualCorrectness = summarizeFactualCorrectness(results); const summary = { + schemaVersion: 3, + evaluationScope: "routing_workflow_and_configured_fixture_claims", + factualCorrectness, verifier_run: await completeVerifierRun(verifierRun), codex, codexVersion, model: optionalModel ?? null, - scenarios: results.map(({ finalMessage: _message, ...result }) => result), + scenarios: results, totals: { scenarios: results.length, naturalUse: results.filter(({ naturalUse }) => naturalUse).length, @@ -205,12 +228,15 @@ try { (total, { cachedInputTokens }) => total + cachedInputTokens, 0, ), - completionQuality: results.filter(({ completionQuality }) => - Boolean(completionQuality), + answerHeuristicsMet: results.filter(({ answerHeuristicsMet }) => + Boolean(answerHeuristicsMet), ).length, - authorityHonesty: results.filter(({ authorityHonesty }) => - Boolean(authorityHonesty), + epistemicCuePresent: results.filter(({ epistemicCuePresent }) => + Boolean(epistemicCuePresent), ).length, + factualScenariosPassed: factualCorrectness.counts.passed, + factualScenariosFailed: factualCorrectness.counts.failed, + factualScenariosNotAssessed: factualCorrectness.counts.notAssessed, }, }; const encoded = `${JSON.stringify(summary, null, 2)}\n`; @@ -221,31 +247,10 @@ try { await writeFile(resolve(reportPath), encoded); } - const failed = results.filter( - ({ - exitCode, - naturalUse, - correctFirstTool, - repeatedCallCount, - inputValidationFailureCount, - requiredToolSubsequenceMet, - inputTokens, - completionQuality, - authorityHonesty, - }) => - exitCode !== 0 || - !naturalUse || - !correctFirstTool || - repeatedCallCount !== 0 || - inputValidationFailureCount !== 0 || - !requiredToolSubsequenceMet || - inputTokens <= 0 || - !completionQuality || - !authorityHonesty, - ); + const failed = results.filter((result) => !agentEvaluationPassed(result)); if (failed.length > 0) throw new Error( - `Codex agent release evaluation failed: ${failed.map(({ id }) => id).join(", ")}`, + `Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`, ); } finally { if (process.env.REA_AGENT_EVAL_KEEP_FIXTURES !== "true") @@ -253,45 +258,7 @@ try { } async function createTargets(root, includeManaged) { - const javascript = join(root, "desktop-app"); - await mkdir(join(javascript, "renderer"), { recursive: true }); - await Promise.all([ - writeFile( - join(javascript, "package.json"), - `${JSON.stringify({ name: "desktop-fixture", version: "1.0.0", main: "main.js" }, null, 2)}\n`, - ), - writeFile( - join(javascript, "main.js"), - 'const { BrowserWindow, ipcMain } = require("electron");\nnew BrowserWindow({ webPreferences: { preload: require("node:path").join(__dirname, "preload.js"), contextIsolation: true, sandbox: true } });\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n', - ), - writeFile( - join(javascript, "preload.js"), - 'const { contextBridge, ipcRenderer } = require("electron");\ncontextBridge.exposeInMainWorld("profileApi", { read: (id) => ipcRenderer.invoke("profile:read", id) });\n', - ), - writeFile( - join(javascript, "renderer/app.js"), - 'globalThis.profileApi.read("fixture");\n', - ), - ]); - const javascriptAsar = join(root, "desktop-app.asar"); - await createPackage(javascript, javascriptAsar); - - const javascriptShapeLeft = join(root, "parser-v1"); - const javascriptShapeRight = join(root, "parser-v2"); - await Promise.all([ - mkdir(javascriptShapeLeft, { recursive: true }), - mkdir(javascriptShapeRight, { recursive: true }), - ]); - await Promise.all([ - writeFile( - join(javascriptShapeLeft, "parser.mjs"), - 'export default function parse() { return { heading: "Title" }; }\n', - ), - writeFile( - join(javascriptShapeRight, "parser.mjs"), - 'export default function parse() { return { heading: { text: "Title", level: 1 } }; }\n', - ), - ]); + const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot); const managedOutput = join(root, "managed-output"); if (includeManaged) { @@ -324,9 +291,7 @@ async function createTargets(root, includeManaged) { } return { native: "/bin/true", - javascript: javascriptAsar, - javascriptShapeLeft, - javascriptShapeRight, + ...fixtures, managed: join(managedOutput, "AgentEval.dll"), }; } diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index 81bf19591..8d8aaa8b5 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -162,7 +162,12 @@ type CapabilityAvailability = }; export type CapabilityDescriptor = CapabilityAvailability & { - /** Live external state must be re-observed, including after snapshot import. */ + /** + * Snapshot opts immutable, profile-bound reads into replay despite incidental + * private filesystem writes. Live state must always be re-observed. Omitting + * this policy retains conservative effect checks; mutation and UI-state + * exclusions apply to every policy. + */ readonly cachePolicy?: "snapshot" | "live"; readonly provider: ProviderIdentity; readonly operation: Exclude; diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 1f66ec8c8..b0007ac8e 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -50,6 +50,11 @@ import type { AnalysisProviderSelector } from "../contracts/providerSelection.js import { artifactInspectionResultSchema } from "../domain/artifactInspection.js"; import { resolveXrefsAddress } from "./XrefsAddressResolution.js"; import { AnalysisInputError } from "../domain/analysisErrorCore.js"; +import { + directAnalysisCleanupFailure, + managedAnalysisCleanupFailure, + withSessionCleanup, +} from "./DirectAnalysisCleanup.js"; type DirectAnalysisTool = | "annotate_native_function" @@ -134,13 +139,15 @@ export const runManagedProviderExecution = async ( ): Promise> => withProcessCancellation(signal, async (operationSignal) => { const session = dependencies.createManagedBinarySession(); - try { - const opened = await session.open(path, { signal: operationSignal }); - if (!opened.ok) return opened; - return await session.execute(tool, {}, { signal: operationSignal }); - } finally { - await session.close(); - } + return withSessionCleanup( + session, + async () => { + const opened = await session.open(path, { signal: operationSignal }); + if (!opened.ok) return opened; + return await session.execute(tool, {}, { signal: operationSignal }); + }, + managedAnalysisCleanupFailure, + ); }); const runManagedProviderAnalysis = async ( @@ -196,119 +203,121 @@ const runAnalysis = async ( const config = parseConfig(options.environment ?? process.env); if (!config.ok) return cliError(config.error); const session = dependencies.createBinarySession(config.value, logger); - try { - const prepared = await prepareSnapshot({ - path, - snapshotPath, - ...(options.formatHint === undefined - ? {} - : { formatHint: options.formatHint }), - }); - if (!prepared.ok) return cliError(prepared.error); - const { snapshot } = prepared.value; - let resolvedTarget: ResolvedSessionOpen | undefined; - if (snapshot !== undefined && prepared.value.target !== undefined) { - const preview = await session.previewTarget(prepared.value.target, { - signal, - snapshot, + return withSessionCleanup( + session, + async () => { + const prepared = await prepareSnapshot({ + path, + snapshotPath, ...(options.formatHint === undefined ? {} : { formatHint: options.formatHint }), + }); + if (!prepared.ok) return cliError(prepared.error); + const { snapshot } = prepared.value; + let resolvedTarget: ResolvedSessionOpen | undefined; + if (snapshot !== undefined && prepared.value.target !== undefined) { + const preview = await session.previewTarget(prepared.value.target, { + signal, + snapshot, + ...(options.formatHint === undefined + ? {} + : { formatHint: options.formatHint }), + ...(options.providerId === undefined + ? {} + : { providerId: options.providerId }), + }); + if (!preview.ok) return cliError(preview.error); + resolvedTarget = preview.value; + const route = preview.value.route; + const bindingProfile = route.profile ?? undefined; + const evidenceProfile = analysisProfileForRoute(route, tool); + if ( + bindingProfile !== undefined && + evidenceProfile !== undefined && + allowsSnapshotReplay(route, tool) + ) { + const cached = snapshotEvidenceForQuery(snapshot, { + target: preview.value.target, + bindingProfile, + operation: tool, + parameters: arguments_, + provider: isWorkflowEvidenceTool(tool) + ? REA_WORKFLOW_PROVIDER + : providerIdentityForRoute(route, tool), + evidenceProfile, + }); + if (cached !== undefined) return cached; + } + } + const openOptions = { + signal, + ...(options.formatHint === undefined + ? {} + : { formatHint: options.formatHint }), + ...(snapshot === undefined ? {} : { snapshot }), ...(options.providerId === undefined ? {} : { providerId: options.providerId }), + }; + const opened = + resolvedTarget === undefined + ? await session.open(path, openOptions) + : await session.openResolvedTarget(resolvedTarget, openOptions); + if (!opened.ok) return cliError(opened.error); + const evidenceProfile = analysisProfileForEvidence(session, tool); + const { output, evidence } = await executeAnalysisTool({ + session, + openedTarget: opened.value, + tool, + arguments: arguments_, + signal, + evidenceProfile, }); - if (!preview.ok) return cliError(preview.error); - resolvedTarget = preview.value; - const route = preview.value.route; - const bindingProfile = route.profile ?? undefined; - const evidenceProfile = analysisProfileForRoute(route, tool); - if ( - bindingProfile !== undefined && - evidenceProfile !== undefined && - allowsSnapshotReplay(route, tool) - ) { - const cached = snapshotEvidenceForQuery(snapshot, { - target: preview.value.target, - bindingProfile, - operation: tool, - parameters: arguments_, - provider: isWorkflowEvidenceTool(tool) - ? REA_WORKFLOW_PROVIDER - : providerIdentityForRoute(route, tool), - evidenceProfile, - }); - if (cached !== undefined) return cached; - } - } - const openOptions = { - signal, - ...(options.formatHint === undefined - ? {} - : { formatHint: options.formatHint }), - ...(snapshot === undefined ? {} : { snapshot }), - ...(options.providerId === undefined - ? {} - : { providerId: options.providerId }), - }; - const opened = - resolvedTarget === undefined - ? await session.open(path, openOptions) - : await session.openResolvedTarget(resolvedTarget, openOptions); - if (!opened.ok) return cliError(opened.error); - const evidenceProfile = analysisProfileForEvidence(session, tool); - const { output, evidence } = await executeAnalysisTool({ - session, - openedTarget: opened.value, - tool, - arguments: arguments_, - signal, - evidenceProfile, - }); - if (evidence !== undefined) { - const recorded = session.recordEvidence(evidence); - if (!recorded.ok) return cliError(recorded.error); - if (isWorkflowEvidenceTool(tool)) { - const unknowns = recordWorkflowUnknowns({ - name: tool, - result: evidence.normalized_result, - evidenceId: evidence.evidence_id, - recordUnknown: (unknown) => session.recordUnknown(unknown), - }); - if (!unknowns.ok) return cliError(unknowns.error); - } - } - if ( - isWorkflowEvidenceTool(tool) && - tool !== "trace_native_ui_action" && - snapshotPath !== undefined && - evidence !== undefined && - session.allowsSnapshotReplay(tool) - ) { - const workflowRecord = workflowSnapshotRecord(evidence, tool); - if (workflowRecord !== undefined) { - const recorded = session.recordWorkflowSnapshot(workflowRecord); + if (evidence !== undefined) { + const recorded = session.recordEvidence(evidence); if (!recorded.ok) return cliError(recorded.error); + if (isWorkflowEvidenceTool(tool)) { + const unknowns = recordWorkflowUnknowns({ + name: tool, + result: evidence.normalized_result, + evidenceId: evidence.evidence_id, + recordUnknown: (unknown) => session.recordUnknown(unknown), + }); + if (!unknowns.ok) return cliError(unknowns.error); + } } - } - if ( - tool !== "trace_native_ui_action" && - snapshotPath !== undefined && - evidence !== undefined - ) { - const snapshot = session.exportAnalysisSnapshot(); - if (!snapshot.ok) return cliError(snapshot.error); - const written = await writeAnalysisSnapshot( - snapshot.value, - snapshotPath, - true, - ); - if (!written.ok) return cliError(written.error); - } - return output; - } finally { - await session.close(); - } + if ( + isWorkflowEvidenceTool(tool) && + tool !== "trace_native_ui_action" && + snapshotPath !== undefined && + evidence !== undefined && + session.allowsSnapshotReplay(tool) + ) { + const workflowRecord = workflowSnapshotRecord(evidence, tool); + if (workflowRecord !== undefined) { + const recorded = session.recordWorkflowSnapshot(workflowRecord); + if (!recorded.ok) return cliError(recorded.error); + } + } + if ( + tool !== "trace_native_ui_action" && + snapshotPath !== undefined && + evidence !== undefined + ) { + const snapshot = session.exportAnalysisSnapshot(); + if (!snapshot.ok) return cliError(snapshot.error); + const written = await writeAnalysisSnapshot( + snapshot.value, + snapshotPath, + true, + ); + if (!written.ok) return cliError(written.error); + } + return output; + }, + directAnalysisCleanupFailure, + ); }; const executeAnalysisTool = async (input: { diff --git a/src/application/DirectAnalysisCleanup.ts b/src/application/DirectAnalysisCleanup.ts new file mode 100644 index 000000000..573dea7c0 --- /dev/null +++ b/src/application/DirectAnalysisCleanup.ts @@ -0,0 +1,97 @@ +import { analysisErrorProjectionSchema } from "../contracts/errorSchemas.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; +import { ProviderAdapterError } from "../domain/providerAdapterError.js"; +import { err, type Result } from "../domain/result.js"; +import type { AnalysisExecution } from "./AnalysisProvider.js"; +import type { BinarySession } from "./binary/BinarySession.js"; + +/** Await one-shot cleanup and retain either the outcome or both failures. */ +export const withSessionCleanup = async ( + session: BinarySession, + operation: () => Promise, + onCleanupFailure: (value: Value, error: AnalysisError) => Value, +): Promise => { + let value: Value; + try { + value = await operation(); + } catch (cause: unknown) { + const closed = await session.close(); + if (!closed.ok) + throw new AggregateError( + [cause, closed.error], + "Analysis and provider cleanup both failed", + { cause }, + ); + throw cause; + } + const closed = await session.close(); + return closed.ok ? value : onCleanupFailure(value, closed.error); +}; + +/** Keep the primary CLI failure or completed Evidence alongside cleanup failure. */ +export const directAnalysisCleanupFailure = ( + output: JsonValue, + error: AnalysisError, +): JsonValue => { + const cleanup = projectAnalysisError(error); + if (typeof output === "object" && output !== null && !Array.isArray(output)) { + const { error: label, ...projection } = output; + const primary = analysisErrorProjectionSchema.safeParse(projection); + if (typeof label === "string" && primary.success) + return { + error: label, + ...primary.data, + code: "cleanup_incomplete", + retryable: false, + details: { + ...primary.data.details, + cleanup: "incomplete", + resources: error.cleanupResources.slice(), + execution_failure: + primary.data.details !== undefined && + "execution_failure" in primary.data.details + ? (primary.data.details.execution_failure ?? primary.data.code) + : primary.data.code, + primary_error: projection, + cleanup_error: cleanup, + }, + }; + } + return { + error: "Analysis failed", + ...cleanup, + code: "cleanup_incomplete", + retryable: false, + details: { + ...cleanup.details, + cleanup: "incomplete", + partial_observation: output, + }, + }; +}; + +/** Preserve managed observations or the original typed failure in diagnostics. */ +export const managedAnalysisCleanupFailure = ( + result: Result, + error: AnalysisError, +): Result => { + const cleanup = projectAnalysisError(error); + return err( + new ProviderAdapterError( + error instanceof ProviderAdapterError ? error.providerId : "rea", + "close_binary", + { + cause: result.ok ? error : result.error, + cleanup: { reason: cleanup.message, resources: error.cleanupResources }, + diagnostics: { + cleanup_error: cleanup, + ...(result.ok + ? { partial_observation: jsonValueSchema.parse(result.value) } + : { primary_error: projectAnalysisError(result.error) }), + }, + }, + ), + ); +}; diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 32776d034..096d50111 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -141,6 +141,74 @@ describe("analysis snapshot cache capacity", () => { }); }); +describe("analysis snapshot cache policy", () => { + it.each<{ + policy?: CapabilityDescriptor["cachePolicy"]; + operation?: CapabilityDescriptor["operation"]; + effects?: Partial; + parameters?: Record; + cacheable: boolean; + }>([ + { cacheable: false }, + { policy: "snapshot", cacheable: true }, + { policy: "live", cacheable: false }, + { + policy: "live", + effects: { mayWriteFilesystem: false }, + cacheable: false, + }, + { + policy: "snapshot", + effects: { mutatesArtifact: true }, + cacheable: false, + }, + { + policy: "snapshot", + effects: { changesPermissions: true }, + cacheable: false, + }, + { policy: "snapshot", operation: "list_documents", cacheable: false }, + { policy: "snapshot", operation: "list_strings", cacheable: false }, + { + policy: "snapshot", + operation: "address_name", + parameters: { document: "fixture" }, + cacheable: false, + }, + ])( + "keeps explicit cache policy within immutable and state-independent boundaries: %j", + ({ + policy, + operation = "analyze_function", + effects, + parameters = {}, + cacheable, + }) => { + const descriptor: CapabilityDescriptor = { + provider: ANALYSIS_SNAPSHOT_PROVIDER, + operation, + available: true, + reason: null, + ...(policy === undefined ? {} : { cachePolicy: policy }), + effects: { + mutatesArtifact: false, + launchesProcess: true, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: true, + changesPermissions: false, + requiresRoot: false, + ...effects, + }, + limitations: [], + }; + expect(isSnapshotCacheable(operation, descriptor, parameters)).toBe( + cacheable, + ); + }, + ); +}); + describe("analysis snapshot cache partitioning", () => { it.each<{ operation: Exclude; diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index 4f39fc970..240bd2b61 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -71,7 +71,8 @@ export const isSnapshotCacheable = ( typeof parameters.address === "string") && descriptor?.effects.mutatesArtifact === false && descriptor.cachePolicy !== "live" && - descriptor.effects.mayWriteFilesystem === false && + (descriptor.cachePolicy === "snapshot" || + descriptor.effects.mayWriteFilesystem === false) && descriptor.effects.changesPermissions === false; /** Bounded in-memory cache for one immutable binary identity. */ diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/src/evaluation/AgentEvaluationFixtures.ts new file mode 100644 index 000000000..1c62c47a8 --- /dev/null +++ b/src/evaluation/AgentEvaluationFixtures.ts @@ -0,0 +1,315 @@ +import { createHash } from "node:crypto"; +import { cp, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { createPackage } from "@electron/asar"; + +import type { JsonValue } from "../domain/jsonValue.js"; +import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; +import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraph.js"; +import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js"; + +/** Source-owned artifact paths used by the configured known-answer benchmarks. */ +export interface AgentEvaluationFixtureTargets { + readonly javascript: string; + readonly javascriptSha256: string; + readonly javascriptShapeLeft: string; + readonly javascriptShapeRight: string; +} + +const desktopFixtureFiles = { + "package.json": `${JSON.stringify({ name: "desktop-fixture", version: "1.0.0", main: "main.js" }, null, 2)}\n`, + "main.js": + 'const { BrowserWindow, ipcMain } = require("electron");\nnew BrowserWindow({ webPreferences: { preload: require("node:path").join(__dirname, "preload.js"), contextIsolation: true, sandbox: true } });\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n', + "preload.js": + 'const { contextBridge, ipcRenderer } = require("electron");\ncontextBridge.exposeInMainWorld("profileApi", { read: (id) => ipcRenderer.invoke("profile:read", id) });\n', + "renderer/app.js": 'globalThis.profileApi.read("fixture");\n', +}; + +/** Materialize the exact packaged and source-owned parser targets used by the runner. */ +export const createAgentEvaluationFixtures = async ( + root: string, + repositoryRoot: string, +): Promise => { + const desktop = join(root, "desktop-app"); + const javascript = join(root, "desktop-app.asar"); + const javascriptShapeLeft = join(root, "parser-v1"); + const javascriptShapeRight = join(root, "parser-v2"); + await Promise.all([ + mkdir(join(desktop, "renderer"), { recursive: true }), + mkdir(javascriptShapeLeft, { recursive: true }), + mkdir(javascriptShapeRight, { recursive: true }), + ]); + await Promise.all([ + ...Object.entries(desktopFixtureFiles).map(([path, source]) => + writeFile(join(desktop, path), source), + ), + cp( + join(repositoryRoot, "tests/fixtures/replay/parser.mjs"), + join(javascriptShapeLeft, "parser.mjs"), + ), + cp( + join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"), + join(javascriptShapeRight, "parser.mjs"), + ), + ]); + await createPackage(desktop, javascript); + return { + javascript, + javascriptSha256: createHash("sha256") + .update(await readFile(javascript)) + .digest("hex"), + javascriptShapeLeft, + javascriptShapeRight, + }; +}; + +type GraphObservation = ApplicationNode["observations"][number]; + +const singleObservation = ( + value: JsonValue, + kind: ApplicationNode["kind"], + include: (observation: GraphObservation) => boolean = () => true, +): GraphObservation | undefined => { + const result = javascriptApplicationAnalysisResultSchema.safeParse(value); + if (!result.success) return undefined; + const candidates = result.data.graph.nodes + .filter((node) => node.kind === kind) + .flatMap((node) => node.observations) + .filter(include); + return candidates.length === 1 ? candidates[0] : undefined; +}; + +const observationSource = ( + observation: GraphObservation, +): string | undefined => { + const location = observation.evidence.location; + return location.available && location.value.kind === "source-range" + ? location.value.source + : undefined; +}; + +const rendererApi = (value: JsonValue): JsonValue | undefined => { + const observation = singleObservation(value, "context-bridge-api"); + if (observation === undefined) return undefined; + const { api_key: key, members, world } = observation.properties; + const source = observationSource(observation); + if ( + key === undefined || + members === undefined || + world === undefined || + source === undefined + ) + return undefined; + return { key, members, world, source }; +}; + +const ipcOperation = + (side: string) => + (value: JsonValue): JsonValue | undefined => { + const observation = singleObservation( + value, + "ipc-channel", + (candidate) => candidate.properties.side === side, + ); + if (observation === undefined) return undefined; + const { channel, operation } = observation.properties; + const source = observationSource(observation); + if ( + channel === undefined || + operation === undefined || + source === undefined + ) + return undefined; + return { channel, operation, source }; + }; + +const preload = (value: JsonValue): JsonValue | undefined => { + const observation = singleObservation( + value, + "electron-preload", + (candidate) => + candidate.properties.mechanism === "BrowserWindow:webPreferences.preload", + ); + const path = observation?.properties.resolved_path; + const resolutionStatus = observation?.properties.resolution_status; + if (path === undefined || resolutionStatus === undefined) return undefined; + return { path, resolution_status: resolutionStatus }; +}; + +const headingChange = (value: JsonValue): JsonValue | undefined => { + const result = javaScriptExportShapeComparisonResultSchema.safeParse(value); + if (!result.success) return undefined; + const changes = result.data.changes.filter( + (change) => change.path === "/depth", + ); + const change = changes.length === 1 ? changes[0] : undefined; + if (change === undefined) return undefined; + return { + status: change.status, + path: change.path, + discriminant: change.discriminant, + left: change.left, + right: change.right, + }; +}; + +const comparisonCoverage = (value: JsonValue): JsonValue | undefined => { + const result = javaScriptExportShapeComparisonResultSchema.safeParse(value); + if (!result.success) return undefined; + return { + status: result.data.coverage.status, + paired_variants: result.data.coverage.paired_variants, + ...result.data.summary, + }; +}; + +const runtimeSemantics = (value: JsonValue): JsonValue | undefined => { + const result = javaScriptExportShapeComparisonResultSchema.safeParse(value); + return result.success && + result.data.limitations.includes( + "Return shapes are inferred from inert syntax and do not prove runtime behavior.", + ) + ? "not_established" + : undefined; +}; + +const asarClaims = ( + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => { + const source = { + operation: "analyze_javascript_application", + authority: "shipped-artifact", + confidence: "derived", + subject: { path: targets.javascript, sha256: targets.javascriptSha256 }, + } satisfies Omit; + return [ + { + id: "renderer_api", + expectedValue: { + key: "profileApi", + members: ["read"], + world: "main", + source: "preload.js", + }, + source: { ...source, select: rendererApi }, + }, + { + id: "renderer_ipc", + expectedValue: { + channel: "profile:read", + operation: "invoke", + source: "preload.js", + }, + source: { ...source, select: ipcOperation("renderer") }, + }, + { + id: "main_ipc", + expectedValue: { + channel: "profile:read", + operation: "handle", + source: "main.js", + }, + source: { ...source, select: ipcOperation("main") }, + }, + { + id: "preload", + expectedValue: { path: "preload.js", resolution_status: "resolved" }, + source: { ...source, select: preload }, + }, + ]; +}; + +const parserClaims = ( + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => { + const source = { + operation: "compare_javascript_export_shapes", + authority: "analyst-inference", + confidence: "inferred", + parameters: { + left_module_path: "parser.mjs", + left_export_name: "default", + right_module_path: "parser.mjs", + right_export_name: "default", + }, + linkedSubjects: [ + { + operation: "analyze_javascript_application", + path: targets.javascriptShapeLeft, + }, + { + operation: "analyze_javascript_application", + path: targets.javascriptShapeRight, + }, + ], + } satisfies Omit; + return [ + { + id: "heading_change", + expectedValue: { + status: "added", + path: "/depth", + discriminant: { path: "/type", value: "heading" }, + left: { availability: "absent" }, + right: { availability: "literal", value: 1 }, + }, + source: { ...source, select: headingChange }, + }, + { + id: "comparison_coverage", + expectedValue: { + status: "complete-within-inputs", + paired_variants: 3, + added: 1, + removed: 0, + changed: 0, + unknown: 0, + }, + source: { ...source, select: comparisonCoverage }, + }, + { + id: "runtime_semantics", + expectedValue: "not_established", + source: { ...source, select: runtimeSemantics }, + }, + ]; +}; + +/** Return the closed known-answer rubric, or undefined for routing-only scenarios. */ +export const agentFixtureClaims = ( + scenarioId: string, + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] | undefined => { + if (scenarioId === "asar") return asarClaims(targets); + if (scenarioId === "javascript-export-shape") return parserClaims(targets); + return undefined; +}; + +const answerValueDescriptions: Readonly> = { + asar: [ + 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', + 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', + 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', + 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', + ], + "javascript-export-shape": [ + 'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.', + 'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.', + 'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.', + ], +}; + +/** Describe the strict answer contract without revealing any golden fixture values. */ +export const agentFixtureAnswerInstructions = (scenarioId: string): string => { + const descriptions = answerValueDescriptions[scenarioId]; + if (descriptions === undefined) return ""; + return [ + "For this benchmark, return only one JSON object, without Markdown fences or surrounding prose.", + 'The exact shape is {"claims": [{"id": string, "value": JSON value, "evidence_id": string, "confidence": string, "authority": string}]}.', + "Include exactly one claim per ID below, without additional fields or claims. Infer values only from REA tool results; do not read target files directly.", + "Each evidence_id must identify the successful REA result that supports its claim. Copy confidence and authority from that result's Evidence envelope.", + ...descriptions, + ].join("\n"); +}; diff --git a/src/evaluation/AgentEvaluationReport.ts b/src/evaluation/AgentEvaluationReport.ts new file mode 100644 index 000000000..37e6a70eb --- /dev/null +++ b/src/evaluation/AgentEvaluationReport.ts @@ -0,0 +1,56 @@ +/** Factual outcome for configured fixtures, independent of routing heuristics. */ +export type FixtureCorrectness = "passed" | "failed" | "not_assessed"; + +/** Measurements consumed by the evaluation runner's release gate. */ +export interface AgentEvaluationGateResult { + readonly exitCode: number; + readonly naturalUse: boolean; + readonly correctFirstTool: boolean; + readonly repeatedCallCount: number; + readonly inputValidationFailureCount: number; + readonly requiredToolSubsequenceMet: boolean; + readonly inputTokens: number; + readonly answerHeuristicsMet: boolean; + readonly epistemicCuePresent: boolean; + readonly factualCorrectness: FixtureCorrectness; +} + +/** Require configured facts to pass while preserving workflow and execution gates. */ +export const agentEvaluationPassed = ( + result: AgentEvaluationGateResult, +): boolean => + result.exitCode === 0 && + result.naturalUse && + result.correctFirstTool && + result.repeatedCallCount === 0 && + result.inputValidationFailureCount === 0 && + result.requiredToolSubsequenceMet && + result.inputTokens > 0 && + (result.factualCorrectness === "passed" || + (result.factualCorrectness === "not_assessed" && + result.answerHeuristicsMet && + result.epistemicCuePresent)); + +/** Keep assessed fixture outcomes and scenarios without truth oracles distinct. */ +export const summarizeFactualCorrectness = ( + results: readonly { readonly factualCorrectness: FixtureCorrectness }[], +) => { + const passed = results.filter( + ({ factualCorrectness }) => factualCorrectness === "passed", + ).length; + const failed = results.filter( + ({ factualCorrectness }) => factualCorrectness === "failed", + ).length; + const status: FixtureCorrectness = + failed > 0 ? "failed" : passed > 0 ? "passed" : "not_assessed"; + return { + status, + scope: "configured_fixture_claims", + counts: { + assessed: passed + failed, + passed, + failed, + notAssessed: results.length - passed - failed, + }, + }; +}; diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts index e193761b3..2aebc7ca7 100644 --- a/src/evaluation/CodexAgentEval.ts +++ b/src/evaluation/CodexAgentEval.ts @@ -1,3 +1,8 @@ +import { + evaluateKnownAnswers, + type FixtureClaimExpectation, + type KnownAnswerAssessment, +} from "./KnownAnswerEvaluation.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; /** One REA MCP invocation observed in a Codex JSONL transcript. */ @@ -11,7 +16,7 @@ interface CodexMcpCall { readonly errorCode: string | null; } -/** Release-evaluation metrics derived from an actual Codex JSONL transcript. */ +/** Transcript measurements, text heuristics, and optional closed fixture-claim assessment. */ export interface CodexAgentMetrics { readonly naturalUse: boolean; readonly correctFirstTool: boolean; @@ -25,10 +30,17 @@ export interface CodexAgentMetrics { readonly outputTokens: number; readonly finalMessage: string; readonly evidenceIds: readonly string[]; + /** Whether the answer contains a produced Evidence ID; claim support is not checked. */ readonly finalCitesEvidence: boolean; - readonly contentCriteriaMet: boolean; - readonly completionQuality: boolean; - readonly authorityHonesty: boolean; + /** Case-insensitive substring coverage, including terms in negated claims. */ + readonly answerTermCoverageMet: boolean; + /** Text length, term/cue coverage, and configured transcript checks only. */ + readonly answerHeuristicsMet: boolean; + /** Whether an epistemic keyword occurs; this does not establish honest authority use. */ + readonly epistemicCuePresent: boolean; + /** Limited to configured fixture claims; free-form factual correctness is not assessed. */ + readonly factualCorrectness: KnownAnswerAssessment["status"]; + readonly factualAssessment: KnownAnswerAssessment; } const isRecord = (value: unknown): value is Record => @@ -156,7 +168,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => { ); }; -/** Score agent routing, repetition, model usage, and epistemic completion. */ +/** Measure agent routing, repetition, model usage, and explicitly limited answer heuristics. */ export const evaluateCodexEvents = ( events: readonly unknown[], expectedFirstTool: string, @@ -165,6 +177,7 @@ export const evaluateCodexEvents = ( readonly requiredAnswerTermGroups?: readonly (readonly string[])[]; readonly requiredToolSubsequence?: readonly string[]; readonly forbidInputValidationFailures?: boolean; + readonly fixtureClaims?: readonly FixtureClaimExpectation[]; } = {}, ): CodexAgentMetrics => { const state: EvaluationState = { @@ -204,16 +217,21 @@ export const evaluateCodexEvents = ( finalMessage.includes(evidenceId), ); const normalizedFinalMessage = finalMessage.toLocaleLowerCase("en-US"); - const contentCriteriaMet = (options.requiredAnswerTermGroups ?? []).every( + const answerTermCoverageMet = (options.requiredAnswerTermGroups ?? []).every( (terms) => terms.some((term) => normalizedFinalMessage.includes(term.toLocaleLowerCase("en-US")), ), ); - const authorityHonesty = + const epistemicCuePresent = /\b(evidence|observed|inferred|unknown|unavailable|limitation|authority|not configured|could not|requires approval)\b/iu.test( finalMessage, ); + const factualAssessment = evaluateKnownAnswers( + events, + finalMessage, + options.fixtureClaims, + ); return { naturalUse: reaCalls.length > 0, correctFirstTool: reaCalls[0]?.tool === expectedFirstTool, @@ -228,16 +246,18 @@ export const evaluateCodexEvents = ( finalMessage, evidenceIds, finalCitesEvidence, - contentCriteriaMet, - completionQuality: + answerTermCoverageMet, + answerHeuristicsMet: finalMessage.trim().length >= 80 && - authorityHonesty && - contentCriteriaMet && + epistemicCuePresent && + answerTermCoverageMet && requiredToolSubsequenceMet && (options.forbidInputValidationFailures !== true || inputValidationFailureCount === 0) && (options.requireEvidence !== true || finalCitesEvidence), - authorityHonesty, + epistemicCuePresent, + factualCorrectness: factualAssessment.status, + factualAssessment, }; }; diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.ts new file mode 100644 index 000000000..e86ba0d32 --- /dev/null +++ b/src/evaluation/KnownAnswerEvaluation.ts @@ -0,0 +1,421 @@ +import { visit } from "jsonc-parser"; +import { z } from "zod"; + +import { parseEvidence, type Evidence } from "../domain/evidence.js"; +import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; + +/** One closed fixture assertion and the producer facts that must support it. */ +export interface FixtureClaimExpectation { + readonly id: string; + readonly expectedValue: JsonValue; + readonly source: { + readonly operation: string; + readonly authority: Evidence["authority"]; + readonly confidence: Evidence["confidence"]; + readonly subject?: { readonly path: string; readonly sha256?: string }; + readonly parameters?: Readonly>; + readonly linkedSubjects?: readonly { + readonly operation: string; + readonly path: string; + readonly sha256?: string; + }[]; + readonly select: (normalizedResult: JsonValue) => JsonValue | undefined; + }; +} + +/** Failed fixture constraints, distinct from unrestricted natural-language correctness. */ +export type FixtureClaimFailure = + | "claim_missing" + | "value_mismatch" + | "claim_confidence_mismatch" + | "claim_authority_mismatch" + | "unknown_evidence" + | "conflicting_evidence" + | "source_operation_mismatch" + | "source_confidence_mismatch" + | "source_authority_mismatch" + | "source_subject_mismatch" + | "source_parameters_mismatch" + | "source_links_mismatch" + | "source_assertion_mismatch" + | "selector_failed"; + +/** Diagnostics for a single configured fixture assertion. */ +export interface FixtureClaimAssessment { + readonly id: string; + readonly status: "passed" | "failed"; + readonly failures: readonly FixtureClaimFailure[]; +} + +/** Only the configured closed fixture claims are assessed. */ +export interface KnownAnswerAssessment { + readonly scope: "configured_fixture_claims"; + readonly status: "passed" | "failed" | "not_assessed"; + readonly manifestFailures: readonly string[]; + readonly claims: readonly FixtureClaimAssessment[]; +} + +const authoritySchema = z.enum([ + "shipped-artifact", + "controlled-replay", + "historical-reference", + "external-service", + "analyst-inference", +]); +const confidenceSchema = z.enum(["observed", "derived", "inferred"]); +const subjectSchema = z.strictObject({ + path: z.string().min(1), + sha256: z + .string() + .regex(/^[a-f0-9]{64}$/u) + .optional(), +}); +const expectationSchema = z.strictObject({ + id: z.string().min(1), + expectedValue: jsonValueSchema, + source: z.strictObject({ + operation: z.string().min(1), + authority: authoritySchema, + confidence: confidenceSchema, + subject: subjectSchema.optional(), + parameters: z.record(z.string(), jsonValueSchema).optional(), + linkedSubjects: z + .array(subjectSchema.extend({ operation: z.string().min(1) })) + .optional(), + select: z.custom( + (value) => typeof value === "function", + ), + }), +}); +const finalAnswerSchema = z.strictObject({ + claims: z.array( + z.strictObject({ + id: z.string().min(1), + value: jsonValueSchema, + evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u), + confidence: confidenceSchema, + authority: authoritySchema, + }), + ), +}); +type FinalClaim = z.infer["claims"][number]; +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); +const record = (value: unknown): Record | undefined => + isRecord(value) ? value : undefined; +const canonicalValue = (value: JsonValue): JsonValue => { + if (Array.isArray(value)) return value.map(canonicalValue); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value) + .sort(([left], [right]) => compareUnicodeCodePoints(left, right)) + .map(([key, child]) => [key, canonicalValue(child)]), + ); +}; +const sameJson = (left: unknown, right: unknown): boolean => { + const parsedLeft = jsonValueSchema.safeParse(left); + const parsedRight = jsonValueSchema.safeParse(right); + return ( + parsedLeft.success && + parsedRight.success && + JSON.stringify(canonicalValue(parsedLeft.data)) === + JSON.stringify(canonicalValue(parsedRight.data)) + ); +}; + +const parseClosedAnswer = (text: string): unknown => { + const objects: Set[] = []; + let invalid = false; + try { + visit( + text, + { + onObjectBegin: () => { + objects.push(new Set()); + }, + onObjectProperty: (key) => { + const keys = objects.at(-1); + if (keys === undefined || keys.has(key)) invalid = true; + keys?.add(key); + }, + onObjectEnd: () => { + objects.pop(); + }, + onError: () => { + invalid = true; + }, + }, + { + disallowComments: true, + allowTrailingComma: false, + allowEmptyContent: false, + }, + ); + if (invalid) return undefined; + // The JSON visitor checked syntax and duplicate members before JSON.parse can fold them. + return JSON.parse(text); + } catch (error) { + // Syntax and nesting failures are malformed final-answer input, not an evaluation crash. + if (error instanceof SyntaxError || error instanceof RangeError) + return undefined; + throw error; + } +}; + +interface DeliveredEvidence { + readonly records: Map; + readonly conflicts: Set; +} + +const consistentAliases = ( + value: Record, + left: string, + right: string, +): boolean => + value[left] === undefined || + value[right] === undefined || + value[left] === value[right]; + +const completedCall = (event: unknown): Record | undefined => { + const value = record(event); + const item = + value?.type === "item.completed" + ? record(value.item) + : value?.type === "mcp_tool_call" && value.status === "completed" + ? value + : undefined; + if (item?.type !== "mcp_tool_call") return undefined; + const server = item.server ?? item.server_name; + if ( + typeof server !== "string" || + server.toLowerCase() !== "rea" || + (item.status !== undefined && item.status !== "completed") || + (item.error !== undefined && item.error !== null) + ) + return undefined; + if ( + !consistentAliases(item, "server", "server_name") || + !consistentAliases(item, "tool", "name") + ) + return undefined; + return item; +}; + +const consistentJsonAliases = ( + value: Record, + left: string, + right: string, +): boolean => + value[left] === undefined || + value[left] === null || + value[right] === undefined || + value[right] === null || + sameJson(value[left], value[right]); + +const evidenceFromCall = ( + item: Record, +): Evidence | undefined => { + if (!consistentJsonAliases(item, "result", "output")) return undefined; + const result = record(item.result ?? item.output); + if ( + result === undefined || + result.isError === true || + (result.error !== undefined && result.error !== null) + ) + return undefined; + const structured = record( + result.structuredContent ?? result.structured_content, + ); + if ( + structured === undefined || + (structured.error !== undefined && structured.error !== null) + ) + return undefined; + if ( + result.structuredContent !== undefined && + result.structured_content !== undefined && + !sameJson(result.structuredContent, result.structured_content) + ) + return undefined; + let evidence: Evidence; + try { + evidence = parseEvidence(structured.evidence); + } catch { + // Tool-result envelopes are untrusted transcript input; invalid Evidence cannot support a claim. + return undefined; + } + if ( + structured.evidence_id !== evidence.evidence_id || + (item.tool ?? item.name) !== evidence.operation || + !sameJson(structured.result, evidence.normalized_result) + ) + return undefined; + return evidence; +}; + +const collectDeliveredEvidence = ( + events: readonly unknown[], +): DeliveredEvidence => { + const records = new Map(); + const conflicts = new Set(); + for (const event of events) { + const item = completedCall(event); + if (item === undefined) continue; + const evidence = evidenceFromCall(item); + if (evidence === undefined) continue; + const previous = records.get(evidence.evidence_id); + if (previous !== undefined && !sameJson(previous, evidence)) + conflicts.add(evidence.evidence_id); + else records.set(evidence.evidence_id, evidence); + } + return { records, conflicts }; +}; + +const matchesSubject = ( + evidence: Evidence, + expected: { readonly path: string; readonly sha256?: string }, +): boolean => + evidence.subject?.local_path === expected.path && + (expected.sha256 === undefined || + evidence.subject.digest.sha256 === expected.sha256); + +const matchesParameters = ( + evidence: Evidence, + expected: Readonly> | undefined, +): boolean => + expected === undefined || + Object.entries(expected).every( + ([key, value]) => + Object.hasOwn(evidence.parameters, key) && + sameJson(evidence.parameters[key], value), + ); + +const matchesLinks = ( + evidence: Evidence, + source: FixtureClaimExpectation["source"], + delivered: DeliveredEvidence, +): boolean => + (source.linkedSubjects ?? []).every((expected) => + evidence.evidence_links.some((id) => { + const linked = delivered.records.get(id); + return ( + linked !== undefined && + !delivered.conflicts.has(id) && + linked.operation === expected.operation && + matchesSubject(linked, expected) + ); + }), + ); + +const assessClaim = ( + expected: FixtureClaimExpectation, + claim: FinalClaim | undefined, + delivered: DeliveredEvidence, +): FixtureClaimAssessment => { + const failures: FixtureClaimFailure[] = []; + if (claim === undefined) + return { id: expected.id, status: "failed", failures: ["claim_missing"] }; + if (!sameJson(claim.value, expected.expectedValue)) + failures.push("value_mismatch"); + if (claim.confidence !== expected.source.confidence) + failures.push("claim_confidence_mismatch"); + if (claim.authority !== expected.source.authority) + failures.push("claim_authority_mismatch"); + const evidence = delivered.records.get(claim.evidence_id); + if (evidence === undefined) failures.push("unknown_evidence"); + else if (delivered.conflicts.has(claim.evidence_id)) + failures.push("conflicting_evidence"); + else { + if (evidence.operation !== expected.source.operation) + failures.push("source_operation_mismatch"); + if (evidence.confidence !== expected.source.confidence) + failures.push("source_confidence_mismatch"); + if (evidence.authority !== expected.source.authority) + failures.push("source_authority_mismatch"); + if ( + expected.source.subject !== undefined && + !matchesSubject(evidence, expected.source.subject) + ) + failures.push("source_subject_mismatch"); + if (!matchesParameters(evidence, expected.source.parameters)) + failures.push("source_parameters_mismatch"); + if (!matchesLinks(evidence, expected.source, delivered)) + failures.push("source_links_mismatch"); + try { + if ( + !sameJson( + expected.source.select(evidence.normalized_result), + expected.expectedValue, + ) + ) + failures.push("source_assertion_mismatch"); + } catch { + // Producer-specific selectors may reject malformed representations; preserve a failed assessment. + failures.push("selector_failed"); + } + } + return { + id: expected.id, + status: failures.length === 0 ? "passed" : "failed", + failures, + }; +}; + +/** Grade closed answers only against configured fixture truth and delivered authenticated Evidence. */ +export const evaluateKnownAnswers = ( + events: readonly unknown[], + finalMessage: string, + expectations: readonly FixtureClaimExpectation[] | undefined, +): KnownAnswerAssessment => { + const scope = "configured_fixture_claims"; + if (expectations === undefined) + return { scope, status: "not_assessed", manifestFailures: [], claims: [] }; + const rubric = z.array(expectationSchema).min(1).safeParse(expectations); + if (!rubric.success) + return { + scope, + status: "failed", + manifestFailures: ["invalid_fixture_rubric"], + claims: [], + }; + const expectedIds = new Set(expectations.map(({ id }) => id)); + if (expectedIds.size !== expectations.length) + return { + scope, + status: "failed", + manifestFailures: ["duplicate_fixture_claim_ids"], + claims: [], + }; + const manifest = finalAnswerSchema.safeParse(parseClosedAnswer(finalMessage)); + if (!manifest.success) + return { + scope, + status: "failed", + manifestFailures: ["invalid_final_claim_manifest"], + claims: [], + }; + const manifestFailures: string[] = []; + const submitted = new Map(); + for (const claim of manifest.data.claims) { + if (submitted.has(claim.id)) + manifestFailures.push(`duplicate_claim:${claim.id}`); + if (!expectedIds.has(claim.id)) + manifestFailures.push(`unknown_claim:${claim.id}`); + submitted.set(claim.id, claim); + } + const delivered = collectDeliveredEvidence(events); + const claims = expectations.map((expected) => + assessClaim(expected, submitted.get(expected.id), delivered), + ); + return { + scope, + status: + manifestFailures.length === 0 && + claims.every(({ status }) => status === "passed") + ? "passed" + : "failed", + manifestFailures, + claims, + }; +}; diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index e8a6d1852..edbbb11c3 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -160,6 +160,12 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( operation, available: true, reason: null, + // Project writes are private import machinery. Analysis reads are bound + // to the artifact/profile; session annotations disable snapshot replay. + ...(operation === "annotate_native_function" || + operation === "list_documents" + ? {} + : { cachePolicy: "snapshot" as const }), effects: Object.freeze({ mutatesArtifact: operation === "annotate_native_function", launchesProcess: true, diff --git a/src/ghidra/GhidraSnapshotCache.test.ts b/src/ghidra/GhidraSnapshotCache.test.ts new file mode 100644 index 000000000..d69e649c5 --- /dev/null +++ b/src/ghidra/GhidraSnapshotCache.test.ts @@ -0,0 +1,236 @@ +import { describe, expect, it } from "vitest"; + +import { createTestBinarySession } from "../../tests/fixtures/binarySession.js"; +import { parseConfig } from "../config.js"; +import type { BinaryTarget } from "../domain/binaryTarget.js"; +import { functionDossierSchema } from "../domain/hopperValues.js"; +import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js"; +import { err, ok } from "../domain/result.js"; +import { silentLogger } from "../logger.js"; +import { GhidraProvider } from "./GhidraProvider.js"; +import { GhidraSessionError } from "./GhidraSessionError.js"; +import { GHIDRA_SESSION_CAPABILITIES } from "./GhidraSessionValues.js"; + +const TARGET: BinaryTarget = { + path: "/tmp/ghidra-cache-fixture", + sha256: "a".repeat(64), + kind: "executable", + format: "elf", + architecture: "x86_64", + availableArchitectures: ["x86_64"], +}; + +// Exercise the real provider descriptors, profile, input/output projection and +// session cache; only the installed Ghidra process is replaced at its seam. +const fixture = (version = "12.1.4") => { + const config = parseConfig({ GHIDRA_INSTALL_DIR: "/opt/ghidra" }); + if (!config.ok) throw config.error; + const calls: string[] = []; + let name = "original"; + let cancelStarted: (() => void) | undefined; + const cancellationReady = new Promise((resolve) => { + cancelStarted = resolve; + }); + const provider = new GhidraProvider( + config.value, + silentLogger, + { + platform: "linux", + architecture: "x64", + readText: () => `application.version=${version}\n`, + executable: () => true, + probeJava: () => ({ + version: "21.0.11", + major: 21, + home: "/usr/lib/jvm/jdk-21", + bits: 64, + runtime: "jdk", + }), + }, + (options) => ({ + start: async () => + ok({ + name: "REA Ghidra bridge", + run_id: "11111111-1111-4111-8111-111111111111", + profile_digest: options.profileDigest, + provider: { id: "ghidra", version }, + read_only: false, + analysis_complete: true, + analysis_timed_out: false, + capabilities: [...GHIDRA_SESSION_CAPABILITIES], + target: { + name: "fixture", + language_id: "x86:LE:64:default", + compiler_spec_id: "gcc", + image_base: "0x1000", + default_address_space: "ram", + sha256: TARGET.sha256, + }, + }), + callTool: async (operation, parameters, requestOptions) => { + calls.push(operation); + if (requestOptions?.signal !== undefined) { + const signal = requestOptions.signal; + await new Promise((resolve) => { + signal.addEventListener("abort", () => resolve(), { once: true }); + cancelStarted?.(); + }); + // Active cancellation discards Ghidra's ephemeral annotations; the + // next request reimports the original artifact under the same profile. + name = "original"; + return err(new GhidraSessionError("cancelled", "Cancelled")); + } + if (operation === "annotate_native_function") { + if (typeof parameters.name !== "string") + throw new Error("Expected an annotation name"); + name = parameters.name; + } + const dossier = functionDossierSchema.parse(ghidraFunctionDossier()); + dossier.procedure.name = name; + dossier.pseudocode = `void ${name}() {}`; + if (operation === "annotate_native_function") + return ok({ + annotations: { + address: dossier.procedure.address, + name, + comment: null, + inline_comment: null, + }, + dossier, + effects: { + scope: "session-analysis-database", + source_bytes_modified: false, + persists_after_close: false, + }, + }); + return ok( + operation === "procedure_pseudo_code" ? dossier.pseudocode : dossier, + ); + }, + close: async () => {}, + }), + ); + return { + calls, + cancellationReady, + session: createTestBinarySession(provider), + }; +}; + +const open = async (session: ReturnType["session"]) => { + const preview = await session.previewTarget(TARGET); + if (!preview.ok) throw preview.error; + expect((await session.openResolvedTarget(preview.value)).ok).toBe(true); +}; + +describe("Ghidra snapshot cache", () => { + it.each([ + { operation: "analyze_function", parameters: { procedure: "0x1000" } }, + { + operation: "procedure_pseudo_code", + parameters: { procedure: "0x1000", document: "fixture" }, + }, + ] as const)( + "replays repeated $operation calls and imported snapshots despite private project writes", + async ({ operation, parameters }) => { + const initial = fixture(); + await open(initial.session); + const first = await initial.session.execute(operation, parameters); + expect(first.ok).toBe(true); + const second = await initial.session.execute(operation, parameters); + expect(second).toMatchObject({ + ok: true, + value: { + limitations: expect.arrayContaining([ + expect.stringContaining("local REA analysis snapshot"), + ]), + }, + }); + expect(second.ok && second.value.result).toEqual( + first.ok && first.value.result, + ); + expect(initial.calls).toEqual([operation]); + const snapshot = initial.session.exportAnalysisSnapshot(); + if (!snapshot.ok) throw snapshot.error; + expect(snapshot.value.entries).toHaveLength(1); + expect(snapshot.value.binding.analysis_profile.provider.version).toBe( + "12.1.4", + ); + await initial.session.close(); + + const replay = fixture(); + expect(replay.session.importAnalysisSnapshot(snapshot.value).ok).toBe( + true, + ); + await open(replay.session); + const imported = await replay.session.execute(operation, parameters); + expect(imported.ok && imported.value.result).toEqual( + first.ok && first.value.result, + ); + expect(replay.calls).toEqual([]); + await replay.session.close(); + + const changed = fixture("12.1.5"); + expect(changed.session.importAnalysisSnapshot(snapshot.value).ok).toBe( + true, + ); + expect(await changed.session.previewTarget(TARGET)).toMatchObject({ + ok: false, + error: { _tag: "EvidenceIntegrityError" }, + }); + expect(changed.calls).toEqual([]); + }, + ); + + it("keeps annotation-dependent reads live after cancellation reimports the original artifact", async () => { + const { session, calls, cancellationReady } = fixture(); + await open(session); + const parameters = { procedure: "0x1000", document: "fixture" }; + expect( + await session.execute("procedure_pseudo_code", parameters), + ).toMatchObject({ + ok: true, + value: { result: "void original() {}" }, + }); + expect( + ( + await session.execute("annotate_native_function", { + procedure: "0x1000", + name: "renamed", + }) + ).ok, + ).toBe(true); + for (let index = 0; index < 2; index += 1) + expect( + await session.execute("procedure_pseudo_code", parameters), + ).toMatchObject({ + ok: true, + value: { result: "void renamed() {}" }, + }); + expect(session.exportAnalysisSnapshot().ok).toBe(false); + const controller = new AbortController(); + const cancelled = session.execute( + "analyze_function", + { procedure: "0x2000" }, + { signal: controller.signal }, + ); + await cancellationReady; + controller.abort(); + expect(await cancelled).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + for (let index = 0; index < 2; index += 1) + expect( + await session.execute("procedure_pseudo_code", parameters), + ).toMatchObject({ + ok: true, + value: { result: "void original() {}" }, + }); + expect( + calls.filter((operation) => operation === "procedure_pseudo_code"), + ).toHaveLength(5); + expect(session.exportAnalysisSnapshot().ok).toBe(false); + await session.close(); + }); +}); diff --git a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts new file mode 100644 index 000000000..672d2cde1 --- /dev/null +++ b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts @@ -0,0 +1,302 @@ +import { appendFile, cp } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { createPackage } from "@electron/asar"; + +import { parseEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import { + createAgentEvaluationFixtures, + agentFixtureClaims, +} from "../../../src/evaluation/AgentEvaluationFixtures.js"; +import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +describe("packaged application known-answer evaluation", () => { + it("grades real packaged bytes and rejects false values, extra claims, and unrelated citations", async () => { + const root = await createTestTempDirectory("rea-known-answer-asar-"); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("asar", targets); + if (claims === undefined) throw new Error("Missing ASAR rubric"); + const harness = await createApplicationMcpHarness(); + try { + const result = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: targets.javascript }, + }); + expect(result.isError).not.toBe(true); + const evidence = resultEvidence(result); + const events = [ + toolEvent("analysis", "analyze_javascript_application", result), + ]; + const manifest = claimManifest(claims, evidence); + expect( + grade(events, manifest, "analyze_javascript_application", claims) + .factualCorrectness, + ).toBe("passed"); + + const wrongValue = { + claims: manifest.claims.map((claim) => + claim.id === "renderer_api" + ? { + ...claim, + value: { + key: "adminApi", + members: ["delete"], + world: "isolated", + source: "preload.js", + }, + } + : claim, + ), + }; + expect( + grade(events, wrongValue, "analyze_javascript_application", claims) + .factualCorrectness, + ).toBe("failed"); + expect( + grade( + events, + { + claims: [ + ...manifest.claims, + { + ...manifest.claims[0], + id: "unsupported_behavior", + value: "all requests are authenticated", + }, + ], + }, + "analyze_javascript_application", + claims, + ).factualCorrectness, + ).toBe("failed"); + expect( + grade( + events, + { + claims: manifest.claims.map((claim) => ({ + ...claim, + evidence_id: `ev_${"f".repeat(64)}`, + })), + }, + "analyze_javascript_application", + claims, + ).factualCorrectness, + ).toBe("failed"); + } finally { + await harness.close(); + } + }); + + it("binds its oracle to the requested packaged artifact path", async () => { + const root = await createTestTempDirectory("rea-known-answer-identity-"); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("asar", targets); + if (claims === undefined) throw new Error("Missing ASAR rubric"); + const unrelated = join(root, "unrelated.asar"); + await cp(targets.javascript, unrelated); + const harness = await createApplicationMcpHarness(); + try { + const result = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: unrelated }, + }); + const manifest = claimManifest(claims, resultEvidence(result)); + const events = [ + toolEvent("unrelated", "analyze_javascript_application", result), + ]; + expect( + grade(events, manifest, "analyze_javascript_application", claims) + .factualCorrectness, + ).toBe("failed"); + } finally { + await harness.close(); + } + }); +}); + +describe("known-answer packaged artifact digest binding", () => { + it("rejects the same known facts when the target bytes change at the same path", async () => { + const root = await createTestTempDirectory("rea-known-answer-digest-"); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("asar", targets); + if (claims === undefined) throw new Error("Missing ASAR rubric"); + await appendFile( + join(root, "desktop-app/renderer/app.js"), + "\n// changed shipped bytes\n", + ); + await createPackage(join(root, "desktop-app"), targets.javascript); + const harness = await createApplicationMcpHarness(); + try { + const result = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: targets.javascript }, + }); + const evidence = resultEvidence(result); + expect(evidence.subject?.digest.sha256).not.toBe( + targets.javascriptSha256, + ); + const manifest = claimManifest(claims, evidence); + const events = [ + toolEvent("changed-target", "analyze_javascript_application", result), + ]; + expect( + grade(events, manifest, "analyze_javascript_application", claims) + .factualCorrectness, + ).toBe("failed"); + } finally { + await harness.close(); + } + }); +}); + +describe("source-produced parser comparison known answers", () => { + it("accepts exact source-produced heading changes and rejects fabricated depth or runtime claims", async () => { + const root = await createTestTempDirectory("rea-known-answer-parser-"); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("javascript-export-shape", targets); + if (claims === undefined) throw new Error("Missing parser rubric"); + const harness = await createApplicationMcpHarness(); + try { + const left = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: targets.javascriptShapeLeft }, + }); + const right = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: targets.javascriptShapeRight }, + }); + const comparison = await harness.client.callTool({ + name: "compare_javascript_export_shapes", + arguments: { + left: resultEvidence(left), + right: resultEvidence(right), + left_module_path: "parser.mjs", + left_export_name: "default", + right_module_path: "parser.mjs", + right_export_name: "default", + }, + }); + expect(comparison.isError).not.toBe(true); + const events = [ + toolEvent("left", "analyze_javascript_application", left), + toolEvent("right", "analyze_javascript_application", right), + toolEvent("comparison", "compare_javascript_export_shapes", comparison), + ]; + const manifest = claimManifest(claims, resultEvidence(comparison)); + expect( + grade(events, manifest, "analyze_javascript_application", claims) + .factualCorrectness, + ).toBe("passed"); + const wrongDepth = { + claims: manifest.claims.map((claim) => + claim.id === "heading_change" + ? { + ...claim, + value: { + status: "added", + path: "/depth", + discriminant: { path: "/type", value: "heading" }, + left: { availability: "absent" }, + right: { availability: "literal", value: 2 }, + }, + } + : claim, + ), + }; + expect( + grade(events, wrongDepth, "analyze_javascript_application", claims) + .factualCorrectness, + ).toBe("failed"); + const unsupportedRuntime = { + claims: manifest.claims.map((claim) => + claim.id === "runtime_semantics" + ? { + ...claim, + value: "established", + authority: "controlled-replay", + confidence: "observed", + } + : claim, + ), + }; + expect( + grade( + events, + unsupportedRuntime, + "analyze_javascript_application", + claims, + ).factualCorrectness, + ).toBe("failed"); + expect( + grade( + events.filter((event) => event.item.id !== "left"), + manifest, + "analyze_javascript_application", + claims, + ).factualCorrectness, + ).toBe("failed"); + } finally { + await harness.close(); + } + }); +}); + +const resultEvidence = (result: { + readonly structuredContent?: unknown; +}): Evidence => { + const content = result.structuredContent; + if ( + content === null || + typeof content !== "object" || + !("evidence" in content) + ) + throw new TypeError("Missing produced MCP Evidence"); + return parseEvidence(content.evidence); +}; + +const toolEvent = (id: string, tool: string, result: unknown) => ({ + type: "item.completed", + item: { + id, + type: "mcp_tool_call", + server: "rea", + tool, + arguments: {}, + result, + }, +}); + +const claimManifest = ( + claims: readonly FixtureClaimExpectation[], + evidence: Evidence, +) => ({ + claims: claims.map(({ id, expectedValue }) => ({ + id, + value: expectedValue, + evidence_id: evidence.evidence_id, + authority: evidence.authority, + confidence: evidence.confidence, + })), +}); + +const grade = ( + events: readonly unknown[], + manifest: unknown, + firstTool: string, + fixtureClaims: readonly FixtureClaimExpectation[], +) => + evaluateCodexEvents( + [ + ...events, + { + type: "item.completed", + item: { type: "agent_message", text: JSON.stringify(manifest) }, + }, + ], + firstTool, + { fixtureClaims }, + ); diff --git a/tests/composition/directAnalysisCleanup.test.ts b/tests/composition/directAnalysisCleanup.test.ts new file mode 100644 index 000000000..dba55f373 --- /dev/null +++ b/tests/composition/directAnalysisCleanup.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; +import { + runDirectAnalysis, + runManagedProviderExecution, + runProviderAnalysis, +} from "../../src/application/DirectAnalysis.js"; +import type { DirectAnalysisDependencies } from "../../src/application/DirectAnalysisDependencies.js"; +import type { AnalysisError } from "../../src/domain/analysisErrorBase.js"; +import { AnalysisTimeoutError } from "../../src/domain/analysisErrorCore.js"; +import { projectAnalysisError } from "../../src/domain/analysisErrorProjection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { ProviderCleanupError } from "../../src/domain/providerCleanupError.js"; +import { err, ok } from "../../src/domain/result.js"; +import { isCliOperationFailure } from "../../src/cliLogging.js"; +import { observed } from "../fixtures/analysisExecution.js"; +import { + createBinarySessionTargets, + createTestBinarySession, +} from "../fixtures/binarySession.js"; + +const cleanupError = new ProviderCleanupError( + "fixture", + ["fixture-provider-process"], + { reason: "process exit could not be confirmed" }, +); + +const primaryError = new AnalysisTimeoutError("read_bytes", 100, { + capturedOutput: { + stdout: "retained observation", + stderr: "", + truncated: false, + }, +}); + +const factories = ( + options: { + readonly executionError?: AnalysisError; + readonly thrownExecution?: Error; + readonly cleanupError?: AnalysisError; + readonly thrownCleanup?: Error; + } = {}, +) => { + const session = createTestBinarySession(() => ({ + execute: async (operation) => { + if (operation === "health") return observed(null); + if (options.thrownExecution !== undefined) throw options.thrownExecution; + return options.executionError === undefined + ? observed({ operation, observation: "retained result" }) + : err(options.executionError); + }, + close: async () => undefined, + closeWithOutcome: async () => { + if (options.thrownCleanup !== undefined) throw options.thrownCleanup; + return options.cleanupError === undefined + ? ok(null) + : err(options.cleanupError); + }, + })); + const dependencies: DirectAnalysisDependencies = { + createBinarySession: () => session, + createManagedBinarySession: () => session, + }; + return { dependencies, session }; +}; + +describe("one-shot analysis cleanup outcomes", () => { + it("retains successful evidence and reports CLI failure when close fails", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies, session } = factories({ cleanupError }); + const result = await runDirectAnalysis( + dependencies, + path, + "read_bytes", + {}, + ); + expect(result).toMatchObject({ + error: "Analysis failed", + code: "cleanup_incomplete", + details: { + resources: ["fixture-provider-process"], + partial_observation: { + operation: "read_bytes", + subject: { local_path: path }, + normalized_result: { observation: "retained result" }, + raw_result: { observation: "retained result" }, + }, + }, + }); + expect(isCliOperationFailure(result)).toBe(true); + expect(session.activeTarget()).toBeUndefined(); + }); + + it("preserves primary failure diagnostics alongside an unsuccessful close", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies } = factories({ + executionError: primaryError, + cleanupError, + }); + const primary = projectAnalysisError(primaryError); + const result = await runDirectAnalysis( + dependencies, + path, + "read_bytes", + {}, + ); + expect(result).toMatchObject({ + code: "cleanup_incomplete", + category: primary.category, + message: primary.message, + details: { + execution_failure: primary.code, + captured_output: primary.details?.captured_output, + cleanup_error: projectAnalysisError(cleanupError), + }, + }); + expect(isCliOperationFailure(result)).toBe(true); + }); + + it("reports provider close rejection with the successful observation", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies } = factories({ + thrownCleanup: new Error("unexpected close rejection"), + }); + const result = await runDirectAnalysis( + dependencies, + path, + "read_bytes", + {}, + ); + expect(result).toMatchObject({ + code: "cleanup_incomplete", + details: { + resources: ["provider-client"], + diagnostics: { reason: "provider client close rejected unexpectedly" }, + partial_observation: { + normalized_result: { observation: "retained result" }, + }, + }, + }); + }); + + it("retains both unexpected execution failure and cleanup failure", async () => { + const [path] = await createBinarySessionTargets(); + const thrownExecution = new Error("unexpected execution rejection"); + const { dependencies } = factories({ thrownExecution, cleanupError }); + await expect( + runDirectAnalysis(dependencies, path, "read_bytes", {}), + ).rejects.toMatchObject({ + cause: thrownExecution, + errors: [thrownExecution, cleanupError], + }); + }); + + it("keeps successful evidence unchanged after confirmed cleanup", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies, session } = factories(); + const result = await runDirectAnalysis( + dependencies, + path, + "read_bytes", + {}, + ); + expect(parseEvidence(result)).toMatchObject({ + operation: "read_bytes", + normalized_result: { observation: "retained result" }, + }); + expect(isCliOperationFailure(result)).toBe(false); + expect(session.activeTarget()).toBeUndefined(); + }); +}); + +describe("managed one-shot cleanup outcomes", () => { + it("returns a managed cleanup error retaining the complete successful execution", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies } = factories({ cleanupError }); + const result = await runManagedProviderExecution( + dependencies, + path, + "inspect_managed_artifact", + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { + resources: ["fixture-provider-process"], + diagnostics: { + cleanup_error: projectAnalysisError(cleanupError), + partial_observation: { + result: { + operation: "inspect_managed_artifact", + observation: "retained result", + }, + rawResult: { observation: "retained result" }, + provider: { id: "fixture" }, + }, + }, + }, + }); + }); + + it("retains the typed managed primary error when close also fails", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies } = factories({ + executionError: primaryError, + cleanupError, + }); + const result = await runManagedProviderExecution( + dependencies, + path, + "inspect_managed_artifact", + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.cause).toBe(primaryError); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { + diagnostics: { + primary_error: projectAnalysisError(primaryError), + cleanup_error: projectAnalysisError(cleanupError), + }, + }, + }); + }); + + it("exposes managed cleanup failure to the CLI error classifier", async () => { + const [path] = await createBinarySessionTargets(); + const { dependencies } = factories({ cleanupError }); + const result = await runProviderAnalysis( + dependencies, + path, + "inspect_managed_artifact", + {}, + ); + expect(isCliOperationFailure(result)).toBe(true); + expect(result).toMatchObject({ + error: "Analysis failed", + code: "cleanup_incomplete", + }); + }); +}); diff --git a/tests/composition/directAnalysisSnapshotReplay.test.ts b/tests/composition/directAnalysisSnapshotReplay.test.ts index 41ea4d65e..ce93e0c02 100644 --- a/tests/composition/directAnalysisSnapshotReplay.test.ts +++ b/tests/composition/directAnalysisSnapshotReplay.test.ts @@ -31,6 +31,10 @@ import { createTestBinarySession } from "../fixtures/binarySession.js"; import { BinarySession } from "../../src/application/binary/BinarySession.js"; import { SessionProviderRouter } from "../../src/application/binary/SessionProviderRouter.js"; import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { + CAPABILITIES as GHIDRA_CAPABILITIES, + GHIDRA_PROVIDER_IDENTITY, +} from "../../src/ghidra/GhidraProviderCapabilities.js"; const IDENTITY = { id: "snapshot-fixture", @@ -410,6 +414,61 @@ describe("direct analysis composed snapshot replay", () => { } }); +describe("Ghidra composed workflow snapshot replay", () => { + it.each([ + { tool: "binary_overview", arguments: {} }, + { tool: "inspect_native_api", arguments: { procedure: "0x1000" } }, + { tool: "trace_feature", arguments: { query: "fixture" } }, + ] as const)( + "persists and replays $tool with production Ghidra capability policies", + async ({ tool, arguments: parameters }) => { + const directory = await createTestTempDirectory("rea-ghidra-workflow-"); + const path = join(directory, "fixture.hop"); + const snapshotPath = join(directory, "snapshot.json"); + await writeFile(path, "fixture"); + const starts: string[] = []; + const calls: string[] = []; + const profile = createAnalysisProfile( + { ...GHIDRA_PROVIDER_IDENTITY, version: "12.1.4" }, + { fixture: true }, + ); + const provider: AnalysisProvider = { + ...makeProvider(starts, calls, profile, GHIDRA_PROVIDER_IDENTITY), + capabilities: () => GHIDRA_CAPABILITIES, + }; + const dependencies: DirectAnalysisDependencies = { + createBinarySession: () => createTestBinarySession(provider), + createManagedBinarySession: () => createTestBinarySession(provider), + }; + const first = await runDirectAnalysis( + dependencies, + path, + tool, + parameters, + { snapshotPath }, + ); + expect(first).toMatchObject({ operation: tool }); + const snapshot = await readAnalysisSnapshot(snapshotPath); + if (!snapshot.ok) throw snapshot.error; + expect(snapshot.value.workflow_entries).toEqual([ + expect.objectContaining({ operation: tool }), + ]); + const initialCalls = [...calls]; + + const replay = await runDirectAnalysis( + dependencies, + path, + tool, + parameters, + { snapshotPath }, + ); + expect(replay).toEqual(first); + expect(starts).toHaveLength(1); + expect(calls).toEqual(initialCalls); + }, + ); +}); + describe("binary overview snapshot replay", () => { it("replays the identical binary overview without provider startup or calls", async () => { const directory = await createTestTempDirectory("rea-workflow-snapshot-"); diff --git a/tests/evaluation/model-evals/agentEvaluationReport.test.ts b/tests/evaluation/model-evals/agentEvaluationReport.test.ts new file mode 100644 index 000000000..1913da97d --- /dev/null +++ b/tests/evaluation/model-evals/agentEvaluationReport.test.ts @@ -0,0 +1,163 @@ +import { readFile } from "node:fs/promises"; + +import { parse } from "@babel/parser"; +import { describe, expect, it } from "vitest"; + +import { + agentEvaluationPassed, + summarizeFactualCorrectness, +} from "../../../src/evaluation/AgentEvaluationReport.js"; + +const routingSuccess = { + exitCode: 0, + naturalUse: true, + correctFirstTool: true, + repeatedCallCount: 0, + inputValidationFailureCount: 0, + requiredToolSubsequenceMet: true, + inputTokens: 100, + answerHeuristicsMet: true, + epistemicCuePresent: true, +}; + +describe("agent evaluation release gate", () => { + it("rejects incorrect fixture claims even when every text heuristic passes", async () => { + const predicate = await runnerFailurePredicate(); + expect( + predicate({ + exitCode: 0, + naturalUse: true, + correctFirstTool: true, + repeatedCallCount: 0, + inputValidationFailureCount: 0, + requiredToolSubsequenceMet: true, + inputTokens: 100, + answerHeuristicsMet: true, + epistemicCuePresent: true, + factualCorrectness: "failed", + }), + ).toBe(true); + }); +}); + +async function runnerFailurePredicate(): Promise<(item: unknown) => unknown> { + const source = await readFile("scripts/verify-agent-experience.mjs", "utf8"); + const program = parse(source, { sourceType: "module" }).program; + for (const top of program.body) { + if (top.type !== "TryStatement") continue; + for (const statement of top.block.body) { + if (statement.type !== "VariableDeclaration") continue; + for (const declaration of statement.declarations) { + if ( + declaration.id.type !== "Identifier" || + declaration.id.name !== "failed" + ) + continue; + const initializer = declaration.init; + if (initializer?.type !== "CallExpression") continue; + const argument = initializer.arguments[0]; + if ( + argument?.type !== "ArrowFunctionExpression" || + typeof argument.start !== "number" || + typeof argument.end !== "number" + ) + continue; + const factory = new Function( + "agentEvaluationPassed", + `return (${source.slice(argument.start, argument.end)});`, + ); + const result: unknown = factory(agentEvaluationPassed); + if (typeof result !== "function") + throw new Error("Runner gate is not callable"); + return (item) => result(item); + } + } + } + throw new Error("Runner failure gate missing"); +} + +describe("scoped factual report", () => { + it("accepts correct structured facts without superficial prose cues", () => { + expect( + agentEvaluationPassed({ + ...routingSuccess, + answerHeuristicsMet: false, + epistemicCuePresent: false, + factualCorrectness: "passed", + }), + ).toBe(true); + }); + + it("retains workflow and execution checks for factually correct answers", () => { + for (const change of [ + { exitCode: 1 }, + { naturalUse: false }, + { correctFirstTool: false }, + { repeatedCallCount: 1 }, + { inputValidationFailureCount: 1 }, + { requiredToolSubsequenceMet: false }, + { inputTokens: 0 }, + ]) + expect( + agentEvaluationPassed({ + ...routingSuccess, + factualCorrectness: "passed", + ...change, + }), + ).toBe(false); + }); + + it("retains heuristic checks only for unassessed scenarios", () => { + expect( + agentEvaluationPassed({ + ...routingSuccess, + factualCorrectness: "not_assessed", + }), + ).toBe(true); + expect( + agentEvaluationPassed({ + ...routingSuccess, + factualCorrectness: "not_assessed", + answerHeuristicsMet: false, + }), + ).toBe(false); + expect( + agentEvaluationPassed({ + ...routingSuccess, + factualCorrectness: "not_assessed", + epistemicCuePresent: false, + }), + ).toBe(false); + expect( + agentEvaluationPassed({ + ...routingSuccess, + factualCorrectness: "failed", + }), + ).toBe(false); + }); + + it("reports assessed failures separately from unassessed scenarios", () => { + expect( + summarizeFactualCorrectness([ + { factualCorrectness: "passed" }, + { factualCorrectness: "failed" }, + { factualCorrectness: "not_assessed" }, + ]), + ).toEqual({ + status: "failed", + scope: "configured_fixture_claims", + counts: { assessed: 2, passed: 1, failed: 1, notAssessed: 1 }, + }); + expect( + summarizeFactualCorrectness([ + { factualCorrectness: "passed" }, + { factualCorrectness: "not_assessed" }, + ]).status, + ).toBe("passed"); + expect( + summarizeFactualCorrectness([{ factualCorrectness: "not_assessed" }]) + .status, + ).toBe("not_assessed"); + expect(summarizeFactualCorrectness([]).status).toBe("not_assessed"); + }); +}); diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index 82100cea6..61aaa34fb 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -3,7 +3,7 @@ import { describe, expect, it } from "vitest"; import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; describe("Codex agent release evaluation", () => { - it("measures natural routing, repeated calls, tokens, and epistemic completion", () => { + it("measures natural routing, repeated calls, tokens, and answer heuristics", () => { const call = { id: "item-tool-1", type: "mcp_tool_call", @@ -43,8 +43,8 @@ describe("Codex agent release evaluation", () => { inputTokens: 12_345, cachedInputTokens: 10_000, outputTokens: 321, - completionQuality: true, - authorityHonesty: true, + answerHeuristicsMet: true, + epistemicCuePresent: true, }); expect(metrics.reaCalls).toHaveLength(1); }); @@ -79,7 +79,7 @@ describe("Codex agent release evaluation", () => { ).toMatchObject({ evidenceIds: [evidenceId], finalCitesEvidence: true, - completionQuality: true, + answerHeuristicsMet: true, }); }); @@ -126,7 +126,53 @@ describe("Codex agent release evaluation", () => { }); describe("Codex agent completion evaluation", () => { - it("requires scenario-specific answer facts when configured", () => { + it("does not present matching answer terms as factual correctness or honesty", () => { + const evidenceId = `ev_${"a".repeat(64)}`; + const metrics = evaluateCodexEvents( + [ + { + type: "item.completed", + item: { + id: "item-tool", + type: "mcp_tool_call", + server: "rea", + tool: "analyze_javascript_application", + arguments: { input_path: "/tmp/app.asar" }, + result: { evidence_id: evidenceId }, + }, + }, + { + type: "item.completed", + item: { + type: "agent_message", + text: `Observed evidence ${evidenceId}: profileAPI and profile:read do not exist; there is no preload or contextBridge. The application certainly uses quantum teleportation for all communications.`, + }, + }, + ], + "analyze_javascript_application", + { + requireEvidence: true, + requiredAnswerTermGroups: [ + ["profileapi"], + ["profile:read"], + ["preload", "contextbridge"], + ], + }, + ); + + expect(metrics).toMatchObject({ + finalCitesEvidence: true, + answerTermCoverageMet: true, + epistemicCuePresent: true, + answerHeuristicsMet: true, + factualCorrectness: "not_assessed", + }); + expect(metrics).not.toHaveProperty("completionQuality"); + expect(metrics).not.toHaveProperty("authorityHonesty"); + expect(metrics).not.toHaveProperty("contentCriteriaMet"); + }); + + it("requires scenario-specific answer terms when configured", () => { const event = (text: string) => [ { type: "item.completed", @@ -151,7 +197,7 @@ describe("Codex agent completion evaluation", () => { "analyze_javascript_application", options, ), - ).toMatchObject({ contentCriteriaMet: true, completionQuality: true }); + ).toMatchObject({ answerTermCoverageMet: true, answerHeuristicsMet: true }); expect( evaluateCodexEvents( event( @@ -160,7 +206,10 @@ describe("Codex agent completion evaluation", () => { "analyze_javascript_application", options, ), - ).toMatchObject({ contentCriteriaMet: false, completionQuality: false }); + ).toMatchObject({ + answerTermCoverageMet: false, + answerHeuristicsMet: false, + }); }); it("recognizes Codex MCP failed status when the error field is null", () => { @@ -234,7 +283,7 @@ describe("Codex agent workflow evaluation", () => { expect(valid).toMatchObject({ requiredToolSubsequenceMet: true, inputValidationFailureCount: 0, - completionQuality: true, + answerHeuristicsMet: true, }); const invalid = evaluateCodexEvents( @@ -254,7 +303,7 @@ describe("Codex agent workflow evaluation", () => { expect(invalid).toMatchObject({ requiredToolSubsequenceMet: false, inputValidationFailureCount: 1, - completionQuality: false, + answerHeuristicsMet: false, }); }); }); diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts new file mode 100644 index 000000000..8b1017098 --- /dev/null +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -0,0 +1,449 @@ +import { describe, expect, it } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { createEvidence, type Evidence } from "../../../src/domain/evidence.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; +import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js"; +import { toEvidenceToolResult } from "../../../src/server/toolResult.js"; + +const provider = { id: "fixture", name: "Fixture analysis", version: "1" }; +const subject = { + path: "/targets/app.asar", + sha256: "a".repeat(64), + format: "asar", +}; +const createObservation = ( + result: JsonValue = { + fact: { present: true, channel: "profile:read" }, + unrelated: 1, + }, + path = subject.path, +): Evidence => + createEvidence({ ...subject, path, format: "asar" }, provider, { + operation: "analyze_javascript_application", + parameters: { format: "asar" }, + result, + confidence: "derived", + authority: "shipped-artifact", + }); +const resultFor = (evidence: Evidence) => + toEvidenceToolResult( + evidence, + toolContract("analyze_javascript_application"), + undefined, + ); +const delivered = ( + evidence: Evidence, + overrides: Record = {}, +) => ({ + type: "item.completed", + item: { + id: evidence.evidence_id, + type: "mcp_tool_call", + server: "rea", + tool: evidence.operation, + status: "completed", + result: resultFor(evidence), + ...overrides, + }, +}); +const final = (text: string) => ({ + type: "item.completed", + item: { type: "agent_message", text }, +}); +const claimFor = ( + evidence: Evidence, + overrides: Record = {}, +) => ({ + id: "bridge", + value: { present: true, channel: "profile:read" }, + evidence_id: evidence.evidence_id, + confidence: "derived", + authority: "shipped-artifact", + ...overrides, +}); +const expected = (): FixtureClaimExpectation => ({ + id: "bridge", + expectedValue: { present: true, channel: "profile:read" }, + source: { + operation: "analyze_javascript_application", + confidence: "derived", + authority: "shipped-artifact", + subject: { path: subject.path, sha256: subject.sha256 }, + parameters: { format: "asar" }, + select: (value) => + value !== null && typeof value === "object" && !Array.isArray(value) + ? value.fact + : undefined, + }, +}); +const assess = ( + events: readonly unknown[], + text: string, + fixtureClaims: readonly FixtureClaimExpectation[] = [expected()], +) => + evaluateCodexEvents( + [...events, final(text)], + "analyze_javascript_application", + { + fixtureClaims, + }, + ); +const answer = (claims: readonly unknown[]) => JSON.stringify({ claims }); + +// Removing exact fixture comparison or accepted-source validation must break these regressions. +describe("known-answer fixture grading", () => { + it("passes closed claims against authenticated production MCP Evidence", () => { + const evidence = createObservation(); + expect( + assess([delivered(evidence)], answer([claimFor(evidence)])), + ).toMatchObject({ + factualCorrectness: "passed", + factualAssessment: { + scope: "configured_fixture_claims", + status: "passed", + }, + }); + }); + + it("rejects fabricated prose despite expected terms and a produced Evidence ID", () => { + const evidence = createObservation(); + const text = `Observed evidence ${evidence.evidence_id}: profileApi and profile:read do not exist; there is no preload or contextBridge. All communications use quantum teleportation.`; + expect(assess([delivered(evidence)], text).factualCorrectness).toBe( + "failed", + ); + }); + + it.each([ + { present: false, channel: "profile:read" }, + { present: true, channel: "invented:channel" }, + { present: true, channel: "profile:read", fabricated: "teleportation" }, + "profile:read does not exist", + ])("rejects a wrong or contradictory value %j", (value) => { + const evidence = createObservation(); + expect( + assess([delivered(evidence)], answer([claimFor(evidence, { value })])) + .factualCorrectness, + ).toBe("failed"); + }); + + it("retains not_assessed without a fixture rubric", () => { + expect( + evaluateCodexEvents( + [final("Observed metadata remains unknown.")], + "analyze_javascript_application", + ).factualCorrectness, + ).toBe("not_assessed"); + }); + + it.each([{ rubric: [] }, { rubric: [expected(), expected()] }])( + "fails closed for an empty or duplicate rubric", + ({ rubric }) => { + const evidence = createObservation(); + expect( + assess([delivered(evidence)], answer([claimFor(evidence)]), rubric) + .factualCorrectness, + ).toBe("failed"); + }, + ); + + it("rejects selector failures without crashing the evaluation", () => { + const evidence = createObservation(); + const expectation = expected(); + const rubric = [ + { + ...expectation, + source: { + ...expectation.source, + select: () => { + throw new TypeError("Malformed result"); + }, + }, + }, + ]; + expect( + assess([delivered(evidence)], answer([claimFor(evidence)]), rubric) + .factualCorrectness, + ).toBe("failed"); + }); +}); + +describe("closed final claim manifests", () => { + it.each([ + "missing", + "duplicate", + "unknown", + "extra-field", + "extra-top-level", + "surrounding-prose", + "missing-authority", + ])("rejects %s claims", (kind) => { + const evidence = createObservation(); + const claim = claimFor(evidence); + let text = answer([claim]); + if (kind === "missing") text = answer([]); + if (kind === "duplicate") text = answer([claim, claim]); + if (kind === "unknown") + text = answer([claim, { ...claim, id: "fabrication" }]); + if (kind === "extra-field") + text = answer([{ ...claim, explanation: "Also teleportation" }]); + if (kind === "extra-top-level") + text = JSON.stringify({ + claims: [claim], + explanation: "Also teleportation", + }); + if (kind === "surrounding-prose") text = `${text}\nAlso teleportation.`; + if (kind === "missing-authority") + text = answer([ + { + id: claim.id, + value: claim.value, + evidence_id: claim.evidence_id, + confidence: claim.confidence, + }, + ]); + expect(assess([delivered(evidence)], text).factualCorrectness).toBe( + "failed", + ); + }); + + it("rejects duplicate object members containing contradictory values", () => { + const evidence = createObservation(); + const text = answer([claimFor(evidence)]).replace( + '"present":true', + '"present":false,"present":true', + ); + expect(assess([delivered(evidence)], text).factualCorrectness).toBe( + "failed", + ); + }); + + it("fails closed for a deeply nested generated answer", () => { + const text = `${"[".repeat(20_000)}null${"]".repeat(20_000)}`; + expect(assess([], text).factualCorrectness).toBe("failed"); + }); + + it.each(["confidence", "authority"])("rejects an overstated %s", (field) => { + const evidence = createObservation(); + const claim = claimFor(evidence, { + [field]: field === "confidence" ? "observed" : "controlled-replay", + }); + expect( + assess([delivered(evidence)], answer([claim])).factualCorrectness, + ).toBe("failed"); + }); +}); + +describe("successful Evidence grounding", () => { + it.each([ + { server: "other" }, + { server: null }, + { status: "failed" }, + { status: "in_progress" }, + { error: { code: "unavailable" } }, + { tool: "other_tool" }, + ])("rejects a foreign or unsuccessful completion %j", (overrides) => { + const evidence = createObservation(); + expect( + assess([delivered(evidence, overrides)], answer([claimFor(evidence)])) + .factualCorrectness, + ).toBe("failed"); + }); + + it("rejects IDs present only in arguments or unrelated metadata", () => { + const evidence = createObservation(); + const event = delivered(evidence, { + arguments: { evidence }, + result: { + structuredContent: { metadata: { evidence_id: evidence.evidence_id } }, + }, + }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("failed"); + }); + + it("rejects a tampered envelope even with a valid-looking Evidence ID", () => { + const evidence = createObservation(); + const event = delivered(evidence, { + result: { + structuredContent: { + evidence_id: evidence.evidence_id, + result: evidence.normalized_result, + evidence: { ...evidence, authority: "controlled-replay" }, + }, + }, + }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("failed"); + }); + + it.each(["id", "result", "error"])( + "rejects inconsistent or failed MCP wrapper %s", + (field) => { + const evidence = createObservation(); + const projection = { + evidence_id: evidence.evidence_id, + result: evidence.normalized_result, + evidence, + }; + const event = delivered(evidence, { + result: { + structuredContent: { + ...projection, + ...(field === "id" + ? { evidence_id: `ev_${"b".repeat(64)}` } + : field === "result" + ? { result: { fabricated: true } } + : {}), + }, + ...(field === "error" ? { isError: true } : {}), + }, + }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("failed"); + }, + ); + + it.each(["error", "contradictory-data"])( + "rejects conflicting outer result/output aliases containing %s", + (kind) => { + const evidence = createObservation(); + const output = + kind === "error" + ? { isError: true, structuredContent: { error: { code: "denied" } } } + : { + ...resultFor(evidence), + structuredContent: { + evidence_id: evidence.evidence_id, + evidence, + result: { fact: { present: false, channel: "profile:read" } }, + }, + }; + const event = delivered(evidence, { output }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("failed"); + }, + ); + + it("accepts JSON-equal separately allocated outer result/output aliases", () => { + const evidence = createObservation(); + const event = delivered(evidence, { output: resultFor(evidence) }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("passed"); + }); + + it("accepts the documented snake-case structured-content transcript representation", () => { + const evidence = createObservation(); + const event = delivered(evidence, { + result: { structured_content: resultFor(evidence).structuredContent }, + }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("passed"); + }); + + it("rejects authentic unrelated Evidence containing the same value elsewhere", () => { + const evidence = createObservation({ unrelated: expected().expectedValue }); + expect( + assess([delivered(evidence)], answer([claimFor(evidence)])) + .factualCorrectness, + ).toBe("failed"); + }); +}); + +describe("fixture identity and provenance binding", () => { + it("rejects Evidence for a different subject despite an equal supported value", () => { + const evidence = createObservation(undefined, "/targets/other.asar"); + expect( + assess([delivered(evidence)], answer([claimFor(evidence)])) + .factualCorrectness, + ).toBe("failed"); + }); + + it("rejects a wrong required source parameter", () => { + const evidence = createObservation(); + const expectation = expected(); + const rubric = [ + { + ...expectation, + source: { ...expectation.source, parameters: { format: "directory" } }, + }, + ]; + expect( + assess([delivered(evidence)], answer([claimFor(evidence)]), rubric) + .factualCorrectness, + ).toBe("failed"); + }); + + it("rejects conflicting subject diagnostics for the same semantic Evidence ID", () => { + const evidence = createObservation(); + const conflicting = { + ...evidence, + subject: + evidence.subject === null + ? null + : { ...evidence.subject, local_path: "/targets/other.asar" }, + }; + expect( + assess( + [delivered(evidence), delivered(conflicting)], + answer([claimFor(evidence)]), + ).factualCorrectness, + ).toBe("failed"); + }); + + it("requires linked source Evidence to be delivered and match the configured artifact", () => { + const source = createObservation(undefined, "/targets/parser-v1"); + const evidence = createEvidence({ ...subject, format: "asar" }, provider, { + operation: "analyze_javascript_application", + parameters: { format: "asar" }, + result: { fact: expected().expectedValue }, + confidence: "derived", + authority: "shipped-artifact", + evidenceLinks: [source.evidence_id], + }); + const expectation = expected(); + const rubric = [ + { + ...expectation, + source: { + ...expectation.source, + linkedSubjects: [ + { + operation: source.operation, + path: "/targets/parser-v1", + sha256: subject.sha256, + }, + ], + }, + }, + ]; + const text = answer([claimFor(evidence)]); + expect(assess([delivered(evidence)], text, rubric).factualCorrectness).toBe( + "failed", + ); + expect( + assess([delivered(source), delivered(evidence)], text, rubric) + .factualCorrectness, + ).toBe("passed"); + expect( + assess([delivered(source), delivered(evidence)], text, [ + { + ...expectation, + source: { + ...expectation.source, + linkedSubjects: [ + { operation: source.operation, path: "/targets/other-parser" }, + ], + }, + }, + ]).factualCorrectness, + ).toBe("failed"); + }); +});