From ac82a1842491b59c72bb516ae14a534e1a7a874a Mon Sep 17 00:00:00 2001 From: "AISOP.dev" Date: Mon, 12 Oct 2026 22:59:45 -0400 Subject: [PATCH] test(agent): grade notes export facts against produced evidence --- scripts/verify-agent-experience.mjs | 12 + src/evaluation/AgentEvaluationFixtures.ts | 157 ++++++++- .../boundary/mcp/agentKnownAnswerMcp.test.ts | 328 ++++++++++++++++++ 3 files changed, 485 insertions(+), 12 deletions(-) diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 99ab060dd..b76e47697 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -154,6 +154,18 @@ try { requiredAnswerTermGroups: [["profileapi"], ["profile:read"]], prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript},說明 renderer 如何透過 preload 讀取 profile,以及 main handler 的關係。不要執行目標或安裝分析引擎;引用完整的產出 Evidence,保留 runtime 未知事項。`, }, + { + id: "notes-export", + expectedFirstTool: "analyze_javascript_application", + requiresEvidence: true, + requiredAnswerTermGroups: [], + requiredToolSubsequence: [ + "analyze_javascript_application", + "trace_application_feature", + "trace_javascript_semantics", + ], + prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`, + }, { id: "javascript-module-view", expectedFirstTool: "analyze_javascript_application", diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/src/evaluation/AgentEvaluationFixtures.ts index b7ef17efe..15c968cec 100644 --- a/src/evaluation/AgentEvaluationFixtures.ts +++ b/src/evaluation/AgentEvaluationFixtures.ts @@ -8,6 +8,12 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js"; import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { + applicationFeatureTraceResultSchema, + featureSeedMatchMode, +} from "../domain/javascript/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js"; /** Source-owned artifact paths used by the configured known-answer benchmarks. */ @@ -16,6 +22,8 @@ export interface AgentEvaluationFixtureTargets { readonly javascriptSha256: string; readonly javascriptShapeLeft: string; readonly javascriptShapeRight: string; + readonly notesElectron: string; + readonly notesElectronSha256: string; } const desktopFixtureFiles = { @@ -36,6 +44,8 @@ export const createAgentEvaluationFixtures = async ( const javascript = join(root, "desktop-app.asar"); const javascriptShapeLeft = join(root, "parser-v1"); const javascriptShapeRight = join(root, "parser-v2"); + const notesDirectory = join(root, "notes-electron"); + const notesElectron = join(root, "notes-electron.asar"); await Promise.all([ mkdir(join(desktop, "renderer"), { recursive: true }), mkdir(javascriptShapeLeft, { recursive: true }), @@ -53,8 +63,14 @@ export const createAgentEvaluationFixtures = async ( join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"), join(javascriptShapeRight, "parser.mjs"), ), + cp( + join(repositoryRoot, "website/public/examples/notes-electron"), + notesDirectory, + { recursive: true }, + ), ]); await createPackage(desktop, javascript); + await createPackage(notesDirectory, notesElectron); return { javascript, javascriptSha256: createHash("sha256") @@ -62,6 +78,10 @@ export const createAgentEvaluationFixtures = async ( .digest("hex"), javascriptShapeLeft, javascriptShapeRight, + notesElectron, + notesElectronSha256: createHash("sha256") + .update(await readFile(notesElectron)) + .digest("hex"), }; }; @@ -175,21 +195,116 @@ const runtimeSemantics = (value: JsonValue): JsonValue | undefined => { : undefined; }; -const asarClaims = ( +const featureModules = (value: JsonValue): JsonValue | undefined => { + const result = applicationFeatureTraceResultSchema.safeParse(value); + if (!result.success || result.data.graph === null) return undefined; + if ( + result.data.seed.kind !== "channel" || + result.data.seed.value !== "notes:export" || + featureSeedMatchMode(result.data.seed) !== "exact" + ) + return undefined; + const paths = result.data.graph.nodes + .filter((node) => node.kind === "javascript-module") + .flatMap((node) => node.observations) + .filter(({ properties }) => properties.semantic_role === "source-module") + .map(({ properties }) => properties.path); + if (paths.length === 0 || !paths.every((path) => typeof path === "string")) + return undefined; + return { + paths: [...new Set(paths)].sort(compareUnicodeCodePoints), + coverage: result.data.coverage.status, + }; +}; + +const semanticBoundary = (value: JsonValue): JsonValue | undefined => { + const result = javaScriptSemanticTraceResultSchema.safeParse(value); + if (!result.success) return undefined; + return { + status: result.data.status, + coverage: result.data.coverage.status, + relevant_unknowns: result.data.summary.relevant_unknowns, + }; +}; + +const notesClaims = ( targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => { + const subject = { + path: targets.notesElectron, + sha256: targets.notesElectronSha256, + }; + const derived = { + authority: "analyst-inference", + confidence: "inferred", + linkedSubjects: [ + { operation: "analyze_javascript_application", ...subject }, + ], + } satisfies Pick< + FixtureClaimExpectation["source"], + "authority" | "confidence" | "linkedSubjects" + >; + // Fixture source establishes bridge/module facts; the semantic claim checks + // the bounded static-query contract, not execution or CSV formatter behavior. + return [ + ...bridgeClaims(subject, "notes", ["exportCsv"], "notes:export"), + { + id: "feature_modules", + expectedValue: { + paths: ["csv.js", "main.js", "preload.js", "renderer.js"], + coverage: "complete-within-source", + }, + source: { + ...derived, + operation: "trace_application_feature", + parameters: { + direction: "both", + }, + select: featureModules, + }, + }, + { + id: "semantic_boundary", + expectedValue: { + status: "ambiguous", + coverage: "partial", + relevant_unknowns: 1, + }, + source: { + ...derived, + operation: "trace_javascript_semantics", + parameters: { + query: { + seed: { kind: "property", name: "invoke" }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: false, + expected: null, + }, + }, + select: semanticBoundary, + }, + }, + ]; +}; + +const bridgeClaims = ( + subject: { readonly path: string; readonly sha256: string }, + key: string, + members: readonly string[], + channel: string, ): readonly FixtureClaimExpectation[] => { const source = { operation: "analyze_javascript_application", authority: "shipped-artifact", confidence: "derived", - subject: { path: targets.javascript, sha256: targets.javascriptSha256 }, + subject, } satisfies Omit; return [ { id: "renderer_api", expectedValue: { - key: "profileApi", - members: ["read"], + key, + members: [...members], world: "main", source: "preload.js", }, @@ -198,7 +313,7 @@ const asarClaims = ( { id: "renderer_ipc", expectedValue: { - channel: "profile:read", + channel, operation: "invoke", source: "preload.js", }, @@ -207,7 +322,7 @@ const asarClaims = ( { id: "main_ipc", expectedValue: { - channel: "profile:read", + channel, operation: "handle", source: "main.js", }, @@ -221,6 +336,16 @@ const asarClaims = ( ]; }; +const asarClaims = ( + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => + bridgeClaims( + { path: targets.javascript, sha256: targets.javascriptSha256 }, + "profileApi", + ["read"], + "profile:read", + ); + const parserClaims = ( targets: AgentEvaluationFixtureTargets, ): readonly FixtureClaimExpectation[] => { @@ -284,21 +409,29 @@ export const agentFixtureClaims = ( ): readonly FixtureClaimExpectation[] | undefined => { if (scenarioId === "asar") return asarClaims(targets); if (scenarioId === "javascript-export-shape") return parserClaims(targets); + if (scenarioId === "notes-export") return notesClaims(targets); return undefined; }; +const bridgeAnswerDescriptions = [ + 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', + 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', + 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', + 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', +]; + const answerValueDescriptions: Readonly> = { - asar: [ - 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', - 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', - 'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.', - 'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.', - ], + asar: bridgeAnswerDescriptions, "javascript-export-shape": [ 'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.', 'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.', 'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.', ], + "notes-export": [ + ...bridgeAnswerDescriptions, + 'feature_modules: {"paths": string[], "coverage": string}, listing the unique source-module observation paths in the feature trace in Unicode code-point order, and its coverage status.', + 'semantic_boundary: {"status": string, "coverage": string, "relevant_unknowns": number}, copying the semantic trace status, coverage status, and summary.relevant_unknowns. Static traces do not establish runtime execution.', + ], }; /** Describe the strict answer contract without revealing any golden fixture values. */ diff --git a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts index b59dd80a8..11e8d365a 100644 --- a/tests/boundary/mcp/agentKnownAnswerMcp.test.ts +++ b/tests/boundary/mcp/agentKnownAnswerMcp.test.ts @@ -245,6 +245,334 @@ describe("source-produced parser comparison known answers", () => { }); }); +describe("Notes export known-answer evaluation", () => { + it("grades real analysis and traces while rejecting false values and unsupported citations", async () => { + const root = await createTestTempDirectory("rea-known-answer-notes-"); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("notes-export", targets); + if (claims === undefined) throw new Error("Missing Notes rubric"); + expect(claims.map(({ id }) => id)).toEqual([ + "renderer_api", + "renderer_ipc", + "main_ipc", + "preload", + "feature_modules", + "semantic_boundary", + ]); + const harness = await createApplicationMcpHarness(); + try { + const workflow = await notesWorkflow(harness, targets.notesElectron); + assertNotesAnswerFailures(workflow, claims); + assertNotesUnsupportedClaims(workflow, claims); + await assertNotesSeedBinding(harness, workflow, claims); + } finally { + await harness.close(); + } + }); + + it("binds Notes analysis and derived traces to the requested artifact path and bytes", async () => { + const root = await createTestTempDirectory( + "rea-known-answer-notes-identity-", + ); + const targets = await createAgentEvaluationFixtures(root, resolve(".")); + const claims = agentFixtureClaims("notes-export", targets); + if (claims === undefined) throw new Error("Missing Notes rubric"); + const foreign = join(root, "other-notes.asar"); + await cp(targets.notesElectron, foreign); + const harness = await createApplicationMcpHarness(); + try { + for (const path of [foreign, targets.notesElectron]) { + if (path === targets.notesElectron) { + await appendFile( + join(root, "notes-electron/renderer.js"), + "\n// changed shipped bytes\n", + ); + await createPackage(join(root, "notes-electron"), path); + } + const workflow = await notesWorkflow(harness, path); + const digest = notesEvidence(workflow, "analyze_javascript_application") + .subject?.digest.sha256; + expect(digest === targets.notesElectronSha256).toBe(path === foreign); + const assessment = grade( + workflow.events, + claimManifestByOperation(claims, workflow.evidence), + "analyze_javascript_application", + claims, + ).factualAssessment; + expect(assessment.status).toBe("failed"); + assertClaimFailure( + assessment, + "renderer_api", + "source_subject_mismatch", + ); + for (const id of ["feature_modules", "semantic_boundary"]) + assertClaimFailure(assessment, id, "source_links_mismatch"); + } + } finally { + await harness.close(); + } + }); +}); + +type NotesWorkflow = Awaited>; + +const notesEvidence = ( + workflow: NotesWorkflow, + operation: string, +): Evidence => { + const evidence = workflow.evidence.get(operation); + if (evidence === undefined) + throw new Error(`Missing Notes ${operation} Evidence`); + return evidence; +}; + +const assertClaimFailure = ( + assessment: ReturnType["factualAssessment"], + id: string, + failure: string, +) => { + expect(assessment.claims).toContainEqual( + expect.objectContaining({ + id, + failures: expect.arrayContaining([failure]), + }), + ); +}; + +const assertNotesAnswerFailures = ( + workflow: NotesWorkflow, + claims: readonly FixtureClaimExpectation[], +) => { + const manifest = claimManifestByOperation(claims, workflow.evidence); + const assess = (answer: unknown, events = workflow.events) => + grade(events, answer, "analyze_javascript_application", claims) + .factualAssessment; + expect(assess(manifest).status).toBe("passed"); + const cases = [ + { + id: "renderer_ipc", + patch: { + value: { + channel: "notes:delete", + operation: "send", + source: "preload.js", + }, + }, + failure: "value_mismatch", + }, + { + id: "semantic_boundary", + patch: { + value: { status: "found", coverage: "complete", relevant_unknowns: 0 }, + }, + failure: "value_mismatch", + }, + { + id: "renderer_api", + patch: { + evidence_id: notesEvidence(workflow, "trace_application_feature") + .evidence_id, + }, + failure: "source_operation_mismatch", + }, + ]; + for (const { id, patch, failure } of cases) { + const answer = { + claims: manifest.claims.map((claim) => + claim.id === id ? { ...claim, ...patch } : claim, + ), + }; + assertClaimFailure(assess(answer), id, failure); + } + const missingParent = assess( + manifest, + workflow.events.filter((event) => event.item.id !== "notes-analysis"), + ); + for (const id of ["feature_modules", "semantic_boundary"]) + assertClaimFailure(missingParent, id, "source_links_mismatch"); +}; + +const assertNotesUnsupportedClaims = ( + workflow: NotesWorkflow, + claims: readonly FixtureClaimExpectation[], +) => { + const manifest = claimManifestByOperation(claims, workflow.evidence); + const evidenceId = notesEvidence( + workflow, + "trace_application_feature", + ).evidence_id; + const unsupportedRuntime = { + claims: [ + ...manifest.claims, + { + id: "runtime_export", + value: "A click wrote notes.csv to the actual Downloads directory.", + evidence_id: evidenceId, + authority: "controlled-replay", + confidence: "observed", + }, + ], + }; + expect( + grade( + workflow.events, + unsupportedRuntime, + "analyze_javascript_application", + claims, + ).factualAssessment.manifestFailures, + ).toContain("unknown_claim:runtime_export"); + const prose = evaluateCodexEvents( + [ + ...workflow.events, + { + type: "item.completed", + item: { + type: "agent_message", + text: `Observed Evidence ${evidenceId}: notes:export, preload.js and main.js do not exist. The export certainly executed and wrote notes.csv; runtime behavior is known without further observation.`, + }, + }, + ], + "analyze_javascript_application", + { + requireEvidence: true, + requiredAnswerTermGroups: [["notes:export"], ["preload.js"], ["main.js"]], + fixtureClaims: claims, + }, + ); + expect(prose).toMatchObject({ + finalCitesEvidence: true, + answerTermCoverageMet: true, + factualCorrectness: "failed", + }); + expect(prose.factualAssessment.manifestFailures).toContain( + "invalid_final_claim_manifest", + ); +}; + +const assertNotesSeedBinding = async ( + harness: Awaited>, + workflow: NotesWorkflow, + claims: readonly FixtureClaimExpectation[], +) => { + const manifest = claimManifestByOperation(claims, workflow.evidence); + for (const match of ["contains", undefined] as const) { + const feature = await harness.client.callTool({ + name: "trace_application_feature", + arguments: { + application: notesEvidence(workflow, "analyze_javascript_application"), + seed: { + kind: "channel", + value: "notes:export", + ...(match === undefined ? {} : { match }), + case_sensitive: false, + }, + direction: "both", + }, + }); + expect(feature.isError).not.toBe(true); + const answer = { + claims: manifest.claims.map((claim) => + claim.id === "feature_modules" + ? { ...claim, evidence_id: resultEvidence(feature).evidence_id } + : claim, + ), + }; + const assessment = grade( + [ + ...workflow.events, + toolEvent( + `notes-feature-${match ?? "default"}`, + "trace_application_feature", + feature, + ), + ], + answer, + "analyze_javascript_application", + claims, + ).factualAssessment; + expect(assessment.status).toBe(match === undefined ? "passed" : "failed"); + if (match === "contains") { + assertClaimFailure( + assessment, + "feature_modules", + "source_assertion_mismatch", + ); + expect( + assessment.claims.find(({ id }) => id === "feature_modules")?.failures, + ).not.toContain("source_parameters_mismatch"); + } + } +}; + +const notesWorkflow = async ( + harness: Awaited>, + inputPath: string, +) => { + const analysis = await harness.client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: inputPath }, + }); + expect(analysis.isError).not.toBe(true); + const application = resultEvidence(analysis); + const feature = await harness.client.callTool({ + name: "trace_application_feature", + arguments: { + application, + seed: { + kind: "channel", + value: "notes:export", + match: "exact", + case_sensitive: false, + }, + direction: "both", + }, + }); + expect(feature.isError).not.toBe(true); + const semantic = await harness.client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application, + query: { + seed: { kind: "property", name: "invoke" }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: false, + expected: null, + }, + }, + }); + expect(semantic.isError).not.toBe(true); + return { + events: [ + toolEvent("notes-analysis", "analyze_javascript_application", analysis), + toolEvent("notes-feature", "trace_application_feature", feature), + toolEvent("notes-semantic", "trace_javascript_semantics", semantic), + ], + evidence: new Map([ + ["analyze_javascript_application", application], + ["trace_application_feature", resultEvidence(feature)], + ["trace_javascript_semantics", resultEvidence(semantic)], + ]), + }; +}; + +const claimManifestByOperation = ( + claims: readonly FixtureClaimExpectation[], + evidenceByOperation: ReadonlyMap, +) => ({ + claims: claims.map(({ id, expectedValue, source }) => { + const evidence = evidenceByOperation.get(source.operation); + if (evidence === undefined) + throw new Error(`Missing producer for claim ${id}`); + return { + id, + value: expectedValue, + evidence_id: evidence.evidence_id, + authority: evidence.authority, + confidence: evidence.confidence, + }; + }), +}); + const resultEvidence = (result: { readonly structuredContent?: unknown; }): Evidence => parseEvidence(result.structuredContent);