Merge branch 'main' into codex/review-fixes-pr-2457

The captured main baseline has an unformatted helper signature. Apply the
repository formatter so the required style gate can run to completion.

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
(cherry picked from commit b5a25fbdda)

docs/testing.md

@@ -389,8 +389,9 @@ adds in-place reparse, breakaway, environment, and token observations.
 The separate `scripts/verify-windows-native-inspection.mjs` lane uses the
 `.cache/windows-native-inspection-fixture` artifact built on Linux with
 `node scripts/build-windows-native.mjs --inspection-fixture`. Its controlled
-first deletion failure checks retained NTFS ownership, later-query recovery,
-successful-admission caching, and environment-teardown cleanup. It neither
+first deletion failure checks retained NTFS ownership across a same-volume
+rename, later-query handle-based recovery, successful-admission caching, and
+environment-teardown cleanup. It neither
 establishes the cause of the intermittent #1701 host failure nor replaces
 verification with the shipped production artifact.
 `npm run verify:ghidra:windows:package` additionally packs and installs REA into

scripts/ci/plan.mjs

@@ -197,6 +197,9 @@ if (tier === "reused") {
         if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
         if (file !== ".github/workflows/ci.yml") continue;
       }
+      // The Notes example is also the source-owned agent evaluation fixture.
+      if (file.startsWith("website/public/examples/notes-electron/"))
+        selected.code = true;
       if (owners.length === 1 && owners[0] === "website") continue;
       if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
       if (file === "package.json") {

scripts/verify-agent-experience.mjs

@@ -155,6 +155,18 @@ try {
       requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
       prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript}，說明 renderer 如何透過 preload 讀取 profile，以及 main handler 的關係。不要執行目標或安裝分析引擎；引用完整的產出 Evidence，保留 runtime 未知事項。`,
     },
+    {
+      id: "notes-export",
+      expectedFirstTool: "analyze_javascript_application",
+      requiresEvidence: true,
+      requiredAnswerTermGroups: [],
+      requiredToolSubsequence: [
+        "analyze_javascript_application",
+        "trace_application_feature",
+        "trace_javascript_semantics",
+      ],
+      prompt: `Use REA on the shipped Electron application at ${targets.notesElectron} to investigate its notes export API, preload exposure and main handler. Do not read target files directly or launch Electron. Use the returned application Evidence to trace the exact notes:export IPC channel in both directions, then query semantic relations for the property invoke with forward influence, excluding ambiguous dynamic edges and without an expected sink. Report the selected static facts, source locations and coverage limits using the closed claim contract.`,
+    },
     {
       id: "javascript-module-view",
       expectedFirstTool: "analyze_javascript_application",

scripts/verify-windows-native-inspection.mjs

@@ -62,8 +62,11 @@ assert.equal(first.retryable,true);
 assert.match(first.reason,/Delete owned object by handle failed.*Win32 5/s);
 const [name,...extra]=readdirSync(temp);assert.ok(name);assert.equal(extra.length,0);
 const path=join(temp,name);const identity=statSync(path,{bigint:true});
-assert.throws(()=>renameSync(path,path+'-replacement'));
-assert.equal(statSync(path,{bigint:true}).ino,identity.ino);
+// This private directory permits rename while its retained handle still owns
+// cleanup. Recovery must follow the object identity after its path changes.
+const replacement=path+'-replacement';renameSync(path,replacement);
+assert.equal(statSync(replacement,{bigint:true}).ino,identity.ino);
+assert.equal(readdirSync(temp).length,1);
 if(process.argv[2]!=='teardown'){
  const second=systemWindowsNativeAuthority();assert.equal(second.available,true);
  assert.equal(readdirSync(temp).length,0);

src/evaluation/AgentEvaluationFixtures.ts

@@ -8,6 +8,11 @@ import type { JsonValue } from "../domain/jsonValue.js";
 import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";
 import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js";
 import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";
+import {
+  applicationFeatureTraceResultSchema,
+  featureSeedMatchMode,
+} from "../domain/javascript/javascriptFeatureTraceSchemas.js";
+import { javaScriptSemanticTraceResultSchema } from "../domain/javascript/javascriptSemanticTraceSchemas.js";
 import {
   managedArtifactInspectionSchema,
   managedMemberInspectionSchema,
@@ -28,6 +33,8 @@ export interface AgentEvaluationFixtureTargets {
   readonly javascriptSha256: string;
   readonly javascriptShapeLeft: string;
   readonly javascriptShapeRight: string;
+  readonly notesElectron: string;
+  readonly notesElectronSha256: string;
   readonly managed?: string;
   readonly managedSha256?: string;
 }
@@ -50,6 +57,8 @@ export const createAgentEvaluationFixtures = async (
   const javascript = join(root, "desktop-app.asar");
   const javascriptShapeLeft = join(root, "parser-v1");
   const javascriptShapeRight = join(root, "parser-v2");
+  const notesDirectory = join(root, "notes-electron");
+  const notesElectron = join(root, "notes-electron.asar");
   await Promise.all([
     mkdir(join(desktop, "renderer"), { recursive: true }),
     mkdir(javascriptShapeLeft, { recursive: true }),
@@ -67,8 +76,14 @@ export const createAgentEvaluationFixtures = async (
       join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"),
       join(javascriptShapeRight, "parser.mjs"),
     ),
+    cp(
+      join(repositoryRoot, "website/public/examples/notes-electron"),
+      notesDirectory,
+      { recursive: true },
+    ),
   ]);
   await createPackage(desktop, javascript);
+  await createPackage(notesDirectory, notesElectron);
   return {
     javascript,
     javascriptSha256: createHash("sha256")
@@ -76,6 +91,10 @@ export const createAgentEvaluationFixtures = async (
       .digest("hex"),
     javascriptShapeLeft,
     javascriptShapeRight,
+    notesElectron,
+    notesElectronSha256: createHash("sha256")
+      .update(await readFile(notesElectron))
+      .digest("hex"),
   };
 };
 
@@ -189,21 +208,116 @@ const runtimeSemantics = (value: JsonValue): JsonValue | undefined => {
     : undefined;
 };
 
-const asarClaims = (
+const featureModules = (value: JsonValue): JsonValue | undefined => {
+  const result = applicationFeatureTraceResultSchema.safeParse(value);
+  if (!result.success || result.data.graph === null) return undefined;
+  if (
+    result.data.seed.kind !== "channel" ||
+    result.data.seed.value !== "notes:export" ||
+    featureSeedMatchMode(result.data.seed) !== "exact"
+  )
+    return undefined;
+  const paths = result.data.graph.nodes
+    .filter((node) => node.kind === "javascript-module")
+    .flatMap((node) => node.observations)
+    .filter(({ properties }) => properties.semantic_role === "source-module")
+    .map(({ properties }) => properties.path);
+  if (paths.length === 0 || !paths.every((path) => typeof path === "string"))
+    return undefined;
+  return {
+    paths: [...new Set(paths)].sort(compareUnicodeCodePoints),
+    coverage: result.data.coverage.status,
+  };
+};
+
+const semanticBoundary = (value: JsonValue): JsonValue | undefined => {
+  const result = javaScriptSemanticTraceResultSchema.safeParse(value);
+  if (!result.success) return undefined;
+  return {
+    status: result.data.status,
+    coverage: result.data.coverage.status,
+    relevant_unknowns: result.data.summary.relevant_unknowns,
+  };
+};
+
+const notesClaims = (
   targets: AgentEvaluationFixtureTargets,
+): readonly FixtureClaimExpectation[] => {
+  const subject = {
+    path: targets.notesElectron,
+    sha256: targets.notesElectronSha256,
+  };
+  const derived = {
+    authority: "analyst-inference",
+    confidence: "inferred",
+    linkedSubjects: [
+      { operation: "analyze_javascript_application", ...subject },
+    ],
+  } satisfies Pick<
+    FixtureClaimExpectation["source"],
+    "authority" | "confidence" | "linkedSubjects"
+  >;
+  // Fixture source establishes bridge/module facts; the semantic claim checks
+  // the bounded static-query contract, not execution or CSV formatter behavior.
+  return [
+    ...bridgeClaims(subject, "notes", ["exportCsv"], "notes:export"),
+    {
+      id: "feature_modules",
+      expectedValue: {
+        paths: ["csv.js", "main.js", "preload.js", "renderer.js"],
+        coverage: "complete-within-source",
+      },
+      source: {
+        ...derived,
+        operation: "trace_application_feature",
+        parameters: {
+          direction: "both",
+        },
+        select: featureModules,
+      },
+    },
+    {
+      id: "semantic_boundary",
+      expectedValue: {
+        status: "ambiguous",
+        coverage: "partial",
+        relevant_unknowns: 1,
+      },
+      source: {
+        ...derived,
+        operation: "trace_javascript_semantics",
+        parameters: {
+          query: {
+            seed: { kind: "property", name: "invoke" },
+            direction: "forward-influence",
+            include_ambiguous_dynamic_edges: false,
+            expected: null,
+          },
+        },
+        select: semanticBoundary,
+      },
+    },
+  ];
+};
+
+const bridgeClaims = (
+  subject: { readonly path: string; readonly sha256: string },
+  key: string,
+  members: readonly string[],
+  channel: string,
 ): readonly FixtureClaimExpectation[] => {
   const source = {
     operation: "analyze_javascript_application",
     authority: "shipped-artifact",
     confidence: "derived",
-    subject: { path: targets.javascript, sha256: targets.javascriptSha256 },
+    subject,
   } satisfies Omit<FixtureClaimExpectation["source"], "select">;
   return [
     {
       id: "renderer_api",
       expectedValue: {
-        key: "profileApi",
-        members: ["read"],
+        key,
+        members: [...members],
         world: "main",
         source: "preload.js",
       },
@@ -212,7 +326,7 @@ const asarClaims = (
     {
       id: "renderer_ipc",
       expectedValue: {
-        channel: "profile:read",
+        channel,
         operation: "invoke",
         source: "preload.js",
       },
@@ -221,7 +335,7 @@ const asarClaims = (
     {
       id: "main_ipc",
       expectedValue: {
-        channel: "profile:read",
+        channel,
         operation: "handle",
         source: "main.js",
       },
@@ -235,6 +349,16 @@ const asarClaims = (
   ];
 };
 
+const asarClaims = (
+  targets: AgentEvaluationFixtureTargets,
+): readonly FixtureClaimExpectation[] =>
+  bridgeClaims(
+    { path: targets.javascript, sha256: targets.javascriptSha256 },
+    "profileApi",
+    ["read"],
+    "profile:read",
+  );
+
 const parserClaims = (
   targets: AgentEvaluationFixtureTargets,
 ): readonly FixtureClaimExpectation[] => {
@@ -399,25 +523,33 @@ export const agentFixtureClaims = (
   if (scenarioId === "asar") return asarClaims(targets);
   if (scenarioId === "managed") return managedClaims(targets);
   if (scenarioId === "javascript-export-shape") return parserClaims(targets);
+  if (scenarioId === "notes-export") return notesClaims(targets);
   return undefined;
 };
 
+const bridgeAnswerDescriptions = [
+  'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
+  'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',
+  'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.',
+  'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.',
+];
+
 const answerValueDescriptions: Readonly<Record<string, readonly string[]>> = {
   managed: [
     "public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).",
     'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.',
   ],
-  asar: [
-    'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
-    'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',
-    'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.',
-    'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.',
-  ],
+  asar: bridgeAnswerDescriptions,
   "javascript-export-shape": [
     'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.',
     'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.',
     'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.',
   ],
+  "notes-export": [
+    ...bridgeAnswerDescriptions,
+    'feature_modules: {"paths": string[], "coverage": string}, listing the unique source-module observation paths in the feature trace in Unicode code-point order, and its coverage status.',
+    'semantic_boundary: {"status": string, "coverage": string, "relevant_unknowns": number}, copying the semantic trace status, coverage status, and summary.relevant_unknowns. Static traces do not establish runtime execution.',
+  ],
 };
 
 /** Describe the strict answer contract without revealing any golden fixture values. */

tests/boundary/filesystem/ciChangeScope.test.ts

@@ -476,6 +476,10 @@ it.each([
     { code: true, runtime: false, browser: false, package: false },
   ],
   ["website/src/index.ts", { code: false, website: true, package: false }],
+  [
+    "website/public/examples/notes-electron/preload.js",
+    { code: true, website: true, package: false },
+  ],
   [
     "website/locales/zh-CN/messages.po",
     { code: false, website: true, package: false },

tests/boundary/mcp/agentKnownAnswerMcp.test.ts

@@ -245,6 +245,334 @@ describe("source-produced parser comparison known answers", () => {
   });
 });
 
+describe("Notes export known-answer evaluation", () => {
+  it("grades real analysis and traces while rejecting false values and unsupported citations", async () => {
+    const root = await createTestTempDirectory("rea-known-answer-notes-");
+    const targets = await createAgentEvaluationFixtures(root, resolve("."));
+    const claims = agentFixtureClaims("notes-export", targets);
+    if (claims === undefined) throw new Error("Missing Notes rubric");
+    expect(claims.map(({ id }) => id)).toEqual([
+      "renderer_api",
+      "renderer_ipc",
+      "main_ipc",
+      "preload",
+      "feature_modules",
+      "semantic_boundary",
+    ]);
+    const harness = await createApplicationMcpHarness();
+    try {
+      const workflow = await notesWorkflow(harness, targets.notesElectron);
+      assertNotesAnswerFailures(workflow, claims);
+      assertNotesUnsupportedClaims(workflow, claims);
+      await assertNotesSeedBinding(harness, workflow, claims);
+    } finally {
+      await harness.close();
+    }
+  });
+
+  it("binds Notes analysis and derived traces to the requested artifact path and bytes", async () => {
+    const root = await createTestTempDirectory(
+      "rea-known-answer-notes-identity-",
+    );
+    const targets = await createAgentEvaluationFixtures(root, resolve("."));
+    const claims = agentFixtureClaims("notes-export", targets);
+    if (claims === undefined) throw new Error("Missing Notes rubric");
+    const foreign = join(root, "other-notes.asar");
+    await cp(targets.notesElectron, foreign);
+    const harness = await createApplicationMcpHarness();
+    try {
+      for (const path of [foreign, targets.notesElectron]) {
+        if (path === targets.notesElectron) {
+          await appendFile(
+            join(root, "notes-electron/renderer.js"),
+            "\n// changed shipped bytes\n",
+          );
+          await createPackage(join(root, "notes-electron"), path);
+        }
+        const workflow = await notesWorkflow(harness, path);
+        const digest = notesEvidence(workflow, "analyze_javascript_application")
+          .subject?.digest.sha256;
+        expect(digest === targets.notesElectronSha256).toBe(path === foreign);
+        const assessment = grade(
+          workflow.events,
+          claimManifestByOperation(claims, workflow.evidence),
+          "analyze_javascript_application",
+          claims,
+        ).factualAssessment;
+        expect(assessment.status).toBe("failed");
+        assertClaimFailure(
+          assessment,
+          "renderer_api",
+          "source_subject_mismatch",
+        );
+        for (const id of ["feature_modules", "semantic_boundary"])
+          assertClaimFailure(assessment, id, "source_links_mismatch");
+      }
+    } finally {
+      await harness.close();
+    }
+  });
+});
+
+type NotesWorkflow = Awaited<ReturnType<typeof notesWorkflow>>;
+
+const notesEvidence = (
+  workflow: NotesWorkflow,
+  operation: string,
+): Evidence => {
+  const evidence = workflow.evidence.get(operation);
+  if (evidence === undefined)
+    throw new Error(`Missing Notes ${operation} Evidence`);
+  return evidence;
+};
+
+const assertClaimFailure = (
+  assessment: ReturnType<typeof grade>["factualAssessment"],
+  id: string,
+  failure: string,
+) => {
+  expect(assessment.claims).toContainEqual(
+    expect.objectContaining({
+      id,
+      failures: expect.arrayContaining([failure]),
+    }),
+  );
+};
+
+const assertNotesAnswerFailures = (
+  workflow: NotesWorkflow,
+  claims: readonly FixtureClaimExpectation[],
+) => {
+  const manifest = claimManifestByOperation(claims, workflow.evidence);
+  const assess = (answer: unknown, events = workflow.events) =>
+    grade(events, answer, "analyze_javascript_application", claims)
+      .factualAssessment;
+  expect(assess(manifest).status).toBe("passed");
+  const cases = [
+    {
+      id: "renderer_ipc",
+      patch: {
+        value: {
+          channel: "notes:delete",
+          operation: "send",
+          source: "preload.js",
+        },
+      },
+      failure: "value_mismatch",
+    },
+    {
+      id: "semantic_boundary",
+      patch: {
+        value: { status: "found", coverage: "complete", relevant_unknowns: 0 },
+      },
+      failure: "value_mismatch",
+    },
+    {
+      id: "renderer_api",
+      patch: {
+        evidence_id: notesEvidence(workflow, "trace_application_feature")
+          .evidence_id,
+      },
+      failure: "source_operation_mismatch",
+    },
+  ];
+  for (const { id, patch, failure } of cases) {
+    const answer = {
+      claims: manifest.claims.map((claim) =>
+        claim.id === id ? { ...claim, ...patch } : claim,
+      ),
+    };
+    assertClaimFailure(assess(answer), id, failure);
+  }
+  const missingParent = assess(
+    manifest,
+    workflow.events.filter((event) => event.item.id !== "notes-analysis"),
+  );
+  for (const id of ["feature_modules", "semantic_boundary"])
+    assertClaimFailure(missingParent, id, "source_links_mismatch");
+};
+
+const assertNotesUnsupportedClaims = (
+  workflow: NotesWorkflow,
+  claims: readonly FixtureClaimExpectation[],
+) => {
+  const manifest = claimManifestByOperation(claims, workflow.evidence);
+  const evidenceId = notesEvidence(
+    workflow,
+    "trace_application_feature",
+  ).evidence_id;
+  const unsupportedRuntime = {
+    claims: [
+      ...manifest.claims,
+      {
+        id: "runtime_export",
+        value: "A click wrote notes.csv to the actual Downloads directory.",
+        evidence_id: evidenceId,
+        authority: "controlled-replay",
+        confidence: "observed",
+      },
+    ],
+  };
+  expect(
+    grade(
+      workflow.events,
+      unsupportedRuntime,
+      "analyze_javascript_application",
+      claims,
+    ).factualAssessment.manifestFailures,
+  ).toContain("unknown_claim:runtime_export");
+  const prose = evaluateCodexEvents(
+    [
+      ...workflow.events,
+      {
+        type: "item.completed",
+        item: {
+          type: "agent_message",
+          text: `Observed Evidence ${evidenceId}: notes:export, preload.js and main.js do not exist. The export certainly executed and wrote notes.csv; runtime behavior is known without further observation.`,
+        },
+      },
+    ],
+    "analyze_javascript_application",
+    {
+      requireEvidence: true,
+      requiredAnswerTermGroups: [["notes:export"], ["preload.js"], ["main.js"]],
+      fixtureClaims: claims,
+    },
+  );
+  expect(prose).toMatchObject({
+    finalCitesEvidence: true,
+    answerTermCoverageMet: true,
+    factualCorrectness: "failed",
+  });
+  expect(prose.factualAssessment.manifestFailures).toContain(
+    "invalid_final_claim_manifest",
+  );
+};
+
+const assertNotesSeedBinding = async (
+  harness: Awaited<ReturnType<typeof createApplicationMcpHarness>>,
+  workflow: NotesWorkflow,
+  claims: readonly FixtureClaimExpectation[],
+) => {
+  const manifest = claimManifestByOperation(claims, workflow.evidence);
+  for (const match of ["contains", undefined] as const) {
+    const feature = await harness.client.callTool({
+      name: "trace_application_feature",
+      arguments: {
+        application: notesEvidence(workflow, "analyze_javascript_application"),
+        seed: {
+          kind: "channel",
+          value: "notes:export",
+          ...(match === undefined ? {} : { match }),
+          case_sensitive: false,
+        },
+        direction: "both",
+      },
+    });
+    expect(feature.isError).not.toBe(true);
+    const answer = {
+      claims: manifest.claims.map((claim) =>
+        claim.id === "feature_modules"
+          ? { ...claim, evidence_id: resultEvidence(feature).evidence_id }
+          : claim,
+      ),
+    };
+    const assessment = grade(
+      [
+        ...workflow.events,
+        toolEvent(
+          `notes-feature-${match ?? "default"}`,
+          "trace_application_feature",
+          feature,
+        ),
+      ],
+      answer,
+      "analyze_javascript_application",
+      claims,
+    ).factualAssessment;
+    expect(assessment.status).toBe(match === undefined ? "passed" : "failed");
+    if (match === "contains") {
+      assertClaimFailure(
+        assessment,
+        "feature_modules",
+        "source_assertion_mismatch",
+      );
+      expect(
+        assessment.claims.find(({ id }) => id === "feature_modules")?.failures,
+      ).not.toContain("source_parameters_mismatch");
+    }
+  }
+};
+
+const notesWorkflow = async (
+  harness: Awaited<ReturnType<typeof createApplicationMcpHarness>>,
+  inputPath: string,
+) => {
+  const analysis = await harness.client.callTool({
+    name: "analyze_javascript_application",
+    arguments: { input_path: inputPath },
+  });
+  expect(analysis.isError).not.toBe(true);
+  const application = resultEvidence(analysis);
+  const feature = await harness.client.callTool({
+    name: "trace_application_feature",
+    arguments: {
+      application,
+      seed: {
+        kind: "channel",
+        value: "notes:export",
+        match: "exact",
+        case_sensitive: false,
+      },
+      direction: "both",
+    },
+  });
+  expect(feature.isError).not.toBe(true);
+  const semantic = await harness.client.callTool({
+    name: "trace_javascript_semantics",
+    arguments: {
+      application,
+      query: {
+        seed: { kind: "property", name: "invoke" },
+        direction: "forward-influence",
+        include_ambiguous_dynamic_edges: false,
+        expected: null,
+      },
+    },
+  });
+  expect(semantic.isError).not.toBe(true);
+  return {
+    events: [
+      toolEvent("notes-analysis", "analyze_javascript_application", analysis),
+      toolEvent("notes-feature", "trace_application_feature", feature),
+      toolEvent("notes-semantic", "trace_javascript_semantics", semantic),
+    ],
+    evidence: new Map<string, Evidence>([
+      ["analyze_javascript_application", application],
+      ["trace_application_feature", resultEvidence(feature)],
+      ["trace_javascript_semantics", resultEvidence(semantic)],
+    ]),
+  };
+};
+
+const claimManifestByOperation = (
+  claims: readonly FixtureClaimExpectation[],
+  evidenceByOperation: ReadonlyMap<string, Evidence>,
+) => ({
+  claims: claims.map(({ id, expectedValue, source }) => {
+    const evidence = evidenceByOperation.get(source.operation);
+    if (evidence === undefined)
+      throw new Error(`Missing producer for claim ${id}`);
+    return {
+      id,
+      value: expectedValue,
+      evidence_id: evidence.evidence_id,
+      authority: evidence.authority,
+      confidence: evidence.confidence,
+    };
+  }),
+});
+
 const resultEvidence = (result: {
   readonly structuredContent?: unknown;
 }): Evidence => parseEvidence(result.structuredContent);

tests/evaluation/model-evals/knownAnswerEvaluation.test.ts

@@ -140,6 +140,8 @@ const managedTranscript = () => {
     javascriptSha256: "",
     javascriptShapeLeft: "",
     javascriptShapeRight: "",
+    notesElectron: "",
+    notesElectronSha256: "",
     managed: target.path,
     managedSha256: target.sha256,
   });