test(evaluation): grade managed agent fixture facts (#1924)

* test(evaluation): grade managed agent fixture facts

* test(evaluation): cover distinct managed artifact deliveries

* fix(evaluation): require unfiltered managed type inventories

* fix(evaluation): preserve selected entry point grounding

---------

Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>

docs/client-verification.md

@@ -458,7 +458,7 @@ artifact path. Its `expectedFirstTool` is `null`, and
 `targetClarificationPassed` grades that limited routing outcome independently
 of analysis prose heuristics. Analyzing a nearby example cannot pass this case.
 
-The packaged desktop and parser-comparison scenarios now assess a closed set of
+The packaged desktop, parser-comparison, and managed scenarios assess a closed set of
 known fixture claims. Each final answer must be a strict JSON manifest containing
 exactly the requested claim IDs, values, Evidence IDs, and Evidence authority and
 confidence. Golden values are not included in the prompt. The evaluator checks
@@ -475,15 +475,24 @@ analyses and use the requested module and export selectors. Boundary tests packa
 the same source-owned desktop fixture and compare the same source-owned parser
 files through actual MCP tool results, then verify that fabricated answers fail.
 
+The managed rubric checks the exact public top-level type set and the CLI entry
+point's declaring type and method name. The exact type-set claim requires an
+unfiltered member inventory: `selection` must be null and the Evidence parameters
+must omit `type_names`. A selected inventory can still ground the entry-point
+claim. Both inspections must describe the freshly built DLL's SHA-256. Entry-point
+resolution additionally requires complete artifact and member metadata, matching
+non-null module MVIDs, and a unique method at the header's metadata token. These are
+static metadata claims, not runtime behavior;
+compiler-generated member counts, token values, and MVIDs are not golden answers.
+
 Per-scenario `factualCorrectness` is `passed`, `failed`, or `not_assessed` within
 the `configured_fixture_claims` scope. A pass establishes only the configured
-claims, not unrestricted factual correctness. Native, managed, browser,
+claims, not unrestricted factual correctness. Native, browser,
 navigation-context, and address-context scenarios have no factual rubric and
-remain `not_assessed`; the managed workflow requires both artifact and member
-inspection to answer its type and entry-point question.
+remain `not_assessed`.
 
 The closed factual scenarios request complete producer Evidence on the initial
-analysis so their source selectors can authenticate all configured facts.
+inspection or analysis so their source selectors can authenticate all configured facts.
 Summary/page/item workflows are exercised separately by `javascript-module-view`;
 its heuristic gate does not establish full-producer factual correctness.
 

scripts/verify-agent-experience.mjs

@@ -1,4 +1,5 @@
 import { spawn } from "node:child_process";
+import { createHash } from "node:crypto";
 import {
   chmod,
   copyFile,
@@ -259,7 +260,7 @@ try {
     const execution = await runCodex(
       answerInstructions.length === 0
         ? scenario.prompt
-        : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
+        : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
       scenario.approvedTools ?? [],
     );
     const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
@@ -440,6 +441,13 @@ async function createTargets(root, includeManaged) {
     ...fixtures,
     largeJavascript,
     managed: join(managedOutput, "AgentEval.dll"),
+    ...(includeManaged
+      ? {
+          managedSha256: createHash("sha256")
+            .update(await readFile(join(managedOutput, "AgentEval.dll")))
+            .digest("hex"),
+        }
+      : {}),
   };
 }
 

src/evaluation/AgentEvaluationFixtures.ts

@@ -8,7 +8,19 @@ import type { JsonValue } from "../domain/jsonValue.js";
 import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";
 import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js";
 import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";
-import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js";
+import {
+  managedArtifactInspectionSchema,
+  managedMemberInspectionSchema,
+} from "../domain/managed/managedArtifact.js";
+import {
+  managedTokenScopesMatch,
+  parseManagedInspectionEvidence,
+} from "../domain/managed/managedInspectionEvidence.js";
+import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
+import type {
+  FixtureClaimContext,
+  FixtureClaimExpectation,
+} from "./KnownAnswerEvaluation.js";
 
 /** Source-owned artifact paths used by the configured known-answer benchmarks. */
 export interface AgentEvaluationFixtureTargets {
@@ -16,6 +28,8 @@ export interface AgentEvaluationFixtureTargets {
   readonly javascriptSha256: string;
   readonly javascriptShapeLeft: string;
   readonly javascriptShapeRight: string;
+  readonly managed?: string;
+  readonly managedSha256?: string;
 }
 
 const desktopFixtureFiles = {
@@ -277,17 +291,122 @@ const parserClaims = (
   ];
 };
 
+const managedMembers = ({ evidence }: FixtureClaimContext) => {
+  const { result } = parseManagedInspectionEvidence(
+    evidence,
+    "inspect_managed_members",
+    managedMemberInspectionSchema,
+  );
+  if (
+    result.coverage.state !== "complete" ||
+    result.metadata.status !== "complete" ||
+    !managedTokenScopesMatch(
+      result.identity_scope.requires_mvid,
+      result.module?.mvid ?? null,
+    )
+  )
+    return undefined;
+  return result;
+};
+
+const managedClaims = (
+  targets: AgentEvaluationFixtureTargets,
+): readonly FixtureClaimExpectation[] => {
+  if (targets.managed === undefined || targets.managedSha256 === undefined)
+    throw new TypeError(
+      "Managed fixture grading requires the built DLL path and SHA-256",
+    );
+  const source = {
+    operation: "inspect_managed_members",
+    authority: "shipped-artifact",
+    confidence: "observed",
+    subject: { path: targets.managed, sha256: targets.managedSha256 },
+  } satisfies Omit<FixtureClaimExpectation["source"], "select">;
+  return [
+    {
+      id: "public_types",
+      expectedValue: ["AgentEval.Profile", "AgentEval.Program"],
+      source: {
+        ...source,
+        select: (_value, context) => {
+          const members = managedMembers(context);
+          if (
+            members === undefined ||
+            members.selection !== null ||
+            Object.hasOwn(context.evidence.parameters, "type_names")
+          )
+            return undefined;
+          return members.types
+            .filter((type) => (type.flags & 7) === 1)
+            .map((type) => type.full_name)
+            .sort(compareUnicodeCodePoints);
+        },
+      },
+    },
+    {
+      id: "entry_point",
+      expectedValue: { declaring_type: "AgentEval.Program", name: "Main" },
+      source: {
+        ...source,
+        select: (_value, context) => {
+          const members = managedMembers(context);
+          if (members === undefined) return undefined;
+          const artifacts = context.deliveredEvidence.filter(
+            (evidence) =>
+              evidence.operation === "inspect_managed_artifact" &&
+              evidence.authority === source.authority &&
+              evidence.confidence === source.confidence &&
+              evidence.subject?.local_path === source.subject.path &&
+              evidence.subject.digest.sha256 === source.subject.sha256,
+          );
+          if (artifacts.length !== 1) return undefined;
+          const { result: artifact } = parseManagedInspectionEvidence(
+            artifacts[0],
+            "inspect_managed_artifact",
+            managedArtifactInspectionSchema,
+          );
+          if (
+            artifact.coverage.state !== "complete" ||
+            artifact.metadata.status !== "complete" ||
+            !managedTokenScopesMatch(
+              artifact.module?.mvid ?? null,
+              members.module?.mvid ?? null,
+            ) ||
+            artifact.pe.cli?.entry_point.kind !== "metadata-token"
+          )
+            return undefined;
+          const methods = members.methods.filter(
+            (method) => method.token === artifact.pe.cli?.entry_point.value,
+          );
+          const method = methods.length === 1 ? methods[0] : undefined;
+          return method === undefined
+            ? undefined
+            : {
+                declaring_type: method.declaring_type,
+                name: method.name,
+              };
+        },
+      },
+    },
+  ];
+};
+
 /** Return the closed known-answer rubric, or undefined for routing-only scenarios. */
 export const agentFixtureClaims = (
   scenarioId: string,
   targets: AgentEvaluationFixtureTargets,
 ): readonly FixtureClaimExpectation[] | undefined => {
   if (scenarioId === "asar") return asarClaims(targets);
+  if (scenarioId === "managed") return managedClaims(targets);
   if (scenarioId === "javascript-export-shape") return parserClaims(targets);
   return undefined;
 };
 
 const answerValueDescriptions: Readonly<Record<string, readonly string[]>> = {
+  managed: [
+    "public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).",
+    'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.',
+  ],
   asar: [
     'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
     'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',

src/evaluation/KnownAnswerEvaluation.ts

@@ -5,6 +5,12 @@ import { parseEvidence, type Evidence } from "../domain/evidence.js";
 import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
 import { canonicalJson } from "../domain/comparisonSemantics.js";
 
+/** Only authenticated, successful, non-conflicting transcript deliveries. */
+export interface FixtureClaimContext {
+  readonly evidence: Evidence;
+  readonly deliveredEvidence: readonly Evidence[];
+}
+
 /** One closed fixture assertion and the producer facts that must support it. */
 export interface FixtureClaimExpectation {
   readonly id: string;
@@ -20,7 +26,10 @@ export interface FixtureClaimExpectation {
       readonly path: string;
       readonly sha256?: string;
     }[];
-    readonly select: (normalizedResult: JsonValue) => JsonValue | undefined;
+    readonly select: (
+      normalizedResult: JsonValue,
+      context: FixtureClaimContext,
+    ) => JsonValue | undefined;
   };
 }
 
@@ -430,7 +439,12 @@ const assessClaim = (
     try {
       if (
         !sameJson(
-          expected.source.select(evidence.normalized_result),
+          expected.source.select(evidence.normalized_result, {
+            evidence,
+            deliveredEvidence: [...delivered.records.values()].filter(
+              (record) => !delivered.conflicts.has(record.evidence_id),
+            ),
+          }),
           expected.expectedValue,
         )
       )

tests/evaluation/model-evals/knownAnswerEvaluation.test.ts

@@ -5,6 +5,16 @@ import { toolContract } from "../../../src/contracts/toolContracts.js";
 import { createEvidence, type Evidence } from "../../../src/domain/evidence.js";
 import type { JsonValue } from "../../../src/domain/jsonValue.js";
 import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js";
+import {
+  agentFixtureAnswerInstructions,
+  agentFixtureClaims,
+} from "../../../src/evaluation/AgentEvaluationFixtures.js";
+import { inspectManagedArtifactBytes } from "../../../src/dotnet/ManagedArtifactInspector.js";
+import { inspectManagedMembersBytes } from "../../../src/dotnet/ManagedMemberInspector.js";
+import {
+  buildManagedPeFixture,
+  managedPeFixtureTarget,
+} from "../../../src/dotnet/ManagedPe.fixture.js";
 import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js";
 import { ToolResultDelivery } from "../../../src/server/toolResult.js";
 
@@ -33,7 +43,13 @@ const createObservation = (
 const resultFor = (evidence: Evidence) =>
   delivery.toEvidenceToolResult(
     evidence,
-    toolContract("analyze_javascript_application"),
+    toolContract(
+      evidence.operation === "inspect_managed_artifact"
+        ? "inspect_managed_artifact"
+        : evidence.operation === "inspect_managed_members"
+          ? "inspect_managed_members"
+          : "analyze_javascript_application",
+    ),
     undefined,
   );
 const delivered = (
@@ -95,6 +111,336 @@ const assess = (
   );
 const answer = (claims: readonly unknown[]) => JSON.stringify({ claims });
 
+// Synthetic transcript cases exercise grading, not an SDK-built assembly oracle.
+const managedTranscript = () => {
+  const bytes = buildManagedPeFixture({
+    typeNamespace: "AgentEval",
+    typeName: "Program",
+    methodName: "Main",
+  });
+  const target = managedPeFixtureTarget(bytes, "/targets/AgentEval.dll");
+  const artifact = inspectManagedArtifactBytes(bytes, target);
+  const members = inspectManagedMembersBytes(bytes, target);
+  const program = members.types[0];
+  if (program === undefined) throw new Error("Fixture type missing");
+  const result = {
+    ...members,
+    types: [
+      program,
+      {
+        ...program,
+        token: "0x02000002",
+        name: "Profile",
+        full_name: "AgentEval.Profile",
+      },
+    ],
+  };
+  const rubric = agentFixtureClaims("managed", {
+    javascript: "",
+    javascriptSha256: "",
+    javascriptShapeLeft: "",
+    javascriptShapeRight: "",
+    managed: target.path,
+    managedSha256: target.sha256,
+  });
+  if (rubric === undefined) throw new Error("Managed rubric missing");
+  const evidenceFor = (
+    operation: string,
+    value: JsonValue,
+    overrides: Record<string, unknown> = {},
+  ) =>
+    createEvidence(target, provider, {
+      operation,
+      parameters: {},
+      result: value,
+      confidence: "observed",
+      authority: "shipped-artifact",
+      ...overrides,
+    });
+  const artifactEvidence = evidenceFor("inspect_managed_artifact", artifact);
+  const memberEvidence = evidenceFor("inspect_managed_members", result);
+  const textFor = (evidence: Evidence) =>
+    answer([
+      {
+        id: "public_types",
+        value: ["AgentEval.Profile", "AgentEval.Program"],
+        evidence_id: evidence.evidence_id,
+        confidence: "observed",
+        authority: "shipped-artifact",
+      },
+      {
+        id: "entry_point",
+        value: { declaring_type: "AgentEval.Program", name: "Main" },
+        evidence_id: evidence.evidence_id,
+        confidence: "observed",
+        authority: "shipped-artifact",
+      },
+    ]);
+  const grade = (
+    events = [delivered(artifactEvidence), delivered(memberEvidence)],
+    text = textFor(memberEvidence),
+  ) =>
+    evaluateCodexEvents([...events, final(text)], "inspect_managed_artifact", {
+      fixtureClaims: rubric,
+    });
+  return {
+    artifact,
+    result,
+    artifactEvidence,
+    memberEvidence,
+    evidenceFor,
+    textFor,
+    grade,
+  };
+};
+
+describe("managed known-answer fixture grading", () => {
+  it("grades public types independent of row order and resolves the header token", () => {
+    expect(managedTranscript().grade().factualCorrectness).toBe("passed");
+  });
+
+  it("does not pin compiler-local entry tokens or MVIDs", () => {
+    const fixture = managedTranscript();
+    const artifact = structuredClone(fixture.artifact);
+    const members = structuredClone(fixture.result);
+    if (
+      artifact.pe.cli === null ||
+      artifact.module === null ||
+      members.module === null ||
+      members.methods[0] === undefined
+    )
+      throw new Error("Fixture identity missing");
+    artifact.pe.cli.entry_point.value = "0x06000009";
+    members.methods[0].token = "0x06000009";
+    const mvid = "11111111-2222-3333-4444-555555555555";
+    artifact.module.mvid = mvid;
+    members.module.mvid = mvid;
+    members.identity_scope.requires_mvid = mvid;
+    members.types.reverse();
+    const artifactEvidence = fixture.evidenceFor(
+      "inspect_managed_artifact",
+      artifact,
+    );
+    const memberEvidence = fixture.evidenceFor(
+      "inspect_managed_members",
+      members,
+    );
+    expect(
+      fixture.grade(
+        [delivered(artifactEvidence), delivered(memberEvidence)],
+        fixture.textFor(memberEvidence),
+      ).factualCorrectness,
+    ).toBe("passed");
+  });
+
+  it.each([
+    "wrong-token",
+    "native-rva",
+    "mvid",
+    "null-mvid",
+    "digest",
+    "partial",
+  ])("rejects an unsupported artifact entry point: %s", (kind) => {
+    const fixture = managedTranscript();
+    const artifact = structuredClone(fixture.artifact);
+    if (artifact.pe.cli === null || artifact.module === null)
+      throw new Error("Fixture identity missing");
+    if (kind === "wrong-token")
+      artifact.pe.cli.entry_point.value = "0x06000002";
+    if (kind === "native-rva") artifact.pe.cli.entry_point.kind = "native-rva";
+    if (kind === "mvid")
+      artifact.module.mvid = "11111111-2222-3333-4444-555555555555";
+    if (kind === "null-mvid") artifact.module.mvid = null;
+    if (kind === "digest") artifact.artifact.sha256 = "b".repeat(64);
+    if (kind === "partial") artifact.coverage.state = "partial";
+    const evidence = fixture.evidenceFor("inspect_managed_artifact", artifact);
+    expect(
+      fixture.grade([delivered(evidence), delivered(fixture.memberEvidence)])
+        .factualCorrectness,
+    ).toBe("failed");
+  });
+});
+
+describe("managed inventory scope", () => {
+  it.each(["selection", "parameters", "both"])(
+    "rejects selected public-type inventories while retaining entry-point grounding: %s",
+    (kind) => {
+      const fixture = managedTranscript();
+      const typeNames = ["AgentEval.Profile", "AgentEval.Program"];
+      const members = {
+        ...fixture.result,
+        selection:
+          kind === "parameters"
+            ? null
+            : { type_names: typeNames, unmatched_type_names: [] },
+      };
+      const evidence = fixture.evidenceFor("inspect_managed_members", members, {
+        parameters: kind === "selection" ? {} : { type_names: typeNames },
+      });
+      expect(
+        fixture.grade(
+          [delivered(fixture.artifactEvidence), delivered(evidence)],
+          fixture.textFor(evidence),
+        ),
+      ).toMatchObject({
+        factualCorrectness: "failed",
+        factualAssessment: {
+          claims: [
+            { id: "public_types", status: "failed" },
+            { id: "entry_point", status: "passed" },
+          ],
+        },
+      });
+    },
+  );
+});
+
+describe("managed member grounding", () => {
+  it.each([
+    "extra-public-type",
+    "missing-public-type",
+    "duplicate-method",
+    "decoy-main",
+    "scope-digest",
+    "scope-mvid",
+    "null-scope",
+    "partial",
+    "unavailable",
+  ])("rejects unsupported member facts: %s", (kind) => {
+    const fixture = managedTranscript();
+    const members = structuredClone(fixture.result);
+    const method = members.methods[0];
+    const type = members.types[0];
+    if (method === undefined || type === undefined)
+      throw new Error("Fixture members missing");
+    if (kind === "extra-public-type")
+      members.types.push({
+        ...type,
+        name: "Extra",
+        full_name: "AgentEval.Extra",
+      });
+    if (kind === "missing-public-type") members.types.pop();
+    if (kind === "duplicate-method") members.methods.push(method);
+    if (kind === "decoy-main") {
+      method.name = "Other";
+      members.methods.push({ ...method, token: "0x06000002", name: "Main" });
+    }
+    if (kind === "scope-digest")
+      members.identity_scope.requires_artifact_sha256 = "b".repeat(64);
+    if (kind === "scope-mvid")
+      members.identity_scope.requires_mvid =
+        "11111111-2222-3333-4444-555555555555";
+    if (kind === "null-scope") members.identity_scope.requires_mvid = null;
+    if (kind === "partial") members.metadata.status = "partial";
+    if (kind === "unavailable") members.coverage.state = "unavailable";
+    const evidence = fixture.evidenceFor("inspect_managed_members", members);
+    expect(
+      fixture.grade(
+        [delivered(fixture.artifactEvidence), delivered(evidence)],
+        fixture.textFor(evidence),
+      ).factualCorrectness,
+    ).toBe("failed");
+  });
+
+  it("rejects distinct authenticated matching artifact deliveries", () => {
+    const fixture = managedTranscript();
+    const secondArtifact = fixture.evidenceFor(
+      "inspect_managed_artifact",
+      fixture.artifact,
+      { parameters: { path: "/targets/AgentEval.dll" } },
+    );
+    expect(secondArtifact.evidence_id).not.toBe(
+      fixture.artifactEvidence.evidence_id,
+    );
+    const member = delivered(fixture.memberEvidence);
+    expect(
+      fixture.grade([delivered(secondArtifact), member]).factualCorrectness,
+    ).toBe("passed");
+    expect(
+      fixture.grade([
+        delivered(fixture.artifactEvidence),
+        delivered(secondArtifact),
+        member,
+      ]).factualCorrectness,
+    ).toBe("failed");
+  });
+
+  it("accepts identical repeated artifact deliveries", () => {
+    const fixture = managedTranscript();
+    expect(
+      fixture.grade([
+        delivered(fixture.artifactEvidence),
+        delivered(fixture.artifactEvidence),
+        delivered(fixture.memberEvidence),
+      ]).factualCorrectness,
+    ).toBe("passed");
+  });
+
+  it("requires companion delivery and rejects conflicting or failed artifact deliveries", () => {
+    const fixture = managedTranscript();
+    const member = delivered(fixture.memberEvidence);
+    expect(fixture.grade([member]).factualCorrectness).toBe("failed");
+    expect(
+      fixture.grade([
+        delivered(fixture.artifactEvidence, { status: "failed" }),
+        member,
+      ]).factualCorrectness,
+    ).toBe("failed");
+    const conflicting = {
+      ...fixture.artifactEvidence,
+      subject:
+        fixture.memberEvidence.subject === null
+          ? null
+          : { ...fixture.memberEvidence.subject, local_path: "/other.dll" },
+    };
+    expect(
+      fixture.grade([
+        delivered(fixture.artifactEvidence),
+        delivered(conflicting),
+        member,
+      ]).factualCorrectness,
+    ).toBe("failed");
+  });
+
+  it("binds the digest, rejects fabricated values and cites the supporting operation", () => {
+    const fixture = managedTranscript();
+    expect(
+      fixture.grade(undefined, fixture.textFor(fixture.artifactEvidence))
+        .factualCorrectness,
+    ).toBe("failed");
+    expect(
+      fixture.grade(
+        undefined,
+        fixture.textFor(fixture.memberEvidence).replace('"Main"', '"Invented"'),
+      ).factualCorrectness,
+    ).toBe("failed");
+    const other = createEvidence(
+      { path: "/targets/AgentEval.dll", sha256: "b".repeat(64), format: "pe" },
+      provider,
+      {
+        operation: "inspect_managed_members",
+        parameters: {},
+        result: fixture.result,
+        confidence: "observed",
+        authority: "shipped-artifact",
+      },
+    );
+    expect(
+      fixture.grade(
+        [delivered(fixture.artifactEvidence), delivered(other)],
+        fixture.textFor(other),
+      ).factualCorrectness,
+    ).toBe("failed");
+  });
+
+  it("keeps golden names out of instructions and preserves routing-only scenarios", () => {
+    const instructions = agentFixtureAnswerInstructions("managed");
+    expect(instructions).toContain("do not read target files directly");
+    expect(instructions).not.toMatch(/AgentEval|Profile|Program|Main/u);
+    expect(agentFixtureAnswerInstructions("native")).toBe("");
+  });
+});
+
 // Removing exact fixture comparison or accepted-source validation must break these regressions.
 describe("known-answer fixture grading", () => {
   it("passes closed claims against authenticated production MCP Evidence", () => {