diff --git a/docs/client-verification.md b/docs/client-verification.md index d9ece9aad..38d0bf399 100644 --- a/docs/client-verification.md +++ b/docs/client-verification.md @@ -458,7 +458,7 @@ artifact path. Its `expectedFirstTool` is `null`, and `targetClarificationPassed` grades that limited routing outcome independently of analysis prose heuristics. Analyzing a nearby example cannot pass this case. -The packaged desktop and parser-comparison scenarios now assess a closed set of +The packaged desktop, parser-comparison, and managed scenarios assess a closed set of known fixture claims. Each final answer must be a strict JSON manifest containing exactly the requested claim IDs, values, Evidence IDs, and Evidence authority and confidence. Golden values are not included in the prompt. The evaluator checks @@ -475,15 +475,24 @@ analyses and use the requested module and export selectors. Boundary tests packa the same source-owned desktop fixture and compare the same source-owned parser files through actual MCP tool results, then verify that fabricated answers fail. +The managed rubric checks the exact public top-level type set and the CLI entry +point's declaring type and method name. The exact type-set claim requires an +unfiltered member inventory: `selection` must be null and the Evidence parameters +must omit `type_names`. A selected inventory can still ground the entry-point +claim. Both inspections must describe the freshly built DLL's SHA-256. Entry-point +resolution additionally requires complete artifact and member metadata, matching +non-null module MVIDs, and a unique method at the header's metadata token. These are +static metadata claims, not runtime behavior; +compiler-generated member counts, token values, and MVIDs are not golden answers. + Per-scenario `factualCorrectness` is `passed`, `failed`, or `not_assessed` within the `configured_fixture_claims` scope. A pass establishes only the configured -claims, not unrestricted factual correctness. Native, managed, browser, +claims, not unrestricted factual correctness. Native, browser, navigation-context, and address-context scenarios have no factual rubric and -remain `not_assessed`; the managed workflow requires both artifact and member -inspection to answer its type and entry-point question. +remain `not_assessed`. The closed factual scenarios request complete producer Evidence on the initial -analysis so their source selectors can authenticate all configured facts. +inspection or analysis so their source selectors can authenticate all configured facts. Summary/page/item workflows are exercised separately by `javascript-module-view`; its heuristic gate does not establish full-producer factual correctness. diff --git a/scripts/verify-agent-experience.mjs b/scripts/verify-agent-experience.mjs index 99ab060dd..71cd6383c 100644 --- a/scripts/verify-agent-experience.mjs +++ b/scripts/verify-agent-experience.mjs @@ -1,4 +1,5 @@ import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; import { chmod, copyFile, @@ -259,7 +260,7 @@ try { const execution = await runCodex( answerInstructions.length === 0 ? scenario.prompt - : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`, + : `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial inspection or analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`, scenario.approvedTools ?? [], ); const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR; @@ -440,6 +441,13 @@ async function createTargets(root, includeManaged) { ...fixtures, largeJavascript, managed: join(managedOutput, "AgentEval.dll"), + ...(includeManaged + ? { + managedSha256: createHash("sha256") + .update(await readFile(join(managedOutput, "AgentEval.dll"))) + .digest("hex"), + } + : {}), }; } diff --git a/src/evaluation/AgentEvaluationFixtures.ts b/src/evaluation/AgentEvaluationFixtures.ts index b7ef17efe..b0535a96e 100644 --- a/src/evaluation/AgentEvaluationFixtures.ts +++ b/src/evaluation/AgentEvaluationFixtures.ts @@ -8,7 +8,19 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js"; import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js"; import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js"; -import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js"; +import { + managedArtifactInspectionSchema, + managedMemberInspectionSchema, +} from "../domain/managed/managedArtifact.js"; +import { + managedTokenScopesMatch, + parseManagedInspectionEvidence, +} from "../domain/managed/managedInspectionEvidence.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import type { + FixtureClaimContext, + FixtureClaimExpectation, +} from "./KnownAnswerEvaluation.js"; /** Source-owned artifact paths used by the configured known-answer benchmarks. */ export interface AgentEvaluationFixtureTargets { @@ -16,6 +28,8 @@ export interface AgentEvaluationFixtureTargets { readonly javascriptSha256: string; readonly javascriptShapeLeft: string; readonly javascriptShapeRight: string; + readonly managed?: string; + readonly managedSha256?: string; } const desktopFixtureFiles = { @@ -277,17 +291,122 @@ const parserClaims = ( ]; }; +const managedMembers = ({ evidence }: FixtureClaimContext) => { + const { result } = parseManagedInspectionEvidence( + evidence, + "inspect_managed_members", + managedMemberInspectionSchema, + ); + if ( + result.coverage.state !== "complete" || + result.metadata.status !== "complete" || + !managedTokenScopesMatch( + result.identity_scope.requires_mvid, + result.module?.mvid ?? null, + ) + ) + return undefined; + return result; +}; + +const managedClaims = ( + targets: AgentEvaluationFixtureTargets, +): readonly FixtureClaimExpectation[] => { + if (targets.managed === undefined || targets.managedSha256 === undefined) + throw new TypeError( + "Managed fixture grading requires the built DLL path and SHA-256", + ); + const source = { + operation: "inspect_managed_members", + authority: "shipped-artifact", + confidence: "observed", + subject: { path: targets.managed, sha256: targets.managedSha256 }, + } satisfies Omit; + return [ + { + id: "public_types", + expectedValue: ["AgentEval.Profile", "AgentEval.Program"], + source: { + ...source, + select: (_value, context) => { + const members = managedMembers(context); + if ( + members === undefined || + members.selection !== null || + Object.hasOwn(context.evidence.parameters, "type_names") + ) + return undefined; + return members.types + .filter((type) => (type.flags & 7) === 1) + .map((type) => type.full_name) + .sort(compareUnicodeCodePoints); + }, + }, + }, + { + id: "entry_point", + expectedValue: { declaring_type: "AgentEval.Program", name: "Main" }, + source: { + ...source, + select: (_value, context) => { + const members = managedMembers(context); + if (members === undefined) return undefined; + const artifacts = context.deliveredEvidence.filter( + (evidence) => + evidence.operation === "inspect_managed_artifact" && + evidence.authority === source.authority && + evidence.confidence === source.confidence && + evidence.subject?.local_path === source.subject.path && + evidence.subject.digest.sha256 === source.subject.sha256, + ); + if (artifacts.length !== 1) return undefined; + const { result: artifact } = parseManagedInspectionEvidence( + artifacts[0], + "inspect_managed_artifact", + managedArtifactInspectionSchema, + ); + if ( + artifact.coverage.state !== "complete" || + artifact.metadata.status !== "complete" || + !managedTokenScopesMatch( + artifact.module?.mvid ?? null, + members.module?.mvid ?? null, + ) || + artifact.pe.cli?.entry_point.kind !== "metadata-token" + ) + return undefined; + const methods = members.methods.filter( + (method) => method.token === artifact.pe.cli?.entry_point.value, + ); + const method = methods.length === 1 ? methods[0] : undefined; + return method === undefined + ? undefined + : { + declaring_type: method.declaring_type, + name: method.name, + }; + }, + }, + }, + ]; +}; + /** Return the closed known-answer rubric, or undefined for routing-only scenarios. */ export const agentFixtureClaims = ( scenarioId: string, targets: AgentEvaluationFixtureTargets, ): readonly FixtureClaimExpectation[] | undefined => { if (scenarioId === "asar") return asarClaims(targets); + if (scenarioId === "managed") return managedClaims(targets); if (scenarioId === "javascript-export-shape") return parserClaims(targets); return undefined; }; const answerValueDescriptions: Readonly> = { + managed: [ + "public_types: string[], listing the exact public top-level type full names in Unicode code-point order from unfiltered member Evidence (omit type_names).", + 'entry_point: {"declaring_type": string, "name": string}, resolving the CLI metadata entry-point token against the member inventory from the same artifact SHA-256 and module MVID. Cite member Evidence; both artifact and member Evidence must be delivered.', + ], asar: [ 'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.', 'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.', diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.ts index c6e4af72b..fe71302eb 100644 --- a/src/evaluation/KnownAnswerEvaluation.ts +++ b/src/evaluation/KnownAnswerEvaluation.ts @@ -5,6 +5,12 @@ import { parseEvidence, type Evidence } from "../domain/evidence.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; import { canonicalJson } from "../domain/comparisonSemantics.js"; +/** Only authenticated, successful, non-conflicting transcript deliveries. */ +export interface FixtureClaimContext { + readonly evidence: Evidence; + readonly deliveredEvidence: readonly Evidence[]; +} + /** One closed fixture assertion and the producer facts that must support it. */ export interface FixtureClaimExpectation { readonly id: string; @@ -20,7 +26,10 @@ export interface FixtureClaimExpectation { readonly path: string; readonly sha256?: string; }[]; - readonly select: (normalizedResult: JsonValue) => JsonValue | undefined; + readonly select: ( + normalizedResult: JsonValue, + context: FixtureClaimContext, + ) => JsonValue | undefined; }; } @@ -430,7 +439,12 @@ const assessClaim = ( try { if ( !sameJson( - expected.source.select(evidence.normalized_result), + expected.source.select(evidence.normalized_result, { + evidence, + deliveredEvidence: [...delivered.records.values()].filter( + (record) => !delivered.conflicts.has(record.evidence_id), + ), + }), expected.expectedValue, ) ) diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts index cbe59dd7b..daed23cac 100644 --- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -5,6 +5,16 @@ import { toolContract } from "../../../src/contracts/toolContracts.js"; import { createEvidence, type Evidence } from "../../../src/domain/evidence.js"; import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +import { + agentFixtureAnswerInstructions, + agentFixtureClaims, +} from "../../../src/evaluation/AgentEvaluationFixtures.js"; +import { inspectManagedArtifactBytes } from "../../../src/dotnet/ManagedArtifactInspector.js"; +import { inspectManagedMembersBytes } from "../../../src/dotnet/ManagedMemberInspector.js"; +import { + buildManagedPeFixture, + managedPeFixtureTarget, +} from "../../../src/dotnet/ManagedPe.fixture.js"; import type { FixtureClaimExpectation } from "../../../src/evaluation/KnownAnswerEvaluation.js"; import { ToolResultDelivery } from "../../../src/server/toolResult.js"; @@ -33,7 +43,13 @@ const createObservation = ( const resultFor = (evidence: Evidence) => delivery.toEvidenceToolResult( evidence, - toolContract("analyze_javascript_application"), + toolContract( + evidence.operation === "inspect_managed_artifact" + ? "inspect_managed_artifact" + : evidence.operation === "inspect_managed_members" + ? "inspect_managed_members" + : "analyze_javascript_application", + ), undefined, ); const delivered = ( @@ -95,6 +111,336 @@ const assess = ( ); const answer = (claims: readonly unknown[]) => JSON.stringify({ claims }); +// Synthetic transcript cases exercise grading, not an SDK-built assembly oracle. +const managedTranscript = () => { + const bytes = buildManagedPeFixture({ + typeNamespace: "AgentEval", + typeName: "Program", + methodName: "Main", + }); + const target = managedPeFixtureTarget(bytes, "/targets/AgentEval.dll"); + const artifact = inspectManagedArtifactBytes(bytes, target); + const members = inspectManagedMembersBytes(bytes, target); + const program = members.types[0]; + if (program === undefined) throw new Error("Fixture type missing"); + const result = { + ...members, + types: [ + program, + { + ...program, + token: "0x02000002", + name: "Profile", + full_name: "AgentEval.Profile", + }, + ], + }; + const rubric = agentFixtureClaims("managed", { + javascript: "", + javascriptSha256: "", + javascriptShapeLeft: "", + javascriptShapeRight: "", + managed: target.path, + managedSha256: target.sha256, + }); + if (rubric === undefined) throw new Error("Managed rubric missing"); + const evidenceFor = ( + operation: string, + value: JsonValue, + overrides: Record = {}, + ) => + createEvidence(target, provider, { + operation, + parameters: {}, + result: value, + confidence: "observed", + authority: "shipped-artifact", + ...overrides, + }); + const artifactEvidence = evidenceFor("inspect_managed_artifact", artifact); + const memberEvidence = evidenceFor("inspect_managed_members", result); + const textFor = (evidence: Evidence) => + answer([ + { + id: "public_types", + value: ["AgentEval.Profile", "AgentEval.Program"], + evidence_id: evidence.evidence_id, + confidence: "observed", + authority: "shipped-artifact", + }, + { + id: "entry_point", + value: { declaring_type: "AgentEval.Program", name: "Main" }, + evidence_id: evidence.evidence_id, + confidence: "observed", + authority: "shipped-artifact", + }, + ]); + const grade = ( + events = [delivered(artifactEvidence), delivered(memberEvidence)], + text = textFor(memberEvidence), + ) => + evaluateCodexEvents([...events, final(text)], "inspect_managed_artifact", { + fixtureClaims: rubric, + }); + return { + artifact, + result, + artifactEvidence, + memberEvidence, + evidenceFor, + textFor, + grade, + }; +}; + +describe("managed known-answer fixture grading", () => { + it("grades public types independent of row order and resolves the header token", () => { + expect(managedTranscript().grade().factualCorrectness).toBe("passed"); + }); + + it("does not pin compiler-local entry tokens or MVIDs", () => { + const fixture = managedTranscript(); + const artifact = structuredClone(fixture.artifact); + const members = structuredClone(fixture.result); + if ( + artifact.pe.cli === null || + artifact.module === null || + members.module === null || + members.methods[0] === undefined + ) + throw new Error("Fixture identity missing"); + artifact.pe.cli.entry_point.value = "0x06000009"; + members.methods[0].token = "0x06000009"; + const mvid = "11111111-2222-3333-4444-555555555555"; + artifact.module.mvid = mvid; + members.module.mvid = mvid; + members.identity_scope.requires_mvid = mvid; + members.types.reverse(); + const artifactEvidence = fixture.evidenceFor( + "inspect_managed_artifact", + artifact, + ); + const memberEvidence = fixture.evidenceFor( + "inspect_managed_members", + members, + ); + expect( + fixture.grade( + [delivered(artifactEvidence), delivered(memberEvidence)], + fixture.textFor(memberEvidence), + ).factualCorrectness, + ).toBe("passed"); + }); + + it.each([ + "wrong-token", + "native-rva", + "mvid", + "null-mvid", + "digest", + "partial", + ])("rejects an unsupported artifact entry point: %s", (kind) => { + const fixture = managedTranscript(); + const artifact = structuredClone(fixture.artifact); + if (artifact.pe.cli === null || artifact.module === null) + throw new Error("Fixture identity missing"); + if (kind === "wrong-token") + artifact.pe.cli.entry_point.value = "0x06000002"; + if (kind === "native-rva") artifact.pe.cli.entry_point.kind = "native-rva"; + if (kind === "mvid") + artifact.module.mvid = "11111111-2222-3333-4444-555555555555"; + if (kind === "null-mvid") artifact.module.mvid = null; + if (kind === "digest") artifact.artifact.sha256 = "b".repeat(64); + if (kind === "partial") artifact.coverage.state = "partial"; + const evidence = fixture.evidenceFor("inspect_managed_artifact", artifact); + expect( + fixture.grade([delivered(evidence), delivered(fixture.memberEvidence)]) + .factualCorrectness, + ).toBe("failed"); + }); +}); + +describe("managed inventory scope", () => { + it.each(["selection", "parameters", "both"])( + "rejects selected public-type inventories while retaining entry-point grounding: %s", + (kind) => { + const fixture = managedTranscript(); + const typeNames = ["AgentEval.Profile", "AgentEval.Program"]; + const members = { + ...fixture.result, + selection: + kind === "parameters" + ? null + : { type_names: typeNames, unmatched_type_names: [] }, + }; + const evidence = fixture.evidenceFor("inspect_managed_members", members, { + parameters: kind === "selection" ? {} : { type_names: typeNames }, + }); + expect( + fixture.grade( + [delivered(fixture.artifactEvidence), delivered(evidence)], + fixture.textFor(evidence), + ), + ).toMatchObject({ + factualCorrectness: "failed", + factualAssessment: { + claims: [ + { id: "public_types", status: "failed" }, + { id: "entry_point", status: "passed" }, + ], + }, + }); + }, + ); +}); + +describe("managed member grounding", () => { + it.each([ + "extra-public-type", + "missing-public-type", + "duplicate-method", + "decoy-main", + "scope-digest", + "scope-mvid", + "null-scope", + "partial", + "unavailable", + ])("rejects unsupported member facts: %s", (kind) => { + const fixture = managedTranscript(); + const members = structuredClone(fixture.result); + const method = members.methods[0]; + const type = members.types[0]; + if (method === undefined || type === undefined) + throw new Error("Fixture members missing"); + if (kind === "extra-public-type") + members.types.push({ + ...type, + name: "Extra", + full_name: "AgentEval.Extra", + }); + if (kind === "missing-public-type") members.types.pop(); + if (kind === "duplicate-method") members.methods.push(method); + if (kind === "decoy-main") { + method.name = "Other"; + members.methods.push({ ...method, token: "0x06000002", name: "Main" }); + } + if (kind === "scope-digest") + members.identity_scope.requires_artifact_sha256 = "b".repeat(64); + if (kind === "scope-mvid") + members.identity_scope.requires_mvid = + "11111111-2222-3333-4444-555555555555"; + if (kind === "null-scope") members.identity_scope.requires_mvid = null; + if (kind === "partial") members.metadata.status = "partial"; + if (kind === "unavailable") members.coverage.state = "unavailable"; + const evidence = fixture.evidenceFor("inspect_managed_members", members); + expect( + fixture.grade( + [delivered(fixture.artifactEvidence), delivered(evidence)], + fixture.textFor(evidence), + ).factualCorrectness, + ).toBe("failed"); + }); + + it("rejects distinct authenticated matching artifact deliveries", () => { + const fixture = managedTranscript(); + const secondArtifact = fixture.evidenceFor( + "inspect_managed_artifact", + fixture.artifact, + { parameters: { path: "/targets/AgentEval.dll" } }, + ); + expect(secondArtifact.evidence_id).not.toBe( + fixture.artifactEvidence.evidence_id, + ); + const member = delivered(fixture.memberEvidence); + expect( + fixture.grade([delivered(secondArtifact), member]).factualCorrectness, + ).toBe("passed"); + expect( + fixture.grade([ + delivered(fixture.artifactEvidence), + delivered(secondArtifact), + member, + ]).factualCorrectness, + ).toBe("failed"); + }); + + it("accepts identical repeated artifact deliveries", () => { + const fixture = managedTranscript(); + expect( + fixture.grade([ + delivered(fixture.artifactEvidence), + delivered(fixture.artifactEvidence), + delivered(fixture.memberEvidence), + ]).factualCorrectness, + ).toBe("passed"); + }); + + it("requires companion delivery and rejects conflicting or failed artifact deliveries", () => { + const fixture = managedTranscript(); + const member = delivered(fixture.memberEvidence); + expect(fixture.grade([member]).factualCorrectness).toBe("failed"); + expect( + fixture.grade([ + delivered(fixture.artifactEvidence, { status: "failed" }), + member, + ]).factualCorrectness, + ).toBe("failed"); + const conflicting = { + ...fixture.artifactEvidence, + subject: + fixture.memberEvidence.subject === null + ? null + : { ...fixture.memberEvidence.subject, local_path: "/other.dll" }, + }; + expect( + fixture.grade([ + delivered(fixture.artifactEvidence), + delivered(conflicting), + member, + ]).factualCorrectness, + ).toBe("failed"); + }); + + it("binds the digest, rejects fabricated values and cites the supporting operation", () => { + const fixture = managedTranscript(); + expect( + fixture.grade(undefined, fixture.textFor(fixture.artifactEvidence)) + .factualCorrectness, + ).toBe("failed"); + expect( + fixture.grade( + undefined, + fixture.textFor(fixture.memberEvidence).replace('"Main"', '"Invented"'), + ).factualCorrectness, + ).toBe("failed"); + const other = createEvidence( + { path: "/targets/AgentEval.dll", sha256: "b".repeat(64), format: "pe" }, + provider, + { + operation: "inspect_managed_members", + parameters: {}, + result: fixture.result, + confidence: "observed", + authority: "shipped-artifact", + }, + ); + expect( + fixture.grade( + [delivered(fixture.artifactEvidence), delivered(other)], + fixture.textFor(other), + ).factualCorrectness, + ).toBe("failed"); + }); + + it("keeps golden names out of instructions and preserves routing-only scenarios", () => { + const instructions = agentFixtureAnswerInstructions("managed"); + expect(instructions).toContain("do not read target files directly"); + expect(instructions).not.toMatch(/AgentEval|Profile|Program|Main/u); + expect(agentFixtureAnswerInstructions("native")).toBe(""); + }); +}); + // Removing exact fixture comparison or accepted-source validation must break these regressions. describe("known-answer fixture grading", () => { it("passes closed claims against authenticated production MCP Evidence", () => {