316 lines
12 KiB
TypeScript
316 lines
12 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { cp, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
|
|
import { createPackage } from "@electron/asar";
|
|
|
|
import type { JsonValue } from "../domain/jsonValue.js";
|
|
import { javascriptApplicationAnalysisResultSchema } from "../domain/javascript/javascriptApplicationAnalysis.js";
|
|
import type { ApplicationNode } from "../domain/javascript/javascriptApplicationGraphSchemas.js";
|
|
import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascript/javascriptExportShapeComparisonSchemas.js";
|
|
import type { FixtureClaimExpectation } from "./KnownAnswerEvaluation.js";
|
|
|
|
/** Source-owned artifact paths used by the configured known-answer benchmarks. */
|
|
export interface AgentEvaluationFixtureTargets {
|
|
readonly javascript: string;
|
|
readonly javascriptSha256: string;
|
|
readonly javascriptShapeLeft: string;
|
|
readonly javascriptShapeRight: string;
|
|
}
|
|
|
|
const desktopFixtureFiles = {
|
|
"package.json": `${JSON.stringify({ name: "desktop-fixture", version: "1.0.0", main: "main.js" }, null, 2)}\n`,
|
|
"main.js":
|
|
'const { BrowserWindow, ipcMain } = require("electron");\nnew BrowserWindow({ webPreferences: { preload: require("node:path").join(__dirname, "preload.js"), contextIsolation: true, sandbox: true } });\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n',
|
|
"preload.js":
|
|
'const { contextBridge, ipcRenderer } = require("electron");\ncontextBridge.exposeInMainWorld("profileApi", { read: (id) => ipcRenderer.invoke("profile:read", id) });\n',
|
|
"renderer/app.js": 'globalThis.profileApi.read("fixture");\n',
|
|
};
|
|
|
|
/** Materialize the exact packaged and source-owned parser targets used by the runner. */
|
|
export const createAgentEvaluationFixtures = async (
|
|
root: string,
|
|
repositoryRoot: string,
|
|
): Promise<AgentEvaluationFixtureTargets> => {
|
|
const desktop = join(root, "desktop-app");
|
|
const javascript = join(root, "desktop-app.asar");
|
|
const javascriptShapeLeft = join(root, "parser-v1");
|
|
const javascriptShapeRight = join(root, "parser-v2");
|
|
await Promise.all([
|
|
mkdir(join(desktop, "renderer"), { recursive: true }),
|
|
mkdir(javascriptShapeLeft, { recursive: true }),
|
|
mkdir(javascriptShapeRight, { recursive: true }),
|
|
]);
|
|
await Promise.all([
|
|
...Object.entries(desktopFixtureFiles).map(([path, source]) =>
|
|
writeFile(join(desktop, path), source),
|
|
),
|
|
cp(
|
|
join(repositoryRoot, "tests/fixtures/replay/parser.mjs"),
|
|
join(javascriptShapeLeft, "parser.mjs"),
|
|
),
|
|
cp(
|
|
join(repositoryRoot, "tests/fixtures/replay/parser-v2.mjs"),
|
|
join(javascriptShapeRight, "parser.mjs"),
|
|
),
|
|
]);
|
|
await createPackage(desktop, javascript);
|
|
return {
|
|
javascript,
|
|
javascriptSha256: createHash("sha256")
|
|
.update(await readFile(javascript))
|
|
.digest("hex"),
|
|
javascriptShapeLeft,
|
|
javascriptShapeRight,
|
|
};
|
|
};
|
|
|
|
type GraphObservation = ApplicationNode["observations"][number];
|
|
|
|
const singleObservation = (
|
|
value: JsonValue,
|
|
kind: ApplicationNode["kind"],
|
|
include: (observation: GraphObservation) => boolean = () => true,
|
|
): GraphObservation | undefined => {
|
|
const result = javascriptApplicationAnalysisResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
const candidates = result.data.graph.nodes
|
|
.filter((node) => node.kind === kind)
|
|
.flatMap((node) => node.observations)
|
|
.filter(include);
|
|
return candidates.length === 1 ? candidates[0] : undefined;
|
|
};
|
|
|
|
const observationSource = (
|
|
observation: GraphObservation,
|
|
): string | undefined => {
|
|
const location = observation.evidence.location;
|
|
return location.available && location.value.kind === "source-range"
|
|
? location.value.source
|
|
: undefined;
|
|
};
|
|
|
|
const rendererApi = (value: JsonValue): JsonValue | undefined => {
|
|
const observation = singleObservation(value, "context-bridge-api");
|
|
if (observation === undefined) return undefined;
|
|
const { api_key: key, members, world } = observation.properties;
|
|
const source = observationSource(observation);
|
|
if (
|
|
key === undefined ||
|
|
members === undefined ||
|
|
world === undefined ||
|
|
source === undefined
|
|
)
|
|
return undefined;
|
|
return { key, members, world, source };
|
|
};
|
|
|
|
const ipcOperation =
|
|
(side: string) =>
|
|
(value: JsonValue): JsonValue | undefined => {
|
|
const observation = singleObservation(
|
|
value,
|
|
"ipc-channel",
|
|
(candidate) => candidate.properties.side === side,
|
|
);
|
|
if (observation === undefined) return undefined;
|
|
const { channel, operation } = observation.properties;
|
|
const source = observationSource(observation);
|
|
if (
|
|
channel === undefined ||
|
|
operation === undefined ||
|
|
source === undefined
|
|
)
|
|
return undefined;
|
|
return { channel, operation, source };
|
|
};
|
|
|
|
const preload = (value: JsonValue): JsonValue | undefined => {
|
|
const observation = singleObservation(
|
|
value,
|
|
"electron-preload",
|
|
(candidate) =>
|
|
candidate.properties.mechanism === "BrowserWindow:webPreferences.preload",
|
|
);
|
|
const path = observation?.properties.resolved_path;
|
|
const resolutionStatus = observation?.properties.resolution_status;
|
|
if (path === undefined || resolutionStatus === undefined) return undefined;
|
|
return { path, resolution_status: resolutionStatus };
|
|
};
|
|
|
|
const headingChange = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
const changes = result.data.changes.filter(
|
|
(change) => change.path === "/depth",
|
|
);
|
|
const change = changes.length === 1 ? changes[0] : undefined;
|
|
if (change === undefined) return undefined;
|
|
return {
|
|
status: change.status,
|
|
path: change.path,
|
|
discriminant: change.discriminant,
|
|
left: change.left,
|
|
right: change.right,
|
|
};
|
|
};
|
|
|
|
const comparisonCoverage = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
|
|
if (!result.success) return undefined;
|
|
return {
|
|
status: result.data.coverage.status,
|
|
paired_variants: result.data.coverage.paired_variants,
|
|
...result.data.summary,
|
|
};
|
|
};
|
|
|
|
const runtimeSemantics = (value: JsonValue): JsonValue | undefined => {
|
|
const result = javaScriptExportShapeComparisonResultSchema.safeParse(value);
|
|
return result.success &&
|
|
result.data.limitations.includes(
|
|
"Return shapes are inferred from inert syntax and do not prove runtime behavior.",
|
|
)
|
|
? "not_established"
|
|
: undefined;
|
|
};
|
|
|
|
const asarClaims = (
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] => {
|
|
const source = {
|
|
operation: "analyze_javascript_application",
|
|
authority: "shipped-artifact",
|
|
confidence: "derived",
|
|
subject: { path: targets.javascript, sha256: targets.javascriptSha256 },
|
|
} satisfies Omit<FixtureClaimExpectation["source"], "select">;
|
|
return [
|
|
{
|
|
id: "renderer_api",
|
|
expectedValue: {
|
|
key: "profileApi",
|
|
members: ["read"],
|
|
world: "main",
|
|
source: "preload.js",
|
|
},
|
|
source: { ...source, select: rendererApi },
|
|
},
|
|
{
|
|
id: "renderer_ipc",
|
|
expectedValue: {
|
|
channel: "profile:read",
|
|
operation: "invoke",
|
|
source: "preload.js",
|
|
},
|
|
source: { ...source, select: ipcOperation("renderer") },
|
|
},
|
|
{
|
|
id: "main_ipc",
|
|
expectedValue: {
|
|
channel: "profile:read",
|
|
operation: "handle",
|
|
source: "main.js",
|
|
},
|
|
source: { ...source, select: ipcOperation("main") },
|
|
},
|
|
{
|
|
id: "preload",
|
|
expectedValue: { path: "preload.js", resolution_status: "resolved" },
|
|
source: { ...source, select: preload },
|
|
},
|
|
];
|
|
};
|
|
|
|
const parserClaims = (
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] => {
|
|
const source = {
|
|
operation: "compare_javascript_export_shapes",
|
|
authority: "analyst-inference",
|
|
confidence: "inferred",
|
|
parameters: {
|
|
left_module_path: "parser.mjs",
|
|
left_export_name: "default",
|
|
right_module_path: "parser.mjs",
|
|
right_export_name: "default",
|
|
},
|
|
linkedSubjects: [
|
|
{
|
|
operation: "analyze_javascript_application",
|
|
path: targets.javascriptShapeLeft,
|
|
},
|
|
{
|
|
operation: "analyze_javascript_application",
|
|
path: targets.javascriptShapeRight,
|
|
},
|
|
],
|
|
} satisfies Omit<FixtureClaimExpectation["source"], "select">;
|
|
return [
|
|
{
|
|
id: "heading_change",
|
|
expectedValue: {
|
|
status: "added",
|
|
path: "/depth",
|
|
discriminant: { path: "/type", value: "heading" },
|
|
left: { availability: "absent" },
|
|
right: { availability: "literal", value: 1 },
|
|
},
|
|
source: { ...source, select: headingChange },
|
|
},
|
|
{
|
|
id: "comparison_coverage",
|
|
expectedValue: {
|
|
status: "complete-within-inputs",
|
|
paired_variants: 3,
|
|
added: 1,
|
|
removed: 0,
|
|
changed: 0,
|
|
unknown: 0,
|
|
},
|
|
source: { ...source, select: comparisonCoverage },
|
|
},
|
|
{
|
|
id: "runtime_semantics",
|
|
expectedValue: "not_established",
|
|
source: { ...source, select: runtimeSemantics },
|
|
},
|
|
];
|
|
};
|
|
|
|
/** Return the closed known-answer rubric, or undefined for routing-only scenarios. */
|
|
export const agentFixtureClaims = (
|
|
scenarioId: string,
|
|
targets: AgentEvaluationFixtureTargets,
|
|
): readonly FixtureClaimExpectation[] | undefined => {
|
|
if (scenarioId === "asar") return asarClaims(targets);
|
|
if (scenarioId === "javascript-export-shape") return parserClaims(targets);
|
|
return undefined;
|
|
};
|
|
|
|
const answerValueDescriptions: Readonly<Record<string, readonly string[]>> = {
|
|
asar: [
|
|
'renderer_api: {"key": string, "members": string[], "world": string, "source": string}, describing the exposed bridge API, its members, world, and source file.',
|
|
'renderer_ipc: {"channel": string, "operation": string, "source": string}, describing the renderer-side IPC operation and its source file.',
|
|
'main_ipc: {"channel": string, "operation": string, "source": string}, describing the main-side IPC operation and its source file.',
|
|
'preload: {"path": string, "resolution_status": string}, describing the BrowserWindow-declared preload resolved within the shipped artifact.',
|
|
],
|
|
"javascript-export-shape": [
|
|
'heading_change: {"status": string, "path": string, "discriminant": object or null, "left": object, "right": object}, copying the exact heading-change fields from the comparison result.',
|
|
'comparison_coverage: {"status": string, "paired_variants": number, "added": number, "removed": number, "changed": number, "unknown": number}, copying comparison coverage and summary counts.',
|
|
'runtime_semantics: "established" or "not_established", stating whether the cited comparison establishes runtime behavior.',
|
|
],
|
|
};
|
|
|
|
/** Describe the strict answer contract without revealing any golden fixture values. */
|
|
export const agentFixtureAnswerInstructions = (scenarioId: string): string => {
|
|
const descriptions = answerValueDescriptions[scenarioId];
|
|
if (descriptions === undefined) return "";
|
|
return [
|
|
"For this benchmark, return only one JSON object, without Markdown fences or surrounding prose.",
|
|
'The exact shape is {"claims": [{"id": string, "value": JSON value, "evidence_id": string, "confidence": string, "authority": string}]}.',
|
|
"Include exactly one claim per ID below, without additional fields or claims. Infer values only from REA tool results; do not read target files directly.",
|
|
"Each evidence_id must identify the successful REA result that supports its claim. Copy confidence and authority from that result's Evidence envelope.",
|
|
...descriptions,
|
|
].join("\n");
|
|
};
|