diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index fb3d4b4be..262779a71 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -74,7 +74,7 @@ on completed output directories. Original replies, catalogs, comparison receipts and exact measured driver source stay private. Repeated deliveries in the same mode also compare the ordered receipt observations, including limitations, view parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific. -Comparator regressions run with `node --test experiments/analysis-efficiency/test_compare.mjs` +Comparator regressions run with `npm run test:local -- tests/evaluation/analysisEfficiencyComparison.test.ts` using synthetic measurement directories; they do not establish provider behavior. ## Actual model tasks @@ -278,3 +278,8 @@ Workflow citation cardinality must equal the planned call count before binding per-call IDs. A shorter or longer answer list cannot silently shorten Python zip’s expected plan; missing URI calls and extra citations are rejected. Original complete/summary model workflows continue to pass unchanged. + +The six comparator CLI regressions now live in the normally discovered Vitest +evaluation suite, so routine local/PR/CI gates execute them. The original six +scenarios and CLI assertions are retained; the former manual Node suite is +removed. They remain synthetic consumer checks, not provider evidence. diff --git a/experiments/analysis-efficiency/test_compare.mjs b/tests/evaluation/analysisEfficiencyComparison.test.ts similarity index 62% rename from experiments/analysis-efficiency/test_compare.mjs rename to tests/evaluation/analysisEfficiencyComparison.test.ts index cfc835717..57a1ffb8c 100644 --- a/experiments/analysis-efficiency/test_compare.mjs +++ b/tests/evaluation/analysisEfficiencyComparison.test.ts @@ -3,11 +3,49 @@ import { spawnSync } from "node:child_process"; import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import test from "node:test"; +import { test } from "vitest"; +import { z } from "zod"; import { fileURLToPath } from "node:url"; +type Delivery = "complete" | "summary"; +type Observation = { + evidence_id: string; + parent_evidence_id?: string; + limitations: string[]; + kind?: string; + view?: { kind: string; facet?: string; offset?: number; limit?: number }; + coverage?: + | string + | { + examined: number; + total: number; + next_offset: number | null; + exhausted: boolean; + }; + unknowns?: string[]; +}; +type Fixture = { + summary: { + artifact: string; + delivery: Delivery; + provider_search_path_sha256: string; + calls: unknown[]; + }; + answer: { + capture: { coverage: string; capture_start: null }; + http_request_uris_first2: string[]; + delivery_observations: Observation[]; + }; +}; +const reportSchema = z.object({ + facts_match: z.boolean(), + settings_match: z.boolean(), + fact_differences: z.array(z.string()), + preserved_run_specific_values: z.array(z.object({ value: z.unknown() })), +}); + // Synthetic measurement directories exercise the comparator CLI, not providers. -const observation = (kind) => ({ +const observation = (kind: string): Observation => ({ evidence_id: `ev_${kind}`, parent_evidence_id: "ev_parent", limitations: ["saved bytes only"], @@ -19,7 +57,7 @@ const observation = (kind) => ({ coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false }, unknowns: ["unrecorded traffic"], }); -const fixture = (delivery, artifact) => ({ +const fixture = (delivery: Delivery, artifact: string): Fixture => ({ summary: { artifact, delivery, @@ -44,22 +82,29 @@ const fixture = (delivery, artifact) => ({ ], }, }); -function compare(left, right) { +function compare(left: Fixture, right: Fixture) { const root = mkdtempSync(join(tmpdir(), "rea-compare-test-")); try { - const directories = ["left", "right"].map((side) => join(root, side)); + const directories = [join(root, "left"), join(root, "right")] as const; for (const [index, value] of [left, right].entries()) { - mkdirSync(directories[index]); - for (const name of ["summary", "answer"]) + const directory = directories[index]; + assert.ok(directory); + mkdirSync(directory); + for (const name of ["summary", "answer"] as const) writeFileSync( - join(directories[index], `${name}.json`), + join(directory, `${name}.json`), JSON.stringify(value[name]), ); } const result = spawnSync( process.execPath, [ - fileURLToPath(new URL("compare.mjs", import.meta.url)), + fileURLToPath( + new URL( + "../../experiments/analysis-efficiency/compare.mjs", + import.meta.url, + ), + ), "--left", directories[0], "--right", @@ -69,7 +114,10 @@ function compare(left, right) { ); assert.ifError(result.error); assert.equal(result.stderr, ""); - return { status: result.status, report: JSON.parse(result.stdout) }; + return { + status: result.status, + report: reportSchema.parse(JSON.parse(result.stdout)), + }; } finally { rmSync(root, { recursive: true, force: true }); } @@ -80,12 +128,12 @@ for (const artifact of [ "wireshark-http_redirects.pcapng", ]) { test(`${artifact}: repeated modes ignore only receipt IDs`, () => { - for (const delivery of ["complete", "summary"]) { + for (const delivery of ["complete", "summary"] as const) { const left = fixture(delivery, artifact); const right = structuredClone(left); for (const value of right.answer.delivery_observations) { value.evidence_id += "_second"; - if (Object.hasOwn(value, "parent_evidence_id")) + if (typeof value.parent_evidence_id === "string") value.parent_evidence_id += "_second"; } const { status, report } = compare(left, right); @@ -99,26 +147,28 @@ for (const artifact of [ }); test(`${artifact}: repeated modes reject stable observation changes`, () => { - for (const delivery of ["complete", "summary"]) { + for (const delivery of ["complete", "summary"] as const) { const left = fixture(delivery, artifact); - const mutations = [ - (values) => values[0].limitations.push("new limitation"), + const mutations: ((values: Observation[]) => void)[] = [ + (values) => values[0]!.limitations.push("new limitation"), (values) => { - values[0].coverage = "partial"; + values[0]!.coverage = "partial"; }, (values) => { - delete values[0].coverage; + delete values[0]!.coverage; }, ]; if (delivery === "summary") mutations.push( (values) => { - values[1].view.offset = 1; + values[1]!.view!.offset = 1; }, (values) => { - values[1].coverage.next_offset = null; + const coverage = values[1]!.coverage; + assert.ok(typeof coverage === "object" && coverage !== null); + coverage.next_offset = null; }, - (values) => values[1].unknowns.push("extra unknown"), + (values) => values[1]!.unknowns!.push("extra unknown"), (values) => values.reverse(), ); for (const mutate of mutations) {