diff --git a/experiments/analysis-efficiency/compare.mjs b/experiments/analysis-efficiency/compare.mjs index 0d22c741b..4e33e5ead 100644 --- a/experiments/analysis-efficiency/compare.mjs +++ b/experiments/analysis-efficiency/compare.mjs @@ -72,8 +72,9 @@ for (const [side, answer] of [ } // PCAP delivery observations preserve each actual receipt's identity, limitations // and view coverage. Summary/page scopes differ from complete source scope; -// retain them as run-specific facts rather than equating page exhaustion with -// capture coverage. The common capture facts and URI prefix still must match. +// retain them separately rather than equating page exhaustion with capture +// coverage. Repeated equivalent deliveries must also match every observation +// field except the receipt and parent IDs, which identify their own invocations. if ( ["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes( left.artifact, @@ -93,7 +94,18 @@ if ( path: ["delivery_observations"], value: answer.delivery_observations, }); - delete answer.delivery_observations; + if (left.delivery === right.delivery) { + answer.delivery_observations = answer.delivery_observations.map( + (observation) => + Object.fromEntries( + Object.entries(observation).filter( + ([key]) => key !== "evidence_id" && key !== "parent_evidence_id", + ), + ), + ); + } else { + delete answer.delivery_observations; + } } } // Binwalk assigns a new opaque region UUID in each invocation. Preserve both diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 1697da82d..9b094ad3a 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -71,7 +71,11 @@ Reproduce with `measure.mjs --artifact wireshark-http.cap` or `wireshark-http_redirects.pcapng`, pinned path/runtime/revision, explicit provider JSON, `--schemas selected`, and `--delivery complete|summary`. Use `compare.mjs` on completed output directories. Original replies, catalogs, comparison receipts -and exact measured driver source stay private. +and exact measured driver source stay private. Repeated deliveries in the same +mode also compare the ordered receipt observations, including limitations, view +parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific. +Comparator regressions run with `node --test experiments/analysis-efficiency/test_compare.mjs` +using synthetic measurement directories; they do not establish provider behavior. ## Actual model tasks diff --git a/experiments/analysis-efficiency/test_compare.mjs b/experiments/analysis-efficiency/test_compare.mjs new file mode 100644 index 000000000..cfc835717 --- /dev/null +++ b/experiments/analysis-efficiency/test_compare.mjs @@ -0,0 +1,149 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +// Synthetic measurement directories exercise the comparator CLI, not providers. +const observation = (kind) => ({ + evidence_id: `ev_${kind}`, + parent_evidence_id: "ev_parent", + limitations: ["saved bytes only"], + kind, + view: + kind === "summary" + ? { kind } + : { kind, facet: "http-uris", offset: 0, limit: 2 }, + coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false }, + unknowns: ["unrecorded traffic"], +}); +const fixture = (delivery, artifact) => ({ + summary: { + artifact, + delivery, + provider_search_path_sha256: "a".repeat(64), + calls: [], + }, + answer: { + capture: { coverage: "complete", capture_start: null }, + http_request_uris_first2: [ + "http://example.test/one", + "http://example.test/two", + ], + delivery_observations: + delivery === "summary" + ? [observation("summary"), observation("pcap")] + : [ + { + evidence_id: "ev_complete", + limitations: ["saved bytes only"], + coverage: "complete", + }, + ], + }, +}); +function compare(left, right) { + const root = mkdtempSync(join(tmpdir(), "rea-compare-test-")); + try { + const directories = ["left", "right"].map((side) => join(root, side)); + for (const [index, value] of [left, right].entries()) { + mkdirSync(directories[index]); + for (const name of ["summary", "answer"]) + writeFileSync( + join(directories[index], `${name}.json`), + JSON.stringify(value[name]), + ); + } + const result = spawnSync( + process.execPath, + [ + fileURLToPath(new URL("compare.mjs", import.meta.url)), + "--left", + directories[0], + "--right", + directories[1], + ], + { encoding: "utf8" }, + ); + assert.ifError(result.error); + assert.equal(result.stderr, ""); + return { status: result.status, report: JSON.parse(result.stdout) }; + } finally { + rmSync(root, { recursive: true, force: true }); + } +} + +for (const artifact of [ + "wireshark-http.cap", + "wireshark-http_redirects.pcapng", +]) { + test(`${artifact}: repeated modes ignore only receipt IDs`, () => { + for (const delivery of ["complete", "summary"]) { + const left = fixture(delivery, artifact); + const right = structuredClone(left); + for (const value of right.answer.delivery_observations) { + value.evidence_id += "_second"; + if (Object.hasOwn(value, "parent_evidence_id")) + value.parent_evidence_id += "_second"; + } + const { status, report } = compare(left, right); + assert.equal(status, 0); + assert.equal(report.facts_match, true); + assert.deepEqual( + report.preserved_run_specific_values.map((value) => value.value), + [left.answer.delivery_observations, right.answer.delivery_observations], + ); + } + }); + + test(`${artifact}: repeated modes reject stable observation changes`, () => { + for (const delivery of ["complete", "summary"]) { + const left = fixture(delivery, artifact); + const mutations = [ + (values) => values[0].limitations.push("new limitation"), + (values) => { + values[0].coverage = "partial"; + }, + (values) => { + delete values[0].coverage; + }, + ]; + if (delivery === "summary") + mutations.push( + (values) => { + values[1].view.offset = 1; + }, + (values) => { + values[1].coverage.next_offset = null; + }, + (values) => values[1].unknowns.push("extra unknown"), + (values) => values.reverse(), + ); + for (const mutate of mutations) { + const right = structuredClone(left); + mutate(right.answer.delivery_observations); + const { status, report } = compare(left, right); + assert.equal(status, 1); + assert.equal(report.settings_match, true); + assert.equal(report.facts_match, false); + assert.deepEqual(report.fact_differences, ["delivery_observations"]); + } + } + }); + + test(`${artifact}: different modes preserve their different receipt scopes`, () => { + const left = fixture("complete", artifact); + const right = fixture("summary", artifact); + const { status, report } = compare(left, right); + assert.equal(status, 0); + assert.equal(report.facts_match, true); + assert.deepEqual( + report.preserved_run_specific_values.map((value) => value.value), + [left.answer.delivery_observations, right.answer.delivery_observations], + ); + right.answer.capture.coverage = "partial"; + assert.equal(compare(left, right).report.facts_match, false); + }); +}