fix(experiments): compare stable PCAP receipt fields across repeated deliveries

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-14 02:40:31 +08:00
committed by morluto
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent 3c36892ab7
commit eac921eb44
3 changed files with 169 additions and 4 deletions
+15 -3
View File
@@ -72,8 +72,9 @@ for (const [side, answer] of [
}
// PCAP delivery observations preserve each actual receipt's identity, limitations
// and view coverage. Summary/page scopes differ from complete source scope;
// retain them as run-specific facts rather than equating page exhaustion with
// capture coverage. The common capture facts and URI prefix still must match.
// retain them separately rather than equating page exhaustion with capture
// coverage. Repeated equivalent deliveries must also match every observation
// field except the receipt and parent IDs, which identify their own invocations.
if (
["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
left.artifact,
@@ -93,7 +94,18 @@ if (
path: ["delivery_observations"],
value: answer.delivery_observations,
});
delete answer.delivery_observations;
if (left.delivery === right.delivery) {
answer.delivery_observations = answer.delivery_observations.map(
(observation) =>
Object.fromEntries(
Object.entries(observation).filter(
([key]) => key !== "evidence_id" && key !== "parent_evidence_id",
),
),
);
} else {
delete answer.delivery_observations;
}
}
}
// Binwalk assigns a new opaque region UUID in each invocation. Preserve both
@@ -71,7 +71,11 @@ Reproduce with `measure.mjs --artifact wireshark-http.cap` or
`wireshark-http_redirects.pcapng`, pinned path/runtime/revision, explicit provider
JSON, `--schemas selected`, and `--delivery complete|summary`. Use `compare.mjs`
on completed output directories. Original replies, catalogs, comparison receipts
and exact measured driver source stay private.
and exact measured driver source stay private. Repeated deliveries in the same
mode also compare the ordered receipt observations, including limitations, view
parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific.
Comparator regressions run with `node --test experiments/analysis-efficiency/test_compare.mjs`
using synthetic measurement directories; they do not establish provider behavior.
## Actual model tasks
@@ -0,0 +1,149 @@
import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
// Synthetic measurement directories exercise the comparator CLI, not providers.
const observation = (kind) => ({
evidence_id: `ev_${kind}`,
parent_evidence_id: "ev_parent",
limitations: ["saved bytes only"],
kind,
view:
kind === "summary"
? { kind }
: { kind, facet: "http-uris", offset: 0, limit: 2 },
coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
unknowns: ["unrecorded traffic"],
});
const fixture = (delivery, artifact) => ({
summary: {
artifact,
delivery,
provider_search_path_sha256: "a".repeat(64),
calls: [],
},
answer: {
capture: { coverage: "complete", capture_start: null },
http_request_uris_first2: [
"http://example.test/one",
"http://example.test/two",
],
delivery_observations:
delivery === "summary"
? [observation("summary"), observation("pcap")]
: [
{
evidence_id: "ev_complete",
limitations: ["saved bytes only"],
coverage: "complete",
},
],
},
});
function compare(left, right) {
const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
try {
const directories = ["left", "right"].map((side) => join(root, side));
for (const [index, value] of [left, right].entries()) {
mkdirSync(directories[index]);
for (const name of ["summary", "answer"])
writeFileSync(
join(directories[index], `${name}.json`),
JSON.stringify(value[name]),
);
}
const result = spawnSync(
process.execPath,
[
fileURLToPath(new URL("compare.mjs", import.meta.url)),
"--left",
directories[0],
"--right",
directories[1],
],
{ encoding: "utf8" },
);
assert.ifError(result.error);
assert.equal(result.stderr, "");
return { status: result.status, report: JSON.parse(result.stdout) };
} finally {
rmSync(root, { recursive: true, force: true });
}
}
for (const artifact of [
"wireshark-http.cap",
"wireshark-http_redirects.pcapng",
]) {
test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
for (const delivery of ["complete", "summary"]) {
const left = fixture(delivery, artifact);
const right = structuredClone(left);
for (const value of right.answer.delivery_observations) {
value.evidence_id += "_second";
if (Object.hasOwn(value, "parent_evidence_id"))
value.parent_evidence_id += "_second";
}
const { status, report } = compare(left, right);
assert.equal(status, 0);
assert.equal(report.facts_match, true);
assert.deepEqual(
report.preserved_run_specific_values.map((value) => value.value),
[left.answer.delivery_observations, right.answer.delivery_observations],
);
}
});
test(`${artifact}: repeated modes reject stable observation changes`, () => {
for (const delivery of ["complete", "summary"]) {
const left = fixture(delivery, artifact);
const mutations = [
(values) => values[0].limitations.push("new limitation"),
(values) => {
values[0].coverage = "partial";
},
(values) => {
delete values[0].coverage;
},
];
if (delivery === "summary")
mutations.push(
(values) => {
values[1].view.offset = 1;
},
(values) => {
values[1].coverage.next_offset = null;
},
(values) => values[1].unknowns.push("extra unknown"),
(values) => values.reverse(),
);
for (const mutate of mutations) {
const right = structuredClone(left);
mutate(right.answer.delivery_observations);
const { status, report } = compare(left, right);
assert.equal(status, 1);
assert.equal(report.settings_match, true);
assert.equal(report.facts_match, false);
assert.deepEqual(report.fact_differences, ["delivery_observations"]);
}
}
});
test(`${artifact}: different modes preserve their different receipt scopes`, () => {
const left = fixture("complete", artifact);
const right = fixture("summary", artifact);
const { status, report } = compare(left, right);
assert.equal(status, 0);
assert.equal(report.facts_match, true);
assert.deepEqual(
report.preserved_run_specific_values.map((value) => value.value),
[left.answer.delivery_observations, right.answer.delivery_observations],
);
right.answer.capture.coverage = "partial";
assert.equal(compare(left, right).report.facts_match, false);
});
}