fix(experiments): compare stable PCAP receipt fields across repeated deliveries
Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
committed by
morluto
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
parent
3c36892ab7
commit
eac921eb44
@@ -72,8 +72,9 @@ for (const [side, answer] of [
|
||||
}
|
||||
// PCAP delivery observations preserve each actual receipt's identity, limitations
|
||||
// and view coverage. Summary/page scopes differ from complete source scope;
|
||||
// retain them as run-specific facts rather than equating page exhaustion with
|
||||
// capture coverage. The common capture facts and URI prefix still must match.
|
||||
// retain them separately rather than equating page exhaustion with capture
|
||||
// coverage. Repeated equivalent deliveries must also match every observation
|
||||
// field except the receipt and parent IDs, which identify their own invocations.
|
||||
if (
|
||||
["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
|
||||
left.artifact,
|
||||
@@ -93,7 +94,18 @@ if (
|
||||
path: ["delivery_observations"],
|
||||
value: answer.delivery_observations,
|
||||
});
|
||||
delete answer.delivery_observations;
|
||||
if (left.delivery === right.delivery) {
|
||||
answer.delivery_observations = answer.delivery_observations.map(
|
||||
(observation) =>
|
||||
Object.fromEntries(
|
||||
Object.entries(observation).filter(
|
||||
([key]) => key !== "evidence_id" && key !== "parent_evidence_id",
|
||||
),
|
||||
),
|
||||
);
|
||||
} else {
|
||||
delete answer.delivery_observations;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Binwalk assigns a new opaque region UUID in each invocation. Preserve both
|
||||
|
||||
@@ -71,7 +71,11 @@ Reproduce with `measure.mjs --artifact wireshark-http.cap` or
|
||||
`wireshark-http_redirects.pcapng`, pinned path/runtime/revision, explicit provider
|
||||
JSON, `--schemas selected`, and `--delivery complete|summary`. Use `compare.mjs`
|
||||
on completed output directories. Original replies, catalogs, comparison receipts
|
||||
and exact measured driver source stay private.
|
||||
and exact measured driver source stay private. Repeated deliveries in the same
|
||||
mode also compare the ordered receipt observations, including limitations, view
|
||||
parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific.
|
||||
Comparator regressions run with `node --test experiments/analysis-efficiency/test_compare.mjs`
|
||||
using synthetic measurement directories; they do not establish provider behavior.
|
||||
|
||||
## Actual model tasks
|
||||
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
// Synthetic measurement directories exercise the comparator CLI, not providers.
|
||||
const observation = (kind) => ({
|
||||
evidence_id: `ev_${kind}`,
|
||||
parent_evidence_id: "ev_parent",
|
||||
limitations: ["saved bytes only"],
|
||||
kind,
|
||||
view:
|
||||
kind === "summary"
|
||||
? { kind }
|
||||
: { kind, facet: "http-uris", offset: 0, limit: 2 },
|
||||
coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
|
||||
unknowns: ["unrecorded traffic"],
|
||||
});
|
||||
const fixture = (delivery, artifact) => ({
|
||||
summary: {
|
||||
artifact,
|
||||
delivery,
|
||||
provider_search_path_sha256: "a".repeat(64),
|
||||
calls: [],
|
||||
},
|
||||
answer: {
|
||||
capture: { coverage: "complete", capture_start: null },
|
||||
http_request_uris_first2: [
|
||||
"http://example.test/one",
|
||||
"http://example.test/two",
|
||||
],
|
||||
delivery_observations:
|
||||
delivery === "summary"
|
||||
? [observation("summary"), observation("pcap")]
|
||||
: [
|
||||
{
|
||||
evidence_id: "ev_complete",
|
||||
limitations: ["saved bytes only"],
|
||||
coverage: "complete",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
function compare(left, right) {
|
||||
const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
|
||||
try {
|
||||
const directories = ["left", "right"].map((side) => join(root, side));
|
||||
for (const [index, value] of [left, right].entries()) {
|
||||
mkdirSync(directories[index]);
|
||||
for (const name of ["summary", "answer"])
|
||||
writeFileSync(
|
||||
join(directories[index], `${name}.json`),
|
||||
JSON.stringify(value[name]),
|
||||
);
|
||||
}
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
fileURLToPath(new URL("compare.mjs", import.meta.url)),
|
||||
"--left",
|
||||
directories[0],
|
||||
"--right",
|
||||
directories[1],
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
assert.ifError(result.error);
|
||||
assert.equal(result.stderr, "");
|
||||
return { status: result.status, report: JSON.parse(result.stdout) };
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const artifact of [
|
||||
"wireshark-http.cap",
|
||||
"wireshark-http_redirects.pcapng",
|
||||
]) {
|
||||
test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
|
||||
for (const delivery of ["complete", "summary"]) {
|
||||
const left = fixture(delivery, artifact);
|
||||
const right = structuredClone(left);
|
||||
for (const value of right.answer.delivery_observations) {
|
||||
value.evidence_id += "_second";
|
||||
if (Object.hasOwn(value, "parent_evidence_id"))
|
||||
value.parent_evidence_id += "_second";
|
||||
}
|
||||
const { status, report } = compare(left, right);
|
||||
assert.equal(status, 0);
|
||||
assert.equal(report.facts_match, true);
|
||||
assert.deepEqual(
|
||||
report.preserved_run_specific_values.map((value) => value.value),
|
||||
[left.answer.delivery_observations, right.answer.delivery_observations],
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test(`${artifact}: repeated modes reject stable observation changes`, () => {
|
||||
for (const delivery of ["complete", "summary"]) {
|
||||
const left = fixture(delivery, artifact);
|
||||
const mutations = [
|
||||
(values) => values[0].limitations.push("new limitation"),
|
||||
(values) => {
|
||||
values[0].coverage = "partial";
|
||||
},
|
||||
(values) => {
|
||||
delete values[0].coverage;
|
||||
},
|
||||
];
|
||||
if (delivery === "summary")
|
||||
mutations.push(
|
||||
(values) => {
|
||||
values[1].view.offset = 1;
|
||||
},
|
||||
(values) => {
|
||||
values[1].coverage.next_offset = null;
|
||||
},
|
||||
(values) => values[1].unknowns.push("extra unknown"),
|
||||
(values) => values.reverse(),
|
||||
);
|
||||
for (const mutate of mutations) {
|
||||
const right = structuredClone(left);
|
||||
mutate(right.answer.delivery_observations);
|
||||
const { status, report } = compare(left, right);
|
||||
assert.equal(status, 1);
|
||||
assert.equal(report.settings_match, true);
|
||||
assert.equal(report.facts_match, false);
|
||||
assert.deepEqual(report.fact_differences, ["delivery_observations"]);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test(`${artifact}: different modes preserve their different receipt scopes`, () => {
|
||||
const left = fixture("complete", artifact);
|
||||
const right = fixture("summary", artifact);
|
||||
const { status, report } = compare(left, right);
|
||||
assert.equal(status, 0);
|
||||
assert.equal(report.facts_match, true);
|
||||
assert.deepEqual(
|
||||
report.preserved_run_specific_values.map((value) => value.value),
|
||||
[left.answer.delivery_observations, right.answer.delivery_observations],
|
||||
);
|
||||
right.answer.capture.coverage = "partial";
|
||||
assert.equal(compare(left, right).report.facts_match, false);
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user