Files
rea/tests/evaluation/analysisEfficiencyComparison.test.ts
T
N0zoM1z0factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>Kaoru0822
4f025b61ae test(perf): discover comparator CLI regressions in Vitest
Migrate the six manual Node comparator scenarios into the existing evaluation
project so routine source, PR and CI suites execute them. Preserve the CLI
assertions and scoped synthetic fixtures, parse required report fields, and
remove the superseded manual suite. Update the exact reproduction command.
No package, workflow, production or provider behavior changes.

Validation: six focused Vitest scenarios pass, and all 1113 configured test
files are discovered exactly once. Unchanged Python/SDK/model tests are not
rerun for this suite-only migration.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-13 19:33:46 +00:00

200 lines
6.2 KiB
TypeScript

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "vitest";
import { z } from "zod";
import { fileURLToPath } from "node:url";
type Delivery = "complete" | "summary";
type Observation = {
evidence_id: string;
parent_evidence_id?: string;
limitations: string[];
kind?: string;
view?: { kind: string; facet?: string; offset?: number; limit?: number };
coverage?:
| string
| {
examined: number;
total: number;
next_offset: number | null;
exhausted: boolean;
};
unknowns?: string[];
};
type Fixture = {
summary: {
artifact: string;
delivery: Delivery;
provider_search_path_sha256: string;
calls: unknown[];
};
answer: {
capture: { coverage: string; capture_start: null };
http_request_uris_first2: string[];
delivery_observations: Observation[];
};
};
const reportSchema = z.object({
facts_match: z.boolean(),
settings_match: z.boolean(),
fact_differences: z.array(z.string()),
preserved_run_specific_values: z.array(z.object({ value: z.unknown() })),
});
// Synthetic measurement directories exercise the comparator CLI, not providers.
const observation = (kind: string): Observation => ({
evidence_id: `ev_${kind}`,
parent_evidence_id: "ev_parent",
limitations: ["saved bytes only"],
kind,
view:
kind === "summary"
? { kind }
: { kind, facet: "http-uris", offset: 0, limit: 2 },
coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
unknowns: ["unrecorded traffic"],
});
const fixture = (delivery: Delivery, artifact: string): Fixture => ({
summary: {
artifact,
delivery,
provider_search_path_sha256: "a".repeat(64),
calls: [],
},
answer: {
capture: { coverage: "complete", capture_start: null },
http_request_uris_first2: [
"http://example.test/one",
"http://example.test/two",
],
delivery_observations:
delivery === "summary"
? [observation("summary"), observation("pcap")]
: [
{
evidence_id: "ev_complete",
limitations: ["saved bytes only"],
coverage: "complete",
},
],
},
});
function compare(left: Fixture, right: Fixture) {
const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
try {
const directories = [join(root, "left"), join(root, "right")] as const;
for (const [index, value] of [left, right].entries()) {
const directory = directories[index];
assert.ok(directory);
mkdirSync(directory);
for (const name of ["summary", "answer"] as const)
writeFileSync(
join(directory, `${name}.json`),
JSON.stringify(value[name]),
);
}
const result = spawnSync(
process.execPath,
[
fileURLToPath(
new URL(
"../../experiments/analysis-efficiency/compare.mjs",
import.meta.url,
),
),
"--left",
directories[0],
"--right",
directories[1],
],
{ encoding: "utf8" },
);
assert.ifError(result.error);
assert.equal(result.stderr, "");
return {
status: result.status,
report: reportSchema.parse(JSON.parse(result.stdout)),
};
} finally {
rmSync(root, { recursive: true, force: true });
}
}
for (const artifact of [
"wireshark-http.cap",
"wireshark-http_redirects.pcapng",
]) {
test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
for (const delivery of ["complete", "summary"] as const) {
const left = fixture(delivery, artifact);
const right = structuredClone(left);
for (const value of right.answer.delivery_observations) {
value.evidence_id += "_second";
if (typeof value.parent_evidence_id === "string")
value.parent_evidence_id += "_second";
}
const { status, report } = compare(left, right);
assert.equal(status, 0);
assert.equal(report.facts_match, true);
assert.deepEqual(
report.preserved_run_specific_values.map((value) => value.value),
[left.answer.delivery_observations, right.answer.delivery_observations],
);
}
});
test(`${artifact}: repeated modes reject stable observation changes`, () => {
for (const delivery of ["complete", "summary"] as const) {
const left = fixture(delivery, artifact);
const mutations: ((values: Observation[]) => void)[] = [
(values) => values[0]!.limitations.push("new limitation"),
(values) => {
values[0]!.coverage = "partial";
},
(values) => {
delete values[0]!.coverage;
},
];
if (delivery === "summary")
mutations.push(
(values) => {
values[1]!.view!.offset = 1;
},
(values) => {
const coverage = values[1]!.coverage;
assert.ok(typeof coverage === "object" && coverage !== null);
coverage.next_offset = null;
},
(values) => values[1]!.unknowns!.push("extra unknown"),
(values) => values.reverse(),
);
for (const mutate of mutations) {
const right = structuredClone(left);
mutate(right.answer.delivery_observations);
const { status, report } = compare(left, right);
assert.equal(status, 1);
assert.equal(report.settings_match, true);
assert.equal(report.facts_match, false);
assert.deepEqual(report.fact_differences, ["delivery_observations"]);
}
}
});
test(`${artifact}: different modes preserve their different receipt scopes`, () => {
const left = fixture("complete", artifact);
const right = fixture("summary", artifact);
const { status, report } = compare(left, right);
assert.equal(status, 0);
assert.equal(report.facts_match, true);
assert.deepEqual(
report.preserved_run_specific_values.map((value) => value.value),
[left.answer.delivery_observations, right.answer.delivery_observations],
);
right.answer.capture.coverage = "partial";
assert.equal(compare(left, right).report.facts_match, false);
});
}