test(perf): discover comparator CLI regressions in Vitest

Migrate the six manual Node comparator scenarios into the existing evaluation
project so routine source, PR and CI suites execute them. Preserve the CLI
assertions and scoped synthetic fixtures, parse required report fields, and
remove the superseded manual suite. Update the exact reproduction command.
No package, workflow, production or provider behavior changes.

Validation: six focused Vitest scenarios pass, and all 1113 configured test
files are discovered exactly once. Unchanged Python/SDK/model tests are not
rerun for this suite-only migration.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-13 19:33:46 +00:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent 57649d3d8d
commit 4f025b61ae
2 changed files with 76 additions and 21 deletions
@@ -74,7 +74,7 @@ on completed output directories. Original replies, catalogs, comparison receipts
and exact measured driver source stay private. Repeated deliveries in the same
mode also compare the ordered receipt observations, including limitations, view
parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific.
Comparator regressions run with `node --test experiments/analysis-efficiency/test_compare.mjs`
Comparator regressions run with `npm run test:local -- tests/evaluation/analysisEfficiencyComparison.test.ts`
using synthetic measurement directories; they do not establish provider behavior.
## Actual model tasks
@@ -278,3 +278,8 @@ Workflow citation cardinality must equal the planned call count before binding
per-call IDs. A shorter or longer answer list cannot silently shorten Python
zip’s expected plan; missing URI calls and extra citations are rejected.
Original complete/summary model workflows continue to pass unchanged.
The six comparator CLI regressions now live in the normally discovered Vitest
evaluation suite, so routine local/PR/CI gates execute them. The original six
scenarios and CLI assertions are retained; the former manual Node suite is
removed. They remain synthetic consumer checks, not provider evidence.
@@ -3,11 +3,49 @@ import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import test from "node:test";
import { test } from "vitest";
import { z } from "zod";
import { fileURLToPath } from "node:url";
type Delivery = "complete" | "summary";
type Observation = {
evidence_id: string;
parent_evidence_id?: string;
limitations: string[];
kind?: string;
view?: { kind: string; facet?: string; offset?: number; limit?: number };
coverage?:
| string
| {
examined: number;
total: number;
next_offset: number | null;
exhausted: boolean;
};
unknowns?: string[];
};
type Fixture = {
summary: {
artifact: string;
delivery: Delivery;
provider_search_path_sha256: string;
calls: unknown[];
};
answer: {
capture: { coverage: string; capture_start: null };
http_request_uris_first2: string[];
delivery_observations: Observation[];
};
};
const reportSchema = z.object({
facts_match: z.boolean(),
settings_match: z.boolean(),
fact_differences: z.array(z.string()),
preserved_run_specific_values: z.array(z.object({ value: z.unknown() })),
});
// Synthetic measurement directories exercise the comparator CLI, not providers.
const observation = (kind) => ({
const observation = (kind: string): Observation => ({
evidence_id: `ev_${kind}`,
parent_evidence_id: "ev_parent",
limitations: ["saved bytes only"],
@@ -19,7 +57,7 @@ const observation = (kind) => ({
coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
unknowns: ["unrecorded traffic"],
});
const fixture = (delivery, artifact) => ({
const fixture = (delivery: Delivery, artifact: string): Fixture => ({
summary: {
artifact,
delivery,
@@ -44,22 +82,29 @@ const fixture = (delivery, artifact) => ({
],
},
});
function compare(left, right) {
function compare(left: Fixture, right: Fixture) {
const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
try {
const directories = ["left", "right"].map((side) => join(root, side));
const directories = [join(root, "left"), join(root, "right")] as const;
for (const [index, value] of [left, right].entries()) {
mkdirSync(directories[index]);
for (const name of ["summary", "answer"])
const directory = directories[index];
assert.ok(directory);
mkdirSync(directory);
for (const name of ["summary", "answer"] as const)
writeFileSync(
join(directories[index], `${name}.json`),
join(directory, `${name}.json`),
JSON.stringify(value[name]),
);
}
const result = spawnSync(
process.execPath,
[
fileURLToPath(new URL("compare.mjs", import.meta.url)),
fileURLToPath(
new URL(
"../../experiments/analysis-efficiency/compare.mjs",
import.meta.url,
),
),
"--left",
directories[0],
"--right",
@@ -69,7 +114,10 @@ function compare(left, right) {
);
assert.ifError(result.error);
assert.equal(result.stderr, "");
return { status: result.status, report: JSON.parse(result.stdout) };
return {
status: result.status,
report: reportSchema.parse(JSON.parse(result.stdout)),
};
} finally {
rmSync(root, { recursive: true, force: true });
}
@@ -80,12 +128,12 @@ for (const artifact of [
"wireshark-http_redirects.pcapng",
]) {
test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
for (const delivery of ["complete", "summary"]) {
for (const delivery of ["complete", "summary"] as const) {
const left = fixture(delivery, artifact);
const right = structuredClone(left);
for (const value of right.answer.delivery_observations) {
value.evidence_id += "_second";
if (Object.hasOwn(value, "parent_evidence_id"))
if (typeof value.parent_evidence_id === "string")
value.parent_evidence_id += "_second";
}
const { status, report } = compare(left, right);
@@ -99,26 +147,28 @@ for (const artifact of [
});
test(`${artifact}: repeated modes reject stable observation changes`, () => {
for (const delivery of ["complete", "summary"]) {
for (const delivery of ["complete", "summary"] as const) {
const left = fixture(delivery, artifact);
const mutations = [
(values) => values[0].limitations.push("new limitation"),
const mutations: ((values: Observation[]) => void)[] = [
(values) => values[0]!.limitations.push("new limitation"),
(values) => {
values[0].coverage = "partial";
values[0]!.coverage = "partial";
},
(values) => {
delete values[0].coverage;
delete values[0]!.coverage;
},
];
if (delivery === "summary")
mutations.push(
(values) => {
values[1].view.offset = 1;
values[1]!.view!.offset = 1;
},
(values) => {
values[1].coverage.next_offset = null;
const coverage = values[1]!.coverage;
assert.ok(typeof coverage === "object" && coverage !== null);
coverage.next_offset = null;
},
(values) => values[1].unknowns.push("extra unknown"),
(values) => values[1]!.unknowns!.push("extra unknown"),
(values) => values.reverse(),
);
for (const mutate of mutations) {