test(perf): discover comparator CLI regressions in Vitest
Migrate the six manual Node comparator scenarios into the existing evaluation project so routine source, PR and CI suites execute them. Preserve the CLI assertions and scoped synthetic fixtures, parse required report fields, and remove the superseded manual suite. Update the exact reproduction command. No package, workflow, production or provider behavior changes. Validation: six focused Vitest scenarios pass, and all 1113 configured test files are discovered exactly once. Unchanged Python/SDK/model tests are not rerun for this suite-only migration. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
parent
57649d3d8d
commit
4f025b61ae
@@ -74,7 +74,7 @@ on completed output directories. Original replies, catalogs, comparison receipts
|
||||
and exact measured driver source stay private. Repeated deliveries in the same
|
||||
mode also compare the ordered receipt observations, including limitations, view
|
||||
parameters, coverage and unknowns; only receipt/parent IDs are invocation-specific.
|
||||
Comparator regressions run with `node --test experiments/analysis-efficiency/test_compare.mjs`
|
||||
Comparator regressions run with `npm run test:local -- tests/evaluation/analysisEfficiencyComparison.test.ts`
|
||||
using synthetic measurement directories; they do not establish provider behavior.
|
||||
|
||||
## Actual model tasks
|
||||
@@ -278,3 +278,8 @@ Workflow citation cardinality must equal the planned call count before binding
|
||||
per-call IDs. A shorter or longer answer list cannot silently shorten Python
|
||||
zip’s expected plan; missing URI calls and extra citations are rejected.
|
||||
Original complete/summary model workflows continue to pass unchanged.
|
||||
|
||||
The six comparator CLI regressions now live in the normally discovered Vitest
|
||||
evaluation suite, so routine local/PR/CI gates execute them. The original six
|
||||
scenarios and CLI assertions are retained; the former manual Node suite is
|
||||
removed. They remain synthetic consumer checks, not provider evidence.
|
||||
|
||||
+70
-20
@@ -3,11 +3,49 @@ import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import test from "node:test";
|
||||
import { test } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
type Delivery = "complete" | "summary";
|
||||
type Observation = {
|
||||
evidence_id: string;
|
||||
parent_evidence_id?: string;
|
||||
limitations: string[];
|
||||
kind?: string;
|
||||
view?: { kind: string; facet?: string; offset?: number; limit?: number };
|
||||
coverage?:
|
||||
| string
|
||||
| {
|
||||
examined: number;
|
||||
total: number;
|
||||
next_offset: number | null;
|
||||
exhausted: boolean;
|
||||
};
|
||||
unknowns?: string[];
|
||||
};
|
||||
type Fixture = {
|
||||
summary: {
|
||||
artifact: string;
|
||||
delivery: Delivery;
|
||||
provider_search_path_sha256: string;
|
||||
calls: unknown[];
|
||||
};
|
||||
answer: {
|
||||
capture: { coverage: string; capture_start: null };
|
||||
http_request_uris_first2: string[];
|
||||
delivery_observations: Observation[];
|
||||
};
|
||||
};
|
||||
const reportSchema = z.object({
|
||||
facts_match: z.boolean(),
|
||||
settings_match: z.boolean(),
|
||||
fact_differences: z.array(z.string()),
|
||||
preserved_run_specific_values: z.array(z.object({ value: z.unknown() })),
|
||||
});
|
||||
|
||||
// Synthetic measurement directories exercise the comparator CLI, not providers.
|
||||
const observation = (kind) => ({
|
||||
const observation = (kind: string): Observation => ({
|
||||
evidence_id: `ev_${kind}`,
|
||||
parent_evidence_id: "ev_parent",
|
||||
limitations: ["saved bytes only"],
|
||||
@@ -19,7 +57,7 @@ const observation = (kind) => ({
|
||||
coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
|
||||
unknowns: ["unrecorded traffic"],
|
||||
});
|
||||
const fixture = (delivery, artifact) => ({
|
||||
const fixture = (delivery: Delivery, artifact: string): Fixture => ({
|
||||
summary: {
|
||||
artifact,
|
||||
delivery,
|
||||
@@ -44,22 +82,29 @@ const fixture = (delivery, artifact) => ({
|
||||
],
|
||||
},
|
||||
});
|
||||
function compare(left, right) {
|
||||
function compare(left: Fixture, right: Fixture) {
|
||||
const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
|
||||
try {
|
||||
const directories = ["left", "right"].map((side) => join(root, side));
|
||||
const directories = [join(root, "left"), join(root, "right")] as const;
|
||||
for (const [index, value] of [left, right].entries()) {
|
||||
mkdirSync(directories[index]);
|
||||
for (const name of ["summary", "answer"])
|
||||
const directory = directories[index];
|
||||
assert.ok(directory);
|
||||
mkdirSync(directory);
|
||||
for (const name of ["summary", "answer"] as const)
|
||||
writeFileSync(
|
||||
join(directories[index], `${name}.json`),
|
||||
join(directory, `${name}.json`),
|
||||
JSON.stringify(value[name]),
|
||||
);
|
||||
}
|
||||
const result = spawnSync(
|
||||
process.execPath,
|
||||
[
|
||||
fileURLToPath(new URL("compare.mjs", import.meta.url)),
|
||||
fileURLToPath(
|
||||
new URL(
|
||||
"../../experiments/analysis-efficiency/compare.mjs",
|
||||
import.meta.url,
|
||||
),
|
||||
),
|
||||
"--left",
|
||||
directories[0],
|
||||
"--right",
|
||||
@@ -69,7 +114,10 @@ function compare(left, right) {
|
||||
);
|
||||
assert.ifError(result.error);
|
||||
assert.equal(result.stderr, "");
|
||||
return { status: result.status, report: JSON.parse(result.stdout) };
|
||||
return {
|
||||
status: result.status,
|
||||
report: reportSchema.parse(JSON.parse(result.stdout)),
|
||||
};
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -80,12 +128,12 @@ for (const artifact of [
|
||||
"wireshark-http_redirects.pcapng",
|
||||
]) {
|
||||
test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
|
||||
for (const delivery of ["complete", "summary"]) {
|
||||
for (const delivery of ["complete", "summary"] as const) {
|
||||
const left = fixture(delivery, artifact);
|
||||
const right = structuredClone(left);
|
||||
for (const value of right.answer.delivery_observations) {
|
||||
value.evidence_id += "_second";
|
||||
if (Object.hasOwn(value, "parent_evidence_id"))
|
||||
if (typeof value.parent_evidence_id === "string")
|
||||
value.parent_evidence_id += "_second";
|
||||
}
|
||||
const { status, report } = compare(left, right);
|
||||
@@ -99,26 +147,28 @@ for (const artifact of [
|
||||
});
|
||||
|
||||
test(`${artifact}: repeated modes reject stable observation changes`, () => {
|
||||
for (const delivery of ["complete", "summary"]) {
|
||||
for (const delivery of ["complete", "summary"] as const) {
|
||||
const left = fixture(delivery, artifact);
|
||||
const mutations = [
|
||||
(values) => values[0].limitations.push("new limitation"),
|
||||
const mutations: ((values: Observation[]) => void)[] = [
|
||||
(values) => values[0]!.limitations.push("new limitation"),
|
||||
(values) => {
|
||||
values[0].coverage = "partial";
|
||||
values[0]!.coverage = "partial";
|
||||
},
|
||||
(values) => {
|
||||
delete values[0].coverage;
|
||||
delete values[0]!.coverage;
|
||||
},
|
||||
];
|
||||
if (delivery === "summary")
|
||||
mutations.push(
|
||||
(values) => {
|
||||
values[1].view.offset = 1;
|
||||
values[1]!.view!.offset = 1;
|
||||
},
|
||||
(values) => {
|
||||
values[1].coverage.next_offset = null;
|
||||
const coverage = values[1]!.coverage;
|
||||
assert.ok(typeof coverage === "object" && coverage !== null);
|
||||
coverage.next_offset = null;
|
||||
},
|
||||
(values) => values[1].unknowns.push("extra unknown"),
|
||||
(values) => values[1]!.unknowns!.push("extra unknown"),
|
||||
(values) => values.reverse(),
|
||||
);
|
||||
for (const mutate of mutations) {
|
||||
Reference in New Issue
Block a user