Files
rea/tests/conformanceReplay.test.ts
T
morluto 4ee30e4145 feat(conformance): add portable conformance export package format
Define a committed scenario/replay/shim/evidence package format for
portable process-fidelity conformance fixtures (#336). The package
format is versioned, path-independent, and consumable from a clean
checkout.

- conformancePackage.ts: top-level package schema with scenario
  manifests, replay plans, shim plans, expected evidence, and
  verifier contracts. Deterministic package ID computed from
  canonical JSON. Cross-reference validation ensures internal
  consistency.
- conformanceTrustGate.ts: trust gates that compare captured runs
  against committed conformance packages (#337). Classifies semantic
  diffs from timing/normalization noise. Unknown/truncated evidence
  is never treated as equivalence.
- conformanceReplay.ts: CI replay engine that runs each scenario,
  evaluates trust gates, and detects drift between captured and
  committed evidence (#338).

Closes #336, #337, #338
2026-07-28 05:40:17 +00:00

191 lines
4.4 KiB
TypeScript

import { describe, expect, it } from "vitest";
import {
CONFORMANCE_PACKAGE_VERSION,
createConformancePackage,
type ConformancePackage,
} from "../src/domain/conformancePackage.js";
import {
replayConformancePackage,
type ScenarioReplayResult,
} from "../src/domain/conformanceReplay.js";
const validPackage = createConformancePackage({
schema_version: CONFORMANCE_PACKAGE_VERSION,
name: "test-fixture",
description: "A test conformance fixture",
created_at: "2026-07-28T00:00:00Z",
scenarios: [
{
scenario_id: "s1",
name: "Simple spawn",
description: "A simple process spawn scenario",
fixture_path: "tests/conformance/c/fixture.c",
expected_exit_code: 0,
expected_patterns: [],
},
],
replay_plans: [
{
scenario_id: "s1",
steps: [
{
step_id: "step1",
action: "run",
arguments: [],
timeout_ms: 1000,
},
],
environment: {},
},
],
shim_plans: [],
expected_evidence: [
{
scenario_id: "s1",
envelopes: [],
bundle: null,
required_dimensions: [],
},
],
verifier_contracts: [
{
scenario_id: "s1",
dimensions: [{ name: "exit_code", required: true, comparison: "exact" }],
timing_tolerance_ms: 0,
},
],
});
async function passingRunner(
scenarioId: string,
): Promise<ScenarioReplayResult> {
return {
scenario_id: scenarioId,
status: "pass",
exit_code: 0,
duration_ms: 10,
output: "ok",
error: null,
};
}
async function failingRunner(
scenarioId: string,
): Promise<ScenarioReplayResult> {
return {
scenario_id: scenarioId,
status: "fail",
exit_code: 1,
duration_ms: 10,
output: "fail",
error: null,
};
}
describe("conformance CI replay", () => {
it("replays a package with passing scenarios", async () => {
const result = await replayConformancePackage(
validPackage,
{},
passingRunner,
);
expect(result.total_scenarios).toBe(1);
expect(result.passed).toBe(1);
expect(result.failed).toBe(0);
expect(result.errored).toBe(0);
});
it("replays a package with failing scenarios", async () => {
const result = await replayConformancePackage(
validPackage,
{},
failingRunner,
);
expect(result.failed).toBe(1);
expect(result.passed).toBe(0);
});
it("does not detect drift when expected and actual match", async () => {
const result = await replayConformancePackage(
validPackage,
{},
passingRunner,
);
expect(result.drift_detected).toBe(false);
});
it("throws on invalid package", async () => {
await expect(replayConformancePackage({}, {})).rejects.toThrow(
/invalid conformance package/u,
);
});
it("counts all result types", async () => {
const base: ConformancePackage = validPackage;
const multiPackage = createConformancePackage({
...base,
scenarios: [
base.scenarios[0]!,
{
scenario_id: "s2",
name: "Second scenario",
description: "Second",
fixture_path: "tests/conformance/c/fixture2.c",
expected_exit_code: 1,
expected_patterns: [],
},
],
replay_plans: [
base.replay_plans[0]!,
{
scenario_id: "s2",
steps: [
{
step_id: "step1",
action: "run",
arguments: [],
timeout_ms: 1000,
},
],
environment: {},
},
],
expected_evidence: [
base.expected_evidence[0]!,
{
scenario_id: "s2",
envelopes: [],
bundle: null,
required_dimensions: [],
},
],
verifier_contracts: [
base.verifier_contracts[0]!,
{
scenario_id: "s2",
dimensions: [
{ name: "exit_code", required: true, comparison: "exact" },
],
timing_tolerance_ms: 0,
},
],
});
const result = await replayConformancePackage(
multiPackage,
{},
async (scenarioId) => ({
scenario_id: scenarioId,
status: scenarioId === "s1" ? "pass" : "fail",
exit_code: scenarioId === "s1" ? 0 : 1,
duration_ms: 10,
output: "",
error: null,
}),
);
expect(result.total_scenarios).toBe(2);
expect(result.passed).toBe(1);
expect(result.failed).toBe(1);
});
});