Define a committed scenario/replay/shim/evidence package format for portable process-fidelity conformance fixtures (#336). The package format is versioned, path-independent, and consumable from a clean checkout. - conformancePackage.ts: top-level package schema with scenario manifests, replay plans, shim plans, expected evidence, and verifier contracts. Deterministic package ID computed from canonical JSON. Cross-reference validation ensures internal consistency. - conformanceTrustGate.ts: trust gates that compare captured runs against committed conformance packages (#337). Classifies semantic diffs from timing/normalization noise. Unknown/truncated evidence is never treated as equivalence. - conformanceReplay.ts: CI replay engine that runs each scenario, evaluates trust gates, and detects drift between captured and committed evidence (#338). Closes #336, #337, #338
191 lines
4.4 KiB
TypeScript
191 lines
4.4 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
CONFORMANCE_PACKAGE_VERSION,
|
|
createConformancePackage,
|
|
type ConformancePackage,
|
|
} from "../src/domain/conformancePackage.js";
|
|
import {
|
|
replayConformancePackage,
|
|
type ScenarioReplayResult,
|
|
} from "../src/domain/conformanceReplay.js";
|
|
|
|
const validPackage = createConformancePackage({
|
|
schema_version: CONFORMANCE_PACKAGE_VERSION,
|
|
name: "test-fixture",
|
|
description: "A test conformance fixture",
|
|
created_at: "2026-07-28T00:00:00Z",
|
|
scenarios: [
|
|
{
|
|
scenario_id: "s1",
|
|
name: "Simple spawn",
|
|
description: "A simple process spawn scenario",
|
|
fixture_path: "tests/conformance/c/fixture.c",
|
|
expected_exit_code: 0,
|
|
expected_patterns: [],
|
|
},
|
|
],
|
|
replay_plans: [
|
|
{
|
|
scenario_id: "s1",
|
|
steps: [
|
|
{
|
|
step_id: "step1",
|
|
action: "run",
|
|
arguments: [],
|
|
timeout_ms: 1000,
|
|
},
|
|
],
|
|
environment: {},
|
|
},
|
|
],
|
|
shim_plans: [],
|
|
expected_evidence: [
|
|
{
|
|
scenario_id: "s1",
|
|
envelopes: [],
|
|
bundle: null,
|
|
required_dimensions: [],
|
|
},
|
|
],
|
|
verifier_contracts: [
|
|
{
|
|
scenario_id: "s1",
|
|
dimensions: [{ name: "exit_code", required: true, comparison: "exact" }],
|
|
timing_tolerance_ms: 0,
|
|
},
|
|
],
|
|
});
|
|
|
|
async function passingRunner(
|
|
scenarioId: string,
|
|
): Promise<ScenarioReplayResult> {
|
|
return {
|
|
scenario_id: scenarioId,
|
|
status: "pass",
|
|
exit_code: 0,
|
|
duration_ms: 10,
|
|
output: "ok",
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
async function failingRunner(
|
|
scenarioId: string,
|
|
): Promise<ScenarioReplayResult> {
|
|
return {
|
|
scenario_id: scenarioId,
|
|
status: "fail",
|
|
exit_code: 1,
|
|
duration_ms: 10,
|
|
output: "fail",
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
describe("conformance CI replay", () => {
|
|
it("replays a package with passing scenarios", async () => {
|
|
const result = await replayConformancePackage(
|
|
validPackage,
|
|
{},
|
|
passingRunner,
|
|
);
|
|
expect(result.total_scenarios).toBe(1);
|
|
expect(result.passed).toBe(1);
|
|
expect(result.failed).toBe(0);
|
|
expect(result.errored).toBe(0);
|
|
});
|
|
|
|
it("replays a package with failing scenarios", async () => {
|
|
const result = await replayConformancePackage(
|
|
validPackage,
|
|
{},
|
|
failingRunner,
|
|
);
|
|
expect(result.failed).toBe(1);
|
|
expect(result.passed).toBe(0);
|
|
});
|
|
|
|
it("does not detect drift when expected and actual match", async () => {
|
|
const result = await replayConformancePackage(
|
|
validPackage,
|
|
{},
|
|
passingRunner,
|
|
);
|
|
expect(result.drift_detected).toBe(false);
|
|
});
|
|
|
|
it("throws on invalid package", async () => {
|
|
await expect(replayConformancePackage({}, {})).rejects.toThrow(
|
|
/invalid conformance package/u,
|
|
);
|
|
});
|
|
|
|
it("counts all result types", async () => {
|
|
const base: ConformancePackage = validPackage;
|
|
const multiPackage = createConformancePackage({
|
|
...base,
|
|
scenarios: [
|
|
base.scenarios[0]!,
|
|
{
|
|
scenario_id: "s2",
|
|
name: "Second scenario",
|
|
description: "Second",
|
|
fixture_path: "tests/conformance/c/fixture2.c",
|
|
expected_exit_code: 1,
|
|
expected_patterns: [],
|
|
},
|
|
],
|
|
replay_plans: [
|
|
base.replay_plans[0]!,
|
|
{
|
|
scenario_id: "s2",
|
|
steps: [
|
|
{
|
|
step_id: "step1",
|
|
action: "run",
|
|
arguments: [],
|
|
timeout_ms: 1000,
|
|
},
|
|
],
|
|
environment: {},
|
|
},
|
|
],
|
|
expected_evidence: [
|
|
base.expected_evidence[0]!,
|
|
{
|
|
scenario_id: "s2",
|
|
envelopes: [],
|
|
bundle: null,
|
|
required_dimensions: [],
|
|
},
|
|
],
|
|
verifier_contracts: [
|
|
base.verifier_contracts[0]!,
|
|
{
|
|
scenario_id: "s2",
|
|
dimensions: [
|
|
{ name: "exit_code", required: true, comparison: "exact" },
|
|
],
|
|
timing_tolerance_ms: 0,
|
|
},
|
|
],
|
|
});
|
|
const result = await replayConformancePackage(
|
|
multiPackage,
|
|
{},
|
|
async (scenarioId) => ({
|
|
scenario_id: scenarioId,
|
|
status: scenarioId === "s1" ? "pass" : "fail",
|
|
exit_code: scenarioId === "s1" ? 0 : 1,
|
|
duration_ms: 10,
|
|
output: "",
|
|
error: null,
|
|
}),
|
|
);
|
|
expect(result.total_scenarios).toBe(2);
|
|
expect(result.passed).toBe(1);
|
|
expect(result.failed).toBe(1);
|
|
});
|
|
});
|