263 lines
6.4 KiB
TypeScript
263 lines
6.4 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
|
|
import {
|
|
createConformancePackage,
|
|
type ConformancePackageInput,
|
|
} from "../../../src/domain/conformancePackage.js";
|
|
import { createEvidence } from "../../../src/domain/evidence.js";
|
|
import {
|
|
packageReplayResultSchema,
|
|
replayConformancePackage,
|
|
type ScenarioReplayResult,
|
|
} from "../../../src/domain/conformanceReplay.js";
|
|
|
|
const validPackageInput: ConformancePackageInput = {
|
|
name: "test-fixture",
|
|
description: "A test conformance fixture",
|
|
created_at: "2026-07-28T00:00:00Z",
|
|
scenarios: [
|
|
{
|
|
scenario_id: "s1",
|
|
name: "Simple spawn",
|
|
description: "A simple process spawn scenario",
|
|
fixture_path: "tests/conformance/c/fixture.c",
|
|
expected_exit_code: 0,
|
|
expected_patterns: [],
|
|
},
|
|
],
|
|
replay_plans: [
|
|
{
|
|
scenario_id: "s1",
|
|
steps: [
|
|
{
|
|
step_id: "step1",
|
|
action: "run",
|
|
arguments: [],
|
|
timeout_ms: 1000,
|
|
},
|
|
],
|
|
environment: {},
|
|
},
|
|
],
|
|
shim_plans: [],
|
|
expected_evidence: [
|
|
{
|
|
scenario_id: "s1",
|
|
envelopes: [],
|
|
bundle: null,
|
|
required_dimensions: [],
|
|
},
|
|
],
|
|
verifier_contracts: [
|
|
{
|
|
scenario_id: "s1",
|
|
dimensions: [{ name: "exit_code", required: true, comparison: "exact" }],
|
|
timing_tolerance_ms: 0,
|
|
},
|
|
],
|
|
};
|
|
|
|
const validPackage = createConformancePackage(validPackageInput);
|
|
|
|
async function passingRunner(
|
|
scenarioId: string,
|
|
): Promise<ScenarioReplayResult> {
|
|
return {
|
|
scenario_id: scenarioId,
|
|
status: "pass",
|
|
exit_code: 0,
|
|
duration_ms: 10,
|
|
output: "ok",
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
async function failingRunner(
|
|
scenarioId: string,
|
|
): Promise<ScenarioReplayResult> {
|
|
return {
|
|
scenario_id: scenarioId,
|
|
status: "fail",
|
|
exit_code: 1,
|
|
duration_ms: 10,
|
|
output: "fail",
|
|
error: null,
|
|
};
|
|
}
|
|
|
|
describe("conformance CI replay", () => {
|
|
it("replays a package with passing scenarios", async () => {
|
|
const result = await replayConformancePackage(
|
|
validPackage,
|
|
{},
|
|
passingRunner,
|
|
);
|
|
expect(result.total_scenarios).toBe(1);
|
|
expect(result.passed).toBe(1);
|
|
expect(result.failed).toBe(0);
|
|
expect(result.errored).toBe(0);
|
|
expect(
|
|
packageReplayResultSchema.safeParse({ ...result, passed: 0 }).success,
|
|
).toBe(false);
|
|
});
|
|
|
|
it("replays a package with failing scenarios", async () => {
|
|
const result = await replayConformancePackage(
|
|
validPackage,
|
|
{},
|
|
failingRunner,
|
|
);
|
|
expect(result.failed).toBe(1);
|
|
expect(result.passed).toBe(0);
|
|
});
|
|
|
|
it("does not detect drift when expected and actual match", async () => {
|
|
const result = await replayConformancePackage(
|
|
validPackage,
|
|
{},
|
|
passingRunner,
|
|
);
|
|
expect(
|
|
packageReplayResultSchema.safeParse({
|
|
...result,
|
|
drift_detected: true,
|
|
}).success,
|
|
).toBe(false);
|
|
expect(result.drift_detected).toBe(false);
|
|
});
|
|
|
|
it("retains the observed Evidence ID on the first drift", async () => {
|
|
const expected = createEvidence(
|
|
undefined,
|
|
{ id: "fixture-provider", name: "Fixture provider", version: "1" },
|
|
{ operation: "run", parameters: {}, result: { exit_code: 0 } },
|
|
);
|
|
const pkg = createConformancePackage({
|
|
...validPackageInput,
|
|
expected_evidence: [
|
|
{
|
|
scenario_id: "s1",
|
|
envelopes: [expected],
|
|
bundle: null,
|
|
required_dimensions: ["exit_code"],
|
|
},
|
|
],
|
|
});
|
|
const actualEvidenceId = `ev_${"3".repeat(64)}`;
|
|
const result = await replayConformancePackage(
|
|
pkg,
|
|
{
|
|
s1: {
|
|
evidence_id: actualEvidenceId,
|
|
normalized_result: { exit_code: 1 },
|
|
},
|
|
},
|
|
passingRunner,
|
|
);
|
|
expect(result.drift_detected).toBe(true);
|
|
expect(result.first_drift).toMatchObject({
|
|
dimension: "exit_code",
|
|
evidence_id: actualEvidenceId,
|
|
});
|
|
});
|
|
|
|
it("throws on invalid package", async () => {
|
|
await expect(replayConformancePackage({}, {})).rejects.toThrow(
|
|
/invalid conformance package/u,
|
|
);
|
|
});
|
|
|
|
it("counts all result types", async () => {
|
|
const multiPackage = createConformancePackage({
|
|
...validPackageInput,
|
|
scenarios: [
|
|
...validPackage.scenarios,
|
|
{
|
|
scenario_id: "s2",
|
|
name: "Second scenario",
|
|
description: "Second",
|
|
fixture_path: "tests/conformance/c/fixture2.c",
|
|
expected_exit_code: 1,
|
|
expected_patterns: [],
|
|
},
|
|
],
|
|
replay_plans: [
|
|
...validPackage.replay_plans,
|
|
{
|
|
scenario_id: "s2",
|
|
steps: [
|
|
{
|
|
step_id: "step1",
|
|
action: "run",
|
|
arguments: [],
|
|
timeout_ms: 1000,
|
|
},
|
|
],
|
|
environment: {},
|
|
},
|
|
],
|
|
expected_evidence: [
|
|
...validPackage.expected_evidence,
|
|
{
|
|
scenario_id: "s2",
|
|
envelopes: [],
|
|
bundle: null,
|
|
required_dimensions: [],
|
|
},
|
|
],
|
|
verifier_contracts: [
|
|
...validPackage.verifier_contracts,
|
|
{
|
|
scenario_id: "s2",
|
|
dimensions: [
|
|
{ name: "exit_code", required: true, comparison: "exact" },
|
|
],
|
|
timing_tolerance_ms: 0,
|
|
},
|
|
],
|
|
});
|
|
const result = await replayConformancePackage(
|
|
multiPackage,
|
|
{},
|
|
async (scenarioId) => ({
|
|
scenario_id: scenarioId,
|
|
status: scenarioId === "s1" ? "pass" : "fail",
|
|
exit_code: scenarioId === "s1" ? 0 : 1,
|
|
duration_ms: 10,
|
|
output: "",
|
|
error: null,
|
|
}),
|
|
);
|
|
expect(result.total_scenarios).toBe(2);
|
|
expect(result.passed).toBe(1);
|
|
expect(result.failed).toBe(1);
|
|
});
|
|
});
|
|
|
|
describe("conformance replay package boundary", () => {
|
|
it("rejects shape-valid packages with broken scenario relations", async () => {
|
|
await expect(
|
|
replayConformancePackage(
|
|
{
|
|
...validPackage,
|
|
replay_plans: [
|
|
{
|
|
scenario_id: "missing",
|
|
steps: [
|
|
{
|
|
step_id: "step1",
|
|
action: "run",
|
|
arguments: [],
|
|
timeout_ms: 1000,
|
|
},
|
|
],
|
|
environment: {},
|
|
},
|
|
],
|
|
},
|
|
{},
|
|
),
|
|
).rejects.toThrow(/replay_plans references unknown scenario missing/u);
|
|
});
|
|
});
|