refactor(process): remove unused paired experiment
This commit is contained in:
@@ -1,260 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
runPairedProcessExperiment,
|
||||
type ProcessCapturePort,
|
||||
} from "./ProcessPairedExperiment.js";
|
||||
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../domain/processCapture.fixture.js";
|
||||
import {
|
||||
parseProcessCapture,
|
||||
type EnabledProcessExecutionPolicy,
|
||||
} from "../domain/processCapture.js";
|
||||
import {
|
||||
analyzeProcessRepeatability,
|
||||
bindProcessScenario,
|
||||
processPairedExperimentSchema,
|
||||
} from "../domain/processPairedExperiment.js";
|
||||
import { ok } from "../domain/result.js";
|
||||
|
||||
const experiment = () =>
|
||||
processPairedExperimentSchema.parse({
|
||||
shared_scenario: {
|
||||
executable: "/bin/template",
|
||||
arguments: ["shared"],
|
||||
working_directory: "/tmp",
|
||||
environment: { SHARED: "yes" },
|
||||
},
|
||||
authority: {
|
||||
id: "authority",
|
||||
executable: "/bin/authority",
|
||||
environment: { SIDE: "authority" },
|
||||
},
|
||||
candidate: {
|
||||
id: "candidate",
|
||||
executable: "/bin/candidate",
|
||||
arguments: ["candidate"],
|
||||
environment: { SIDE: "candidate" },
|
||||
},
|
||||
repeat_count: 3,
|
||||
freshness_policy: { max_capture_age_ms: 60_000 },
|
||||
required_dimensions: [
|
||||
"terminal",
|
||||
"interaction",
|
||||
"exit",
|
||||
"filesystem",
|
||||
"protocol",
|
||||
"process",
|
||||
"shim",
|
||||
],
|
||||
});
|
||||
|
||||
const policy: EnabledProcessExecutionPolicy = {
|
||||
status: "enabled",
|
||||
executableRoots: ["/bin"],
|
||||
workingRoots: ["/tmp"],
|
||||
allowedEnvironment: ["SHARED", "SIDE"],
|
||||
networkAccess: "none",
|
||||
};
|
||||
|
||||
const cloneCapture = (
|
||||
capture: ReturnType<typeof parseProcessCapture>,
|
||||
): ReturnType<typeof parseProcessCapture> =>
|
||||
parseProcessCapture(structuredClone(capture));
|
||||
|
||||
describe("paired process experiments", () => {
|
||||
it("binds only declared side inputs onto the shared observation contract", () => {
|
||||
const parsed = experiment();
|
||||
const authority = bindProcessScenario(
|
||||
parsed.shared_scenario,
|
||||
parsed.authority,
|
||||
);
|
||||
const candidate = bindProcessScenario(
|
||||
parsed.shared_scenario,
|
||||
parsed.candidate,
|
||||
);
|
||||
expect(authority).toMatchObject({
|
||||
executable: "/bin/authority",
|
||||
arguments: ["shared"],
|
||||
environment: { SHARED: "yes", SIDE: "authority" },
|
||||
});
|
||||
expect(candidate).toMatchObject({
|
||||
executable: "/bin/candidate",
|
||||
arguments: ["candidate"],
|
||||
environment: { SHARED: "yes", SIDE: "candidate" },
|
||||
});
|
||||
});
|
||||
|
||||
it("runs same-side comparisons first and compares stable sides", async () => {
|
||||
const observedExecutables: string[] = [];
|
||||
const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
|
||||
const capturePort: ProcessCapturePort = {
|
||||
capture: (scenario) => {
|
||||
observedExecutables.push(scenario.executable);
|
||||
return Promise.resolve(ok(cloneCapture(capture)));
|
||||
},
|
||||
};
|
||||
const result = await runPairedProcessExperiment(experiment(), policy, {
|
||||
capturePort,
|
||||
now: () => Date.parse("2026-01-01T00:00:01.000Z"),
|
||||
});
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
expect(observedExecutables).toEqual([
|
||||
"/bin/authority",
|
||||
"/bin/authority",
|
||||
"/bin/authority",
|
||||
"/bin/candidate",
|
||||
"/bin/candidate",
|
||||
"/bin/candidate",
|
||||
]);
|
||||
expect(result.value.authority_repeatability.stable).toBe(true);
|
||||
expect(result.value.candidate_repeatability.stable).toBe(true);
|
||||
expect(result.value.cross_side?.status).toBe("unchanged");
|
||||
expect(result.value.cross_side_blocked_reason).toBeNull();
|
||||
});
|
||||
|
||||
it("blocks cross-side claims when a candidate has a flaky exit", async () => {
|
||||
const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
|
||||
let candidateRun = 0;
|
||||
const capturePort: ProcessCapturePort = {
|
||||
capture: (scenario) => {
|
||||
let capture = cloneCapture(base);
|
||||
if (scenario.executable === "/bin/candidate") {
|
||||
candidateRun += 1;
|
||||
if (candidateRun === 2)
|
||||
capture = parseProcessCapture({
|
||||
...capture,
|
||||
exit: { code: 1, signal: null, reason: "exited" },
|
||||
});
|
||||
}
|
||||
return Promise.resolve(ok(capture));
|
||||
},
|
||||
};
|
||||
const result = await runPairedProcessExperiment(experiment(), policy, {
|
||||
capturePort,
|
||||
now: () => Date.parse("2026-01-01T00:00:01.000Z"),
|
||||
});
|
||||
expect(result.ok).toBe(true);
|
||||
if (!result.ok) return;
|
||||
expect(result.value.candidate_repeatability).toMatchObject({
|
||||
stable: false,
|
||||
classification: "flaky_exit",
|
||||
unstable_dimensions: ["exit"],
|
||||
});
|
||||
expect(result.value.cross_side).toBeNull();
|
||||
expect(result.value.cross_side_blocked_reason).toMatch(/unstable/u);
|
||||
});
|
||||
});
|
||||
|
||||
describe("paired process experiment comparisons", () => {
|
||||
it("distinguishes normalized timing variance from semantic output drift", () => {
|
||||
const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
|
||||
const early = parseProcessCapture({
|
||||
...base,
|
||||
frames: [{ sequence: 0, at_ms: 10, data: "ready" }],
|
||||
});
|
||||
const late = parseProcessCapture({
|
||||
...base,
|
||||
frames: [{ sequence: 0, at_ms: 20, data: "ready" }],
|
||||
});
|
||||
expect(
|
||||
analyzeProcessRepeatability([early, late], ["terminal"]),
|
||||
).toMatchObject({
|
||||
stable: false,
|
||||
classification: "timing_variance",
|
||||
unstable_dimensions: ["terminal"],
|
||||
});
|
||||
const changed = parseProcessCapture({
|
||||
...base,
|
||||
frames: [{ sequence: 0, at_ms: 20, data: "failed" }],
|
||||
});
|
||||
expect(
|
||||
analyzeProcessRepeatability([early, changed], ["terminal"]),
|
||||
).toMatchObject({ classification: "nondeterministic_output" });
|
||||
});
|
||||
|
||||
it("stops before capture when the experiment is cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const capturePort: ProcessCapturePort = {
|
||||
capture: () => {
|
||||
throw new Error("capture must not start after cancellation");
|
||||
},
|
||||
};
|
||||
|
||||
const result = await runPairedProcessExperiment(experiment(), policy, {
|
||||
capturePort,
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "ProcessCaptureError", reason: "cancelled" },
|
||||
});
|
||||
});
|
||||
|
||||
it("blocks full cross-side output when only a subset was repeatable", async () => {
|
||||
const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
|
||||
const capturePort: ProcessCapturePort = {
|
||||
capture: () => Promise.resolve(ok(cloneCapture(capture))),
|
||||
};
|
||||
const subset = processPairedExperimentSchema.parse({
|
||||
...experiment(),
|
||||
required_dimensions: ["exit"],
|
||||
});
|
||||
|
||||
const result = await runPairedProcessExperiment(subset, policy, {
|
||||
capturePort,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
cross_side: null,
|
||||
cross_side_blocked_reason: expect.stringMatching(/every returned/u),
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("paired process experiment freshness", () => {
|
||||
it("compares the newest stable captures under the freshness policy", async () => {
|
||||
const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
|
||||
let captureIndex = 0;
|
||||
const capturePort: ProcessCapturePort = {
|
||||
capture: () => {
|
||||
const completedAt =
|
||||
captureIndex === 0
|
||||
? "2026-01-01T00:00:00.000Z"
|
||||
: "2026-01-01T00:00:09.500Z";
|
||||
captureIndex += 1;
|
||||
return Promise.resolve(
|
||||
ok(
|
||||
parseProcessCapture({
|
||||
...structuredClone(base),
|
||||
manifest: {
|
||||
...base.manifest,
|
||||
started_at: completedAt,
|
||||
completed_at: completedAt,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
},
|
||||
};
|
||||
const shortFreshness = processPairedExperimentSchema.parse({
|
||||
...experiment(),
|
||||
freshness_policy: { max_capture_age_ms: 1_000 },
|
||||
});
|
||||
|
||||
const result = await runPairedProcessExperiment(shortFreshness, policy, {
|
||||
capturePort,
|
||||
now: () => Date.parse("2026-01-01T00:00:10.000Z"),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: true,
|
||||
value: { cross_side: { status: "unchanged" } },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,152 +0,0 @@
|
||||
import {
|
||||
processCaptureCancelled,
|
||||
type ProcessCaptureError,
|
||||
} from "./ProcessCaptureError.js";
|
||||
import { captureProcessScenario } from "./ProcessHarness.js";
|
||||
import type {
|
||||
ProcessCapture,
|
||||
ProcessExecutionPolicy,
|
||||
ProcessScenario,
|
||||
} from "../domain/processCapture.js";
|
||||
import { compareProcessCaptures } from "../domain/processComparison.js";
|
||||
import { PROCESS_COMPARISON_DIMENSIONS } from "../domain/processComparison.js";
|
||||
import {
|
||||
analyzeProcessRepeatability,
|
||||
bindProcessScenario,
|
||||
type ProcessPairedExperiment,
|
||||
} from "../domain/processPairedExperiment.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
/** Capture seam used by paired process experiments. */
|
||||
export interface ProcessCapturePort {
|
||||
capture(
|
||||
scenario: ProcessScenario,
|
||||
policy: ProcessExecutionPolicy,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<ProcessCapture, ProcessCaptureError>>;
|
||||
}
|
||||
|
||||
interface PairedProcessExperimentEvidence {
|
||||
readonly authority_runs: readonly ProcessCapture[];
|
||||
readonly candidate_runs: readonly ProcessCapture[];
|
||||
readonly authority_repeatability: ReturnType<
|
||||
typeof analyzeProcessRepeatability
|
||||
>;
|
||||
readonly candidate_repeatability: ReturnType<
|
||||
typeof analyzeProcessRepeatability
|
||||
>;
|
||||
}
|
||||
|
||||
/** Repeatability evidence and either a stable comparison or its blocking reason. */
|
||||
export type PairedProcessExperimentResult = PairedProcessExperimentEvidence &
|
||||
(
|
||||
| {
|
||||
readonly cross_side: ReturnType<typeof compareProcessCaptures>;
|
||||
readonly cross_side_blocked_reason: null;
|
||||
}
|
||||
| {
|
||||
readonly cross_side: null;
|
||||
readonly cross_side_blocked_reason: string;
|
||||
}
|
||||
);
|
||||
|
||||
const productionCapturePort: ProcessCapturePort = {
|
||||
capture: captureProcessScenario,
|
||||
};
|
||||
|
||||
const captureRepeats = async (input: {
|
||||
readonly scenario: ProcessScenario;
|
||||
readonly repeatCount: number;
|
||||
readonly policy: ProcessExecutionPolicy;
|
||||
readonly port: ProcessCapturePort;
|
||||
readonly signal?: AbortSignal;
|
||||
}): Promise<Result<readonly ProcessCapture[], ProcessCaptureError>> => {
|
||||
const captures: ProcessCapture[] = [];
|
||||
for (let index = 0; index < input.repeatCount; index += 1) {
|
||||
if (input.signal?.aborted === true) return err(processCaptureCancelled());
|
||||
const capture = await input.port.capture(
|
||||
input.scenario,
|
||||
input.policy,
|
||||
input.signal,
|
||||
);
|
||||
if (!capture.ok) return err(capture.error);
|
||||
captures.push(capture.value);
|
||||
}
|
||||
return ok(captures);
|
||||
};
|
||||
|
||||
/** Execute repeatability-first authority/candidate captures under one contract. */
|
||||
export const runPairedProcessExperiment = async (
|
||||
experiment: ProcessPairedExperiment,
|
||||
policy: ProcessExecutionPolicy,
|
||||
options: {
|
||||
readonly signal?: AbortSignal;
|
||||
readonly capturePort?: ProcessCapturePort;
|
||||
readonly now?: () => number;
|
||||
} = {},
|
||||
): Promise<Result<PairedProcessExperimentResult, ProcessCaptureError>> => {
|
||||
const port = options.capturePort ?? productionCapturePort;
|
||||
const authorityScenario = bindProcessScenario(
|
||||
experiment.shared_scenario,
|
||||
experiment.authority,
|
||||
);
|
||||
const candidateScenario = bindProcessScenario(
|
||||
experiment.shared_scenario,
|
||||
experiment.candidate,
|
||||
);
|
||||
const authorityRuns = await captureRepeats({
|
||||
scenario: authorityScenario,
|
||||
repeatCount: experiment.repeat_count,
|
||||
policy,
|
||||
port,
|
||||
...(options.signal === undefined ? {} : { signal: options.signal }),
|
||||
});
|
||||
if (!authorityRuns.ok) return authorityRuns;
|
||||
const candidateRuns = await captureRepeats({
|
||||
scenario: candidateScenario,
|
||||
repeatCount: experiment.repeat_count,
|
||||
policy,
|
||||
port,
|
||||
...(options.signal === undefined ? {} : { signal: options.signal }),
|
||||
});
|
||||
if (!candidateRuns.ok) return candidateRuns;
|
||||
const authorityRepeatability = analyzeProcessRepeatability(
|
||||
authorityRuns.value,
|
||||
experiment.required_dimensions,
|
||||
);
|
||||
const candidateRepeatability = analyzeProcessRepeatability(
|
||||
candidateRuns.value,
|
||||
experiment.required_dimensions,
|
||||
);
|
||||
const stable = authorityRepeatability.stable && candidateRepeatability.stable;
|
||||
const allDimensionsRequired = PROCESS_COMPARISON_DIMENSIONS.every(
|
||||
(dimension) => experiment.required_dimensions.includes(dimension),
|
||||
);
|
||||
const authorityLatest = authorityRuns.value.at(-1);
|
||||
const candidateLatest = candidateRuns.value.at(-1);
|
||||
const crossSideOutcome =
|
||||
stable &&
|
||||
allDimensionsRequired &&
|
||||
authorityLatest !== undefined &&
|
||||
candidateLatest !== undefined
|
||||
? {
|
||||
cross_side: compareProcessCaptures(authorityLatest, candidateLatest, {
|
||||
maxCaptureAgeMs: experiment.freshness_policy.max_capture_age_ms,
|
||||
...(options.now === undefined ? {} : { now: options.now }),
|
||||
}),
|
||||
cross_side_blocked_reason: null,
|
||||
}
|
||||
: {
|
||||
cross_side: null,
|
||||
cross_side_blocked_reason: !allDimensionsRequired
|
||||
? "Cross-side comparison is blocked until every returned dimension is required and repeatable."
|
||||
: "Cross-side comparison is blocked because a required same-side dimension is unstable.",
|
||||
};
|
||||
return ok({
|
||||
authority_runs: authorityRuns.value,
|
||||
candidate_runs: candidateRuns.value,
|
||||
authority_repeatability: authorityRepeatability,
|
||||
candidate_repeatability: candidateRepeatability,
|
||||
...crossSideOutcome,
|
||||
});
|
||||
};
|
||||
Reference in New Issue
Block a user