feat(process): compare repeatable paired experiments

This commit is contained in:
morluto
2026-07-20 13:02:52 +00:00
parent c6a1a93a9a
commit 9ab7aa320c
4 changed files with 619 additions and 1 deletions
+140
View File
@@ -0,0 +1,140 @@
import {
processCaptureCancelled,
type ProcessCaptureError,
} from "./ProcessCaptureError.js";
import { captureProcessScenario } from "./ProcessHarness.js";
import type {
ProcessCapture,
ProcessExecutionPolicy,
ProcessScenario,
} from "../domain/processCapture.js";
import { compareProcessCaptures } from "../domain/processComparison.js";
import { PROCESS_COMPARISON_DIMENSIONS } from "../domain/processComparison.js";
import {
analyzeProcessRepeatability,
bindProcessScenario,
type ProcessPairedExperiment,
} from "../domain/processPairedExperiment.js";
import { err, ok, type Result } from "../domain/result.js";
/** Capture seam used by paired process experiments. */
export interface ProcessCapturePort {
capture(
scenario: ProcessScenario,
policy: ProcessExecutionPolicy,
signal?: AbortSignal,
): Promise<Result<ProcessCapture, ProcessCaptureError>>;
}
/** Repeatability evidence and an optional stable cross-side comparison. */
export interface PairedProcessExperimentResult {
readonly authority_runs: readonly ProcessCapture[];
readonly candidate_runs: readonly ProcessCapture[];
readonly authority_repeatability: ReturnType<
typeof analyzeProcessRepeatability
>;
readonly candidate_repeatability: ReturnType<
typeof analyzeProcessRepeatability
>;
readonly cross_side: ReturnType<typeof compareProcessCaptures> | null;
readonly cross_side_blocked_reason: string | null;
}
const productionCapturePort: ProcessCapturePort = {
capture: captureProcessScenario,
};
const captureRepeats = async (input: {
readonly scenario: ProcessScenario;
readonly repeatCount: number;
readonly policy: ProcessExecutionPolicy;
readonly port: ProcessCapturePort;
readonly signal?: AbortSignal;
}): Promise<Result<readonly ProcessCapture[], ProcessCaptureError>> => {
const captures: ProcessCapture[] = [];
for (let index = 0; index < input.repeatCount; index += 1) {
if (input.signal?.aborted === true) return err(processCaptureCancelled());
const capture = await input.port.capture(
input.scenario,
input.policy,
input.signal,
);
if (!capture.ok) return err(capture.error);
captures.push(capture.value);
}
return ok(captures);
};
/** Execute repeatability-first authority/candidate captures under one contract. */
export const runPairedProcessExperiment = async (
experiment: ProcessPairedExperiment,
policy: ProcessExecutionPolicy,
options: {
readonly signal?: AbortSignal;
readonly capturePort?: ProcessCapturePort;
readonly now?: () => number;
} = {},
): Promise<Result<PairedProcessExperimentResult, ProcessCaptureError>> => {
const port = options.capturePort ?? productionCapturePort;
const authorityScenario = bindProcessScenario(
experiment.shared_scenario,
experiment.authority,
);
const candidateScenario = bindProcessScenario(
experiment.shared_scenario,
experiment.candidate,
);
const authorityRuns = await captureRepeats({
scenario: authorityScenario,
repeatCount: experiment.repeat_count,
policy,
port,
...(options.signal === undefined ? {} : { signal: options.signal }),
});
if (!authorityRuns.ok) return authorityRuns;
const candidateRuns = await captureRepeats({
scenario: candidateScenario,
repeatCount: experiment.repeat_count,
policy,
port,
...(options.signal === undefined ? {} : { signal: options.signal }),
});
if (!candidateRuns.ok) return candidateRuns;
const authorityRepeatability = analyzeProcessRepeatability(
authorityRuns.value,
experiment.required_dimensions,
);
const candidateRepeatability = analyzeProcessRepeatability(
candidateRuns.value,
experiment.required_dimensions,
);
const stable = authorityRepeatability.stable && candidateRepeatability.stable;
const allDimensionsRequired = PROCESS_COMPARISON_DIMENSIONS.every(
(dimension) => experiment.required_dimensions.includes(dimension),
);
const authorityLatest = authorityRuns.value.at(-1);
const candidateLatest = candidateRuns.value.at(-1);
const crossSide =
stable &&
allDimensionsRequired &&
authorityLatest !== undefined &&
candidateLatest !== undefined
? compareProcessCaptures(authorityLatest, candidateLatest, {
maxCaptureAgeMs: experiment.freshness_policy.max_capture_age_ms,
...(options.now === undefined ? {} : { now: options.now }),
})
: null;
return ok({
authority_runs: authorityRuns.value,
candidate_runs: candidateRuns.value,
authority_repeatability: authorityRepeatability,
candidate_repeatability: candidateRepeatability,
cross_side: crossSide,
cross_side_blocked_reason:
crossSide !== null
? null
: !allDimensionsRequired
? "Cross-side comparison is blocked until every returned dimension is required and repeatable."
: "Cross-side comparison is blocked because a required same-side dimension is unstable.",
});
};
+26 -1
View File
@@ -86,7 +86,9 @@ export const processCaptureComparisonSchema = z
path: ["status"],
});
});
type ProcessCaptureComparison = z.infer<typeof processCaptureComparisonSchema>;
export type ProcessCaptureComparison = z.infer<
typeof processCaptureComparisonSchema
>;
const terminalObservations = (capture: ProcessCapture): readonly unknown[] =>
[
@@ -113,6 +115,29 @@ const filesystemObservations = (
},
];
/** Canonical observations for one process-comparison dimension. */
export const processDimensionObservations = (
capture: ProcessCapture,
dimension: (typeof PROCESS_COMPARISON_DIMENSIONS)[number],
): readonly unknown[] => {
switch (dimension) {
case "terminal":
return terminalObservations(capture);
case "interaction":
return capture.interaction_events;
case "exit":
return [{ ...capture.exit, settlement: capture.settlement }];
case "filesystem":
return filesystemObservations(capture);
case "protocol":
return capture.protocol_events;
case "process":
return capture.process_samples;
case "shim":
return capture.shim_events;
}
};
const classifyCollection = (
left: readonly unknown[],
right: readonly unknown[],
+206
View File
@@ -0,0 +1,206 @@
import { z } from "zod";
import type { ProcessCapture } from "./processCapture.js";
import {
PROCESS_COMPARISON_DIMENSIONS,
compareProcessCaptures,
processDimensionObservations,
type ProcessCaptureComparison,
} from "./processComparison.js";
import {
digestProcessCommitment,
processScenarioSchema,
type ProcessScenario,
} from "./processScenario.js";
const sideIdentifierSchema = z
.string()
.regex(/^[A-Za-z][A-Za-z0-9._-]{0,127}$/u);
const environmentNameSchema = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/u);
const processDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS);
const sideBindingSchema = z
.object({
id: sideIdentifierSchema,
executable: z.string().startsWith("/"),
arguments: z.array(z.string()).max(256).optional(),
environment: z.record(environmentNameSchema, z.string()).default({}),
})
.strict();
/** Boundary contract for one bounded authority/candidate process experiment. */
export const processPairedExperimentSchema = z
.object({
approved: z.literal(true),
shared_scenario: processScenarioSchema,
authority: sideBindingSchema,
candidate: sideBindingSchema,
repeat_count: z.number().int().min(2).max(10).default(3),
freshness_policy: z
.object({
max_capture_age_ms: z.number().int().positive().max(86_400_000),
})
.strict(),
required_dimensions: z
.array(processDimensionSchema)
.min(1)
.max(PROCESS_COMPARISON_DIMENSIONS.length)
.default([...PROCESS_COMPARISON_DIMENSIONS]),
})
.strict()
.superRefine((experiment, context) => {
if (experiment.authority.id === experiment.candidate.id)
context.addIssue({
code: "custom",
message: "authority and candidate identities must differ",
path: ["candidate", "id"],
});
if (
new Set(experiment.required_dimensions).size !==
experiment.required_dimensions.length
)
context.addIssue({
code: "custom",
message: "required process dimensions must be unique",
path: ["required_dimensions"],
});
});
export type ProcessPairedExperiment = z.infer<
typeof processPairedExperimentSchema
>;
/** Executable and environment values applied to one experiment side. */
export type ProcessSideBinding = ProcessPairedExperiment[
| "authority"
| "candidate"];
type ProcessRepeatabilityClassification =
| "stable"
| "timing_variance"
| "nondeterministic_output"
| "flaky_exit"
| "environmental_drift"
| "insufficient_samples";
/** Stability classification for repeated captures of one experiment side. */
export interface ProcessSideRepeatability {
readonly classification: ProcessRepeatabilityClassification;
readonly stable: boolean;
readonly digest: string;
readonly comparisons: readonly ProcessCaptureComparison[];
readonly unstable_dimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][];
}
/** Apply a side binding without changing the shared observation contract. */
export const bindProcessScenario = (
shared: ProcessScenario,
side: ProcessSideBinding,
): ProcessScenario =>
processScenarioSchema.parse({
...shared,
executable: side.executable,
arguments: side.arguments ?? shared.arguments,
environment: { ...shared.environment, ...side.environment },
});
const classifyInstability = (
dimensions: ReadonlySet<(typeof PROCESS_COMPARISON_DIMENSIONS)[number]>,
): ProcessRepeatabilityClassification => {
if (dimensions.has("exit")) return "flaky_exit";
if (dimensions.has("filesystem") || dimensions.has("process"))
return "environmental_drift";
return "nondeterministic_output";
};
const timingKeys = new Set([
"at_ms",
"scheduled_at_ms",
"dispatched_at_ms",
"elapsed_ms",
]);
const withoutTiming = (value: unknown): unknown =>
typeof value !== "object" || value === null || Array.isArray(value)
? value
: Object.fromEntries(
Object.entries(value).filter(([key]) => !timingKeys.has(key)),
);
const observationsWithoutTiming = (
capture: ProcessCapture,
dimension: (typeof PROCESS_COMPARISON_DIMENSIONS)[number],
): readonly unknown[] =>
dimension === "exit"
? [
{
...capture.exit,
settlement: {
state: capture.settlement.state,
cleanup_outcome: capture.settlement.cleanup_outcome,
},
},
]
: processDimensionObservations(capture, dimension).map(withoutTiming);
const hasOnlyTimingVariance = (
captures: readonly ProcessCapture[],
dimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][],
): boolean =>
dimensions.every((dimension) => {
const first = captures[0];
if (first === undefined) return false;
const baseline = JSON.stringify(
observationsWithoutTiming(first, dimension),
);
return captures
.slice(1)
.every(
(capture) =>
JSON.stringify(observationsWithoutTiming(capture, dimension)) ===
baseline,
);
});
/** Compare same-side repeats before permitting a cross-side claim. */
export const analyzeProcessRepeatability = (
captures: readonly ProcessCapture[],
requiredDimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][],
): ProcessSideRepeatability => {
if (captures.length < 2)
return {
classification: "insufficient_samples",
stable: false,
digest: digestProcessCommitment({ captures: [], requiredDimensions }),
comparisons: [],
unstable_dimensions: [],
};
const baseline = captures[0];
if (baseline === undefined)
throw new TypeError("Repeatability analysis lost its baseline capture");
const comparisons = captures
.slice(1)
.map((capture) => compareProcessCaptures(baseline, capture));
const unstable = new Set(
requiredDimensions.filter((dimension) =>
comparisons.some((comparison) => comparison[dimension] !== "unchanged"),
),
);
const unstableDimensions = PROCESS_COMPARISON_DIMENSIONS.filter((dimension) =>
unstable.has(dimension),
);
return {
classification:
unstable.size === 0
? "stable"
: hasOnlyTimingVariance(captures, unstableDimensions)
? "timing_variance"
: classifyInstability(unstable),
stable: unstable.size === 0,
digest: digestProcessCommitment({
requiredDimensions,
comparisons,
}),
comparisons,
unstable_dimensions: unstableDimensions,
};
};