diff --git a/src/domain/processTraceComparison.ts b/src/domain/processTraceComparison.ts new file mode 100644 index 000000000..c03895388 --- /dev/null +++ b/src/domain/processTraceComparison.ts @@ -0,0 +1,53 @@ +import type { ProcessCapture } from "./processCapture.js"; +import { + evaluateProcessTraceSide, + processTraceComparisonResultSchema, + type ProcessTraceComparisonResult, +} from "./processTraceEvaluation.js"; +import { + processTraceSpecificationSchema, + type ProcessTraceSpecification, +} from "./processTraceSpecification.js"; + +export { + processTraceComparisonResultSchema, + type ProcessTraceComparisonResult, + type ProcessTraceLocation, +} from "./processTraceEvaluation.js"; +export { + processTraceSourceSchema, + processTraceSpecificationSchema, + type ProcessTraceSource, + type ProcessTraceSpecification, +} from "./processTraceSpecification.js"; + +/** Compare two captures against an explicit finite trace language. */ +export const compareProcessTraces = ( + left: ProcessCapture, + right: ProcessCapture, + specificationInput: ProcessTraceSpecification, +): ProcessTraceComparisonResult => { + const specification = + processTraceSpecificationSchema.parse(specificationInput); + const leftEvaluation = evaluateProcessTraceSide(left, specification); + const rightEvaluation = evaluateProcessTraceSide(right, specification); + const diagnostic = + leftEvaluation.diagnostic === null + ? rightEvaluation.diagnostic === null + ? null + : { side: "right" as const, ...rightEvaluation.diagnostic } + : { side: "left" as const, ...leftEvaluation.diagnostic }; + return processTraceComparisonResultSchema.parse({ + verdict: + leftEvaluation.result.status === "unknown" || + rightEvaluation.result.status === "unknown" + ? "unknown" + : leftEvaluation.result.status === "pass" && + rightEvaluation.result.status === "pass" + ? "equivalent" + : "different", + left: leftEvaluation.result, + right: rightEvaluation.result, + diagnostic, + }); +}; diff --git a/src/domain/processTraceEvaluation.ts b/src/domain/processTraceEvaluation.ts new file mode 100644 index 000000000..56843dbdc --- /dev/null +++ b/src/domain/processTraceEvaluation.ts @@ -0,0 +1,433 @@ +import { z } from "zod"; + +import { jsonValueSchema, type JsonValue } from "./jsonValue.js"; +import type { ProcessCapture } from "./processCapture.js"; +import { + canonicalTraceJson, + processTraceCardinalityBounds, + type ProcessTraceSource, + type ProcessTraceSpecification, +} from "./processTraceSpecification.js"; + +const identifierSchema = z.string().regex(/^[A-Za-z][A-Za-z0-9._-]{0,63}$/u); + +export type ProcessTraceLocation = { + readonly collection: + | "frames" + | "rendered_frames" + | "interaction_events" + | "lifecycle" + | "process_samples" + | "filesystem_checkpoints" + | "shim_events" + | "protocol_events" + | "replay_transitions"; + readonly index: number; + readonly capture_order: number; +}; + +const locationSchema = z.strictObject({ + collection: z.enum([ + "frames", + "rendered_frames", + "interaction_events", + "lifecycle", + "process_samples", + "filesystem_checkpoints", + "shim_events", + "protocol_events", + "replay_transitions", + ]), + index: z.number().int().nonnegative(), + capture_order: z.number().int().nonnegative(), +}); + +const matchedEventSchema = z.strictObject({ + event_id: identifierSchema, + location: locationSchema, +}); + +const sideResultSchema = z.strictObject({ + status: z.enum(["pass", "fail", "unknown"]), + matched_variant: identifierSchema.nullable(), + satisfied_constraints: z.array(z.string()), + raw_trace: z.array(matchedEventSchema), +}); + +const diagnosticSchema = z.strictObject({ + kind: z.enum([ + "journal", + "predicate", + "cardinality", + "edge", + "prefix", + "suffix", + "trace", + ]), + side: z.enum(["left", "right"]), + message: z.string(), + event_ids: z.array(identifierSchema), + locations: z.array(locationSchema), +}); + +/** Structured verdict for one declared process trace language. */ +export const processTraceComparisonResultSchema = z.strictObject({ + verdict: z.enum(["equivalent", "different", "unknown"]), + left: sideResultSchema, + right: sideResultSchema, + diagnostic: diagnosticSchema.nullable(), +}); +export type ProcessTraceComparisonResult = z.infer< + typeof processTraceComparisonResultSchema +>; + +type TraceRecord = { + readonly source: ProcessTraceSource; + readonly payload: JsonValue; + readonly location: ProcessTraceLocation; +}; + +const recordAt = ( + capture: ProcessCapture, + location: ProcessTraceLocation, +): Omit | null => { + const value = (() => { + switch (location.collection) { + case "frames": + return { + source: "terminal_raw" as const, + value: capture.frames[location.index], + }; + case "rendered_frames": + return { + source: "terminal_rendered" as const, + value: capture.rendered_frames[location.index], + }; + case "interaction_events": + return { + source: "interaction" as const, + value: capture.interaction_events[location.index], + }; + case "lifecycle": + return location.index === 0 + ? { + source: "lifecycle" as const, + value: { event: "exit", ...capture.exit }, + } + : location.index === 1 + ? { + source: "lifecycle" as const, + value: { event: "settlement", ...capture.settlement }, + } + : null; + case "process_samples": + return { + source: "process" as const, + value: capture.process_samples[location.index], + }; + case "filesystem_checkpoints": + return { + source: "filesystem" as const, + value: capture.filesystem_checkpoints[location.index], + }; + case "shim_events": + return { + source: "shim" as const, + value: capture.shim_events[location.index], + }; + case "protocol_events": { + const event = capture.protocol_events[location.index]; + return event === undefined + ? null + : { source: event.protocol, value: event }; + } + case "replay_transitions": + return { + source: "replay_transition" as const, + value: capture.replay_transitions[location.index], + }; + } + })(); + if (value === null || value.value === undefined) return null; + return { source: value.source, payload: jsonValueSchema.parse(value.value) }; +}; + +const scopesFor = ( + sources: ReadonlySet, +): ReadonlySet => + new Set( + [...sources].map((source) => { + if (source.startsWith("terminal_")) return "terminal"; + if (source === "lifecycle") return "exit"; + if ( + source === "http" || + source === "websocket" || + source === "replay_transition" + ) + return "protocol"; + if (source === "filesystem") return "filesystem"; + if (source === "process") return "process"; + if (source === "shim") return "shim"; + return "interaction"; + }), + ); + +const unknownSide = (): z.infer => ({ + status: "unknown", + matched_variant: null, + satisfied_constraints: [], + raw_trace: [], +}); + +export type EvaluatedSide = { + readonly result: z.infer; + readonly diagnostic: Omit, "side"> | null; +}; + +type FailureInput = { + readonly kind: z.infer["kind"]; + readonly message: string; + readonly eventIds: readonly string[]; + readonly locations: readonly ProcessTraceLocation[]; + readonly rawTrace: z.infer[]; +}; + +const failure = (input: FailureInput): EvaluatedSide => ({ + result: { + status: "fail", + matched_variant: null, + satisfied_constraints: [], + raw_trace: input.rawTrace, + }, + diagnostic: { + kind: input.kind, + message: input.message, + event_ids: [...input.eventIds], + locations: [...input.locations], + }, +}); + +const evaluateFiniteTrace = ( + ids: readonly string[], + specification: ProcessTraceSpecification, +): string | null => { + if (specification.language.kind !== "finite_traces") return null; + for (const variant of specification.language.variants) { + let offset = 0; + let accepted = true; + for (const token of variant.trace) { + if (typeof token === "string") { + if (ids[offset] !== token) accepted = false; + offset += 1; + continue; + } + const actual = ids.slice(offset, offset + token.unordered.length).sort(); + const expected = [...token.unordered].sort(); + if (canonicalTraceJson(actual) !== canonicalTraceJson(expected)) + accepted = false; + offset += token.unordered.length; + } + if (accepted && offset === ids.length) return variant.id; + } + return null; +}; + +type MatchedTrace = { + readonly rawTrace: z.infer[]; + readonly locationsById: ReadonlyMap; +}; + +const collectRecords = ( + capture: ProcessCapture, + sources: ReadonlySet, +): readonly TraceRecord[] => { + const records: TraceRecord[] = []; + for (const location of capture.event_journal ?? []) { + const record = recordAt(capture, location); + if (record !== null && sources.has(record.source)) + records.push({ ...record, location }); + } + return records; +}; + +const matchRecords = ( + records: readonly TraceRecord[], + specification: ProcessTraceSpecification, +): MatchedTrace | EvaluatedSide => { + const rawTrace: z.infer[] = []; + const locationsById = new Map(); + for (const record of records) { + const matches = specification.events.filter( + (event) => + event.source === record.source && + canonicalTraceJson(event.exact) === canonicalTraceJson(record.payload), + ); + if (matches.length !== 1) + return failure({ + kind: "predicate", + message: + matches.length === 0 + ? "Observed event does not match any declared exact predicate." + : "Observed event matches multiple predicates.", + eventIds: matches.map(({ id }) => id), + locations: [record.location], + rawTrace, + }); + const match = matches[0]; + if (match === undefined) + throw new TypeError("Unique trace match is missing"); + rawTrace.push({ event_id: match.id, location: record.location }); + const locations = locationsById.get(match.id) ?? []; + locations.push(record.location); + locationsById.set(match.id, locations); + } + return { rawTrace, locationsById }; +}; + +const cardinalityFailure = ( + trace: MatchedTrace, + specification: ProcessTraceSpecification, +): EvaluatedSide | null => { + for (const event of specification.events) { + const [minimum, maximum] = processTraceCardinalityBounds(event.cardinality); + const locations = trace.locationsById.get(event.id) ?? []; + if (locations.length < minimum || locations.length > maximum) + return failure({ + kind: "cardinality", + message: `${event.id} observed ${String(locations.length)} times; expected ${String(minimum)}..${String(maximum)}.`, + eventIds: [event.id], + locations, + rawTrace: trace.rawTrace, + }); + } + return null; +}; + +const evaluatePartialOrder = ( + trace: MatchedTrace, + specification: ProcessTraceSpecification, +): EvaluatedSide => { + if (specification.language.kind !== "partial_order") + throw new TypeError("Expected a partial-order trace specification"); + const satisfied: string[] = []; + for (const edge of specification.language.happens_before) { + const before = trace.locationsById.get(edge.before) ?? []; + const after = trace.locationsById.get(edge.after) ?? []; + const lastBefore = before.at(-1); + const firstAfter = after[0]; + if ( + lastBefore !== undefined && + firstAfter !== undefined && + lastBefore.capture_order >= firstAfter.capture_order + ) + return failure({ + kind: "edge", + message: `${edge.before} must happen before ${edge.after}.`, + eventIds: [edge.before, edge.after], + locations: [lastBefore, firstAfter], + rawTrace: trace.rawTrace, + }); + satisfied.push(`happens_before:${edge.before}:${edge.after}`); + } + const ids = trace.rawTrace.map(({ event_id }) => event_id); + const prefix = specification.language.prefix; + if ( + canonicalTraceJson(ids.slice(0, prefix.length)) !== + canonicalTraceJson(prefix) + ) + return failure({ + kind: "prefix", + message: "Observed trace does not satisfy the declared prefix.", + eventIds: prefix, + locations: trace.rawTrace + .slice(0, prefix.length) + .map(({ location }) => location), + rawTrace: trace.rawTrace, + }); + const suffix = specification.language.suffix; + if ( + canonicalTraceJson(suffix.length === 0 ? [] : ids.slice(-suffix.length)) !== + canonicalTraceJson(suffix) + ) + return failure({ + kind: "suffix", + message: "Observed trace does not satisfy the declared suffix.", + eventIds: suffix, + locations: trace.rawTrace + .slice(-suffix.length) + .map(({ location }) => location), + rawTrace: trace.rawTrace, + }); + satisfied.push( + ...specification.language.unordered_groups.map( + ({ events }) => `unordered:${events.join(":")}`, + ), + ); + if (prefix.length > 0) satisfied.push(`prefix:${prefix.join(":")}`); + if (suffix.length > 0) satisfied.push(`suffix:${suffix.join(":")}`); + return { + result: { + status: "pass", + matched_variant: null, + satisfied_constraints: satisfied, + raw_trace: trace.rawTrace, + }, + diagnostic: null, + }; +}; + +const evaluateMatchedTrace = ( + trace: MatchedTrace, + specification: ProcessTraceSpecification, +): EvaluatedSide => { + const invalidCardinality = cardinalityFailure(trace, specification); + if (invalidCardinality !== null) return invalidCardinality; + if (specification.language.kind === "partial_order") + return evaluatePartialOrder(trace, specification); + const ids = trace.rawTrace.map(({ event_id }) => event_id); + const variant = evaluateFiniteTrace(ids, specification); + return variant === null + ? failure({ + kind: "trace", + message: "Observed events do not match any finite trace variant.", + eventIds: ids, + locations: trace.rawTrace.map(({ location }) => location), + rawTrace: trace.rawTrace, + }) + : { + result: { + status: "pass", + matched_variant: variant, + satisfied_constraints: [`variant:${variant}`], + raw_trace: trace.rawTrace, + }, + diagnostic: null, + }; +}; + +export const evaluateProcessTraceSide = ( + capture: ProcessCapture, + specification: ProcessTraceSpecification, +): EvaluatedSide => { + const sources = new Set(specification.events.map(({ source }) => source)); + const relevantScopes = scopesFor(sources); + if ( + capture.truncated || + capture.residual_unknowns.some(({ scope }) => relevantScopes.has(scope)) + ) + return { result: unknownSide(), diagnostic: null }; + if ((capture.event_journal ?? []).length === 0) + return { + result: unknownSide(), + diagnostic: { + kind: "journal", + message: "Capture has no cross-source event journal.", + event_ids: [], + locations: [], + }, + }; + const matched = matchRecords(collectRecords(capture, sources), specification); + return "rawTrace" in matched + ? evaluateMatchedTrace(matched, specification) + : matched; +}; diff --git a/src/domain/processTraceSpecification.ts b/src/domain/processTraceSpecification.ts new file mode 100644 index 000000000..92a06d44f --- /dev/null +++ b/src/domain/processTraceSpecification.ts @@ -0,0 +1,374 @@ +import canonicalize from "canonicalize"; +import { z } from "zod"; + +import { jsonValueSchema } from "./jsonValue.js"; + +const identifierSchema = z.string().regex(/^[A-Za-z][A-Za-z0-9._-]{0,63}$/u); + +/** Process observation families admitted by a declared trace specification. */ +export const processTraceSourceSchema = z.enum([ + "terminal_raw", + "terminal_rendered", + "interaction", + "lifecycle", + "process", + "filesystem", + "http", + "websocket", + "shim", + "replay_transition", +]); +export type ProcessTraceSource = z.infer; + +const cardinalitySchema = z.discriminatedUnion("kind", [ + z.strictObject({ kind: z.literal("required") }), + z.strictObject({ kind: z.literal("optional") }), + z.strictObject({ + kind: z.literal("exact"), + count: z.number().int().min(0).max(10_000), + }), + z + .strictObject({ + kind: z.literal("range"), + min: z.number().int().min(0).max(10_000), + max: z.number().int().min(0).max(10_000), + }) + .refine(({ min, max }) => min <= max, { + message: "cardinality min must not exceed max", + path: ["min"], + }), +]); + +const eventDeclarationSchema = z.strictObject({ + id: identifierSchema, + source: processTraceSourceSchema, + exact: jsonValueSchema, + cardinality: cardinalitySchema.default({ kind: "required" }), +}); + +const partialOrderLanguageSchema = z.strictObject({ + kind: z.literal("partial_order"), + happens_before: z + .array( + z.strictObject({ + before: identifierSchema, + after: identifierSchema, + }), + ) + .max(4_096) + .default([]), + unordered_groups: z + .array( + z.strictObject({ events: z.array(identifierSchema).min(2).max(256) }), + ) + .max(256) + .default([]), + prefix: z.array(identifierSchema).max(4_096).default([]), + suffix: z.array(identifierSchema).max(4_096).default([]), +}); + +const finiteTraceTokenSchema = z.union([ + identifierSchema, + z.strictObject({ unordered: z.array(identifierSchema).min(2).max(256) }), +]); + +const finiteTraceLanguageSchema = z.strictObject({ + kind: z.literal("finite_traces"), + variants: z + .array( + z.strictObject({ + id: identifierSchema, + trace: z.array(finiteTraceTokenSchema).max(4_096), + }), + ) + .min(1) + .max(64), +}); + +const specificationShapeSchema = z.strictObject({ + version: z.literal(1), + events: z.array(eventDeclarationSchema).min(1).max(256), + language: z.discriminatedUnion("kind", [ + partialOrderLanguageSchema, + finiteTraceLanguageSchema, + ]), +}); + +export type ProcessTraceSpecification = z.infer< + typeof specificationShapeSchema +>; +type PartialOrderLanguage = z.infer; + +export const canonicalTraceJson = (value: unknown): string => { + const serialized = canonicalize(value); + if (serialized === undefined) + throw new TypeError("Trace specification contains a non-JSON value"); + return serialized; +}; + +export const processTraceCardinalityBounds = ( + cardinality: ProcessTraceSpecification["events"][number]["cardinality"], +): readonly [number, number] => { + switch (cardinality.kind) { + case "required": + return [1, 1]; + case "optional": + return [0, 1]; + case "exact": + return [cardinality.count, cardinality.count]; + case "range": + return [cardinality.min, cardinality.max]; + } +}; + +const addReferenceIssue = ( + context: z.RefinementCtx, + known: ReadonlySet, + id: string, + path: PropertyKey[], +): void => { + if (!known.has(id)) + context.addIssue({ + code: "custom", + message: `unknown event id: ${id}`, + path, + }); +}; + +const pairKey = (left: string, right: string): string => + [left, right].sort().join("\0"); + +const buildAdjacency = ( + language: PartialOrderLanguage, + context: z.RefinementCtx, + known: ReadonlySet, +): ReadonlyMap> => { + const adjacency = new Map([...known].map((id) => [id, new Set()])); + for (const [index, edge] of language.happens_before.entries()) { + const path = ["language", "happens_before", index]; + addReferenceIssue(context, known, edge.before, [...path, "before"]); + addReferenceIssue(context, known, edge.after, [...path, "after"]); + if (edge.before === edge.after) + context.addIssue({ + code: "custom", + message: "happens-before edges must reference distinct events", + path, + }); + adjacency.get(edge.before)?.add(edge.after); + } + return adjacency; +}; + +const graphHasCycle = ( + ids: readonly string[], + adjacency: ReadonlyMap>, +): boolean => { + const active = new Set(); + const complete = new Set(); + const visit = (current: string): boolean => { + if (active.has(current)) return true; + if (complete.has(current)) return false; + active.add(current); + const cyclic = [...(adjacency.get(current) ?? [])].some(visit); + active.delete(current); + complete.add(current); + return cyclic; + }; + return ids.some(visit); +}; + +const computeReachability = ( + ids: readonly string[], + adjacency: ReadonlyMap>, +): ReadonlyMap> => { + const reachable = new Map>(); + for (const origin of ids) { + const seen = new Set(); + const pending = [...(adjacency.get(origin) ?? [])]; + while (pending.length > 0) { + const current = pending.pop(); + if (current === undefined || seen.has(current)) continue; + seen.add(current); + pending.push(...(adjacency.get(current) ?? [])); + } + reachable.set(origin, seen); + } + return reachable; +}; + +const isOrdered = ( + reachable: ReadonlyMap>, + left: string, + right: string, +): boolean => + reachable.get(left)?.has(right) === true || + reachable.get(right)?.has(left) === true; + +const validateUnorderedGroups = ( + language: PartialOrderLanguage, + context: z.RefinementCtx, + known: ReadonlySet, + reachable: ReadonlyMap>, +): ReadonlySet => { + const pairs = new Set(); + for (const [groupIndex, group] of language.unordered_groups.entries()) { + const path = ["language", "unordered_groups", groupIndex]; + if (new Set(group.events).size !== group.events.length) + context.addIssue({ + code: "custom", + message: "unordered groups must not repeat event ids", + path: [...path, "events"], + }); + for (const [eventIndex, id] of group.events.entries()) + addReferenceIssue(context, known, id, [...path, "events", eventIndex]); + for (const [leftIndex, leftId] of group.events.entries()) { + for (const rightId of group.events.slice(leftIndex + 1)) { + const key = pairKey(leftId, rightId); + if (pairs.has(key)) + context.addIssue({ + code: "custom", + message: "unordered event pairs must be declared once", + path, + }); + pairs.add(key); + if (isOrdered(reachable, leftId, rightId)) + context.addIssue({ + code: "custom", + message: "ordered events cannot also be declared unordered", + path, + }); + } + } + } + return pairs; +}; + +const validateCompletePairDeclarations = ( + ids: readonly string[], + reachable: ReadonlyMap>, + unorderedPairs: ReadonlySet, + context: z.RefinementCtx, +): void => { + for (const [leftIndex, leftId] of ids.entries()) + for (const rightId of ids.slice(leftIndex + 1)) + if ( + !isOrdered(reachable, leftId, rightId) && + !unorderedPairs.has(pairKey(leftId, rightId)) + ) + context.addIssue({ + code: "custom", + message: `event pair ${leftId}/${rightId} must be explicitly ordered or unordered`, + path: ["language"], + }); +}; + +const validatePartialOrder = ( + specification: ProcessTraceSpecification, + context: z.RefinementCtx, +): void => { + if (specification.language.kind !== "partial_order") return; + const ids = specification.events.map(({ id }) => id); + const known = new Set(ids); + const adjacency = buildAdjacency(specification.language, context, known); + if (graphHasCycle(ids, adjacency)) + context.addIssue({ + code: "custom", + message: "happens-before graph must be acyclic", + path: ["language", "happens_before"], + }); + const reachable = computeReachability(ids, adjacency); + const unorderedPairs = validateUnorderedGroups( + specification.language, + context, + known, + reachable, + ); + validateCompletePairDeclarations(ids, reachable, unorderedPairs, context); + for (const [kind, values] of [ + ["prefix", specification.language.prefix], + ["suffix", specification.language.suffix], + ] as const) + for (const [index, id] of values.entries()) + addReferenceIssue(context, known, id, ["language", kind, index]); +}; + +const countVariantEvents = ( + trace: readonly (string | { readonly unordered: readonly string[] })[], +): ReadonlyMap => { + const counts = new Map(); + for (const token of trace) + for (const id of typeof token === "string" ? [token] : token.unordered) + counts.set(id, (counts.get(id) ?? 0) + 1); + return counts; +}; + +const validateFiniteTraces = ( + specification: ProcessTraceSpecification, + context: z.RefinementCtx, +): void => { + if (specification.language.kind !== "finite_traces") return; + const known = new Set(specification.events.map(({ id }) => id)); + const variantIds = new Set(); + for (const [ + variantIndex, + variant, + ] of specification.language.variants.entries()) { + const variantPath = ["language", "variants", variantIndex]; + if (variantIds.has(variant.id)) + context.addIssue({ + code: "custom", + message: "finite trace variant ids must be unique", + path: [...variantPath, "id"], + }); + variantIds.add(variant.id); + for (const [tokenIndex, token] of variant.trace.entries()) { + const values = typeof token === "string" ? [token] : token.unordered; + for (const [eventIndex, id] of values.entries()) + addReferenceIssue(context, known, id, [ + ...variantPath, + "trace", + tokenIndex, + ...(typeof token === "string" ? [] : ["unordered", eventIndex]), + ]); + } + const counts = countVariantEvents(variant.trace); + for (const event of specification.events) { + const [minimum, maximum] = processTraceCardinalityBounds( + event.cardinality, + ); + const count = counts.get(event.id) ?? 0; + if (count < minimum || count > maximum) + context.addIssue({ + code: "custom", + message: `variant ${variant.id} violates ${event.id} cardinality`, + path: [...variantPath, "trace"], + }); + } + } +}; + +/** Strict, bounded concurrency model for Process Capture observations. */ +export const processTraceSpecificationSchema = + specificationShapeSchema.superRefine((specification, context) => { + const ids = new Set(); + const predicates = new Set(); + for (const [index, event] of specification.events.entries()) { + if (ids.has(event.id)) + context.addIssue({ + code: "custom", + message: "event ids must be unique", + path: ["events", index, "id"], + }); + ids.add(event.id); + const predicate = `${event.source}\0${canonicalTraceJson(event.exact)}`; + if (predicates.has(predicate)) + context.addIssue({ + code: "custom", + message: "exact event predicates must not overlap", + path: ["events", index], + }); + predicates.add(predicate); + } + validatePartialOrder(specification, context); + validateFiniteTraces(specification, context); + });