From ee4d284597a32b4acc4279b74083d9bf4e32d08b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:03:41 +0800 Subject: [PATCH 1/8] feat(process): introduce evidence-safe capture v4 --- src/application/CommandShimReplay.ts | 2 + src/application/EvidenceBundleFiles.ts | 13 +- src/application/ProcessCli.ts | 16 +- src/application/ProcessEvidence.ts | 19 +- src/application/ProcessHarness.ts | 147 +++++++++- src/application/ProcessOwnership.ts | 41 +++ src/cliProcessCommands.ts | 6 +- src/contracts/investigationExamples.ts | 6 +- src/contracts/processCaptureExample.ts | 56 +++- src/contracts/toolContracts.ts | 5 +- src/domain/changedBehavior.ts | 4 +- src/domain/processCapture.ts | 288 +++++++++++++++++++- src/domain/processComparison.ts | 41 ++- src/domain/reconstructionVerification.ts | 6 +- src/domain/staticRuntimeCorrelation.ts | 4 +- src/identity.ts | 1 + src/server/registerProcessComparisonTool.ts | 30 +- tests/changedBehavior.test.ts | 4 +- tests/integration/serverIntegration.test.ts | 2 +- tests/processCapture.test.ts | 175 +++++++++++- tests/reconstructionVerification.test.ts | 6 +- tests/staticRuntimeCorrelation.test.ts | 6 +- 22 files changed, 793 insertions(+), 85 deletions(-) diff --git a/src/application/CommandShimReplay.ts b/src/application/CommandShimReplay.ts index 7d766da3d..ecd5868ec 100644 --- a/src/application/CommandShimReplay.ts +++ b/src/application/CommandShimReplay.ts @@ -99,6 +99,8 @@ export const startCommandShimReplay = async ( sequence: events.length, at_ms: Math.max(0, Date.now() - started), command: invocation.command, + route_index: + routeIndex !== undefined && routeIndex >= 0 ? routeIndex : null, arguments: invocation.arguments, working_directory: invocation.working_directory, outcome, diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.ts index f6e50449d..d3b4a3886 100644 --- a/src/application/EvidenceBundleFiles.ts +++ b/src/application/EvidenceBundleFiles.ts @@ -11,6 +11,10 @@ import { } from "../domain/evidenceBundle.js"; import { EvidenceFileError, EvidenceIntegrityError } from "../domain/errors.js"; import { err, ok, type Result } from "../domain/result.js"; +import { + LEGACY_PROCESS_CAPTURE_MESSAGE, + parseProcessCapture, +} from "../domain/processCapture.js"; type EvidenceReadFailure = EvidenceFileError | EvidenceIntegrityError; type EvidenceWriteFailure = EvidenceFileError | EvidenceIntegrityError; @@ -42,7 +46,14 @@ export const readEvidenceBundle = async ( const limits = validateJsonLimits(decoded, policy); if (!limits.ok) return limits; try { - return ok(parseEvidenceBundle(decoded)); + const bundle = parseEvidenceBundle(decoded); + for (const record of bundle.records) { + if (record.predicate_type === "rea.process-capture/v3") + throw new TypeError(LEGACY_PROCESS_CAPTURE_MESSAGE); + if (record.predicate_type === "rea.process-capture/v4") + parseProcessCapture(record.normalized_result); + } + return ok(bundle); } catch (cause: unknown) { return err( new EvidenceIntegrityError("Evidence bundle validation failed", { diff --git a/src/application/ProcessCli.ts b/src/application/ProcessCli.ts index d329821a2..75ee57637 100644 --- a/src/application/ProcessCli.ts +++ b/src/application/ProcessCli.ts @@ -5,8 +5,9 @@ import { createEvidence, parseEvidence } from "../domain/evidence.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; import { compareProcessCaptures, + LEGACY_PROCESS_CAPTURE_MESSAGE, parseProcessScenario, - processCaptureSchema, + parseProcessCapture, } from "../domain/processCapture.js"; import { captureProcessScenario } from "./ProcessHarness.js"; import { @@ -38,7 +39,7 @@ export const compareProcessEvidenceFiles = async ( const right = parseCaptureEvidence(await readJson(rightPath)); const comparison = compareProcessCaptures(left.capture, right.capture); return createEvidence(undefined, PROCESS_PROVIDER, { - predicateType: "rea.process-comparison/v2", + predicateType: "rea.process-comparison/v3", operation: "compare_process_captures", parameters: { left_evidence_id: left.id, @@ -58,14 +59,17 @@ const parseCaptureEvidence = (input: unknown) => { const evidence = parseEvidence(input); if ( evidence.operation !== "capture_process_scenario" || - evidence.predicate_type !== "rea.process-capture/v3" || + evidence.predicate_type !== "rea.process-capture/v4" || evidence.provider.id !== PROCESS_PROVIDER.id || evidence.provider.version !== PROCESS_PROVIDER.version - ) - throw new TypeError("Expected REA Process Capture v3 Evidence"); + ) { + if (evidence.predicate_type === "rea.process-capture/v3") + throw new TypeError(LEGACY_PROCESS_CAPTURE_MESSAGE); + throw new TypeError("Expected REA Process Capture v4 Evidence"); + } return { id: evidence.evidence_id, - capture: processCaptureSchema.parse(evidence.normalized_result), + capture: parseProcessCapture(evidence.normalized_result), }; }; diff --git a/src/application/ProcessEvidence.ts b/src/application/ProcessEvidence.ts index 4819caf7f..4ab635ebe 100644 --- a/src/application/ProcessEvidence.ts +++ b/src/application/ProcessEvidence.ts @@ -5,12 +5,13 @@ import type { ProcessCapture, ProcessScenario, } from "../domain/processCapture.js"; +import { parseProcessCapture } from "../domain/processCapture.js"; -/** Provider identity for controlled Process Capture v3 evidence. */ +/** Provider identity for controlled Process Capture v4 evidence. */ export const PROCESS_PROVIDER = { id: "rea-process", name: "REA deterministic process harness", - version: "2", + version: "3", } as const; /** Project one process scenario into secret-free Evidence parameters. */ @@ -26,16 +27,17 @@ const processEvidenceParameters = ( normalization: scenario.normalization, }); -/** Create one canonical observed Process Capture v3 Evidence record. */ +/** Create one canonical observed Process Capture v4 Evidence record. */ export const createProcessCaptureEvidence = ( scenario: ProcessScenario, capture: ProcessCapture, -) => - createEvidence(undefined, PROCESS_PROVIDER, { - predicateType: "rea.process-capture/v3", +) => { + const validatedCapture = parseProcessCapture(capture); + return createEvidence(undefined, PROCESS_PROVIDER, { + predicateType: "rea.process-capture/v4", operation: "capture_process_scenario", parameters: processEvidenceParameters(scenario), - result: jsonValueSchema.parse(capture), + result: jsonValueSchema.parse(validatedCapture), confidence: "observed", authority: "controlled-replay", environment: { @@ -44,5 +46,6 @@ export const createProcessCaptureEvidence = ( architecture: process.arch, isolation: "process", }, - limitations: capture.limitations, + limitations: validatedCapture.limitations, }); +}; diff --git a/src/application/ProcessHarness.ts b/src/application/ProcessHarness.ts index d1a6c84b3..21e37e025 100644 --- a/src/application/ProcessHarness.ts +++ b/src/application/ProcessHarness.ts @@ -1,4 +1,5 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; import { mkdtemp, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -12,11 +13,21 @@ import type { ProcessScenario, TerminalFrame, } from "../domain/processCapture.js"; -import { authorizeProcessScenario } from "../domain/processCapture.js"; +import { + authorizeProcessScenario, + digestProcessCommitment, + processComparisonContract, + processScenarioCommitment, + validateProcessCapture, +} from "../domain/processCapture.js"; import { err, ok, type Result } from "../domain/result.js"; import { AnalysisError } from "../domain/errors.js"; +import { PRODUCT_IDENTITY } from "../identity.js"; import { startLoopbackReplay, type LoopbackReplay } from "./LoopbackReplay.js"; -import { cleanupOwnedProcessGroup } from "./ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + observeOwnedProcessGroup, +} from "./ProcessOwnership.js"; import { startProcessSampler } from "./ProcessSampling.js"; import { snapshotRoots, type SnapshotResult } from "./FilesystemSnapshot.js"; import { @@ -33,6 +44,7 @@ import { normalizeProcessText, redactProtocolEvents, } from "./ProcessNormalization.js"; +import { PROCESS_PROVIDER } from "./ProcessEvidence.js"; const isWithin = (candidate: string, root: string): boolean => candidate === root || @@ -249,19 +261,31 @@ interface CaptureResultOptions { readonly interactions: readonly InteractionEvent[]; readonly checkpoints: readonly FilesystemCheckpoint[]; readonly shimEvents: ProcessCapture["shim_events"]; + readonly settlement: ProcessCapture["settlement"]; + readonly manifest: ProcessCapture["manifest"]; } const buildCaptureResult = (options: CaptureResultOptions): ProcessCapture => ({ - schema_version: 3, + schema_version: 4, + manifest: options.manifest, normalization: options.scenario.normalization, frames: options.frames, - rendered_frames: options.renderedFrames, + rendered_frames: [...options.renderedFrames] + .sort( + (left, right) => + left.at_ms - right.at_ms || left.sequence - right.sequence, + ) + .map((frame, sequence) => ({ ...frame, sequence })), interaction_events: options.interactions, exit: { - code: options.exit.exitCode < 0 ? null : options.exit.exitCode, + code: + options.exit.reason === "exited" && options.exit.exitCode >= 0 + ? options.exit.exitCode + : null, signal: options.exit.signal ?? null, reason: options.exit.reason, }, + settlement: options.settlement, process_samples: normalizeProcessSamples( options.samples, options.scenario, @@ -304,6 +328,78 @@ const buildCaptureResult = (options: CaptureResultOptions): ProcessCapture => ({ }, }); +const hashFile = async (path: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +}; + +const createRunManifest = async ( + scenario: ProcessScenario, + startedAt: Date, + completedAt: Date, +): Promise => { + const executableSha256 = await hashFile(scenario.executable); + const scenarioCommitment = processScenarioCommitment( + scenario, + executableSha256, + ); + const comparisonContract = processComparisonContract(scenario); + return { + rea_version: PRODUCT_IDENTITY.packageVersion, + provider_version: PROCESS_PROVIDER.version, + platform: process.platform, + architecture: process.arch, + pty_backend: "node-pty", + started_at: startedAt.toISOString(), + completed_at: completedAt.toISOString(), + scenario: scenarioCommitment, + comparison_contract: comparisonContract, + shim_plan: scenario.command_shims, + replay_plan: scenario.replay, + full_scenario_sha256: digestProcessCommitment(scenarioCommitment), + comparison_contract_sha256: digestProcessCommitment(comparisonContract), + executable_sha256: executableSha256, + normalization_sha256: digestProcessCommitment(scenario.normalization), + shim_plan_sha256: digestProcessCommitment(scenario.command_shims), + replay_plan_sha256: digestProcessCommitment(scenario.replay), + }; +}; + +const observeSettlement = async ( + runId: string, + processGroupIds: readonly number[], + settleMs: number, +): Promise> => { + if (process.platform === "win32") + return { state: "unverifiable", elapsed_ms: 0 }; + const started = Date.now(); + let consecutiveEmpty = 0; + let deadlineReached = false; + while (!deadlineReached) { + const observations = await Promise.all( + [...new Set(processGroupIds)].map((processGroupId) => + observeOwnedProcessGroup({ + runId, + leaderPid: processGroupId, + processGroupId, + }), + ), + ); + if (observations.some(({ state }) => state === "unverifiable")) + return { state: "unverifiable", elapsed_ms: Date.now() - started }; + if (observations.every(({ state }) => state === "empty")) { + consecutiveEmpty += 1; + if (consecutiveEmpty >= 2) + return { state: "quiesced", elapsed_ms: Date.now() - started }; + } else consecutiveEmpty = 0; + deadlineReached = Date.now() - started >= settleMs; + if (deadlineReached) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return { state: "alive_at_deadline", elapsed_ms: Date.now() - started }; +}; + const awaitTerminalExit = async ({ terminal, scenario, @@ -523,6 +619,7 @@ const runProcessScenario = async ( let checkpoints: ProcessCheckpoints | undefined; let shimReplay: CommandShimReplay | undefined; let started = 0; + let startedAt = new Date(0); let lastOutput = 0; const timers = new Set(); let capture: ProcessCapture | undefined; @@ -530,11 +627,16 @@ const runProcessScenario = async ( let framesTruncated = (): boolean => false; let stopSampler = async () => ({ partial: false }); let samplingPartial = false; + let settlement: Omit = { + state: "unverifiable", + elapsed_ms: 0, + }; const interactions: InteractionEvent[] = []; const dispatchedEventIndexes = new Set(); try { replay = await startLoopbackReplay(scenario); + startedAt = new Date(); started = Date.now(); lastOutput = started; renderer = new TerminalRenderer({ @@ -601,8 +703,15 @@ const runProcessScenario = async ( checkpoints.trigger("root_exit"); if (exitReason === "cancelled") throw new ProcessCaptureError("process capture was cancelled"); - await new Promise((resolveSettle) => - setTimeout(resolveSettle, scenario.settle_ms), + settlement = await observeSettlement( + runId, + [ + terminal.pid, + ...samples.flatMap(({ process_group_id }) => + process_group_id === null ? [] : [process_group_id], + ), + ], + scenario.settle_ms, ); checkpoints.trigger("settled"); samplingPartial = (await stopSampler()).partial; @@ -610,6 +719,7 @@ const runProcessScenario = async ( const after = await snapshotRoots(scenario, signal); const renderedFrames = await renderer.frames(); const filesystemCheckpoints = await checkpoints.finish(after); + const manifest = await createRunManifest(scenario, startedAt, new Date()); truncated ||= after.truncated || replay.truncated || @@ -649,6 +759,8 @@ const runProcessScenario = async ( terminal?.pid ?? -1, ), })), + settlement: { ...settlement, cleanup_outcome: "not_required" }, + manifest, }); } catch (cause: unknown) { terminal?.kill("SIGKILL"); @@ -668,6 +780,25 @@ const runProcessScenario = async ( process_group_id === null ? [] : [process_group_id], ), }); + if (capture !== undefined) { + capture = { + ...capture, + settlement: { + ...capture.settlement, + cleanup_outcome: + capture.settlement.state === "quiesced" + ? "not_required" + : cleanupFailure === undefined + ? "cleaned" + : "failed", + }, + }; + const issues = validateProcessCapture(capture); + if (issues.length > 0) + executionFailure = new ProcessCaptureError( + `process capture validation failed: ${issues[0]!.path}: ${issues[0]!.message}`, + ); + } return resolveProcessResult(capture, executionFailure, cleanupFailure); }; diff --git a/src/application/ProcessOwnership.ts b/src/application/ProcessOwnership.ts index 85e633d1e..7ce6ac02a 100644 --- a/src/application/ProcessOwnership.ts +++ b/src/application/ProcessOwnership.ts @@ -34,6 +34,12 @@ export type ProcessCleanupResult = | { readonly cleaned: true; readonly signaled: boolean } | { readonly cleaned: false; readonly reason: string }; +/** Token-verified liveness result used by post-root-exit settlement. */ +export type ProcessGroupObservation = + | { readonly state: "empty" } + | { readonly state: "alive" } + | { readonly state: "unverifiable"; readonly reason: string }; + const parseEnvironment = (value: string): Readonly> => Object.fromEntries( value @@ -146,6 +152,41 @@ export const cleanupOwnedProcessGroup = async ( return { cleaned: true, signaled: true }; }; +/** Observe one group without signaling it, failing closed on identity doubt. */ +export const observeOwnedProcessGroup = async ( + ownership: OwnedProcessGroup, + host: ProcessOwnershipHost = systemHost, +): Promise => { + let members: readonly ProcessGroupMember[]; + try { + members = await host.listMembers(ownership.processGroupId); + } catch { + return { + state: "unverifiable", + reason: "process group could not be inspected", + }; + } + if (members.length === 0) return { state: "empty" }; + for (const member of members) { + try { + if ( + (await host.environment(member.pid)).REA_PROCESS_RUN_ID !== + ownership.runId + ) + return { + state: "unverifiable", + reason: "process ownership did not match", + }; + } catch { + return { + state: "unverifiable", + reason: "process ownership could not be revalidated", + }; + } + } + return { state: "alive" }; +}; + const commandMatches = (actual: string, expected: string): boolean => { const normalizedActual = actual.trim(); const normalizedExpected = expected.trim(); diff --git a/src/cliProcessCommands.ts b/src/cliProcessCommands.ts index a7beefcff..2c7696e28 100644 --- a/src/cliProcessCommands.ts +++ b/src/cliProcessCommands.ts @@ -7,13 +7,13 @@ import { import { logCliCommand } from "./cliLogging.js"; import type { Logger } from "./logger.js"; -/** Register Process Capture v3 one-shot commands through shared application services. */ +/** Register Process Capture v4 one-shot commands through shared application services. */ export const registerProcessCommands = ( cli: ReturnType, logger: Logger, ): void => { cli.command("capture-process", { - description: "Capture one approved Process Capture v3 JSON scenario", + description: "Capture one approved Process Capture v4 JSON scenario", args: z.object({ scenario: z.string().describe("Scenario JSON path") }), run: ({ args }) => logCliCommand(logger, "capture-process", () => @@ -21,7 +21,7 @@ export const registerProcessCommands = ( ), }); cli.command("compare-process-captures", { - description: "Compare two Process Capture v3 Evidence JSON files", + description: "Compare two Process Capture v4 Evidence JSON files", args: z.object({ left: z.string().describe("Left capture Evidence JSON path"), right: z.string().describe("Right capture Evidence JSON path"), diff --git a/src/contracts/investigationExamples.ts b/src/contracts/investigationExamples.ts index 0c4da92cf..6b74affc9 100644 --- a/src/contracts/investigationExamples.ts +++ b/src/contracts/investigationExamples.ts @@ -45,12 +45,12 @@ export const FUNCTION_COMPARISON_EVIDENCE = createEvidence( const PROCESS_PROVIDER = { id: "rea-process", name: "REA deterministic process harness", - version: "2", + version: "3", } as const; const capture = processCaptureSchema.parse(EMPTY_PROCESS_CAPTURE_EXAMPLE); const captureEvidence = (scenario: string) => createEvidence(undefined, PROCESS_PROVIDER, { - predicateType: "rea.process-capture/v3", + predicateType: "rea.process-capture/v4", operation: "capture_process_scenario", parameters: { scenario }, result: jsonValueSchema.parse(capture), @@ -69,7 +69,7 @@ export const PROCESS_COMPARISON_EVIDENCE = createEvidence( undefined, PROCESS_PROVIDER, { - predicateType: "rea.process-comparison/v2", + predicateType: "rea.process-comparison/v3", operation: "compare_process_captures", parameters: { left_evidence_id: PROCESS_CAPTURE_REFERENCE.evidence_id, diff --git a/src/contracts/processCaptureExample.ts b/src/contracts/processCaptureExample.ts index 6bc8420ac..4572fda7d 100644 --- a/src/contracts/processCaptureExample.ts +++ b/src/contracts/processCaptureExample.ts @@ -1,21 +1,61 @@ import type { JsonValue } from "../domain/jsonValue.js"; +import { digestProcessCommitment } from "../domain/processCapture.js"; + +const normalization = { + paths: true, + pids: true, + ports: true, + time_bucket_ms: 10, + patterns: [], +}; +const scenario = { executable_sha256: "0".repeat(64) }; +const comparisonContract = {}; +const shimPlan: JsonValue[] = []; +const replayPlan = {}; /** Minimal valid process capture used only in public contract examples. */ export const EMPTY_PROCESS_CAPTURE_EXAMPLE = { - schema_version: 3, - normalization: { - paths: true, - pids: true, - ports: true, - time_bucket_ms: 10, - patterns: [], + schema_version: 4, + manifest: { + rea_version: "1.1.0", + provider_version: "3", + platform: "fixture", + architecture: "fixture", + pty_backend: "node-pty", + started_at: "2026-01-01T00:00:00.000Z", + completed_at: "2026-01-01T00:00:00.001Z", + scenario, + comparison_contract: comparisonContract, + shim_plan: shimPlan, + replay_plan: replayPlan, + full_scenario_sha256: digestProcessCommitment(scenario), + comparison_contract_sha256: digestProcessCommitment(comparisonContract), + executable_sha256: "0".repeat(64), + normalization_sha256: digestProcessCommitment(normalization), + shim_plan_sha256: digestProcessCommitment(shimPlan), + replay_plan_sha256: digestProcessCommitment(replayPlan), }, + normalization, frames: [], rendered_frames: [], interaction_events: [], exit: { code: 0, signal: null, reason: "exited" }, + settlement: { + state: "quiesced", + elapsed_ms: 50, + cleanup_outcome: "not_required", + }, process_samples: [], - filesystem_checkpoints: [], + filesystem_checkpoints: [ + { name: "before", at_ms: 0, files: [], effects: [], truncated: false }, + { + name: "after_settlement", + at_ms: 50, + files: [], + effects: [], + truncated: false, + }, + ], shim_events: [], protocol_events: [], files_before: [], diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index a990ec056..b2c4f0e9f 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -432,17 +432,18 @@ export const SESSION_TOOL_CONTRACTS = [ ), session( "capture_process_scenario", - "Run one bounded process under a PTY using operator-approved executable and working roots. Requires approved: true; unknown_registry_approved: true separately records capture residuals. Captures raw and xterm-rendered terminal frames, scripted interactions, lifecycle filesystem checkpoints, process ownership, declarative command shims, and loopback replay. Disabled unless operator policy enables it; not a security sandbox.", + "Run one bounded process under a PTY using operator-approved executable and working roots. Produces Process Capture v4; legacy v3 captures cannot be upgraded and must be recaptured with this tool. Requires approved: true; unknown_registry_approved: true separately records capture residuals. Captures raw and xterm-rendered terminal frames, scripted interactions, lifecycle filesystem checkpoints, process ownership, declarative command shims, and loopback replay. Disabled unless operator policy enables it; not a security sandbox.", processScenarioSchema, ), session( "compare_process_captures", - "Compare two Process Capture v3 observations across terminal, interaction, exit, process, filesystem, command-shim, HTTP, and WebSocket evidence, returning the first bounded divergence. Missing or truncated observations are never treated as equivalent.", + "Compare two compatible Process Capture v4 observations across terminal, interaction, exit, settlement, process, filesystem, command-shim, HTTP, and WebSocket evidence, returning the first bounded divergence. Process Capture v3 is unsupported and must be recaptured with capture_process_scenario. Missing or truncated observations are never treated as equivalent.", z.object({ left_evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u), left: processCaptureSchema, right_evidence_id: z.string().regex(/^ev_[a-f0-9]{64}$/u), right: processCaptureSchema, + max_capture_age_ms: z.number().int().nonnegative().optional(), unknown_registry_approved: z .literal(true) .optional() diff --git a/src/domain/changedBehavior.ts b/src/domain/changedBehavior.ts index 09170ee35..b64e56cba 100644 --- a/src/domain/changedBehavior.ts +++ b/src/domain/changedBehavior.ts @@ -64,10 +64,10 @@ type RuntimeStatus = z.infer["status"]; const EXPECTED_PROVIDERS = { compare_process_captures: { - predicate: "rea.process-comparison/v2", + predicate: "rea.process-comparison/v3", id: "rea-process", name: "REA deterministic process harness", - version: "2", + version: "3", }, compare_artifacts: { predicate: "rea.artifact-comparison/v1", diff --git a/src/domain/processCapture.ts b/src/domain/processCapture.ts index d172c331a..eb0becbc9 100644 --- a/src/domain/processCapture.ts +++ b/src/domain/processCapture.ts @@ -1,4 +1,6 @@ +import { createHash } from "node:crypto"; import { resolve } from "node:path"; +import canonicalize from "canonicalize"; import { z } from "zod"; const positiveBudget = z.number().int().positive(); @@ -36,7 +38,57 @@ const outputChunkSchema = z.object({ data: z.string().max(1_000_000), }); -/** Exact boundary schema for bounded dynamic process scenarios. */ +/** Actionable refusal shared by CLI, MCP, and Evidence import boundaries. */ +export const LEGACY_PROCESS_CAPTURE_MESSAGE = + "Process Capture v3 is unsupported. Re-run the scenario with this REA version using capture_process_scenario to produce Process Capture v4; captures cannot be upgraded because v4 requires new manifest and settlement evidence."; + +const canonicalSha256 = (value: unknown): string => { + const serialized = canonicalize(value); + if (serialized === undefined) + throw new TypeError("Process capture commitment is not canonical JSON"); + return createHash("sha256").update(serialized).digest("hex"); +}; + +/** Build the secret-safe scenario projection committed by Process Capture v4. */ +export const processScenarioCommitment = ( + scenario: ProcessScenario, + executableSha256?: string, +): Readonly> => ({ + ...scenario, + environment: Object.fromEntries( + Object.entries(scenario.environment).map(([name, value]) => [ + name, + scenario.secret_aliases.includes(name) ? "" : value, + ]), + ), + unknown_registry_approved: scenario.unknown_registry_approved === true, + executable_sha256: executableSha256 ?? null, +}); + +/** Project observation settings shared by authority and reconstruction. */ +export const processComparisonContract = ( + scenario: ProcessScenario, +): Readonly> => ({ + working_directory: scenario.working_directory, + inherit_environment: scenario.inherit_environment, + secret_aliases: scenario.secret_aliases, + network_access: scenario.network_access, + filesystem_roots: scenario.filesystem_roots, + terminal: scenario.terminal, + checkpoints: scenario.checkpoints, + events: scenario.events, + timeout_ms: scenario.timeout_ms, + idle_timeout_ms: scenario.idle_timeout_ms, + settle_ms: scenario.settle_ms, + limits: scenario.limits, + normalization: scenario.normalization, + command_shims: scenario.command_shims, + replay: scenario.replay, +}); + +/** Compute a canonical SHA-256 commitment independent of object key order. */ +export const digestProcessCommitment = canonicalSha256; + /** * Boundary schema for one explicitly approved, bounded process experiment. * Defaults are part of the evidence contract and must remain deterministic. @@ -313,7 +365,6 @@ export const processScenarioSchema = z }); }); -/** A parsed, bounded dynamic process observation scenario. */ /** Parsed instructions and resource bounds for one process experiment. */ export type ProcessScenario = z.infer; @@ -321,7 +372,6 @@ export type ProcessScenario = z.infer; export const parseProcessScenario = (input: unknown): ProcessScenario => processScenarioSchema.parse(input); -/** Operator-owned policy for process capture. */ /** Operator-owned ceiling applied in addition to per-scenario approval. */ export interface ProcessExecutionPolicy { readonly enabled: boolean; @@ -331,7 +381,6 @@ export interface ProcessExecutionPolicy { readonly allowExternalNetwork: boolean; } -/** A safe, caller-visible process-policy decision. */ /** Explicit authorization result; denial reasons are safe to show callers. */ export type ProcessPolicyDecision = | { readonly allowed: true } @@ -395,7 +444,6 @@ export const authorizeProcessScenario = ( return { allowed: true }; }; -/** One bounded terminal observation. */ /** Normalized raw PTY chunk, preserving transport-level output differences. */ export interface TerminalFrame { readonly sequence: number; @@ -403,7 +451,6 @@ export interface TerminalFrame { readonly data: string; } -/** One rendered terminal state after xterm has parsed a PTY chunk or resize. */ /** Serialized terminal state after interpreting control and resize sequences. */ export interface RenderedTerminalFrame { readonly sequence: number; @@ -417,7 +464,6 @@ export interface RenderedTerminalFrame { readonly serialized_state: string; } -/** One attempted scripted interaction with the PTY. */ /** Scheduled input, resize, or signal with its observed dispatch outcome. */ export interface InteractionEvent { readonly sequence: number; @@ -455,7 +501,6 @@ export interface ProcessSample { readonly session_id: number | null; } -/** A named filesystem observation captured during a process lifecycle. */ /** * Named filesystem state whose effects are relative to the prior checkpoint. */ @@ -467,12 +512,12 @@ export interface FilesystemCheckpoint { readonly truncated: boolean; } -/** One invocation observed by the declarative command-shim replay adapter. */ /** Recorded deterministic dependency invocation and route-match outcome. */ export interface ShimEvent { readonly sequence: number; readonly at_ms: number; readonly command: string; + readonly route_index: number | null; readonly arguments: readonly string[]; readonly working_directory: string; readonly outcome: "matched" | "unmatched" | "exhausted"; @@ -494,15 +539,33 @@ export interface ProtocolEvent { | "disconnected"; } -/** A bounded process observation with explicit incompleteness metadata. */ /** - * Process Capture v3 observation set. + * Process Capture v4 observation set. * * `truncated` and `residual_unknowns` are semantic evidence: consumers must not * infer equivalence from matching bounded observations when either is present. */ export interface ProcessCapture { - readonly schema_version: 3; + readonly schema_version: 4; + readonly manifest: { + readonly rea_version: string; + readonly provider_version: string; + readonly platform: string; + readonly architecture: string; + readonly pty_backend: "node-pty"; + readonly started_at: string; + readonly completed_at: string; + readonly scenario: Readonly>; + readonly comparison_contract: Readonly>; + readonly shim_plan: readonly unknown[]; + readonly replay_plan: Readonly>; + readonly full_scenario_sha256: string; + readonly comparison_contract_sha256: string; + readonly executable_sha256: string; + readonly normalization_sha256: string; + readonly shim_plan_sha256: string; + readonly replay_plan_sha256: string; + }; readonly normalization: z.infer; readonly frames: readonly TerminalFrame[]; readonly rendered_frames: readonly RenderedTerminalFrame[]; @@ -512,6 +575,11 @@ export interface ProcessCapture { readonly signal: number | null; readonly reason: "exited" | "timeout" | "idle_timeout"; }; + readonly settlement: { + readonly state: "quiesced" | "alive_at_deadline" | "unverifiable"; + readonly elapsed_ms: number; + readonly cleanup_outcome: "not_required" | "cleaned" | "failed"; + }; readonly process_samples: readonly ProcessSample[]; readonly filesystem_checkpoints: readonly FilesystemCheckpoint[]; readonly shim_events: readonly ShimEvent[]; @@ -558,7 +626,28 @@ const fileEffectSchema = z.object({ /** Exact serialized shape of a bounded process capture. */ export const processCaptureSchema: z.ZodType = z.object({ - schema_version: z.literal(3), + schema_version: z.literal(4, { + error: LEGACY_PROCESS_CAPTURE_MESSAGE, + }), + manifest: z.object({ + rea_version: z.string().min(1), + provider_version: z.string().min(1), + platform: z.string().min(1), + architecture: z.string().min(1), + pty_backend: z.literal("node-pty"), + started_at: z.iso.datetime(), + completed_at: z.iso.datetime(), + scenario: z.record(z.string(), z.unknown()), + comparison_contract: z.record(z.string(), z.unknown()), + shim_plan: z.array(z.unknown()), + replay_plan: z.record(z.string(), z.unknown()), + full_scenario_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + comparison_contract_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + executable_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + normalization_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + shim_plan_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + replay_plan_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + }), normalization: normalizationSchema, frames: z.array( z.object({ @@ -595,6 +684,11 @@ export const processCaptureSchema: z.ZodType = z.object({ signal: z.number().int().nullable(), reason: z.enum(["exited", "timeout", "idle_timeout"]), }), + settlement: z.object({ + state: z.enum(["quiesced", "alive_at_deadline", "unverifiable"]), + elapsed_ms: z.number().int().nonnegative(), + cleanup_outcome: z.enum(["not_required", "cleaned", "failed"]), + }), process_samples: z.array( z.object({ at_ms: z.number().int().nonnegative(), @@ -619,6 +713,7 @@ export const processCaptureSchema: z.ZodType = z.object({ sequence: z.number().int().nonnegative(), at_ms: z.number().int().nonnegative(), command: z.string(), + route_index: z.number().int().nonnegative().nullable(), arguments: z.array(z.string()), working_directory: z.string(), outcome: z.enum(["matched", "unmatched", "exhausted"]), @@ -666,6 +761,173 @@ export const processCaptureSchema: z.ZodType = z.object({ }), }); +/** One pure semantic validation failure in an otherwise shaped v4 capture. */ +export interface ProcessCaptureValidationIssue { + readonly path: string; + readonly message: string; +} + +const orderedBy = ( + values: readonly Value[], + project: (value: Value) => number, +): boolean => + values.every( + (value, index) => + index === 0 || project(value) >= project(values[index - 1]!), + ); + +const orderedTimestamps = ( + values: readonly { readonly at_ms: number }[], +): boolean => orderedBy(values, ({ at_ms }) => at_ms); + +/** Recompute v4 commitments and cross-field invariants without side effects. */ +export const validateProcessCapture = ( + capture: ProcessCapture, +): readonly ProcessCaptureValidationIssue[] => { + const issues: ProcessCaptureValidationIssue[] = []; + const require = (condition: boolean, path: string, message: string): void => { + if (!condition) issues.push({ path, message }); + }; + const { manifest } = capture; + require(manifest.scenario.executable_sha256 === + manifest.executable_sha256, "manifest.executable_sha256", "executable commitment does not match the scenario projection"); + const commitments = [ + ["full_scenario_sha256", manifest.scenario], + ["comparison_contract_sha256", manifest.comparison_contract], + ["normalization_sha256", capture.normalization], + ["shim_plan_sha256", manifest.shim_plan], + ["replay_plan_sha256", manifest.replay_plan], + ] as const; + for (const [field, value] of commitments) + require(manifest[field] === + canonicalSha256( + value, + ), `manifest.${field}`, "commitment does not match its canonical value"); + require(Date.parse(manifest.started_at) <= + Date.parse( + manifest.completed_at, + ), "manifest.completed_at", "completion precedes start"); + const sequences = [ + ["frames", capture.frames], + ["rendered_frames", capture.rendered_frames], + ["interaction_events", capture.interaction_events], + ["shim_events", capture.shim_events], + ["protocol_events", capture.protocol_events], + ] as const; + for (const [name, values] of sequences) { + require(values.every( + ({ sequence }, index) => sequence === index, + ), name, "sequence values must be contiguous from zero"); + } + for (const [name, values] of [ + ["frames", capture.frames], + ["rendered_frames", capture.rendered_frames], + ["process_samples", capture.process_samples], + ["filesystem_checkpoints", capture.filesystem_checkpoints], + ["shim_events", capture.shim_events], + ["protocol_events", capture.protocol_events], + ] as const) + require(orderedTimestamps(values), name, "timestamps must be ordered"); + const checkpointNames = capture.filesystem_checkpoints.map( + ({ name }) => name, + ); + require(new Set(checkpointNames).size === + checkpointNames.length, "filesystem_checkpoints", "checkpoint names must be unique"); + require(checkpointNames[0] === + "before", "filesystem_checkpoints", "first checkpoint must be before"); + require(checkpointNames.at(-1) === + "after_settlement", "filesystem_checkpoints", "last checkpoint must be after_settlement"); + require(!capture.filesystem_checkpoints.some(({ truncated }) => truncated) || + capture.truncated, "truncated", "checkpoint truncation must propagate to the capture"); + require(capture.exit.reason === "exited" || + capture.exit.code === + null, "exit", "deadline termination cannot declare a normal exit code"); + require(capture.settlement.state !== "quiesced" || + capture.settlement.cleanup_outcome === + "not_required", "settlement.cleanup_outcome", "quiesced settlement cannot require cleanup"); + require(capture.settlement.state === "quiesced" || + capture.settlement.cleanup_outcome !== + "not_required", "settlement.cleanup_outcome", "non-quiesced settlement must report cleanup"); + const shimPlans = manifest.shim_plan.flatMap((shim) => + typeof shim === "object" && + shim !== null && + "name" in shim && + "routes" in shim && + typeof shim.name === "string" && + Array.isArray(shim.routes) + ? shim.routes.map((route, routeIndex) => ({ + command: shim.name as string, + route, + routeIndex, + })) + : [], + ); + for (const event of capture.shim_events) { + const declaredRoute = shimPlans.find( + ({ command, routeIndex }) => + command === event.command && routeIndex === event.route_index, + ); + require(event.outcome === "unmatched" + ? event.route_index === null + : declaredRoute !== + undefined, "shim_events", "shim event has no declared route"); + } + for (const plan of shimPlans) { + const maximum = + typeof plan.route === "object" && + plan.route !== null && + "max_calls" in plan.route && + typeof plan.route.max_calls === "number" + ? plan.route.max_calls + : 0; + require(capture.shim_events.filter( + (event) => + event.command === plan.command && + event.route_index === plan.routeIndex && + event.outcome === "matched", + ).length <= + maximum, "shim_events", "matched shim events exceed the declared route max_calls"); + } + const replayHttp = + "http" in manifest.replay_plan && Array.isArray(manifest.replay_plan.http) + ? manifest.replay_plan.http + : []; + for (const event of capture.protocol_events) + if ( + event.protocol === "http" && + event.direction === "request" && + event.outcome === "matched" + ) + require(replayHttp.some( + (route) => + typeof route === "object" && + route !== null && + "method" in route && + route.method === event.method && + "path" in route && + route.path === event.path, + ), "protocol_events", "matched HTTP event has no declared replay route"); + return issues; +}; + +/** Parse unknown input as v4 and reject invalid commitments or semantics. */ +export const parseProcessCapture = (input: unknown): ProcessCapture => { + if ( + typeof input === "object" && + input !== null && + "schema_version" in input && + input.schema_version === 3 + ) + throw new TypeError(LEGACY_PROCESS_CAPTURE_MESSAGE); + const capture = processCaptureSchema.parse(input); + const issues = validateProcessCapture(capture); + if (issues.length > 0) + throw new TypeError( + `Invalid Process Capture v4: ${issues.map(({ path, message }) => `${path}: ${message}`).join("; ")}`, + ); + return capture; +}; + export { compareProcessCaptures, comparisonStatusSchema, diff --git a/src/domain/processComparison.ts b/src/domain/processComparison.ts index e81a5939c..9f25e11a9 100644 --- a/src/domain/processComparison.ts +++ b/src/domain/processComparison.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import type { ProcessCapture } from "./processCapture.js"; +import { validateProcessCapture } from "./processCapture.js"; /** Comparison classification that never equates incomplete evidence. */ export const comparisonStatusSchema = z.enum([ @@ -198,7 +199,7 @@ const chooseFirstDivergence = ( /** Compare bounded captures without equating missing or incompatible evidence. */ /** - * Compare two Process Capture v3 observations without treating absence as proof. + * Compare two Process Capture v4 observations without treating absence as proof. * * Observed differences outrank residual unknowns. A capture may therefore be * definitively changed in one dimension while completeness remains unknown. @@ -206,7 +207,36 @@ const chooseFirstDivergence = ( export const compareProcessCaptures = ( left: ProcessCapture, right: ProcessCapture, + options: { + readonly maxCaptureAgeMs?: number; + readonly now?: () => number; + } = {}, ): ProcessCaptureComparison => { + const invalid = [ + ...validateProcessCapture(left), + ...validateProcessCapture(right), + ]; + if (invalid.length > 0) + throw new TypeError(`Invalid Process Capture v4: ${invalid[0]!.path}`); + if ( + left.schema_version !== right.schema_version || + left.manifest.comparison_contract_sha256 !== + right.manifest.comparison_contract_sha256 + ) + throw new TypeError( + "Process captures have incompatible comparison contracts", + ); + if (options.maxCaptureAgeMs !== undefined) { + const maxCaptureAgeMs = options.maxCaptureAgeMs; + const now = (options.now ?? Date.now)(); + if ( + [left, right].some( + ({ manifest }) => + now - Date.parse(manifest.completed_at) > maxCaptureAgeMs, + ) + ) + throw new TypeError("Process capture exceeds max_capture_age_ms"); + } if (left.truncated || right.truncated) { return { status: "truncated", @@ -247,7 +277,8 @@ export const compareProcessCaptures = ( const exit = hasUnknown(left, "exit") || hasUnknown(right, "exit") ? "unknown" - : JSON.stringify(left.exit) === JSON.stringify(right.exit) + : JSON.stringify({ exit: left.exit, settlement: left.settlement }) === + JSON.stringify({ exit: right.exit, settlement: right.settlement }) ? "unchanged" : "changed"; const filesystem = classify( @@ -279,7 +310,11 @@ export const compareProcessCaptures = ( left.interaction_events, right.interaction_events, ), - firstCollectionDivergence("exit", [left.exit], [right.exit]), + firstCollectionDivergence( + "exit", + [{ ...left.exit, settlement: left.settlement }], + [{ ...right.exit, settlement: right.settlement }], + ), firstCollectionDivergence( "filesystem", filesystemObservations(left), diff --git a/src/domain/reconstructionVerification.ts b/src/domain/reconstructionVerification.ts index c6d8a98b8..c9090b74b 100644 --- a/src/domain/reconstructionVerification.ts +++ b/src/domain/reconstructionVerification.ts @@ -44,7 +44,7 @@ const MAX_CANONICAL_BYTES = 32 * 1024 * 1024; const providers = { behavioral: { operation: "compare_process_captures", - predicate: "rea.process-comparison/v2", + predicate: "rea.process-comparison/v3", id: "rea-process", name: "REA deterministic process harness", }, @@ -199,7 +199,7 @@ const validateSourceKinds = ( ): void => { const expected = claim.kind === "behavioral" - ? ["capture_process_scenario", "rea.process-capture/v3"] + ? ["capture_process_scenario", "rea.process-capture/v4"] : claim.kind === "structural-function" ? ["analyze_function", "rea.analysis/v2"] : ["inventory_artifact", "rea.analysis/v2"]; @@ -230,7 +230,7 @@ const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => { evidence.predicate_type !== expected.predicate || evidence.provider.id !== expected.id || evidence.provider.name !== expected.name || - evidence.provider.version !== (claim.kind === "behavioral" ? "2" : "1") || + evidence.provider.version !== (claim.kind === "behavioral" ? "3" : "1") || evidence.confidence !== "derived" || evidence.authority !== "analyst-inference" || evidence.subject !== null diff --git a/src/domain/staticRuntimeCorrelation.ts b/src/domain/staticRuntimeCorrelation.ts index 59d716536..eaea50856 100644 --- a/src/domain/staticRuntimeCorrelation.ts +++ b/src/domain/staticRuntimeCorrelation.ts @@ -158,7 +158,7 @@ const EXPECTED = { "REA function comparison", ], compare_process_captures: [ - "rea.process-comparison/v2", + "rea.process-comparison/v3", "rea-process", "REA deterministic process harness", ], @@ -403,7 +403,7 @@ const assertComparisonIdentity = (evidence: Evidence): void => { evidence.provider.id !== expected[1] || evidence.provider.name !== expected[2] || evidence.provider.version !== - (evidence.operation === "compare_process_captures" ? "2" : "1") || + (evidence.operation === "compare_process_captures" ? "3" : "1") || evidence.confidence !== "derived" || evidence.authority !== "analyst-inference" || evidence.subject !== null || diff --git a/src/identity.ts b/src/identity.ts index dc9889101..474cdc35d 100644 --- a/src/identity.ts +++ b/src/identity.ts @@ -6,6 +6,7 @@ export const PRODUCT_IDENTITY = { displayName: "REA", packageName: "rea-agents", + packageVersion: "1.1.0", cliBinary: "rea", mcpCommand: "npx -y rea-agents mcp", mcpServerKey: "rea", diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.ts index 5089d9841..49c8c2cc9 100644 --- a/src/server/registerProcessComparisonTool.ts +++ b/src/server/registerProcessComparisonTool.ts @@ -11,6 +11,7 @@ import type { RecordUnknownInput } from "../domain/residualUnknown.js"; import { compareProcessCaptures, processCaptureSchema, + validateProcessCapture, } from "../domain/processCapture.js"; import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; import { err } from "../domain/result.js"; @@ -23,6 +24,7 @@ export const registerProcessComparisonTool = ( server: McpServer, session: BinarySessionPort, contract: (typeof SESSION_TOOL_CONTRACTS)[6], + now: () => number = Date.now, ): void => { server.registerTool( contract.name, @@ -34,14 +36,34 @@ export const registerProcessComparisonTool = ( left: processCaptureSchema, right_evidence_id: z.string(), right: processCaptureSchema, + max_capture_age_ms: z.number().int().nonnegative().optional(), unknown_registry_approved: z.literal(true).optional(), }) .parse(input); const validated = validateCaptureSources(session, parsed); if (!validated.ok) return toCallToolResult(validated, contract); - const comparison = compareProcessCaptures(parsed.left, parsed.right); + let comparison: ReturnType; + try { + comparison = compareProcessCaptures(parsed.left, parsed.right, { + ...(parsed.max_capture_age_ms === undefined + ? {} + : { maxCaptureAgeMs: parsed.max_capture_age_ms }), + now, + }); + } catch (cause: unknown) { + return toCallToolResult( + err( + new EvidenceIntegrityError( + cause instanceof Error + ? cause.message + : "Invalid Process Capture v4", + ), + ), + contract, + ); + } const evidence = createEvidence(undefined, PROCESS_PROVIDER, { - predicateType: "rea.process-comparison/v2", + predicateType: "rea.process-comparison/v3", operation: contract.name, parameters: { left_evidence_id: parsed.left_evidence_id, @@ -85,13 +107,15 @@ const validateCaptureSources = ( if ( evidence === undefined || evidence.operation !== "capture_process_scenario" || - evidence.predicate_type !== "rea.process-capture/v3" || + evidence.predicate_type !== "rea.process-capture/v4" || evidence.provider.id !== PROCESS_PROVIDER.id || evidence.provider.name !== PROCESS_PROVIDER.name || evidence.provider.version !== PROCESS_PROVIDER.version || evidence.confidence !== "observed" || evidence.authority !== "controlled-replay" || !result.success || + (result.success && validateProcessCapture(result.data).length > 0) || + validateProcessCapture(capture).length > 0 || canonicalJson(result.data) !== canonicalJson(capture) ) return err( diff --git a/tests/changedBehavior.test.ts b/tests/changedBehavior.test.ts index 6929cbfd7..57f6ebb40 100644 --- a/tests/changedBehavior.test.ts +++ b/tests/changedBehavior.test.ts @@ -33,11 +33,11 @@ const comparison = ( name: process ? "REA deterministic process harness" : "REA artifact comparison", - version: operation === "compare_process_captures" ? "2" : "1", + version: operation === "compare_process_captures" ? "3" : "1", }, { predicateType: process - ? "rea.process-comparison/v2" + ? "rea.process-comparison/v3" : "rea.artifact-comparison/v1", operation, parameters: {}, diff --git a/tests/integration/serverIntegration.test.ts b/tests/integration/serverIntegration.test.ts index a6ebb8054..45cf5153e 100644 --- a/tests/integration/serverIntegration.test.ts +++ b/tests/integration/serverIntegration.test.ts @@ -283,7 +283,7 @@ describe("full MCP integration with multi-tool sequences", () => { }); const captureEvidence = (capture: typeof left) => createEvidence(undefined, PROCESS_PROVIDER, { - predicateType: "rea.process-capture/v3", + predicateType: "rea.process-capture/v4", operation: "capture_process_scenario", parameters: {}, result: jsonValueSchema.parse(capture), diff --git a/tests/processCapture.test.ts b/tests/processCapture.test.ts index e3edd43bf..7c8804810 100644 --- a/tests/processCapture.test.ts +++ b/tests/processCapture.test.ts @@ -16,7 +16,12 @@ import { snapshotRoots } from "../src/application/FilesystemSnapshot.js"; import { authorizeProcessScenario, compareProcessCaptures, + digestProcessCommitment, + LEGACY_PROCESS_CAPTURE_MESSAGE, + parseProcessCapture, parseProcessScenario, + processCaptureSchema, + type ProcessCapture, type ProcessExecutionPolicy, } from "../src/domain/processCapture.js"; @@ -25,6 +30,75 @@ const processFixture = fileURLToPath( ); const execFileAsync = promisify(execFile); +const emptyCapture = (): ProcessCapture => { + const normalization = { + paths: true, + pids: true, + ports: true, + time_bucket_ms: 10, + patterns: [], + }; + const scenario = { executable_sha256: "0".repeat(64) }; + const comparisonContract = {}; + const shimPlan: readonly unknown[] = []; + const replayPlan = {}; + return { + schema_version: 4, + manifest: { + rea_version: "test", + provider_version: "3", + platform: process.platform, + architecture: process.arch, + pty_backend: "node-pty", + started_at: "2026-01-01T00:00:00.000Z", + completed_at: "2026-01-01T00:00:00.001Z", + scenario, + comparison_contract: comparisonContract, + shim_plan: shimPlan, + replay_plan: replayPlan, + full_scenario_sha256: digestProcessCommitment(scenario), + comparison_contract_sha256: digestProcessCommitment(comparisonContract), + executable_sha256: "0".repeat(64), + normalization_sha256: digestProcessCommitment(normalization), + shim_plan_sha256: digestProcessCommitment(shimPlan), + replay_plan_sha256: digestProcessCommitment(replayPlan), + }, + normalization, + frames: [], + rendered_frames: [], + interaction_events: [], + exit: { code: 0, signal: null, reason: "exited" }, + settlement: { + state: "quiesced", + elapsed_ms: 50, + cleanup_outcome: "not_required", + }, + process_samples: [], + filesystem_checkpoints: [ + { name: "before", at_ms: 0, files: [], effects: [], truncated: false }, + { + name: "after_settlement", + at_ms: 50, + files: [], + effects: [], + truncated: false, + }, + ], + shim_events: [], + protocol_events: [], + files_before: [], + files_after: [], + filesystem_effects: [], + truncated: false, + limitations: [], + residual_unknowns: [], + cleanup: { + owned_process_group: "verified", + temporary_root: "removed", + }, + }; +}; + describe("process capture domain", () => { const base = { approved: true as const, @@ -107,7 +181,9 @@ describe("process capture domain", () => { it("never considers truncated captures equivalent", () => { const capture = { - schema_version: 3 as const, + schema_version: 4 as const, + manifest: emptyCapture().manifest, + settlement: emptyCapture().settlement, normalization: { paths: true, pids: true, @@ -120,7 +196,7 @@ describe("process capture domain", () => { interaction_events: [], exit: { code: 0, signal: null, reason: "exited" as const }, process_samples: [], - filesystem_checkpoints: [], + filesystem_checkpoints: emptyCapture().filesystem_checkpoints, shim_events: [], protocol_events: [], files_before: [], @@ -137,9 +213,72 @@ describe("process capture domain", () => { expect(compareProcessCaptures(capture, capture).status).toBe("truncated"); }); + it("rejects altered v4 commitments and accepts canonical key reordering", () => { + const capture = emptyCapture(); + expect(parseProcessCapture(capture)).toEqual(capture); + expect(digestProcessCommitment({ second: 2, first: 1 })).toBe( + digestProcessCommitment({ first: 1, second: 2 }), + ); + expect(() => + parseProcessCapture({ + ...capture, + manifest: { + ...capture.manifest, + normalization_sha256: "f".repeat(64), + }, + }), + ).toThrow("normalization_sha256"); + expect(() => + parseProcessCapture({ + ...capture, + manifest: { + ...capture.manifest, + executable_sha256: "f".repeat(64), + }, + }), + ).toThrow("executable_sha256"); + }); + + it("tells agents and users to recapture unsupported v3 evidence", () => { + const legacy = { ...emptyCapture(), schema_version: 3 }; + expect(() => parseProcessCapture(legacy)).toThrow( + LEGACY_PROCESS_CAPTURE_MESSAGE, + ); + const parsed = processCaptureSchema.safeParse(legacy); + expect(parsed.success).toBe(false); + if (parsed.success) throw new Error("expected legacy capture rejection"); + expect(parsed.error.issues[0]?.message).toBe( + LEGACY_PROCESS_CAPTURE_MESSAGE, + ); + }); + + it("requires compatible contracts and enforces capture age through a clock seam", () => { + const capture = emptyCapture(); + expect(() => + compareProcessCaptures(capture, { + ...capture, + manifest: { + ...capture.manifest, + comparison_contract: { changed: true }, + comparison_contract_sha256: digestProcessCommitment({ + changed: true, + }), + }, + }), + ).toThrow("incompatible comparison contracts"); + expect(() => + compareProcessCaptures(capture, capture, { + maxCaptureAgeMs: 1, + now: () => Date.parse("2026-01-01T00:00:01.000Z"), + }), + ).toThrow("max_capture_age_ms"); + }); + it("classifies missing observations as unknown and one-sided evidence as added", () => { const base = { - schema_version: 3 as const, + schema_version: 4 as const, + manifest: emptyCapture().manifest, + settlement: emptyCapture().settlement, normalization: { paths: true, pids: true, @@ -152,7 +291,7 @@ describe("process capture domain", () => { interaction_events: [], exit: { code: 0, signal: null, reason: "exited" as const }, process_samples: [], - filesystem_checkpoints: [], + filesystem_checkpoints: emptyCapture().filesystem_checkpoints, shim_events: [], protocol_events: [], files_before: [], @@ -196,7 +335,9 @@ describe("process capture domain", () => { it("compares raw terminal chunks even when rendered states agree", () => { const capture = { - schema_version: 3 as const, + schema_version: 4 as const, + manifest: emptyCapture().manifest, + settlement: emptyCapture().settlement, normalization: { paths: true, pids: true, @@ -209,7 +350,7 @@ describe("process capture domain", () => { interaction_events: [], exit: { code: 0, signal: null, reason: "exited" as const }, process_samples: [], - filesystem_checkpoints: [], + filesystem_checkpoints: emptyCapture().filesystem_checkpoints, shim_events: [], protocol_events: [], files_before: [], @@ -237,7 +378,9 @@ describe("process capture domain", () => { it("keeps filesystem evidence unknown when stable snapshots match", () => { const capture = { - schema_version: 3 as const, + schema_version: 4 as const, + manifest: emptyCapture().manifest, + settlement: emptyCapture().settlement, normalization: { paths: true, pids: true, @@ -250,7 +393,7 @@ describe("process capture domain", () => { interaction_events: [], exit: { code: 0, signal: null, reason: "exited" as const }, process_samples: [], - filesystem_checkpoints: [], + filesystem_checkpoints: emptyCapture().filesystem_checkpoints, shim_events: [], protocol_events: [], files_before: [], @@ -275,6 +418,7 @@ describe("process capture domain", () => { const transient = compareProcessCaptures(complete, { ...complete, filesystem_checkpoints: [ + { name: "before", at_ms: 0, files: [], effects: [], truncated: false }, { name: "during_run", at_ms: 10, @@ -282,6 +426,13 @@ describe("process capture domain", () => { effects: [], truncated: false, }, + { + name: "after_settlement", + at_ms: 50, + files: [], + effects: [], + truncated: false, + }, ], }); expect(transient.filesystem).toBe("changed"); @@ -290,7 +441,9 @@ describe("process capture domain", () => { it("detects changes in normalized process sample metadata", () => { const capture = { - schema_version: 3 as const, + schema_version: 4 as const, + manifest: emptyCapture().manifest, + settlement: emptyCapture().settlement, normalization: { paths: true, pids: true, @@ -312,7 +465,7 @@ describe("process capture domain", () => { command: "worker", }, ], - filesystem_checkpoints: [], + filesystem_checkpoints: emptyCapture().filesystem_checkpoints, shim_events: [], protocol_events: [], files_before: [], @@ -618,7 +771,7 @@ describe("process capture adapter", () => { ); expect(result.ok).toBe(true); if (!result.ok) throw result.error; - expect(result.value.schema_version).toBe(3); + expect(result.value.schema_version).toBe(4); expect(result.value.rendered_frames.at(-1)?.lines.join("\n")).toContain( "probe:codex 1.2.3", ); diff --git a/tests/reconstructionVerification.test.ts b/tests/reconstructionVerification.test.ts index e3d85dfcf..f362116f6 100644 --- a/tests/reconstructionVerification.test.ts +++ b/tests/reconstructionVerification.test.ts @@ -33,7 +33,7 @@ const source = ( { predicateType: authority === "controlled-replay" - ? "rea.process-capture/v3" + ? "rea.process-capture/v4" : "rea.analysis/v2", operation: authority === "controlled-replay" @@ -89,10 +89,10 @@ const processComparison = ( { id: "rea-process", name: "REA deterministic process harness", - version: "2", + version: "3", }, { - predicateType: "rea.process-comparison/v2", + predicateType: "rea.process-comparison/v3", operation: "compare_process_captures", parameters: { left_evidence_id: left.evidence_id, diff --git a/tests/staticRuntimeCorrelation.test.ts b/tests/staticRuntimeCorrelation.test.ts index ee8c727c1..35640663d 100644 --- a/tests/staticRuntimeCorrelation.test.ts +++ b/tests/staticRuntimeCorrelation.test.ts @@ -18,7 +18,7 @@ const capture = (digit: string): Evidence => undefined, { id: "fixture", name: "Fixture", version: "1" }, { - predicateType: "rea.process-capture/v3", + predicateType: "rea.process-capture/v4", operation: "capture_process_scenario", parameters: {}, result: { digit }, @@ -68,10 +68,10 @@ const processComparison = ( { id: "rea-process", name: "REA deterministic process harness", - version: "2", + version: "3", }, { - predicateType: "rea.process-comparison/v2", + predicateType: "rea.process-comparison/v3", operation: "compare_process_captures", parameters: { left_evidence_id: leftCapture.evidence_id, From 24d000a76e3fb98c69d47ab26ac1322a4c87f901 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:04:00 +0800 Subject: [PATCH 2/8] feat(artifacts): add approved native DMG traversal --- src/application/ArtifactInventory.ts | 184 +++++++++++------- src/application/runtime.ts | 2 +- src/artifacts/ArtifactProvider.ts | 17 +- src/artifacts/NativeDmgArtifactReader.ts | 227 +++++++++++++++++++++++ src/config.ts | 4 + src/contracts/artifactToolContracts.ts | 1 + tests/config.test.ts | 1 + tests/nativeDmgArtifactReader.test.ts | 80 ++++++++ 8 files changed, 447 insertions(+), 69 deletions(-) create mode 100644 src/artifacts/NativeDmgArtifactReader.ts create mode 100644 tests/nativeDmgArtifactReader.test.ts diff --git a/src/application/ArtifactInventory.ts b/src/application/ArtifactInventory.ts index 2147a5282..32a66ad9f 100644 --- a/src/application/ArtifactInventory.ts +++ b/src/application/ArtifactInventory.ts @@ -17,6 +17,7 @@ import { import { DirectoryArtifactReader } from "../artifacts/DirectoryArtifactReader.js"; import { ZipArtifactReader } from "../artifacts/ZipArtifactReader.js"; import { MachOSliceArtifactReader } from "../artifacts/MachOSliceArtifactReader.js"; +import { NativeDmgArtifactReader } from "../artifacts/NativeDmgArtifactReader.js"; import { streamChunkToBuffer } from "../artifacts/StreamBytes.js"; import { artifactInventoryResultSchema, @@ -49,6 +50,17 @@ interface InventoryPageInput { readonly edgeLimit: number; } +/** Per-request authority combined with operator-owned native mount policy. */ +export interface ArtifactNativeMountPolicy { + readonly nativeMountApproved: boolean; + readonly nativeMountEnabled: boolean; +} + +const NATIVE_MOUNT_DISABLED: ArtifactNativeMountPolicy = { + nativeMountApproved: false, + nativeMountEnabled: false, +}; + /** Immutable inventory produced by one complete artifact scan. */ export interface ArtifactInventorySnapshot { readonly manifest: ArtifactInventoryResult["manifest"]; @@ -68,8 +80,14 @@ export const inventoryArtifact = async ( limits: ArtifactLimits, page: InventoryPageInput, signal?: AbortSignal, + nativeMount: ArtifactNativeMountPolicy = NATIVE_MOUNT_DISABLED, ): Promise => { - const snapshot = await scanArtifactInventory(inputPath, limits, signal); + const snapshot = await scanArtifactInventory( + inputPath, + limits, + signal, + nativeMount, + ); return paginateArtifactInventory(snapshot, page); }; @@ -78,6 +96,7 @@ export const scanArtifactInventory = async ( inputPath: string, limits: ArtifactLimits, signal?: AbortSignal, + nativeMount: ArtifactNativeMountPolicy = NATIVE_MOUNT_DISABLED, ): Promise => { abortIfNeeded(signal); const path = await realpath(inputPath); @@ -86,7 +105,7 @@ export const scanArtifactInventory = async ( const rootDigest = metadata.isDirectory() ? null : await hashReadable(createReadStream(path), limits.maxTotalBytes, signal); - const reader = await createReader(path, rootFormat); + const reader = await createReader(path, rootFormat, nativeMount, signal); try { const { nodes, occurrences } = await scanReader(reader, limits, signal); @@ -202,6 +221,8 @@ const inventoryLimitations = ( const createReader = async ( path: string, format: ArtifactNode["format"], + nativeMount: ArtifactNativeMountPolicy, + signal?: AbortSignal, ): Promise => { switch (format) { case "directory": @@ -216,6 +237,14 @@ const createReader = async ( return process.platform === "darwin" ? new MachOSliceArtifactReader(path) : undefined; + case "dmg": + if (!nativeMount.nativeMountApproved) return undefined; + if (!nativeMount.nativeMountEnabled) + throw new ArtifactReaderFailure( + "unavailable", + "Native DMG mounting is disabled by operator policy", + ); + return NativeDmgArtifactReader.create(path, signal); default: return undefined; } @@ -235,75 +264,102 @@ const scanReader = async ( const occurrenceByPath = new Map(); const registry = new ArtifactPathRegistry(); let totalBytes = 0; - for await (const entry of reader.entries(signal)) { - if (occurrences.length >= limits.maxEntries) - throw new ArtifactReaderFailure("limit", "Artifact entry limit exceeded"); - const logicalPath = normalizeArtifactPath(entry.path, limits); - registry.add(logicalPath, entry.kind); - preflightEntry(entry, limits); - const parent = nearestParent(logicalPath, occurrenceByPath); - const occurrence = createOccurrence( - entry, - logicalPath, - parent?.occurrence_id ?? null, - ); - if ((entry.kind === "file" || entry.kind === "slice") && !entry.encrypted) { - const remainingBytes = limits.maxTotalBytes - totalBytes; - if (remainingBytes <= 0) + const visit = async ( + currentReader: ArtifactReader, + prefix: string, + ): Promise => { + for await (const entry of currentReader.entries(signal)) { + if (occurrences.length >= limits.maxEntries) throw new ArtifactReaderFailure( "limit", - "Artifact total byte limit exceeded", + "Artifact entry limit exceeded", ); - if (entry.declaredSize !== null && entry.declaredSize > remainingBytes) - throw new ArtifactReaderFailure( - "limit", - "Declared artifact bytes exceed remaining cumulative limit", - ); - const digest = await hashReadable( - await reader.open(entry, signal), - Math.min(limits.maxEntryBytes, remainingBytes), - signal, + const logicalPath = normalizeArtifactPath( + prefix.length === 0 ? entry.path : `${prefix}/${entry.path}`, + limits, + ); + const expandableAsar = + entry.kind === "file" && + logicalPath.toLowerCase().endsWith(".asar") && + entry.adapterKey.startsWith("/"); + registry.add(logicalPath, expandableAsar ? "directory" : entry.kind); + preflightEntry(entry, limits); + const parent = nearestParent(logicalPath, occurrenceByPath); + const occurrence = createOccurrence( + entry, + logicalPath, + parent?.occurrence_id ?? null, ); - totalBytes += digest.bytes; if ( - entry.declaredSha256 !== null && - entry.declaredSha256 !== digest.sha256 - ) - throw new ArtifactReaderFailure( - "integrity", - `Artifact integrity metadata disagrees with content: ${logicalPath}`, - undefined, - { - logicalPath, - declaredSha256: entry.declaredSha256, - calculatedSha256: digest.sha256, - unpacked: entry.unpacked, - }, + (entry.kind === "file" || entry.kind === "slice") && + !entry.encrypted + ) { + const remainingBytes = limits.maxTotalBytes - totalBytes; + if (remainingBytes <= 0) + throw new ArtifactReaderFailure( + "limit", + "Artifact total byte limit exceeded", + ); + if (entry.declaredSize !== null && entry.declaredSize > remainingBytes) + throw new ArtifactReaderFailure( + "limit", + "Declared artifact bytes exceed remaining cumulative limit", + ); + const digest = await hashReadable( + await currentReader.open(entry, signal), + Math.min(limits.maxEntryBytes, remainingBytes), + signal, ); - if (totalBytes > limits.maxTotalBytes) - throw new ArtifactReaderFailure( - "limit", - "Artifact total byte limit exceeded", - ); - const classified = - entry.kind === "slice" - ? ({ kind: "universal-slice", format: "mach-o" } as const) - : classifyArtifactContent(logicalPath, digest.prefix); - const node = createArtifactNode({ - sha256: digest.sha256, - size: digest.bytes, - kind: classified.kind, - format: classified.format, - executable: entry.executable, - contentState: "embedded", - }); - nodes.set(node.artifact_id, node); - occurrence.artifact_id = node.artifact_id; - occurrence.hash_status = "verified"; + totalBytes += digest.bytes; + if ( + entry.declaredSha256 !== null && + entry.declaredSha256 !== digest.sha256 + ) + throw new ArtifactReaderFailure( + "integrity", + `Artifact integrity metadata disagrees with content: ${logicalPath}`, + undefined, + { + logicalPath, + declaredSha256: entry.declaredSha256, + calculatedSha256: digest.sha256, + unpacked: entry.unpacked, + }, + ); + if (totalBytes > limits.maxTotalBytes) + throw new ArtifactReaderFailure( + "limit", + "Artifact total byte limit exceeded", + ); + const classified = + entry.kind === "slice" + ? ({ kind: "universal-slice", format: "mach-o" } as const) + : classifyArtifactContent(logicalPath, digest.prefix); + const node = createArtifactNode({ + sha256: digest.sha256, + size: digest.bytes, + kind: classified.kind, + format: classified.format, + executable: entry.executable, + contentState: "embedded", + }); + nodes.set(node.artifact_id, node); + occurrence.artifact_id = node.artifact_id; + occurrence.hash_status = "verified"; + } + occurrences.push(occurrence); + occurrenceByPath.set(logicalPath, occurrence); + if (expandableAsar) { + const nested = new AsarArtifactReader(entry.adapterKey); + try { + await visit(nested, logicalPath); + } finally { + await nested.close(); + } + } } - occurrences.push(occurrence); - occurrenceByPath.set(logicalPath, occurrence); - } + }; + await visit(reader, ""); return { nodes, occurrences }; }; diff --git a/src/application/runtime.ts b/src/application/runtime.ts index dcef83779..9dafb2330 100644 --- a/src/application/runtime.ts +++ b/src/application/runtime.ts @@ -17,7 +17,7 @@ export const createBinarySession = ( ): BinarySession => { return new BinarySession( new CompositeProvider([ - new ArtifactProvider(), + new ArtifactProvider(config.artifactNativeMountEnabled), new NativeMacOSProvider(), new HopperProvider(config, logger), ]), diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index 02a42f5ea..c9d7cc76b 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -35,6 +35,8 @@ const IDENTITY: ProviderIdentity = Object.freeze({ /** Read-only inventory and exclusively owned extraction provider. */ export class ArtifactProvider implements AnalysisProvider { + constructor(private readonly nativeMountEnabled = false) {} + readonly #capabilities: readonly CapabilityDescriptor[] = Object.freeze( ARTIFACT_TOOL_CONTRACTS.map((contract) => Object.freeze({ @@ -61,8 +63,8 @@ export class ArtifactProvider implements AnalysisProvider { timeoutMs: 120_000, }), limitations: Object.freeze([ - "DMG and PKG child inventory requires a future native read-only adapter.", - "Nested containers are recorded but not recursively expanded implicitly.", + "DMG child inventory is macOS-only and requires per-call approval plus operator policy; PKG remains root-hash-only.", + "ASAR files discovered in filesystem-backed inventories are expanded without bulk extraction; other nested containers remain recorded only.", ]), }), ), @@ -77,12 +79,15 @@ export class ArtifactProvider implements AnalysisProvider { } createClient(target: BinaryTarget): AnalysisClient { - return new ArtifactClient(target); + return new ArtifactClient(target, this.nativeMountEnabled); } } class ArtifactClient implements AnalysisClient { - constructor(private readonly target: BinaryTarget) {} + constructor( + private readonly target: BinaryTarget, + private readonly nativeMountEnabled: boolean, + ) {} async execute( operation: AnalysisOperation, @@ -144,6 +149,10 @@ class ArtifactClient implements AnalysisClient { edgeLimit: parsed.edge_limit, }, options?.signal, + { + nativeMountApproved: parsed.native_mount_approved === true, + nativeMountEnabled: this.nativeMountEnabled, + }, ); return ok( createAnalysisExecution(result, IDENTITY, { diff --git a/src/artifacts/NativeDmgArtifactReader.ts b/src/artifacts/NativeDmgArtifactReader.ts new file mode 100644 index 000000000..03b23c8be --- /dev/null +++ b/src/artifacts/NativeDmgArtifactReader.ts @@ -0,0 +1,227 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import type { Readable } from "node:stream"; +import { promisify } from "node:util"; + +import { parse } from "plist"; +import { z } from "zod"; + +import type { ArtifactCommand } from "../domain/artifactGraph.js"; +import { + ArtifactReaderFailure, + type ArtifactEntry, + type ArtifactReader, +} from "./ArtifactReader.js"; +import { DirectoryArtifactReader } from "./DirectoryArtifactReader.js"; + +const execFileAsync = promisify(execFile); +const attachOutputSchema = z.object({ + "system-entities": z.array( + z.object({ + "dev-entry": z.string().startsWith("/dev/"), + "mount-point": z.string().optional(), + }), + ), +}); + +/** Narrow host seam for tested, shell-free hdiutil lifecycle operations. */ +export interface NativeDmgHost { + run( + arguments_: readonly string[], + signal?: AbortSignal, + ): Promise<{ readonly stdout: string; readonly exitCode: number }>; +} + +const systemHost: NativeDmgHost = { + async run(arguments_, signal) { + try { + const { stdout } = await execFileAsync( + "/usr/bin/hdiutil", + [...arguments_], + { + encoding: "utf8", + timeout: 120_000, + ...(signal === undefined ? {} : { signal }), + }, + ); + return { stdout, exitCode: 0 }; + } catch (cause: unknown) { + if (cause instanceof Error && cause.name === "AbortError") + throw new ArtifactReaderFailure("cancelled", "DMG operation cancelled"); + throw new ArtifactReaderFailure("format", "hdiutil rejected the DMG", { + cause, + }); + } + }, +}; + +/** Read-only macOS DMG adapter that owns attachment and reverse-order detach. */ +export class NativeDmgArtifactReader implements ArtifactReader { + readonly format = "file" as const; + readonly #provenance: ArtifactCommand[] = []; + #directory: DirectoryArtifactReader | undefined; + #devices: string[] = []; + #mountRoot: string | undefined; + + private constructor( + private readonly path: string, + private readonly host: NativeDmgHost, + ) {} + + /** Verify and attach one image beneath an exclusively owned temporary root. */ + static async create( + path: string, + signal?: AbortSignal, + host: NativeDmgHost = systemHost, + ): Promise { + if (process.platform !== "darwin") + throw new ArtifactReaderFailure( + "unavailable", + "Native DMG traversal is available only on macOS", + ); + const reader = new NativeDmgArtifactReader(path, host); + await reader.attach(signal); + return reader; + } + + async *entries(signal?: AbortSignal): AsyncIterable { + if (this.#directory === undefined) + throw new ArtifactReaderFailure("unavailable", "DMG is not attached"); + const prefix = basename(this.path); + for await (const entry of this.#directory.entries(signal)) + yield { ...entry, path: `${prefix}/${entry.path}` }; + } + + open(entry: ArtifactEntry, signal?: AbortSignal): Promise { + if (this.#directory === undefined) + return Promise.reject( + new ArtifactReaderFailure("unavailable", "DMG is not attached"), + ); + return this.#directory.open(entry, signal); + } + + provenance(): readonly ArtifactCommand[] { + return this.#provenance; + } + + async close(): Promise { + let detachFailure: unknown; + for (const device of [...this.#devices].reverse()) { + try { + await runChecked(this.host, ["detach", device]); + this.#provenance.push(command(["detach", device], ["mount"])); + } catch (cause: unknown) { + detachFailure ??= cause; + } + } + this.#devices = []; + if (this.#mountRoot !== undefined) + await rm(this.#mountRoot, { recursive: true, force: true }).catch( + (cause: unknown) => { + detachFailure ??= cause; + }, + ); + if (detachFailure !== undefined) + throw new ArtifactReaderFailure( + "unavailable", + "DMG detach or mount-root cleanup failed", + { cause: detachFailure }, + ); + } + + async attach(signal?: AbortSignal): Promise { + await runChecked(this.host, ["verify", this.path], signal); + this.#provenance.push(command(["verify", this.path], ["read"])); + this.#mountRoot = await mkdtemp(join(tmpdir(), "rea-dmg-")); + try { + const attached = await runChecked( + this.host, + [ + "attach", + "-readonly", + "-nobrowse", + "-plist", + "-mountroot", + this.#mountRoot, + this.path, + ], + signal, + ); + const parsed = attachOutputSchema.parse(parse(attached.stdout)); + this.#devices = parsed["system-entities"].map( + (entity) => entity["dev-entry"], + ); + if (this.#devices.length === 0) + throw new ArtifactReaderFailure( + "format", + "hdiutil returned no attached devices", + ); + for (const entity of parsed["system-entities"]) + if ( + entity["mount-point"] !== undefined && + !entity["mount-point"].startsWith(`${this.#mountRoot}/`) + ) + throw new ArtifactReaderFailure( + "path", + "hdiutil mounted outside the owned root", + ); + this.#provenance.push( + command( + [ + "attach", + "-readonly", + "-nobrowse", + "-plist", + "-mountroot", + this.#mountRoot, + this.path, + ], + ["read", "mount"], + ), + ); + this.#directory = new DirectoryArtifactReader(this.#mountRoot); + } catch (cause: unknown) { + let cleanupFailure: unknown; + try { + await this.close(); + } catch (cleanupCause: unknown) { + cleanupFailure = cleanupCause; + } + if (cleanupFailure !== undefined) + throw new ArtifactReaderFailure( + "unavailable", + "DMG attach failed and cleanup could not detach every device", + { cause: new AggregateError([cause, cleanupFailure]) }, + ); + throw cause; + } + } +} + +const runChecked = async ( + host: NativeDmgHost, + arguments_: readonly string[], + signal?: AbortSignal, +): Promise<{ readonly stdout: string; readonly exitCode: 0 }> => { + const result = await host.run(arguments_, signal); + if (result.exitCode !== 0) + throw new ArtifactReaderFailure( + "unavailable", + `hdiutil ${arguments_[0] ?? "operation"} failed with a non-zero exit code`, + ); + return { stdout: result.stdout, exitCode: 0 }; +}; + +const command = ( + arguments_: readonly string[], + effects: ArtifactCommand["effects"], +): ArtifactCommand => ({ + tool: "/usr/bin/hdiutil", + arguments: [...arguments_], + tool_version: null, + executable_sha256: null, + exit_code: 0, + effects, +}); diff --git a/src/config.ts b/src/config.ts index 62e952170..6554bbcf0 100644 --- a/src/config.ts +++ b/src/config.ts @@ -21,6 +21,7 @@ export interface AppConfig { readonly hopperLoaderArgs: readonly string[]; readonly logLevel: LogLevel; readonly processExecutionPolicy: ProcessExecutionPolicy; + readonly artifactNativeMountEnabled: boolean; readonly evidenceFilePolicy: EvidenceFilePolicy; readonly referenceSourcePolicy: ReferenceSourcePolicy; } @@ -34,6 +35,7 @@ const environmentSchema = z.object({ .enum(["trace", "debug", "info", "warn", "error", "fatal", "silent"]) .default("info"), REA_PROCESS_CAPTURE_ENABLED: z.enum(["true", "false"]).default("false"), + REA_ARTIFACT_NATIVE_MOUNT_ENABLED: z.enum(["true", "false"]).default("false"), REA_PROCESS_ALLOW_EXTERNAL_NETWORK: z .enum(["true", "false"]) .default("false"), @@ -151,6 +153,8 @@ export const parseConfig = ( allowExternalNetwork: parsedEnvironment.data.REA_PROCESS_ALLOW_EXTERNAL_NETWORK === "true", }, + artifactNativeMountEnabled: + parsedEnvironment.data.REA_ARTIFACT_NATIVE_MOUNT_ENABLED === "true", evidenceFilePolicy: { roots: evidenceRoots.value, maxBytes: 64 * 1024 * 1024, diff --git a/src/contracts/artifactToolContracts.ts b/src/contracts/artifactToolContracts.ts index c42af930d..bfe9694df 100644 --- a/src/contracts/artifactToolContracts.ts +++ b/src/contracts/artifactToolContracts.ts @@ -34,6 +34,7 @@ const traversalLimitsInput = { /** Exact caller boundary for deterministic artifact inventory. */ export const artifactInventoryInputSchema = z.object({ + native_mount_approved: z.boolean().default(false), ...pageInput, ...traversalLimitsInput, }); diff --git a/tests/config.test.ts b/tests/config.test.ts index ab86eadce..4c2322e7a 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -47,6 +47,7 @@ describe("runtime configuration", () => { hopperTargetKind: "database", hopperLoaderArgs: ["-l", "FAT", "--aarch64", "-l", "Mach-O"], logLevel: "info", + artifactNativeMountEnabled: false, processExecutionPolicy: { enabled: false, executableRoots: [], diff --git a/tests/nativeDmgArtifactReader.test.ts b/tests/nativeDmgArtifactReader.test.ts new file mode 100644 index 000000000..f5cae14b2 --- /dev/null +++ b/tests/nativeDmgArtifactReader.test.ts @@ -0,0 +1,80 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { build } from "plist"; +import { describe, expect, it } from "vitest"; + +import { + NativeDmgArtifactReader, + type NativeDmgHost, +} from "../src/artifacts/NativeDmgArtifactReader.js"; + +describe("native DMG artifact reader", () => { + it("uses plist attachment metadata and detaches returned devices", async () => { + if (process.platform !== "darwin") return; + const calls: string[][] = []; + const host: NativeDmgHost = { + async run(arguments_) { + const args = [...arguments_]; + calls.push(args); + if (args[0] !== "attach") return { stdout: "", exitCode: 0 }; + const mountRoot = args[args.indexOf("-mountroot") + 1]; + if (mountRoot === undefined) throw new Error("missing mount root"); + const mountPoint = join(mountRoot, "Fixture"); + await mkdir(mountPoint); + await writeFile(join(mountPoint, "hello.txt"), "hello"); + return { + stdout: build({ + "system-entities": [ + { "dev-entry": "/dev/disk-fixture", "mount-point": mountPoint }, + ], + }), + exitCode: 0, + }; + }, + }; + const reader = await NativeDmgArtifactReader.create( + "/tmp/image.dmg", + undefined, + host, + ); + const entries = []; + for await (const entry of reader.entries()) entries.push(entry.path); + expect(entries).toContain("image.dmg/Fixture/hello.txt"); + await reader.close(); + expect(calls).toContainEqual(["verify", "/tmp/image.dmg"]); + expect(calls).toContainEqual(["detach", "/dev/disk-fixture"]); + }); + + it("rejects non-zero results and surfaces detach failure during attach cleanup", async () => { + if (process.platform !== "darwin") return; + await expect( + NativeDmgArtifactReader.create("/tmp/image.dmg", undefined, { + run: () => Promise.resolve({ stdout: "", exitCode: 1 }), + }), + ).rejects.toThrow("non-zero exit code"); + + const host: NativeDmgHost = { + run(arguments_) { + if (arguments_[0] === "detach") + return Promise.reject(new Error("detach failed")); + if (arguments_[0] !== "attach") + return Promise.resolve({ stdout: "", exitCode: 0 }); + return Promise.resolve({ + stdout: build({ + "system-entities": [ + { + "dev-entry": "/dev/disk-fixture", + "mount-point": "/tmp/not-owned", + }, + ], + }), + exitCode: 0, + }); + }, + }; + await expect( + NativeDmgArtifactReader.create("/tmp/image.dmg", undefined, host), + ).rejects.toThrow("cleanup could not detach every device"); + }); +}); From aca518bb885825b912567eba95eb64a776a5624c Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:04:08 +0800 Subject: [PATCH 3/8] docs: document capture v4 and native mounts --- README.md | 8 ++++---- docs/process-capture.md | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b6f6ea563..7d93940d3 100644 --- a/README.md +++ b/README.md @@ -268,12 +268,12 @@ The public interface describes what the agent is trying to learn. Providers deci REA is already useful for native application investigation on macOS: - Open Mach-O, ELF, PE, `.app`, ZIP, APK, IPA, ASAR, plist, JavaScript, source-map, and Hopper database targets. -- Traverse content-addressed artifact graphs without extraction; materialize only approved occurrences into absent output roots. +- Traverse content-addressed artifact graphs without extraction; on macOS, read-only DMG traversal additionally requires `native_mount_approved: true` and `REA_ARTIFACT_NATIVE_MOUNT_ENABLED=true`. Materialize only approved occurrences into absent output roots. - Build bounded function dossiers with pseudocode, assembly, CFG edges, comments, calls, references, strings, and names. - Search and trace features across symbols, strings, metadata, references, and call paths. - Record every successful result as deterministic Evidence v2 with artifact and provider identity, confidence, authority, limitations, and locations. - Export and import evidence bundles across sessions. -- Capture approved PTY scenarios as Process Capture v3 Evidence, including raw and rendered terminal frames, scripted interactions, child-process lifecycle, named filesystem checkpoints, deterministic command shims, and loopback HTTP/WebSocket exchanges. +- Capture approved PTY scenarios as Process Capture v4 Evidence, including committed run manifests, raw and rendered terminal frames, scripted interactions, descendant settlement, named filesystem checkpoints, deterministic command shims, and loopback HTTP/WebSocket exchanges. - Compare complete artifact inventories by stable path, content, metadata, and relations; incomplete evidence never implies equivalence. - Compare explicit function dossiers across text, calls, references, strings, and address-normalized CFG topology with per-facet unknowns. - Compare canonical Evidence bundles by exact membership, explicit observation pairs, and residual-unknown histories without turning omissions into behavioral absence. @@ -390,7 +390,7 @@ environment allowlist in `REA_PROCESS_ALLOWED_ENV_JSON`. Because the current PTY adapter uses host networking, it also requires `REA_PROCESS_ALLOW_EXTERNAL_NETWORK=true`. -Capture a scenario or compare two saved Process Capture v3 Evidence records: +Capture a scenario or compare two saved Process Capture v4 Evidence records: ```bash rea capture-process ./scenario.json > authority.json @@ -400,7 +400,7 @@ rea compare-process-captures authority.json reconstruction.json The comparison reports each observed dimension separately and identifies the first terminal, interaction, exit, filesystem, protocol, process, or shim -divergence. See [Process Capture v3](docs/process-capture.md) for scenario +divergence. See [Process Capture v4](docs/process-capture.md) for scenario fields, command-shim replay, checkpoint triggers, limits, and safety behavior. If the native PTY backend is unavailable, install Xcode command-line tools and diff --git a/docs/process-capture.md b/docs/process-capture.md index 6bc19465f..95470a068 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -1,4 +1,4 @@ -# Process Capture v3 +# Process Capture v4 Process Capture records one approved command as deterministic Evidence. It is intended for authority-versus-reconstruction checks where terminal behavior, @@ -11,6 +11,13 @@ HTTP/WebSocket exchanges, and process exit ownership. Missing or bounded observations remain explicit; a truncated capture is never treated as equivalent to another capture. +Every v4 capture includes a run manifest with canonical SHA-256 commitments for +the secret-safe full scenario projection, comparison contract, executable, +normalization rules, command-shim plan, and replay plan. The manifest also +records the REA/provider versions, platform, architecture, PTY backend, and UTC +start/completion timestamps. Capture creation, Evidence import, and comparison +recompute the self-contained commitments and reject invalid lifecycle data. + ## Enable the capability Process execution is disabled until the operator supplies policy roots and @@ -31,7 +38,7 @@ not a security sandbox: the target runs with the current user's permissions. ## Capture a scenario -`rea capture-process` reads a JSON scenario and writes Process Capture v3 +`rea capture-process` reads a JSON scenario and writes Process Capture v4 Evidence to stdout: ```bash @@ -115,6 +122,10 @@ process, and shim behavior. `first_divergence` points to the earliest observed difference. Residual unknowns prevent a claim of complete equivalence, but they do not hide an observed difference in another dimension. +Captures must have equal comparison-contract commitments, but may identify +different scenarios and executables. MCP callers may set `max_capture_age_ms`; +the application clock then rejects stale captures before comparison. + ## Limits and lifecycle behavior Scenarios bound output bytes, frames, files, file bytes, process observations, @@ -128,3 +139,9 @@ revalidates that identifier before signaling sampled process groups, including groups detached from the original parent. Temporary HOME, replay, shim, terminal, checkpoint, and sampling resources are released on success, failure, timeout, and cancellation. + +After root exit, REA checks token-owned process groups every 50 ms. Two +consecutive empty observations establish `quiesced`; the settlement deadline +otherwise produces `alive_at_deadline`, while inspection or identity failures +produce `unverifiable`. Sampling can still miss short-lived or early-detached +descendants, which remains a residual unknown. From ac78de0a74b09a01efd91d9f88a16fde4f0ed9d9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:20:17 +0800 Subject: [PATCH 4/8] fix(process): harden capture validation and cleanup --- src/application/ProcessCaptureAuthority.ts | 48 + src/application/ProcessCaptureCapability.ts | 38 + src/application/ProcessCaptureError.ts | 6 + src/application/ProcessCaptureLifecycle.ts | 417 +++++++++ src/application/ProcessHarness.ts | 932 +++++++------------- src/domain/processCapture.ts | 598 +------------ src/domain/processCaptureValidation.ts | 190 ++++ src/domain/processComparison.ts | 212 ++--- src/domain/processScenario.ts | 445 ++++++++++ src/server/registerProcessComparisonTool.ts | 15 +- 10 files changed, 1589 insertions(+), 1312 deletions(-) create mode 100644 src/application/ProcessCaptureAuthority.ts create mode 100644 src/application/ProcessCaptureCapability.ts create mode 100644 src/application/ProcessCaptureError.ts create mode 100644 src/application/ProcessCaptureLifecycle.ts create mode 100644 src/domain/processCaptureValidation.ts create mode 100644 src/domain/processScenario.ts diff --git a/src/application/ProcessCaptureAuthority.ts b/src/application/ProcessCaptureAuthority.ts new file mode 100644 index 000000000..c33ce2954 --- /dev/null +++ b/src/application/ProcessCaptureAuthority.ts @@ -0,0 +1,48 @@ +import { realpath } from "node:fs/promises"; + +import type { + ProcessExecutionPolicy, + ProcessScenario, +} from "../domain/processCapture.js"; +import { ProcessCaptureError } from "./ProcessCaptureError.js"; + +const isWithin = (candidate: string, root: string): boolean => + candidate === root || + candidate.startsWith(`${root.endsWith("/") ? root.slice(0, -1) : root}/`); + +export const assertRealPathAuthority = async ( + scenario: ProcessScenario, + policy: ProcessExecutionPolicy, +): Promise => { + const executable = await realpath(scenario.executable); + const executableRoots = await Promise.all( + policy.executableRoots.map((root) => realpath(root)), + ); + if (!executableRoots.some((root) => isWithin(executable, root))) + throw new ProcessCaptureError( + "resolved executable is outside approved roots", + ); + const workingDirectory = await realpath(scenario.working_directory); + const workingRoots = await Promise.all( + policy.workingRoots.map((root) => realpath(root)), + ); + if (!workingRoots.some((root) => isWithin(workingDirectory, root))) + throw new ProcessCaptureError( + "resolved working directory is outside approved roots", + ); + for (const root of scenario.filesystem_roots) { + const resolvedRoot = await realpath(root); + if (!workingRoots.some((approved) => isWithin(resolvedRoot, approved))) + throw new ProcessCaptureError( + "resolved filesystem root is outside approved roots", + ); + } +}; + +/** Expected refusal or runtime failure from the process capture adapter. */ +export const assertNotCancelled = (signal: AbortSignal | undefined): void => { + if (signal?.aborted === true) + throw new ProcessCaptureError("process capture was cancelled"); +}; + +/** Runtime availability of the native PTY adapter on this host. */ diff --git a/src/application/ProcessCaptureCapability.ts b/src/application/ProcessCaptureCapability.ts new file mode 100644 index 000000000..542fe9845 --- /dev/null +++ b/src/application/ProcessCaptureCapability.ts @@ -0,0 +1,38 @@ +import { tmpdir } from "node:os"; +import { spawn } from "node-pty"; + +export type ProcessCaptureCapability = + | { readonly available: true; readonly backend: "node-pty" } + | { + readonly available: false; + readonly backend: "node-pty"; + readonly reason: string; + }; + +/** Probe the actual native PTY seam instead of inferring support from the OS name. */ +export const probeProcessCaptureCapability = + async (): Promise => { + try { + const terminal = spawn( + process.platform === "win32" ? "cmd.exe" : "/bin/sh", + process.platform === "win32" ? ["/c", "exit", "0"] : ["-c", "exit 0"], + { + cwd: tmpdir(), + env: { HOME: tmpdir(), TERM: "xterm-256color" }, + cols: 80, + rows: 24, + name: "xterm-256color", + }, + ); + await new Promise((resolveExit) => + terminal.onExit(() => resolveExit()), + ); + return { available: true, backend: "node-pty" }; + } catch { + return { + available: false, + backend: "node-pty", + reason: "the native PTY backend could not start a probe process", + }; + } + }; diff --git a/src/application/ProcessCaptureError.ts b/src/application/ProcessCaptureError.ts new file mode 100644 index 000000000..17593e7d1 --- /dev/null +++ b/src/application/ProcessCaptureError.ts @@ -0,0 +1,6 @@ +import { AnalysisError } from "../domain/errors.js"; + +/** Typed application failure produced by controlled process capture. */ +export class ProcessCaptureError extends AnalysisError { + readonly _tag = "ProcessCaptureError"; +} diff --git a/src/application/ProcessCaptureLifecycle.ts b/src/application/ProcessCaptureLifecycle.ts new file mode 100644 index 000000000..7470079fd --- /dev/null +++ b/src/application/ProcessCaptureLifecycle.ts @@ -0,0 +1,417 @@ +import { createHash, randomUUID } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { IPty } from "node-pty"; + +import type { + FilesystemCheckpoint, + InteractionEvent, + ProcessCapture, + ProcessExecutionPolicy, + ProcessSample, + ProcessScenario, + TerminalFrame, +} from "../domain/processCapture.js"; +import { + authorizeProcessScenario, + digestProcessCommitment, + processComparisonContract, + processScenarioCommitment, +} from "../domain/processCapture.js"; +import { PRODUCT_IDENTITY } from "../identity.js"; +import type { SnapshotResult } from "./FilesystemSnapshot.js"; +import { snapshotRoots } from "./FilesystemSnapshot.js"; +import type { LoopbackReplay } from "./LoopbackReplay.js"; +import { + cleanupOwnedProcessGroup, + observeOwnedProcessGroup, +} from "./ProcessOwnership.js"; +import { + classifyFilesystemEffects, + ProcessCheckpoints, +} from "./ProcessCheckpoints.js"; +import { ProcessCaptureError } from "./ProcessCaptureError.js"; +import { + assertNotCancelled, + assertRealPathAuthority, +} from "./ProcessCaptureAuthority.js"; +import { + normalizeProcessSamples, + normalizeProcessText, + redactProtocolEvents, +} from "./ProcessNormalization.js"; +import type { CommandShimReplay } from "./CommandShimReplay.js"; +import { PROCESS_PROVIDER } from "./ProcessEvidence.js"; +import { TerminalRenderer } from "./TerminalRenderer.js"; + +interface TerminalExitOptions { + readonly terminal: IPty; + readonly scenario: ProcessScenario; + readonly started: number; + readonly lastOutput: () => number; + readonly signal: AbortSignal | undefined; + readonly timers: Set; + readonly interactions: InteractionEvent[]; + readonly dispatchedEventIndexes: ReadonlySet; +} + +interface CaptureResultOptions { + readonly frames: readonly TerminalFrame[]; + readonly exit: { + readonly exitCode: number; + readonly signal?: number; + readonly reason: "exited" | "timeout" | "idle_timeout"; + }; + readonly samples: readonly ProcessSample[]; + readonly replay: LoopbackReplay; + readonly before: SnapshotResult; + readonly after: SnapshotResult; + readonly truncated: boolean; + readonly scenario: ProcessScenario; + readonly rootPid: number; + readonly samplingPartial: boolean; + readonly renderedFrames: ProcessCapture["rendered_frames"]; + readonly interactions: readonly InteractionEvent[]; + readonly checkpoints: readonly FilesystemCheckpoint[]; + readonly shimEvents: ProcessCapture["shim_events"]; + readonly settlement: ProcessCapture["settlement"]; + readonly manifest: ProcessCapture["manifest"]; +} + +export const buildCaptureResult = ( + options: CaptureResultOptions, +): ProcessCapture => ({ + schema_version: 4, + manifest: options.manifest, + normalization: options.scenario.normalization, + frames: options.frames, + rendered_frames: [...options.renderedFrames] + .sort( + (left, right) => + left.at_ms - right.at_ms || left.sequence - right.sequence, + ) + .map((frame, sequence) => ({ ...frame, sequence })), + interaction_events: options.interactions, + exit: { + code: + options.exit.reason === "exited" && options.exit.exitCode >= 0 + ? options.exit.exitCode + : null, + signal: options.exit.signal ?? null, + reason: options.exit.reason, + }, + settlement: options.settlement, + process_samples: normalizeProcessSamples( + options.samples, + options.scenario, + options.rootPid, + ), + filesystem_checkpoints: options.checkpoints, + shim_events: options.shimEvents, + protocol_events: redactProtocolEvents( + options.replay.events, + options.scenario, + ), + files_before: options.before.files, + files_after: options.after.files, + filesystem_effects: classifyFilesystemEffects( + options.before.files, + options.after.files, + ), + truncated: options.truncated, + limitations: [ + "Process trees are sampled and may omit short-lived descendants.", + ...(options.samplingPartial + ? ["Process-tree sampling ended with an incomplete observation."] + : []), + "Filesystem observations are before/after snapshots, not syscall traces.", + "The harness does not enforce external network isolation.", + ], + residual_unknowns: [ + { + scope: "process", + reason: "Process trees are sampled and may omit short-lived descendants.", + }, + { + scope: "network", + reason: "External network isolation is not enforced by this adapter.", + }, + ], + cleanup: { + owned_process_group: "verified", + temporary_root: "removed", + }, +}); + +const hashFile = async (path: string): Promise => { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +}; + +export const createRunManifest = async ( + scenario: ProcessScenario, + startedAt: Date, + completedAt: Date, +): Promise => { + const executableSha256 = await hashFile(scenario.executable); + const scenarioCommitment = processScenarioCommitment( + scenario, + executableSha256, + ); + const comparisonContract = processComparisonContract(scenario); + return { + rea_version: PRODUCT_IDENTITY.packageVersion, + provider_version: PROCESS_PROVIDER.version, + platform: process.platform, + architecture: process.arch, + pty_backend: "node-pty", + started_at: startedAt.toISOString(), + completed_at: completedAt.toISOString(), + scenario: scenarioCommitment, + comparison_contract: comparisonContract, + shim_plan: scenario.command_shims, + replay_plan: scenario.replay, + full_scenario_sha256: digestProcessCommitment(scenarioCommitment), + comparison_contract_sha256: digestProcessCommitment(comparisonContract), + executable_sha256: executableSha256, + normalization_sha256: digestProcessCommitment(scenario.normalization), + shim_plan_sha256: digestProcessCommitment(scenario.command_shims), + replay_plan_sha256: digestProcessCommitment(scenario.replay), + }; +}; + +export const observeSettlement = async ( + runId: string, + processGroupIds: readonly number[], + settleMs: number, +): Promise> => { + if (process.platform === "win32") + return { state: "unverifiable", elapsed_ms: 0 }; + const started = Date.now(); + let consecutiveEmpty = 0; + let deadlineReached = false; + while (!deadlineReached) { + const observations = await Promise.all( + [...new Set(processGroupIds)].map((processGroupId) => + observeOwnedProcessGroup({ + runId, + leaderPid: processGroupId, + processGroupId, + }), + ), + ); + if (observations.some(({ state }) => state === "unverifiable")) + return { state: "unverifiable", elapsed_ms: Date.now() - started }; + if (observations.every(({ state }) => state === "empty")) { + consecutiveEmpty += 1; + if (consecutiveEmpty >= 2) + return { state: "quiesced", elapsed_ms: Date.now() - started }; + } else consecutiveEmpty = 0; + deadlineReached = Date.now() - started >= settleMs; + if (deadlineReached) break; + await new Promise((resolve) => setTimeout(resolve, 50)); + } + return { state: "alive_at_deadline", elapsed_ms: Date.now() - started }; +}; + +export const awaitTerminalExit = async ({ + terminal, + scenario, + started, + lastOutput, + signal, + timers, + interactions, + dispatchedEventIndexes, +}: TerminalExitOptions): Promise<{ + exitCode: number; + signal?: number; + reason: "exited" | "timeout" | "idle_timeout" | "cancelled"; +}> => + new Promise((resolveExit) => { + // The kill caused by a deadline is observed later as an ordinary PTY exit. + // Keep the initiating lifecycle reason so comparisons distinguish a target + // exit from harness-owned timeout, idle-timeout, and cancellation cleanup. + let reason: "exited" | "timeout" | "idle_timeout" | "cancelled" = "exited"; + terminal.onExit((exit) => { + for (const [eventIndex, event] of scenario.events.entries()) { + if (dispatchedEventIndexes.has(eventIndex)) continue; + interactions.push({ + sequence: interactions.length, + scheduled_at_ms: event.at_ms, + dispatched_at_ms: Math.max(0, Date.now() - started), + type: event.type, + data: + event.type === "input" + ? "" + : event.type === "resize" + ? `${String(event.columns)}x${String(event.rows)}` + : event.signal, + outcome: "target_exited", + }); + } + for (const timer of timers) { + clearTimeout(timer); + clearInterval(timer); + } + timers.clear(); + resolveExit({ ...exit, reason }); + }); + const timeout = setInterval(() => { + if (signal?.aborted === true) { + reason = "cancelled"; + terminal.kill("SIGKILL"); + } else if (Date.now() - started >= scenario.timeout_ms) { + reason = "timeout"; + terminal.kill("SIGKILL"); + } else if (Date.now() - lastOutput() >= scenario.idle_timeout_ms) { + reason = "idle_timeout"; + terminal.kill("SIGKILL"); + } + }, 20); + timers.add(timeout); + }); + +export const releaseProcessResources = async (options: { + readonly timers: ReadonlySet; + readonly replay: LoopbackReplay | undefined; + readonly terminal: IPty | undefined; + readonly renderer: TerminalRenderer | undefined; + readonly shimReplay: CommandShimReplay | undefined; + readonly checkpoints: ProcessCheckpoints | undefined; + readonly runId: string; + readonly temporaryRoot: string; + readonly sampledProcessGroupIds: readonly number[]; +}): Promise => { + for (const timer of options.timers) clearTimeout(timer); + let failure: string | undefined; + try { + await options.replay?.close(); + } catch { + failure = "loopback replay cleanup failed"; + } + try { + await options.shimReplay?.close(); + } catch { + failure ??= "command shim replay cleanup failed"; + } + try { + await options.renderer?.dispose(); + } catch { + failure ??= "terminal renderer cleanup failed"; + } + try { + await options.checkpoints?.dispose(); + } catch { + failure ??= "filesystem checkpoint cleanup failed"; + } + if (options.terminal !== undefined && process.platform !== "win32") { + const processGroupIds = new Set([ + options.terminal.pid, + ...options.sampledProcessGroupIds, + ]); + for (const processGroupId of processGroupIds) { + const cleaned = await cleanupOwnedProcessGroup({ + runId: options.runId, + leaderPid: processGroupId, + processGroupId, + }); + if (!cleaned.cleaned) failure ??= cleaned.reason; + } + } + try { + await rm(options.temporaryRoot, { recursive: true, force: true }); + } catch { + failure ??= "temporary process root cleanup failed"; + } + return failure; +}; + +export const captureTerminalFrames = (options: { + readonly terminal: IPty; + readonly scenario: ProcessScenario; + readonly frames: TerminalFrame[]; + readonly started: number; + readonly temporaryRoot: string; + readonly onOutput: () => void; + readonly onFirstOutput: () => void; + readonly renderer: TerminalRenderer; + readonly checkpoints: ProcessCheckpoints; +}): (() => boolean) => { + let outputBytes = 0; + let truncated = false; + options.terminal.onData((data) => { + options.onFirstOutput(); + options.onOutput(); + const bytes = Buffer.byteLength(data); + if ( + options.frames.length >= options.scenario.limits.frames || + outputBytes + bytes > options.scenario.limits.output_bytes + ) { + truncated = true; + return; + } + outputBytes += bytes; + const atMs = + Math.floor( + (Date.now() - options.started) / + options.scenario.normalization.time_bucket_ms, + ) * options.scenario.normalization.time_bucket_ms; + const normalized = normalizeProcessText( + data, + options.scenario, + options.temporaryRoot, + options.terminal.pid, + ); + options.frames.push({ + sequence: options.frames.length, + at_ms: atMs, + data: normalized, + }); + options.renderer.write(data, atMs); + options.checkpoints.observeTerminal(normalized); + }); + return () => truncated; +}; + +export const resolveProcessResult = ( + capture: ProcessCapture | undefined, + executionFailure: unknown, + cleanupFailure: string | undefined, +): ProcessCapture => { + if (executionFailure instanceof ProcessCaptureError) throw executionFailure; + if (executionFailure !== undefined) + throw new ProcessCaptureError("process capture failed", { + cause: executionFailure, + }); + if (cleanupFailure !== undefined) + throw new ProcessCaptureError(cleanupFailure); + if (capture === undefined) + throw new ProcessCaptureError("process capture produced no result"); + return capture; +}; + +export const prepareProcessCapture = async ( + scenario: ProcessScenario, + policy: ProcessExecutionPolicy, + signal: AbortSignal | undefined, +): Promise<{ + readonly temporaryRoot: string; + readonly runId: string; + readonly home: string; + readonly before: SnapshotResult; +}> => { + const decision = authorizeProcessScenario(scenario, policy); + if (!decision.allowed) throw new ProcessCaptureError(decision.reason); + await assertRealPathAuthority(scenario, policy); + assertNotCancelled(signal); + const before = await snapshotRoots(scenario, signal); + const temporaryRoot = await mkdtemp(join(tmpdir(), "rea-process-")); + const runId = randomUUID(); + const home = join(temporaryRoot, "home"); + await import("node:fs/promises").then(({ mkdir }) => mkdir(home)); + return { temporaryRoot, runId, home, before }; +}; diff --git a/src/application/ProcessHarness.ts b/src/application/ProcessHarness.ts index 21e37e025..986867a53 100644 --- a/src/application/ProcessHarness.ts +++ b/src/application/ProcessHarness.ts @@ -1,11 +1,5 @@ -import { createHash, randomUUID } from "node:crypto"; -import { createReadStream } from "node:fs"; -import { mkdtemp, realpath, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; import { spawn, type IPty } from "node-pty"; import type { - FilesystemCheckpoint, InteractionEvent, ProcessCapture, ProcessExecutionPolicy, @@ -13,118 +7,35 @@ import type { ProcessScenario, TerminalFrame, } from "../domain/processCapture.js"; -import { - authorizeProcessScenario, - digestProcessCommitment, - processComparisonContract, - processScenarioCommitment, - validateProcessCapture, -} from "../domain/processCapture.js"; +import { validateProcessCapture } from "../domain/processCapture.js"; import { err, ok, type Result } from "../domain/result.js"; -import { AnalysisError } from "../domain/errors.js"; -import { PRODUCT_IDENTITY } from "../identity.js"; +import { ProcessCaptureError } from "./ProcessCaptureError.js"; + +export { ProcessCaptureError } from "./ProcessCaptureError.js"; import { startLoopbackReplay, type LoopbackReplay } from "./LoopbackReplay.js"; -import { - cleanupOwnedProcessGroup, - observeOwnedProcessGroup, -} from "./ProcessOwnership.js"; import { startProcessSampler } from "./ProcessSampling.js"; -import { snapshotRoots, type SnapshotResult } from "./FilesystemSnapshot.js"; -import { - classifyFilesystemEffects, - ProcessCheckpoints, -} from "./ProcessCheckpoints.js"; +import { snapshotRoots } from "./FilesystemSnapshot.js"; +import { ProcessCheckpoints } from "./ProcessCheckpoints.js"; import { TerminalRenderer } from "./TerminalRenderer.js"; import { startCommandShimReplay, type CommandShimReplay, } from "./CommandShimReplay.js"; +import { normalizeProcessText } from "./ProcessNormalization.js"; import { - normalizeProcessSamples, - normalizeProcessText, - redactProtocolEvents, -} from "./ProcessNormalization.js"; -import { PROCESS_PROVIDER } from "./ProcessEvidence.js"; + awaitTerminalExit, + buildCaptureResult, + captureTerminalFrames, + createRunManifest, + observeSettlement, + prepareProcessCapture, + releaseProcessResources, + resolveProcessResult, +} from "./ProcessCaptureLifecycle.js"; +import { assertNotCancelled } from "./ProcessCaptureAuthority.js"; -const isWithin = (candidate: string, root: string): boolean => - candidate === root || - candidate.startsWith(`${root.endsWith("/") ? root.slice(0, -1) : root}/`); - -const assertRealPathAuthority = async ( - scenario: ProcessScenario, - policy: ProcessExecutionPolicy, -): Promise => { - const executable = await realpath(scenario.executable); - const executableRoots = await Promise.all( - policy.executableRoots.map((root) => realpath(root)), - ); - if (!executableRoots.some((root) => isWithin(executable, root))) - throw new ProcessCaptureError( - "resolved executable is outside approved roots", - ); - const workingDirectory = await realpath(scenario.working_directory); - const workingRoots = await Promise.all( - policy.workingRoots.map((root) => realpath(root)), - ); - if (!workingRoots.some((root) => isWithin(workingDirectory, root))) - throw new ProcessCaptureError( - "resolved working directory is outside approved roots", - ); - for (const root of scenario.filesystem_roots) { - const resolvedRoot = await realpath(root); - if (!workingRoots.some((approved) => isWithin(resolvedRoot, approved))) - throw new ProcessCaptureError( - "resolved filesystem root is outside approved roots", - ); - } -}; - -/** Expected refusal or runtime failure from the process capture adapter. */ -export class ProcessCaptureError extends AnalysisError { - readonly _tag = "ProcessCaptureError"; -} - -const assertNotCancelled = (signal: AbortSignal | undefined): void => { - if (signal?.aborted === true) - throw new ProcessCaptureError("process capture was cancelled"); -}; - -/** Runtime availability of the native PTY adapter on this host. */ -export type ProcessCaptureCapability = - | { readonly available: true; readonly backend: "node-pty" } - | { - readonly available: false; - readonly backend: "node-pty"; - readonly reason: string; - }; - -/** Probe the actual native PTY seam instead of inferring support from the OS name. */ -export const probeProcessCaptureCapability = - async (): Promise => { - try { - const terminal = spawn( - process.platform === "win32" ? "cmd.exe" : "/bin/sh", - process.platform === "win32" ? ["/c", "exit", "0"] : ["-c", "exit 0"], - { - cwd: tmpdir(), - env: { HOME: tmpdir(), TERM: "xterm-256color" }, - cols: 80, - rows: 24, - name: "xterm-256color", - }, - ); - await new Promise((resolveExit) => - terminal.onExit(() => resolveExit()), - ); - return { available: true, backend: "node-pty" }; - } catch { - return { - available: false, - backend: "node-pty", - reason: "the native PTY backend could not start a probe process", - }; - } - }; +export { probeProcessCaptureCapability } from "./ProcessCaptureCapability.js"; +export type { ProcessCaptureCapability } from "./ProcessCaptureCapability.js"; interface EnvironmentOptions { readonly scenario: ProcessScenario; @@ -231,235 +142,22 @@ const eventsRequireReadiness = (scenario: ProcessScenario): boolean => (event) => event.type === "input" || event.type === "resize", ); -interface TerminalExitOptions { - readonly terminal: IPty; - readonly scenario: ProcessScenario; - readonly started: number; - readonly lastOutput: () => number; - readonly signal: AbortSignal | undefined; - readonly timers: Set; - readonly interactions: InteractionEvent[]; - readonly dispatchedEventIndexes: ReadonlySet; -} - -interface CaptureResultOptions { - readonly frames: readonly TerminalFrame[]; - readonly exit: { - readonly exitCode: number; - readonly signal?: number; - readonly reason: "exited" | "timeout" | "idle_timeout"; - }; - readonly samples: readonly ProcessSample[]; +interface StartedCaptureRuntime { readonly replay: LoopbackReplay; - readonly before: SnapshotResult; - readonly after: SnapshotResult; - readonly truncated: boolean; - readonly scenario: ProcessScenario; - readonly rootPid: number; - readonly samplingPartial: boolean; - readonly renderedFrames: ProcessCapture["rendered_frames"]; - readonly interactions: readonly InteractionEvent[]; - readonly checkpoints: readonly FilesystemCheckpoint[]; - readonly shimEvents: ProcessCapture["shim_events"]; - readonly settlement: ProcessCapture["settlement"]; - readonly manifest: ProcessCapture["manifest"]; + readonly renderer: TerminalRenderer; + readonly checkpoints: ProcessCheckpoints; + readonly shimReplay: CommandShimReplay; + readonly terminal: IPty; + readonly started: number; + readonly startedAt: Date; + readonly lastOutput: () => number; + readonly framesTruncated: () => boolean; + readonly stopSampler: () => Promise<{ readonly partial: boolean }>; } -const buildCaptureResult = (options: CaptureResultOptions): ProcessCapture => ({ - schema_version: 4, - manifest: options.manifest, - normalization: options.scenario.normalization, - frames: options.frames, - rendered_frames: [...options.renderedFrames] - .sort( - (left, right) => - left.at_ms - right.at_ms || left.sequence - right.sequence, - ) - .map((frame, sequence) => ({ ...frame, sequence })), - interaction_events: options.interactions, - exit: { - code: - options.exit.reason === "exited" && options.exit.exitCode >= 0 - ? options.exit.exitCode - : null, - signal: options.exit.signal ?? null, - reason: options.exit.reason, - }, - settlement: options.settlement, - process_samples: normalizeProcessSamples( - options.samples, - options.scenario, - options.rootPid, - ), - filesystem_checkpoints: options.checkpoints, - shim_events: options.shimEvents, - protocol_events: redactProtocolEvents( - options.replay.events, - options.scenario, - ), - files_before: options.before.files, - files_after: options.after.files, - filesystem_effects: classifyFilesystemEffects( - options.before.files, - options.after.files, - ), - truncated: options.truncated, - limitations: [ - "Process trees are sampled and may omit short-lived descendants.", - ...(options.samplingPartial - ? ["Process-tree sampling ended with an incomplete observation."] - : []), - "Filesystem observations are before/after snapshots, not syscall traces.", - "The harness does not enforce external network isolation.", - ], - residual_unknowns: [ - { - scope: "process", - reason: "Process trees are sampled and may omit short-lived descendants.", - }, - { - scope: "network", - reason: "External network isolation is not enforced by this adapter.", - }, - ], - cleanup: { - owned_process_group: "verified", - temporary_root: "removed", - }, -}); - -const hashFile = async (path: string): Promise => { - const hash = createHash("sha256"); - for await (const chunk of createReadStream(path)) hash.update(chunk); - return hash.digest("hex"); -}; - -const createRunManifest = async ( - scenario: ProcessScenario, - startedAt: Date, - completedAt: Date, -): Promise => { - const executableSha256 = await hashFile(scenario.executable); - const scenarioCommitment = processScenarioCommitment( - scenario, - executableSha256, - ); - const comparisonContract = processComparisonContract(scenario); - return { - rea_version: PRODUCT_IDENTITY.packageVersion, - provider_version: PROCESS_PROVIDER.version, - platform: process.platform, - architecture: process.arch, - pty_backend: "node-pty", - started_at: startedAt.toISOString(), - completed_at: completedAt.toISOString(), - scenario: scenarioCommitment, - comparison_contract: comparisonContract, - shim_plan: scenario.command_shims, - replay_plan: scenario.replay, - full_scenario_sha256: digestProcessCommitment(scenarioCommitment), - comparison_contract_sha256: digestProcessCommitment(comparisonContract), - executable_sha256: executableSha256, - normalization_sha256: digestProcessCommitment(scenario.normalization), - shim_plan_sha256: digestProcessCommitment(scenario.command_shims), - replay_plan_sha256: digestProcessCommitment(scenario.replay), - }; -}; - -const observeSettlement = async ( - runId: string, - processGroupIds: readonly number[], - settleMs: number, -): Promise> => { - if (process.platform === "win32") - return { state: "unverifiable", elapsed_ms: 0 }; - const started = Date.now(); - let consecutiveEmpty = 0; - let deadlineReached = false; - while (!deadlineReached) { - const observations = await Promise.all( - [...new Set(processGroupIds)].map((processGroupId) => - observeOwnedProcessGroup({ - runId, - leaderPid: processGroupId, - processGroupId, - }), - ), - ); - if (observations.some(({ state }) => state === "unverifiable")) - return { state: "unverifiable", elapsed_ms: Date.now() - started }; - if (observations.every(({ state }) => state === "empty")) { - consecutiveEmpty += 1; - if (consecutiveEmpty >= 2) - return { state: "quiesced", elapsed_ms: Date.now() - started }; - } else consecutiveEmpty = 0; - deadlineReached = Date.now() - started >= settleMs; - if (deadlineReached) break; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - return { state: "alive_at_deadline", elapsed_ms: Date.now() - started }; -}; - -const awaitTerminalExit = async ({ - terminal, - scenario, - started, - lastOutput, - signal, - timers, - interactions, - dispatchedEventIndexes, -}: TerminalExitOptions): Promise<{ - exitCode: number; - signal?: number; - reason: "exited" | "timeout" | "idle_timeout" | "cancelled"; -}> => - new Promise((resolveExit) => { - // The kill caused by a deadline is observed later as an ordinary PTY exit. - // Keep the initiating lifecycle reason so comparisons distinguish a target - // exit from harness-owned timeout, idle-timeout, and cancellation cleanup. - let reason: "exited" | "timeout" | "idle_timeout" | "cancelled" = "exited"; - terminal.onExit((exit) => { - for (const [eventIndex, event] of scenario.events.entries()) { - if (dispatchedEventIndexes.has(eventIndex)) continue; - interactions.push({ - sequence: interactions.length, - scheduled_at_ms: event.at_ms, - dispatched_at_ms: Math.max(0, Date.now() - started), - type: event.type, - data: - event.type === "input" - ? "" - : event.type === "resize" - ? `${String(event.columns)}x${String(event.rows)}` - : event.signal, - outcome: "target_exited", - }); - } - for (const timer of timers) { - clearTimeout(timer); - clearInterval(timer); - } - timers.clear(); - resolveExit({ ...exit, reason }); - }); - const timeout = setInterval(() => { - if (signal?.aborted === true) { - reason = "cancelled"; - terminal.kill("SIGKILL"); - } else if (Date.now() - started >= scenario.timeout_ms) { - reason = "timeout"; - terminal.kill("SIGKILL"); - } else if (Date.now() - lastOutput() >= scenario.idle_timeout_ms) { - reason = "idle_timeout"; - terminal.kill("SIGKILL"); - } - }, 20); - timers.add(timeout); - }); - -const releaseProcessResources = async (options: { - readonly timers: ReadonlySet; +const cleanupFailedStartup = async (options: { + readonly cause: unknown; + readonly timers: Set; readonly replay: LoopbackReplay | undefined; readonly terminal: IPty | undefined; readonly renderer: TerminalRenderer | undefined; @@ -467,136 +165,264 @@ const releaseProcessResources = async (options: { readonly checkpoints: ProcessCheckpoints | undefined; readonly runId: string; readonly temporaryRoot: string; - readonly sampledProcessGroupIds: readonly number[]; -}): Promise => { - for (const timer of options.timers) clearTimeout(timer); - let failure: string | undefined; - try { - await options.replay?.close(); - } catch { - failure = "loopback replay cleanup failed"; - } - try { - await options.shimReplay?.close(); - } catch { - failure ??= "command shim replay cleanup failed"; - } - try { - await options.renderer?.dispose(); - } catch { - failure ??= "terminal renderer cleanup failed"; - } - try { - await options.checkpoints?.dispose(); - } catch { - failure ??= "filesystem checkpoint cleanup failed"; - } - if (options.terminal !== undefined && process.platform !== "win32") { - const processGroupIds = new Set([ - options.terminal.pid, - ...options.sampledProcessGroupIds, - ]); - for (const processGroupId of processGroupIds) { - const cleaned = await cleanupOwnedProcessGroup({ - runId: options.runId, - leaderPid: processGroupId, - processGroupId, - }); - if (!cleaned.cleaned) failure ??= cleaned.reason; - } - } - try { - await rm(options.temporaryRoot, { recursive: true, force: true }); - } catch { - failure ??= "temporary process root cleanup failed"; - } - return failure; -}; - -const captureTerminalFrames = (options: { - readonly terminal: IPty; - readonly scenario: ProcessScenario; - readonly frames: TerminalFrame[]; - readonly started: number; - readonly temporaryRoot: string; - readonly onOutput: () => void; - readonly onFirstOutput: () => void; - readonly renderer: TerminalRenderer; - readonly checkpoints: ProcessCheckpoints; -}): (() => boolean) => { - let outputBytes = 0; - let truncated = false; - options.terminal.onData((data) => { - options.onFirstOutput(); - options.onOutput(); - const bytes = Buffer.byteLength(data); - if ( - options.frames.length >= options.scenario.limits.frames || - outputBytes + bytes > options.scenario.limits.output_bytes - ) { - truncated = true; - return; - } - outputBytes += bytes; - const atMs = - Math.floor( - (Date.now() - options.started) / - options.scenario.normalization.time_bucket_ms, - ) * options.scenario.normalization.time_bucket_ms; - const normalized = normalizeProcessText( - data, - options.scenario, - options.temporaryRoot, - options.terminal.pid, - ); - options.frames.push({ - sequence: options.frames.length, - at_ms: atMs, - data: normalized, - }); - options.renderer.write(data, atMs); - options.checkpoints.observeTerminal(normalized); +}): Promise => { + options.terminal?.kill("SIGKILL"); + const cleanupFailure = await releaseProcessResources({ + ...options, + sampledProcessGroupIds: [], }); - return () => truncated; -}; - -const resolveProcessResult = ( - capture: ProcessCapture | undefined, - executionFailure: unknown, - cleanupFailure: string | undefined, -): ProcessCapture => { - if (executionFailure instanceof ProcessCaptureError) throw executionFailure; - if (executionFailure !== undefined) - throw new ProcessCaptureError("process capture failed", { - cause: executionFailure, - }); if (cleanupFailure !== undefined) - throw new ProcessCaptureError(cleanupFailure); - if (capture === undefined) - throw new ProcessCaptureError("process capture produced no result"); - return capture; + throw new ProcessCaptureError(cleanupFailure, { cause: options.cause }); + throw options.cause; }; -const prepareProcessCapture = async ( +const createTerminalRenderer = ( scenario: ProcessScenario, - policy: ProcessExecutionPolicy, - signal: AbortSignal | undefined, -): Promise<{ + temporaryRoot: string, + terminalPid: () => number, +): TerminalRenderer => + new TerminalRenderer({ + columns: scenario.terminal.columns, + rows: scenario.terminal.rows, + scrollback: scenario.terminal.scrollback, + maxFrames: scenario.limits.frames, + maxBytes: scenario.limits.output_bytes, + normalize: (value) => + normalizeProcessText(value, scenario, temporaryRoot, terminalPid()), + }); + +const startCaptureRuntime = async (options: { + readonly scenario: ProcessScenario; + readonly home: string; readonly temporaryRoot: string; readonly runId: string; - readonly home: string; - readonly before: SnapshotResult; -}> => { - const decision = authorizeProcessScenario(scenario, policy); - if (!decision.allowed) throw new ProcessCaptureError(decision.reason); - await assertRealPathAuthority(scenario, policy); - assertNotCancelled(signal); - const before = await snapshotRoots(scenario, signal); - const temporaryRoot = await mkdtemp(join(tmpdir(), "rea-process-")); - const runId = randomUUID(); - const home = join(temporaryRoot, "home"); - await import("node:fs/promises").then(({ mkdir }) => mkdir(home)); - return { temporaryRoot, runId, home, before }; + readonly before: Awaited>; + readonly frames: TerminalFrame[]; + readonly samples: ProcessSample[]; + readonly interactions: InteractionEvent[]; + readonly timers: Set; + readonly dispatchedEventIndexes: Set; + readonly signal?: AbortSignal; +}): Promise => { + const { scenario } = options; + let replay: LoopbackReplay | undefined; + let renderer: TerminalRenderer | undefined; + let checkpoints: ProcessCheckpoints | undefined; + let shimReplay: CommandShimReplay | undefined; + let terminal: IPty | undefined; + try { + replay = await startLoopbackReplay(scenario); + const startedAt = new Date(); + const started = Date.now(); + let lastOutput = started; + renderer = createTerminalRenderer( + scenario, + options.temporaryRoot, + () => terminal?.pid ?? -1, + ); + checkpoints = new ProcessCheckpoints( + scenario, + started, + options.before, + options.signal, + ); + shimReplay = await startCommandShimReplay( + scenario, + options.temporaryRoot, + started, + ); + terminal = spawn(scenario.executable, [...scenario.arguments], { + cwd: scenario.working_directory, + env: makeEnvironment({ ...options, replay, shimReplay }), + cols: scenario.terminal.columns, + rows: scenario.terminal.rows, + name: "xterm-256color", + }); + const startEvents = startScenarioEvents({ + ...options, + getTerminal: () => terminal, + renderer, + started, + }); + if (!eventsRequireReadiness(scenario)) startEvents(); + const framesTruncated = captureTerminalFrames({ + ...options, + terminal, + started, + onOutput: () => (lastOutput = Date.now()), + onFirstOutput: startEvents, + renderer, + checkpoints, + }); + const stopSampler = startProcessSampler( + terminal.pid, + started, + scenario.limits.processes, + options.samples, + ); + return { + replay, + renderer, + checkpoints, + shimReplay, + terminal, + started, + startedAt, + lastOutput: () => lastOutput, + framesTruncated, + stopSampler, + }; + } catch (cause: unknown) { + return cleanupFailedStartup({ + cause, + timers: options.timers, + replay, + terminal, + renderer, + shimReplay, + checkpoints, + runId: options.runId, + temporaryRoot: options.temporaryRoot, + }); + } +}; + +const normalizedShimEvents = ( + runtime: StartedCaptureRuntime, + scenario: ProcessScenario, + temporaryRoot: string, +): ProcessCapture["shim_events"] => + runtime.shimReplay.events.map((event) => ({ + ...event, + arguments: event.arguments.map((argument) => + normalizeProcessText( + argument, + scenario, + temporaryRoot, + runtime.terminal.pid, + ), + ), + working_directory: normalizeProcessText( + event.working_directory, + scenario, + temporaryRoot, + runtime.terminal.pid, + ), + })); + +const finishProcessRun = async (options: { + readonly runtime: StartedCaptureRuntime | undefined; + readonly timers: Set; + readonly runId: string; + readonly temporaryRoot: string; + readonly samples: readonly ProcessSample[]; + readonly stopSampler: () => Promise<{ readonly partial: boolean }>; + readonly capture: ProcessCapture | undefined; + readonly executionFailure: unknown; +}): Promise => { + await options.stopSampler(); + const cleanupFailure = await releaseProcessResources({ + timers: options.timers, + replay: options.runtime?.replay, + terminal: options.runtime?.terminal, + renderer: options.runtime?.renderer, + shimReplay: options.runtime?.shimReplay, + checkpoints: options.runtime?.checkpoints, + runId: options.runId, + temporaryRoot: options.temporaryRoot, + sampledProcessGroupIds: options.samples.flatMap(({ process_group_id }) => + process_group_id === null ? [] : [process_group_id], + ), + }); + let { capture, executionFailure } = options; + if (capture !== undefined) { + capture = { + ...capture, + settlement: { + ...capture.settlement, + cleanup_outcome: + capture.settlement.state === "quiesced" + ? "not_required" + : cleanupFailure === undefined + ? "cleaned" + : "failed", + }, + }; + const issue = validateProcessCapture(capture)[0]; + if (issue !== undefined) + executionFailure = new ProcessCaptureError( + `process capture validation failed: ${issue.path}: ${issue.message}`, + ); + } + return resolveProcessResult(capture, executionFailure, cleanupFailure); +}; + +const completeCapture = async (options: { + readonly scenario: ProcessScenario; + readonly runtime: StartedCaptureRuntime; + readonly runId: string; + readonly temporaryRoot: string; + readonly before: Awaited>; + readonly frames: readonly TerminalFrame[]; + readonly samples: readonly ProcessSample[]; + readonly interactions: readonly InteractionEvent[]; + readonly exit: Awaited>; + readonly signal?: AbortSignal; + readonly initiallyTruncated: boolean; +}): Promise => { + const { runtime, scenario } = options; + runtime.checkpoints.trigger("root_exit"); + const { reason } = options.exit; + if (reason === "cancelled") + throw new ProcessCaptureError("process capture was cancelled"); + const settlement = await observeSettlement( + options.runId, + [ + runtime.terminal.pid, + ...options.samples.flatMap(({ process_group_id }) => + process_group_id === null ? [] : [process_group_id], + ), + ], + scenario.settle_ms, + ); + runtime.checkpoints.trigger("settled"); + const samplingPartial = (await runtime.stopSampler()).partial; + assertNotCancelled(options.signal); + const after = await snapshotRoots(scenario, options.signal); + const renderedFrames = await runtime.renderer.frames(); + const checkpoints = await runtime.checkpoints.finish(after); + const manifest = await createRunManifest( + scenario, + runtime.startedAt, + new Date(), + ); + const truncated = + options.initiallyTruncated || + after.truncated || + runtime.replay.truncated || + runtime.shimReplay.truncated || + runtime.framesTruncated() || + checkpoints.some(({ truncated: partial }) => partial) || + samplingPartial || + runtime.renderer.truncated(); + return buildCaptureResult({ + frames: options.frames, + exit: { ...options.exit, reason }, + samples: options.samples, + replay: runtime.replay, + before: options.before, + after, + truncated, + scenario, + rootPid: runtime.terminal.pid, + samplingPartial, + renderedFrames, + interactions: options.interactions, + checkpoints, + shimEvents: normalizedShimEvents(runtime, scenario, options.temporaryRoot), + settlement: { ...settlement, cleanup_outcome: "not_required" }, + manifest, + }); }; /** Execute one authorized scenario and return bounded observations. */ @@ -612,194 +438,66 @@ const runProcessScenario = async ( ); const frames: TerminalFrame[] = []; const samples: ProcessSample[] = []; - let truncated = before.truncated; - let terminal: IPty | undefined; - let replay: LoopbackReplay | undefined; - let renderer: TerminalRenderer | undefined; - let checkpoints: ProcessCheckpoints | undefined; - let shimReplay: CommandShimReplay | undefined; - let started = 0; - let startedAt = new Date(0); - let lastOutput = 0; + let runtime: StartedCaptureRuntime | undefined; const timers = new Set(); let capture: ProcessCapture | undefined; let executionFailure: unknown; - let framesTruncated = (): boolean => false; let stopSampler = async () => ({ partial: false }); - let samplingPartial = false; - let settlement: Omit = { - state: "unverifiable", - elapsed_ms: 0, - }; const interactions: InteractionEvent[] = []; const dispatchedEventIndexes = new Set(); try { - replay = await startLoopbackReplay(scenario); - startedAt = new Date(); - started = Date.now(); - lastOutput = started; - renderer = new TerminalRenderer({ - columns: scenario.terminal.columns, - rows: scenario.terminal.rows, - scrollback: scenario.terminal.scrollback, - maxFrames: scenario.limits.frames, - maxBytes: scenario.limits.output_bytes, - normalize: (value) => - normalizeProcessText( - value, - scenario, - temporaryRoot, - terminal?.pid ?? -1, - ), - }); - checkpoints = new ProcessCheckpoints(scenario, started, before, signal); - shimReplay = await startCommandShimReplay(scenario, temporaryRoot, started); - terminal = spawn(scenario.executable, [...scenario.arguments], { - cwd: scenario.working_directory, - env: makeEnvironment({ scenario, home, replay, shimReplay, runId }), - cols: scenario.terminal.columns, - rows: scenario.terminal.rows, - name: "xterm-256color", - }); - const startEvents = startScenarioEvents({ + runtime = await startCaptureRuntime({ scenario, - getTerminal: () => terminal, - timers, - interactions, - renderer, - started, - dispatchedEventIndexes, - }); - if (!eventsRequireReadiness(scenario)) startEvents(); - framesTruncated = captureTerminalFrames({ - terminal, - scenario, - frames, - started, + home, temporaryRoot, - onOutput: () => (lastOutput = Date.now()), - onFirstOutput: startEvents, - renderer, - checkpoints, - }); - stopSampler = startProcessSampler( - terminal.pid, - started, - scenario.limits.processes, + runId, + before, + frames, samples, - ); + interactions, + timers, + dispatchedEventIndexes, + ...(signal === undefined ? {} : { signal }), + }); + stopSampler = runtime.stopSampler; const exit = await awaitTerminalExit({ - terminal, + terminal: runtime.terminal, scenario, - started, - lastOutput: () => lastOutput, + started: runtime.started, + lastOutput: runtime.lastOutput, signal, timers, interactions, dispatchedEventIndexes, }); - const exitReason = exit.reason; - checkpoints.trigger("root_exit"); - if (exitReason === "cancelled") - throw new ProcessCaptureError("process capture was cancelled"); - settlement = await observeSettlement( - runId, - [ - terminal.pid, - ...samples.flatMap(({ process_group_id }) => - process_group_id === null ? [] : [process_group_id], - ), - ], - scenario.settle_ms, - ); - checkpoints.trigger("settled"); - samplingPartial = (await stopSampler()).partial; - assertNotCancelled(signal); - const after = await snapshotRoots(scenario, signal); - const renderedFrames = await renderer.frames(); - const filesystemCheckpoints = await checkpoints.finish(after); - const manifest = await createRunManifest(scenario, startedAt, new Date()); - truncated ||= - after.truncated || - replay.truncated || - shimReplay.truncated || - framesTruncated() || - filesystemCheckpoints.some(({ truncated: partial }) => partial) || - samplingPartial; - truncated ||= renderer.truncated(); - capture = buildCaptureResult({ - frames, - exit: { ...exit, reason: exitReason }, - samples, - replay, - before, - after, - truncated, + capture = await completeCapture({ scenario, - rootPid: terminal.pid, - samplingPartial, - renderedFrames, + runtime, + runId, + temporaryRoot, + before, + frames, + samples, interactions, - checkpoints: filesystemCheckpoints, - shimEvents: shimReplay.events.map((event) => ({ - ...event, - arguments: event.arguments.map((argument) => - normalizeProcessText( - argument, - scenario, - temporaryRoot, - terminal?.pid ?? -1, - ), - ), - working_directory: normalizeProcessText( - event.working_directory, - scenario, - temporaryRoot, - terminal?.pid ?? -1, - ), - })), - settlement: { ...settlement, cleanup_outcome: "not_required" }, - manifest, + exit, + initiallyTruncated: before.truncated, + ...(signal === undefined ? {} : { signal }), }); } catch (cause: unknown) { - terminal?.kill("SIGKILL"); + runtime?.terminal.kill("SIGKILL"); executionFailure = cause; } - samplingPartial ||= (await stopSampler()).partial; - const cleanupFailure = await releaseProcessResources({ + return finishProcessRun({ + runtime, timers, - replay, - terminal, - renderer, - shimReplay, - checkpoints, runId, temporaryRoot, - sampledProcessGroupIds: samples.flatMap(({ process_group_id }) => - process_group_id === null ? [] : [process_group_id], - ), + samples, + stopSampler, + capture, + executionFailure, }); - if (capture !== undefined) { - capture = { - ...capture, - settlement: { - ...capture.settlement, - cleanup_outcome: - capture.settlement.state === "quiesced" - ? "not_required" - : cleanupFailure === undefined - ? "cleaned" - : "failed", - }, - }; - const issues = validateProcessCapture(capture); - if (issues.length > 0) - executionFailure = new ProcessCaptureError( - `process capture validation failed: ${issues[0]!.path}: ${issues[0]!.message}`, - ); - } - return resolveProcessResult(capture, executionFailure, cleanupFailure); }; /** Execute one scenario through a typed expected-failure channel. */ diff --git a/src/domain/processCapture.ts b/src/domain/processCapture.ts index eb0becbc9..03d451408 100644 --- a/src/domain/processCapture.ts +++ b/src/domain/processCapture.ts @@ -1,448 +1,11 @@ -import { createHash } from "node:crypto"; -import { resolve } from "node:path"; -import canonicalize from "canonicalize"; import { z } from "zod"; -const positiveBudget = z.number().int().positive(); -const timedEventBase = { at_ms: z.number().int().nonnegative() }; -const environmentName = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/u); -const reservedEnvironment = new Set([ - "HOME", - "TERM", - "REA_PROCESS_RUN_ID", - "REA_REPLAY_HTTP_URL", - "REA_REPLAY_WEBSOCKET_URL", - "REA_SHIM_LEDGER_URL", -]); -const normalizationSchema = z.object({ - paths: z.boolean(), - pids: z.boolean(), - ports: z.boolean(), - time_bucket_ms: positiveBudget.max(60_000), - patterns: z.array( - z.object({ - pattern: z.string().max(500), - replacement: z.string().max(100), - }), - ), -}); -const checkpointNameSchema = z - .string() - .regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/u); -const commandNameSchema = z - .string() - .regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u); -const outputChunkSchema = z.object({ - at_ms: z.number().int().nonnegative().max(300_000), - stream: z.enum(["stdout", "stderr"]), - data: z.string().max(1_000_000), -}); +import { + LEGACY_PROCESS_CAPTURE_MESSAGE, + normalizationSchema, +} from "./processScenario.js"; -/** Actionable refusal shared by CLI, MCP, and Evidence import boundaries. */ -export const LEGACY_PROCESS_CAPTURE_MESSAGE = - "Process Capture v3 is unsupported. Re-run the scenario with this REA version using capture_process_scenario to produce Process Capture v4; captures cannot be upgraded because v4 requires new manifest and settlement evidence."; - -const canonicalSha256 = (value: unknown): string => { - const serialized = canonicalize(value); - if (serialized === undefined) - throw new TypeError("Process capture commitment is not canonical JSON"); - return createHash("sha256").update(serialized).digest("hex"); -}; - -/** Build the secret-safe scenario projection committed by Process Capture v4. */ -export const processScenarioCommitment = ( - scenario: ProcessScenario, - executableSha256?: string, -): Readonly> => ({ - ...scenario, - environment: Object.fromEntries( - Object.entries(scenario.environment).map(([name, value]) => [ - name, - scenario.secret_aliases.includes(name) ? "" : value, - ]), - ), - unknown_registry_approved: scenario.unknown_registry_approved === true, - executable_sha256: executableSha256 ?? null, -}); - -/** Project observation settings shared by authority and reconstruction. */ -export const processComparisonContract = ( - scenario: ProcessScenario, -): Readonly> => ({ - working_directory: scenario.working_directory, - inherit_environment: scenario.inherit_environment, - secret_aliases: scenario.secret_aliases, - network_access: scenario.network_access, - filesystem_roots: scenario.filesystem_roots, - terminal: scenario.terminal, - checkpoints: scenario.checkpoints, - events: scenario.events, - timeout_ms: scenario.timeout_ms, - idle_timeout_ms: scenario.idle_timeout_ms, - settle_ms: scenario.settle_ms, - limits: scenario.limits, - normalization: scenario.normalization, - command_shims: scenario.command_shims, - replay: scenario.replay, -}); - -/** Compute a canonical SHA-256 commitment independent of object key order. */ -export const digestProcessCommitment = canonicalSha256; - -/** - * Boundary schema for one explicitly approved, bounded process experiment. - * Defaults are part of the evidence contract and must remain deterministic. - */ -export const processScenarioSchema = z - .object({ - approved: z - .literal(true) - .describe( - "Explicit per-call acknowledgement that this operation launches the target", - ), - unknown_registry_approved: z - .literal(true) - .optional() - .describe("Explicit approval to record capture residuals durably"), - executable: z.string().startsWith("/"), - arguments: z.array(z.string()).max(256).default([]), - working_directory: z.string().startsWith("/"), - environment: z.record(environmentName, z.string()).default({}), - inherit_environment: z.array(environmentName).max(64).default([]), - secret_aliases: z.array(environmentName).max(64).default([]), - network_access: z.literal("host").default("host"), - filesystem_roots: z.array(z.string().startsWith("/")).max(16).default([]), - terminal: z - .object({ - columns: z.number().int().min(1).max(1_000).default(80), - rows: z.number().int().min(1).max(1_000).default(24), - scrollback: z.number().int().min(0).max(10_000).default(1_000), - }) - .default({ columns: 80, rows: 24, scrollback: 1_000 }), - checkpoints: z - .array( - z.object({ - name: checkpointNameSchema, - trigger: z.discriminatedUnion("type", [ - z.object({ - type: z.literal("time"), - at_ms: z.number().int().nonnegative().max(300_000), - }), - z.object({ - type: z.literal("terminal_literal"), - value: z.string().min(1).max(10_000), - occurrence: z.number().int().positive().max(1_000).default(1), - }), - z.object({ type: z.literal("root_exit") }), - z.object({ type: z.literal("settled") }), - ]), - }), - ) - .max(64) - .default([]), - command_shims: z - .array( - z.object({ - name: commandNameSchema, - routes: z - .array( - z.object({ - arguments: z.array(z.string()).max(256), - outputs: z.array(outputChunkSchema).max(1_000).default([]), - termination: z.discriminatedUnion("type", [ - z.object({ - type: z.literal("exit"), - code: z.number().int().min(0).max(255), - }), - z.object({ - type: z.literal("signal"), - signal: z.enum(["SIGINT", "SIGTERM", "SIGKILL"]), - }), - ]), - max_calls: z.number().int().positive().max(100).default(1), - }), - ) - .min(1) - .max(100), - }), - ) - .max(32) - .default([]), - events: z - .array( - z.discriminatedUnion("type", [ - z.object({ - ...timedEventBase, - type: z.literal("input"), - data: z.string(), - sensitive: z.boolean().default(false), - }), - z.object({ - ...timedEventBase, - type: z.literal("resize"), - columns: z.number().int().min(1).max(1_000), - rows: z.number().int().min(1).max(1_000), - }), - z.object({ - ...timedEventBase, - type: z.literal("signal"), - signal: z.enum(["SIGINT", "SIGTERM", "SIGKILL"]), - }), - ]), - ) - .max(1_000) - .default([]), - timeout_ms: positiveBudget.max(300_000).default(30_000), - idle_timeout_ms: positiveBudget.max(300_000).default(30_000), - settle_ms: z.number().int().nonnegative().max(10_000).default(100), - limits: z - .object({ - output_bytes: positiveBudget.max(10_000_000).default(1_000_000), - frames: positiveBudget.max(100_000).default(10_000), - files: positiveBudget.max(100_000).default(10_000), - file_bytes: positiveBudget.max(100_000_000).default(10_000_000), - processes: positiveBudget.max(10_000).default(1_000), - protocol_events: positiveBudget.max(100_000).default(10_000), - protocol_body_bytes: positiveBudget.max(10_000_000).default(1_000_000), - connections: positiveBudget.max(1_000).default(100), - filesystem_depth: z.number().int().min(1).max(64).default(16), - }) - .default({ - output_bytes: 1_000_000, - frames: 10_000, - files: 10_000, - file_bytes: 10_000_000, - processes: 1_000, - protocol_events: 10_000, - protocol_body_bytes: 1_000_000, - connections: 100, - filesystem_depth: 16, - }), - normalization: z - .object({ - paths: z.boolean().default(true), - pids: z.boolean().default(true), - ports: z.boolean().default(true), - time_bucket_ms: positiveBudget.max(60_000).default(10), - patterns: z - .array( - z.object({ - pattern: z.string().max(500), - replacement: z.string().max(100), - }), - ) - .max(32) - .default([]), - }) - .default({ - paths: true, - pids: true, - ports: true, - time_bucket_ms: 10, - patterns: [], - }), - replay: z - .object({ - http: z - .array( - z.object({ - method: z.string().max(16), - path: z.string().startsWith("/"), - status: z.number().int().min(100).max(599), - body: z.string().max(1_000_000), - request_headers: z.record(z.string(), z.string()).default({}), - request_body: z.string().max(1_000_000).optional(), - response_headers: z.record(z.string(), z.string()).default({}), - delay_ms: z.number().int().nonnegative().max(30_000).default(0), - disconnect: z.boolean().default(false), - max_calls: z.number().int().min(1).max(100).default(1), - }), - ) - .max(100) - .default([]), - websocket_messages: z - .array(z.string().max(1_000_000)) - .max(100) - .default([]), - websocket_connections: z - .array( - z.object({ - messages: z - .array( - z.object({ - data: z.string().max(1_000_000), - delay_ms: z - .number() - .int() - .nonnegative() - .max(30_000) - .default(0), - }), - ) - .max(100), - disconnect_after: z.boolean().default(false), - }), - ) - .max(100) - .default([]), - }) - .default({ - http: [], - websocket_messages: [], - websocket_connections: [], - }), - }) - .strict() - .superRefine((scenario, context) => { - for (let index = 1; index < scenario.events.length; index += 1) { - if (scenario.events[index]!.at_ms < scenario.events[index - 1]!.at_ms) { - context.addIssue({ - code: "custom", - message: "events must be ordered by at_ms", - path: ["events", index], - }); - } - } - const secrets = new Set(scenario.secret_aliases); - for (const alias of secrets) { - if (!(alias in scenario.environment)) { - context.addIssue({ - code: "custom", - message: "secret alias has no environment value", - path: ["secret_aliases"], - }); - } - } - const explicit = new Set(Object.keys(scenario.environment)); - for (const name of [...explicit, ...scenario.inherit_environment]) { - if (reservedEnvironment.has(name)) { - context.addIssue({ - code: "custom", - message: `${name} is reserved by the process adapter`, - path: ["environment", name], - }); - } - } - for (const name of scenario.inherit_environment) { - if (explicit.has(name)) { - context.addIssue({ - code: "custom", - message: "an environment name cannot be explicit and inherited", - path: ["inherit_environment"], - }); - } - } - for (const event of scenario.events) { - if (event.at_ms > scenario.timeout_ms) { - context.addIssue({ - code: "custom", - message: "event occurs after the scenario timeout", - path: ["events"], - }); - } - } - const checkpointNames = scenario.checkpoints.map(({ name }) => name); - for (const [index, name] of checkpointNames.entries()) { - if (name === "before" || name === "after_settlement") - context.addIssue({ - code: "custom", - message: "checkpoint name is reserved by the capture lifecycle", - path: ["checkpoints", index, "name"], - }); - } - if (new Set(checkpointNames).size !== checkpointNames.length) - context.addIssue({ - code: "custom", - message: "checkpoint names must be unique", - path: ["checkpoints"], - }); - const shimNames = scenario.command_shims.map(({ name }) => name); - if (new Set(shimNames).size !== shimNames.length) - context.addIssue({ - code: "custom", - message: "command shim names must be unique", - path: ["command_shims"], - }); - }); - -/** Parsed instructions and resource bounds for one process experiment. */ -export type ProcessScenario = z.infer; - -/** Parse untrusted process scenario input and apply safe default budgets. */ -export const parseProcessScenario = (input: unknown): ProcessScenario => - processScenarioSchema.parse(input); - -/** Operator-owned ceiling applied in addition to per-scenario approval. */ -export interface ProcessExecutionPolicy { - readonly enabled: boolean; - readonly executableRoots: readonly string[]; - readonly workingRoots: readonly string[]; - readonly allowedEnvironment: readonly string[]; - readonly allowExternalNetwork: boolean; -} - -/** Explicit authorization result; denial reasons are safe to show callers. */ -export type ProcessPolicyDecision = - | { readonly allowed: true } - | { readonly allowed: false; readonly reason: string }; - -const isWithin = (candidate: string, root: string): boolean => - resolve(candidate) === resolve(root) || - resolve(candidate).startsWith( - `${resolve(root)}${resolve(root).endsWith("/") ? "" : "/"}`, - ); - -/** Evaluate scenario authority before any process or filesystem side effect occurs. */ -export const authorizeProcessScenario = ( - scenario: ProcessScenario, - policy: ProcessExecutionPolicy, -): ProcessPolicyDecision => { - if (!policy.enabled) - return { allowed: false, reason: "process capture is disabled" }; - if (scenario.network_access === "host" && !policy.allowExternalNetwork) - return { - allowed: false, - reason: "host network access is not approved by operator policy", - }; - if ( - !policy.executableRoots.some((root) => isWithin(scenario.executable, root)) - ) { - return { allowed: false, reason: "executable is outside approved roots" }; - } - if ( - !policy.workingRoots.some((root) => - isWithin(scenario.working_directory, root), - ) - ) { - return { - allowed: false, - reason: "working directory is outside approved roots", - }; - } - const requestedNames = [ - ...Object.keys(scenario.environment), - ...scenario.inherit_environment, - ]; - if ( - requestedNames.some((name) => !policy.allowedEnvironment.includes(name)) - ) { - return { - allowed: false, - reason: "scenario requests an environment variable not allowed by policy", - }; - } - if ( - scenario.filesystem_roots.some( - (path) => !policy.workingRoots.some((root) => isWithin(path, root)), - ) - ) { - return { - allowed: false, - reason: "filesystem root is outside approved roots", - }; - } - return { allowed: true }; -}; +export * from "./processScenario.js"; /** Normalized raw PTY chunk, preserving transport-level output differences. */ export interface TerminalFrame { @@ -761,154 +324,9 @@ export const processCaptureSchema: z.ZodType = z.object({ }), }); -/** One pure semantic validation failure in an otherwise shaped v4 capture. */ -export interface ProcessCaptureValidationIssue { - readonly path: string; - readonly message: string; -} - -const orderedBy = ( - values: readonly Value[], - project: (value: Value) => number, -): boolean => - values.every( - (value, index) => - index === 0 || project(value) >= project(values[index - 1]!), - ); - -const orderedTimestamps = ( - values: readonly { readonly at_ms: number }[], -): boolean => orderedBy(values, ({ at_ms }) => at_ms); - -/** Recompute v4 commitments and cross-field invariants without side effects. */ -export const validateProcessCapture = ( - capture: ProcessCapture, -): readonly ProcessCaptureValidationIssue[] => { - const issues: ProcessCaptureValidationIssue[] = []; - const require = (condition: boolean, path: string, message: string): void => { - if (!condition) issues.push({ path, message }); - }; - const { manifest } = capture; - require(manifest.scenario.executable_sha256 === - manifest.executable_sha256, "manifest.executable_sha256", "executable commitment does not match the scenario projection"); - const commitments = [ - ["full_scenario_sha256", manifest.scenario], - ["comparison_contract_sha256", manifest.comparison_contract], - ["normalization_sha256", capture.normalization], - ["shim_plan_sha256", manifest.shim_plan], - ["replay_plan_sha256", manifest.replay_plan], - ] as const; - for (const [field, value] of commitments) - require(manifest[field] === - canonicalSha256( - value, - ), `manifest.${field}`, "commitment does not match its canonical value"); - require(Date.parse(manifest.started_at) <= - Date.parse( - manifest.completed_at, - ), "manifest.completed_at", "completion precedes start"); - const sequences = [ - ["frames", capture.frames], - ["rendered_frames", capture.rendered_frames], - ["interaction_events", capture.interaction_events], - ["shim_events", capture.shim_events], - ["protocol_events", capture.protocol_events], - ] as const; - for (const [name, values] of sequences) { - require(values.every( - ({ sequence }, index) => sequence === index, - ), name, "sequence values must be contiguous from zero"); - } - for (const [name, values] of [ - ["frames", capture.frames], - ["rendered_frames", capture.rendered_frames], - ["process_samples", capture.process_samples], - ["filesystem_checkpoints", capture.filesystem_checkpoints], - ["shim_events", capture.shim_events], - ["protocol_events", capture.protocol_events], - ] as const) - require(orderedTimestamps(values), name, "timestamps must be ordered"); - const checkpointNames = capture.filesystem_checkpoints.map( - ({ name }) => name, - ); - require(new Set(checkpointNames).size === - checkpointNames.length, "filesystem_checkpoints", "checkpoint names must be unique"); - require(checkpointNames[0] === - "before", "filesystem_checkpoints", "first checkpoint must be before"); - require(checkpointNames.at(-1) === - "after_settlement", "filesystem_checkpoints", "last checkpoint must be after_settlement"); - require(!capture.filesystem_checkpoints.some(({ truncated }) => truncated) || - capture.truncated, "truncated", "checkpoint truncation must propagate to the capture"); - require(capture.exit.reason === "exited" || - capture.exit.code === - null, "exit", "deadline termination cannot declare a normal exit code"); - require(capture.settlement.state !== "quiesced" || - capture.settlement.cleanup_outcome === - "not_required", "settlement.cleanup_outcome", "quiesced settlement cannot require cleanup"); - require(capture.settlement.state === "quiesced" || - capture.settlement.cleanup_outcome !== - "not_required", "settlement.cleanup_outcome", "non-quiesced settlement must report cleanup"); - const shimPlans = manifest.shim_plan.flatMap((shim) => - typeof shim === "object" && - shim !== null && - "name" in shim && - "routes" in shim && - typeof shim.name === "string" && - Array.isArray(shim.routes) - ? shim.routes.map((route, routeIndex) => ({ - command: shim.name as string, - route, - routeIndex, - })) - : [], - ); - for (const event of capture.shim_events) { - const declaredRoute = shimPlans.find( - ({ command, routeIndex }) => - command === event.command && routeIndex === event.route_index, - ); - require(event.outcome === "unmatched" - ? event.route_index === null - : declaredRoute !== - undefined, "shim_events", "shim event has no declared route"); - } - for (const plan of shimPlans) { - const maximum = - typeof plan.route === "object" && - plan.route !== null && - "max_calls" in plan.route && - typeof plan.route.max_calls === "number" - ? plan.route.max_calls - : 0; - require(capture.shim_events.filter( - (event) => - event.command === plan.command && - event.route_index === plan.routeIndex && - event.outcome === "matched", - ).length <= - maximum, "shim_events", "matched shim events exceed the declared route max_calls"); - } - const replayHttp = - "http" in manifest.replay_plan && Array.isArray(manifest.replay_plan.http) - ? manifest.replay_plan.http - : []; - for (const event of capture.protocol_events) - if ( - event.protocol === "http" && - event.direction === "request" && - event.outcome === "matched" - ) - require(replayHttp.some( - (route) => - typeof route === "object" && - route !== null && - "method" in route && - route.method === event.method && - "path" in route && - route.path === event.path, - ), "protocol_events", "matched HTTP event has no declared replay route"); - return issues; -}; +export { validateProcessCapture } from "./processCaptureValidation.js"; +export type { ProcessCaptureValidationIssue } from "./processCaptureValidation.js"; +import { validateProcessCapture } from "./processCaptureValidation.js"; /** Parse unknown input as v4 and reject invalid commitments or semantics. */ export const parseProcessCapture = (input: unknown): ProcessCapture => { diff --git a/src/domain/processCaptureValidation.ts b/src/domain/processCaptureValidation.ts new file mode 100644 index 000000000..571617746 --- /dev/null +++ b/src/domain/processCaptureValidation.ts @@ -0,0 +1,190 @@ +import type { ProcessCapture } from "./processCapture.js"; +import { canonicalProcessSha256 } from "./processScenario.js"; + +/** One pure semantic validation failure in an otherwise shaped v4 capture. */ +export interface ProcessCaptureValidationIssue { + readonly path: string; + readonly message: string; +} + +type RequireInvariant = ( + condition: boolean, + path: string, + message: string, +) => void; + +const orderedTimestamps = ( + values: readonly { readonly at_ms: number }[], +): boolean => + values.every( + (value, index) => index === 0 || value.at_ms >= values[index - 1]!.at_ms, + ); + +const validateCommitments = ( + capture: ProcessCapture, + require: RequireInvariant, +): void => { + const { manifest } = capture; + require(manifest.scenario.executable_sha256 === + manifest.executable_sha256, "manifest.executable_sha256", "executable commitment does not match the scenario projection"); + for (const [field, value] of [ + ["full_scenario_sha256", manifest.scenario], + ["comparison_contract_sha256", manifest.comparison_contract], + ["normalization_sha256", capture.normalization], + ["shim_plan_sha256", manifest.shim_plan], + ["replay_plan_sha256", manifest.replay_plan], + ] as const) + require(manifest[field] === + canonicalProcessSha256( + value, + ), `manifest.${field}`, "commitment does not match its canonical value"); + require(Date.parse(manifest.started_at) <= + Date.parse( + manifest.completed_at, + ), "manifest.completed_at", "completion precedes start"); +}; + +const validateOrdering = ( + capture: ProcessCapture, + require: RequireInvariant, +): void => { + for (const [name, values] of [ + ["frames", capture.frames], + ["rendered_frames", capture.rendered_frames], + ["interaction_events", capture.interaction_events], + ["shim_events", capture.shim_events], + ["protocol_events", capture.protocol_events], + ] as const) + require(values.every( + ({ sequence }, index) => sequence === index, + ), name, "sequence values must be contiguous from zero"); + for (const [name, values] of [ + ["frames", capture.frames], + ["rendered_frames", capture.rendered_frames], + ["process_samples", capture.process_samples], + ["filesystem_checkpoints", capture.filesystem_checkpoints], + ["shim_events", capture.shim_events], + ["protocol_events", capture.protocol_events], + ] as const) + require(orderedTimestamps(values), name, "timestamps must be ordered"); +}; + +const validateLifecycle = ( + capture: ProcessCapture, + require: RequireInvariant, +): void => { + const checkpointNames = capture.filesystem_checkpoints.map( + ({ name }) => name, + ); + require(new Set(checkpointNames).size === + checkpointNames.length, "filesystem_checkpoints", "checkpoint names must be unique"); + require(checkpointNames[0] === + "before", "filesystem_checkpoints", "first checkpoint must be before"); + require(checkpointNames.at(-1) === + "after_settlement", "filesystem_checkpoints", "last checkpoint must be after_settlement"); + require(!capture.filesystem_checkpoints.some(({ truncated }) => truncated) || + capture.truncated, "truncated", "checkpoint truncation must propagate to the capture"); + require(capture.exit.reason === "exited" || + capture.exit.code === + null, "exit", "deadline termination cannot declare a normal exit code"); + require(capture.settlement.state !== "quiesced" || + capture.settlement.cleanup_outcome === + "not_required", "settlement.cleanup_outcome", "quiesced settlement cannot require cleanup"); + require(capture.settlement.state === "quiesced" || + capture.settlement.cleanup_outcome !== + "not_required", "settlement.cleanup_outcome", "non-quiesced settlement must report cleanup"); +}; + +const shimPlans = (capture: ProcessCapture) => + capture.manifest.shim_plan.flatMap((shim) => + typeof shim === "object" && + shim !== null && + "name" in shim && + "routes" in shim && + typeof shim.name === "string" && + Array.isArray(shim.routes) + ? shim.routes.map((route, routeIndex) => ({ + command: shim.name as string, + route, + routeIndex, + })) + : [], + ); + +const validateShimEvents = ( + capture: ProcessCapture, + require: RequireInvariant, +): void => { + const plans = shimPlans(capture); + for (const event of capture.shim_events) { + const declared = plans.some( + ({ command, routeIndex }) => + command === event.command && routeIndex === event.route_index, + ); + require(event.outcome === "unmatched" + ? event.route_index === null + : declared, "shim_events", "shim event has no declared route"); + } + for (const plan of plans) { + const maximum = + typeof plan.route === "object" && + plan.route !== null && + "max_calls" in plan.route && + typeof plan.route.max_calls === "number" + ? plan.route.max_calls + : 0; + const matches = capture.shim_events.filter( + (event) => + event.command === plan.command && + event.route_index === plan.routeIndex && + event.outcome === "matched", + ).length; + require(matches <= + maximum, "shim_events", "matched shim events exceed the declared route max_calls"); + } +}; + +const validateReplayEvents = ( + capture: ProcessCapture, + require: RequireInvariant, +): void => { + const { replay_plan: replayPlan } = capture.manifest; + const routes = + "http" in replayPlan && Array.isArray(replayPlan.http) + ? replayPlan.http + : []; + for (const event of capture.protocol_events) { + if ( + event.protocol !== "http" || + event.direction !== "request" || + event.outcome !== "matched" + ) + continue; + const declared = routes.some( + (route) => + typeof route === "object" && + route !== null && + "method" in route && + route.method === event.method && + "path" in route && + route.path === event.path, + ); + require(declared, "protocol_events", "matched HTTP event has no declared replay route"); + } +}; + +/** Recompute v4 commitments and cross-field invariants without side effects. */ +export const validateProcessCapture = ( + capture: ProcessCapture, +): readonly ProcessCaptureValidationIssue[] => { + const issues: ProcessCaptureValidationIssue[] = []; + const require: RequireInvariant = (condition, path, message) => { + if (!condition) issues.push({ path, message }); + }; + validateCommitments(capture, require); + validateOrdering(capture, require); + validateLifecycle(capture, require); + validateShimEvents(capture, require); + validateReplayEvents(capture, require); + return issues; +}; diff --git a/src/domain/processComparison.ts b/src/domain/processComparison.ts index 9f25e11a9..0c9b3e105 100644 --- a/src/domain/processComparison.ts +++ b/src/domain/processComparison.ts @@ -197,6 +197,108 @@ const chooseFirstDivergence = ( ); }; +const assertComparable = ( + left: ProcessCapture, + right: ProcessCapture, + options: { readonly maxCaptureAgeMs?: number; readonly now?: () => number }, +): void => { + const invalid = [ + ...validateProcessCapture(left), + ...validateProcessCapture(right), + ]; + if (invalid.length > 0) + throw new TypeError(`Invalid Process Capture v4: ${invalid[0]!.path}`); + if ( + left.schema_version !== right.schema_version || + left.manifest.comparison_contract_sha256 !== + right.manifest.comparison_contract_sha256 + ) + throw new TypeError( + "Process captures have incompatible comparison contracts", + ); + if (options.maxCaptureAgeMs === undefined) return; + const maxCaptureAgeMs = options.maxCaptureAgeMs; + const now = (options.now ?? Date.now)(); + if ( + [left, right].some( + ({ manifest }) => + now - Date.parse(manifest.completed_at) > maxCaptureAgeMs, + ) + ) + throw new TypeError("Process capture exceeds max_capture_age_ms"); +}; + +const truncatedComparison = (): ProcessCaptureComparison => ({ + status: "truncated", + terminal: "truncated", + interaction: "truncated", + exit: "truncated", + filesystem: "truncated", + protocol: "truncated", + process: "truncated", + shim: "truncated", + first_divergence: { + status: "unknown", + reason: "At least one capture is truncated.", + }, + limitations: ["At least one capture is truncated."], +}); + +type ComparisonDimensions = Pick< + ProcessCaptureComparison, + | "terminal" + | "interaction" + | "exit" + | "filesystem" + | "protocol" + | "process" + | "shim" +>; + +const compareDimensions = ( + left: ProcessCapture, + right: ProcessCapture, + sameNormalization: boolean, +): ComparisonDimensions => { + const classify = ( + scope: ProcessCapture["residual_unknowns"][number]["scope"], + leftValues: readonly unknown[], + rightValues: readonly unknown[], + ): ComparisonStatus => + !sameNormalization || hasUnknown(left, scope) || hasUnknown(right, scope) + ? "unknown" + : classifyCollection(leftValues, rightValues); + return { + terminal: classify( + "terminal", + terminalObservations(left), + terminalObservations(right), + ), + interaction: classify( + "terminal", + left.interaction_events, + right.interaction_events, + ), + exit: + hasUnknown(left, "exit") || hasUnknown(right, "exit") + ? "unknown" + : JSON.stringify({ exit: left.exit, settlement: left.settlement }) === + JSON.stringify({ exit: right.exit, settlement: right.settlement }) + ? "unchanged" + : "changed", + filesystem: classify( + "filesystem", + filesystemObservations(left), + filesystemObservations(right), + ), + protocol: classify("protocol", left.protocol_events, right.protocol_events), + process: classify("process", left.process_samples, right.process_samples), + shim: sameNormalization + ? classifyCollection(left.shim_events, right.shim_events) + : "unknown", + }; +}; + /** Compare bounded captures without equating missing or incompatible evidence. */ /** * Compare two Process Capture v4 observations without treating absence as proof. @@ -212,93 +314,11 @@ export const compareProcessCaptures = ( readonly now?: () => number; } = {}, ): ProcessCaptureComparison => { - const invalid = [ - ...validateProcessCapture(left), - ...validateProcessCapture(right), - ]; - if (invalid.length > 0) - throw new TypeError(`Invalid Process Capture v4: ${invalid[0]!.path}`); - if ( - left.schema_version !== right.schema_version || - left.manifest.comparison_contract_sha256 !== - right.manifest.comparison_contract_sha256 - ) - throw new TypeError( - "Process captures have incompatible comparison contracts", - ); - if (options.maxCaptureAgeMs !== undefined) { - const maxCaptureAgeMs = options.maxCaptureAgeMs; - const now = (options.now ?? Date.now)(); - if ( - [left, right].some( - ({ manifest }) => - now - Date.parse(manifest.completed_at) > maxCaptureAgeMs, - ) - ) - throw new TypeError("Process capture exceeds max_capture_age_ms"); - } - if (left.truncated || right.truncated) { - return { - status: "truncated", - terminal: "truncated", - interaction: "truncated", - exit: "truncated", - filesystem: "truncated", - protocol: "truncated", - process: "truncated", - shim: "truncated", - first_divergence: { - status: "unknown", - reason: "At least one capture is truncated.", - }, - limitations: ["At least one capture is truncated."], - }; - } + assertComparable(left, right, options); + if (left.truncated || right.truncated) return truncatedComparison(); const sameNormalization = JSON.stringify(left.normalization) === JSON.stringify(right.normalization); - const classify = ( - scope: ProcessCapture["residual_unknowns"][number]["scope"], - leftValues: readonly unknown[], - rightValues: readonly unknown[], - ): ComparisonStatus => - !sameNormalization || hasUnknown(left, scope) || hasUnknown(right, scope) - ? "unknown" - : classifyCollection(leftValues, rightValues); - const terminal = classify( - "terminal", - terminalObservations(left), - terminalObservations(right), - ); - const interaction = classify( - "terminal", - left.interaction_events, - right.interaction_events, - ); - const exit = - hasUnknown(left, "exit") || hasUnknown(right, "exit") - ? "unknown" - : JSON.stringify({ exit: left.exit, settlement: left.settlement }) === - JSON.stringify({ exit: right.exit, settlement: right.settlement }) - ? "unchanged" - : "changed"; - const filesystem = classify( - "filesystem", - filesystemObservations(left), - filesystemObservations(right), - ); - const protocol = classify( - "protocol", - left.protocol_events, - right.protocol_events, - ); - const process = classify( - "process", - left.process_samples, - right.process_samples, - ); - const shim = sameNormalization - ? classifyCollection(left.shim_events, right.shim_events) - : "unknown"; + const dimensions = compareDimensions(left, right, sameNormalization); const observedFirstDivergence = chooseFirstDivergence([ firstCollectionDivergence( "terminal", @@ -346,21 +366,15 @@ export const compareProcessCaptures = ( : observedFirstDivergence; return { status: comparisonStatus([ - terminal, - interaction, - exit, - filesystem, - protocol, - process, - shim, + dimensions.terminal, + dimensions.interaction, + dimensions.exit, + dimensions.filesystem, + dimensions.protocol, + dimensions.process, + dimensions.shim, ]), - terminal, - interaction, - exit, - filesystem, - protocol, - process, - shim, + ...dimensions, first_divergence: firstDivergence, limitations: [ ...left.limitations, diff --git a/src/domain/processScenario.ts b/src/domain/processScenario.ts new file mode 100644 index 000000000..5d7d5cea1 --- /dev/null +++ b/src/domain/processScenario.ts @@ -0,0 +1,445 @@ +import { createHash } from "node:crypto"; +import { resolve } from "node:path"; +import canonicalize from "canonicalize"; +import { z } from "zod"; + +const positiveBudget = z.number().int().positive(); +const timedEventBase = { at_ms: z.number().int().nonnegative() }; +const environmentName = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/u); +const reservedEnvironment = new Set([ + "HOME", + "TERM", + "REA_PROCESS_RUN_ID", + "REA_REPLAY_HTTP_URL", + "REA_REPLAY_WEBSOCKET_URL", + "REA_SHIM_LEDGER_URL", +]); +export const normalizationSchema = z.object({ + paths: z.boolean(), + pids: z.boolean(), + ports: z.boolean(), + time_bucket_ms: positiveBudget.max(60_000), + patterns: z.array( + z.object({ + pattern: z.string().max(500), + replacement: z.string().max(100), + }), + ), +}); +const checkpointNameSchema = z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/u); +const commandNameSchema = z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u); +const outputChunkSchema = z.object({ + at_ms: z.number().int().nonnegative().max(300_000), + stream: z.enum(["stdout", "stderr"]), + data: z.string().max(1_000_000), +}); + +/** Actionable refusal shared by CLI, MCP, and Evidence import boundaries. */ +export const LEGACY_PROCESS_CAPTURE_MESSAGE = + "Process Capture v3 is unsupported. Re-run the scenario with this REA version using capture_process_scenario to produce Process Capture v4; captures cannot be upgraded because v4 requires new manifest and settlement evidence."; + +export const canonicalProcessSha256 = (value: unknown): string => { + const serialized = canonicalize(value); + if (serialized === undefined) + throw new TypeError("Process capture commitment is not canonical JSON"); + return createHash("sha256").update(serialized).digest("hex"); +}; + +/** Build the secret-safe scenario projection committed by Process Capture v4. */ +export const processScenarioCommitment = ( + scenario: ProcessScenario, + executableSha256?: string, +): Readonly> => ({ + ...scenario, + environment: Object.fromEntries( + Object.entries(scenario.environment).map(([name, value]) => [ + name, + scenario.secret_aliases.includes(name) ? "" : value, + ]), + ), + unknown_registry_approved: scenario.unknown_registry_approved === true, + executable_sha256: executableSha256 ?? null, +}); + +/** Project observation settings shared by authority and reconstruction. */ +export const processComparisonContract = ( + scenario: ProcessScenario, +): Readonly> => ({ + working_directory: scenario.working_directory, + inherit_environment: scenario.inherit_environment, + secret_aliases: scenario.secret_aliases, + network_access: scenario.network_access, + filesystem_roots: scenario.filesystem_roots, + terminal: scenario.terminal, + checkpoints: scenario.checkpoints, + events: scenario.events, + timeout_ms: scenario.timeout_ms, + idle_timeout_ms: scenario.idle_timeout_ms, + settle_ms: scenario.settle_ms, + limits: scenario.limits, + normalization: scenario.normalization, + command_shims: scenario.command_shims, + replay: scenario.replay, +}); + +/** Compute a canonical SHA-256 commitment independent of object key order. */ +export const digestProcessCommitment = canonicalProcessSha256; + +/** + * Boundary schema for one explicitly approved, bounded process experiment. + * Defaults are part of the evidence contract and must remain deterministic. + */ +export const processScenarioSchema = z + .object({ + approved: z + .literal(true) + .describe( + "Explicit per-call acknowledgement that this operation launches the target", + ), + unknown_registry_approved: z + .literal(true) + .optional() + .describe("Explicit approval to record capture residuals durably"), + executable: z.string().startsWith("/"), + arguments: z.array(z.string()).max(256).default([]), + working_directory: z.string().startsWith("/"), + environment: z.record(environmentName, z.string()).default({}), + inherit_environment: z.array(environmentName).max(64).default([]), + secret_aliases: z.array(environmentName).max(64).default([]), + network_access: z.literal("host").default("host"), + filesystem_roots: z.array(z.string().startsWith("/")).max(16).default([]), + terminal: z + .object({ + columns: z.number().int().min(1).max(1_000).default(80), + rows: z.number().int().min(1).max(1_000).default(24), + scrollback: z.number().int().min(0).max(10_000).default(1_000), + }) + .default({ columns: 80, rows: 24, scrollback: 1_000 }), + checkpoints: z + .array( + z.object({ + name: checkpointNameSchema, + trigger: z.discriminatedUnion("type", [ + z.object({ + type: z.literal("time"), + at_ms: z.number().int().nonnegative().max(300_000), + }), + z.object({ + type: z.literal("terminal_literal"), + value: z.string().min(1).max(10_000), + occurrence: z.number().int().positive().max(1_000).default(1), + }), + z.object({ type: z.literal("root_exit") }), + z.object({ type: z.literal("settled") }), + ]), + }), + ) + .max(64) + .default([]), + command_shims: z + .array( + z.object({ + name: commandNameSchema, + routes: z + .array( + z.object({ + arguments: z.array(z.string()).max(256), + outputs: z.array(outputChunkSchema).max(1_000).default([]), + termination: z.discriminatedUnion("type", [ + z.object({ + type: z.literal("exit"), + code: z.number().int().min(0).max(255), + }), + z.object({ + type: z.literal("signal"), + signal: z.enum(["SIGINT", "SIGTERM", "SIGKILL"]), + }), + ]), + max_calls: z.number().int().positive().max(100).default(1), + }), + ) + .min(1) + .max(100), + }), + ) + .max(32) + .default([]), + events: z + .array( + z.discriminatedUnion("type", [ + z.object({ + ...timedEventBase, + type: z.literal("input"), + data: z.string(), + sensitive: z.boolean().default(false), + }), + z.object({ + ...timedEventBase, + type: z.literal("resize"), + columns: z.number().int().min(1).max(1_000), + rows: z.number().int().min(1).max(1_000), + }), + z.object({ + ...timedEventBase, + type: z.literal("signal"), + signal: z.enum(["SIGINT", "SIGTERM", "SIGKILL"]), + }), + ]), + ) + .max(1_000) + .default([]), + timeout_ms: positiveBudget.max(300_000).default(30_000), + idle_timeout_ms: positiveBudget.max(300_000).default(30_000), + settle_ms: z.number().int().nonnegative().max(10_000).default(100), + limits: z + .object({ + output_bytes: positiveBudget.max(10_000_000).default(1_000_000), + frames: positiveBudget.max(100_000).default(10_000), + files: positiveBudget.max(100_000).default(10_000), + file_bytes: positiveBudget.max(100_000_000).default(10_000_000), + processes: positiveBudget.max(10_000).default(1_000), + protocol_events: positiveBudget.max(100_000).default(10_000), + protocol_body_bytes: positiveBudget.max(10_000_000).default(1_000_000), + connections: positiveBudget.max(1_000).default(100), + filesystem_depth: z.number().int().min(1).max(64).default(16), + }) + .default({ + output_bytes: 1_000_000, + frames: 10_000, + files: 10_000, + file_bytes: 10_000_000, + processes: 1_000, + protocol_events: 10_000, + protocol_body_bytes: 1_000_000, + connections: 100, + filesystem_depth: 16, + }), + normalization: z + .object({ + paths: z.boolean().default(true), + pids: z.boolean().default(true), + ports: z.boolean().default(true), + time_bucket_ms: positiveBudget.max(60_000).default(10), + patterns: z + .array( + z.object({ + pattern: z.string().max(500), + replacement: z.string().max(100), + }), + ) + .max(32) + .default([]), + }) + .default({ + paths: true, + pids: true, + ports: true, + time_bucket_ms: 10, + patterns: [], + }), + replay: z + .object({ + http: z + .array( + z.object({ + method: z.string().max(16), + path: z.string().startsWith("/"), + status: z.number().int().min(100).max(599), + body: z.string().max(1_000_000), + request_headers: z.record(z.string(), z.string()).default({}), + request_body: z.string().max(1_000_000).optional(), + response_headers: z.record(z.string(), z.string()).default({}), + delay_ms: z.number().int().nonnegative().max(30_000).default(0), + disconnect: z.boolean().default(false), + max_calls: z.number().int().min(1).max(100).default(1), + }), + ) + .max(100) + .default([]), + websocket_messages: z + .array(z.string().max(1_000_000)) + .max(100) + .default([]), + websocket_connections: z + .array( + z.object({ + messages: z + .array( + z.object({ + data: z.string().max(1_000_000), + delay_ms: z + .number() + .int() + .nonnegative() + .max(30_000) + .default(0), + }), + ) + .max(100), + disconnect_after: z.boolean().default(false), + }), + ) + .max(100) + .default([]), + }) + .default({ + http: [], + websocket_messages: [], + websocket_connections: [], + }), + }) + .strict() + .superRefine((scenario, context) => { + for (let index = 1; index < scenario.events.length; index += 1) { + if (scenario.events[index]!.at_ms < scenario.events[index - 1]!.at_ms) { + context.addIssue({ + code: "custom", + message: "events must be ordered by at_ms", + path: ["events", index], + }); + } + } + const secrets = new Set(scenario.secret_aliases); + for (const alias of secrets) { + if (!(alias in scenario.environment)) { + context.addIssue({ + code: "custom", + message: "secret alias has no environment value", + path: ["secret_aliases"], + }); + } + } + const explicit = new Set(Object.keys(scenario.environment)); + for (const name of [...explicit, ...scenario.inherit_environment]) { + if (reservedEnvironment.has(name)) { + context.addIssue({ + code: "custom", + message: `${name} is reserved by the process adapter`, + path: ["environment", name], + }); + } + } + for (const name of scenario.inherit_environment) { + if (explicit.has(name)) { + context.addIssue({ + code: "custom", + message: "an environment name cannot be explicit and inherited", + path: ["inherit_environment"], + }); + } + } + for (const event of scenario.events) { + if (event.at_ms > scenario.timeout_ms) { + context.addIssue({ + code: "custom", + message: "event occurs after the scenario timeout", + path: ["events"], + }); + } + } + const checkpointNames = scenario.checkpoints.map(({ name }) => name); + for (const [index, name] of checkpointNames.entries()) { + if (name === "before" || name === "after_settlement") + context.addIssue({ + code: "custom", + message: "checkpoint name is reserved by the capture lifecycle", + path: ["checkpoints", index, "name"], + }); + } + if (new Set(checkpointNames).size !== checkpointNames.length) + context.addIssue({ + code: "custom", + message: "checkpoint names must be unique", + path: ["checkpoints"], + }); + const shimNames = scenario.command_shims.map(({ name }) => name); + if (new Set(shimNames).size !== shimNames.length) + context.addIssue({ + code: "custom", + message: "command shim names must be unique", + path: ["command_shims"], + }); + }); + +/** Parsed instructions and resource bounds for one process experiment. */ +export type ProcessScenario = z.infer; + +/** Parse untrusted process scenario input and apply safe default budgets. */ +export const parseProcessScenario = (input: unknown): ProcessScenario => + processScenarioSchema.parse(input); + +/** Operator-owned ceiling applied in addition to per-scenario approval. */ +export interface ProcessExecutionPolicy { + readonly enabled: boolean; + readonly executableRoots: readonly string[]; + readonly workingRoots: readonly string[]; + readonly allowedEnvironment: readonly string[]; + readonly allowExternalNetwork: boolean; +} + +/** Explicit authorization result; denial reasons are safe to show callers. */ +export type ProcessPolicyDecision = + | { readonly allowed: true } + | { readonly allowed: false; readonly reason: string }; + +const isWithin = (candidate: string, root: string): boolean => + resolve(candidate) === resolve(root) || + resolve(candidate).startsWith( + `${resolve(root)}${resolve(root).endsWith("/") ? "" : "/"}`, + ); + +/** Evaluate scenario authority before any process or filesystem side effect occurs. */ +export const authorizeProcessScenario = ( + scenario: ProcessScenario, + policy: ProcessExecutionPolicy, +): ProcessPolicyDecision => { + if (!policy.enabled) + return { allowed: false, reason: "process capture is disabled" }; + if (scenario.network_access === "host" && !policy.allowExternalNetwork) + return { + allowed: false, + reason: "host network access is not approved by operator policy", + }; + if ( + !policy.executableRoots.some((root) => isWithin(scenario.executable, root)) + ) { + return { allowed: false, reason: "executable is outside approved roots" }; + } + if ( + !policy.workingRoots.some((root) => + isWithin(scenario.working_directory, root), + ) + ) { + return { + allowed: false, + reason: "working directory is outside approved roots", + }; + } + const requestedNames = [ + ...Object.keys(scenario.environment), + ...scenario.inherit_environment, + ]; + if ( + requestedNames.some((name) => !policy.allowedEnvironment.includes(name)) + ) { + return { + allowed: false, + reason: "scenario requests an environment variable not allowed by policy", + }; + } + if ( + scenario.filesystem_roots.some( + (path) => !policy.workingRoots.some((root) => isWithin(path, root)), + ) + ) { + return { + allowed: false, + reason: "filesystem root is outside approved roots", + }; + } + return { allowed: true }; +}; diff --git a/src/server/registerProcessComparisonTool.ts b/src/server/registerProcessComparisonTool.ts index 49c8c2cc9..708859e51 100644 --- a/src/server/registerProcessComparisonTool.ts +++ b/src/server/registerProcessComparisonTool.ts @@ -11,7 +11,7 @@ import type { RecordUnknownInput } from "../domain/residualUnknown.js"; import { compareProcessCaptures, processCaptureSchema, - validateProcessCapture, + parseProcessCapture, } from "../domain/processCapture.js"; import { recordDerivedEvidence } from "./recordDerivedEvidence.js"; import { err } from "../domain/result.js"; @@ -103,7 +103,12 @@ const validateCaptureSources = ( [parsed.right_evidence_id, parsed.right], ] as const) { const evidence = session.evidenceById(evidenceId); - const result = processCaptureSchema.safeParse(evidence?.normalized_result); + let evidenceCapture: z.infer | undefined; + try { + evidenceCapture = parseProcessCapture(evidence?.normalized_result); + } catch { + evidenceCapture = undefined; + } if ( evidence === undefined || evidence.operation !== "capture_process_scenario" || @@ -113,10 +118,8 @@ const validateCaptureSources = ( evidence.provider.version !== PROCESS_PROVIDER.version || evidence.confidence !== "observed" || evidence.authority !== "controlled-replay" || - !result.success || - (result.success && validateProcessCapture(result.data).length > 0) || - validateProcessCapture(capture).length > 0 || - canonicalJson(result.data) !== canonicalJson(capture) + evidenceCapture === undefined || + canonicalJson(evidenceCapture) !== canonicalJson(capture) ) return err( new EvidenceIntegrityError( From c8d762e6ffe0a2e23fb539590d7b1850ef1ce2a7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:20:30 +0800 Subject: [PATCH 5/8] refactor(artifacts): simplify inventory traversal --- src/application/ArtifactInventory.ts | 155 +++++++++++++++------------ src/artifacts/ArtifactProvider.ts | 8 +- 2 files changed, 89 insertions(+), 74 deletions(-) diff --git a/src/application/ArtifactInventory.ts b/src/application/ArtifactInventory.ts index 32a66ad9f..ed5b60be3 100644 --- a/src/application/ArtifactInventory.ts +++ b/src/application/ArtifactInventory.ts @@ -79,14 +79,16 @@ export const inventoryArtifact = async ( inputPath: string, limits: ArtifactLimits, page: InventoryPageInput, - signal?: AbortSignal, - nativeMount: ArtifactNativeMountPolicy = NATIVE_MOUNT_DISABLED, + options: { + readonly signal?: AbortSignal; + readonly nativeMount?: ArtifactNativeMountPolicy; + } = {}, ): Promise => { const snapshot = await scanArtifactInventory( inputPath, limits, - signal, - nativeMount, + options.signal, + options.nativeMount ?? NATIVE_MOUNT_DISABLED, ); return paginateArtifactInventory(snapshot, page); }; @@ -250,6 +252,29 @@ const createReader = async ( } }; +const visitNestedAsar = async ( + adapterKey: string, + logicalPath: string, + visit: (reader: ArtifactReader, prefix: string) => Promise, +): Promise => { + const nested = new AsarArtifactReader(adapterKey); + try { + await visit(nested, logicalPath); + } finally { + await nested.close(); + } +}; + +const isExpandableAsar = (entry: ArtifactEntry, logicalPath: string): boolean => + entry.kind === "file" && + logicalPath.toLowerCase().endsWith(".asar") && + entry.adapterKey.startsWith("/"); + +const emptyScan = (): { + readonly nodes: Map; + readonly occurrences: MutableOccurrence[]; +} => ({ nodes: new Map(), occurrences: [] }); + const scanReader = async ( reader: ArtifactReader | undefined, limits: ArtifactLimits, @@ -258,12 +283,60 @@ const scanReader = async ( readonly nodes: Map; readonly occurrences: MutableOccurrence[]; }> => { - const nodes = new Map(); - const occurrences: MutableOccurrence[] = []; + const { nodes, occurrences } = emptyScan(); if (reader === undefined) return { nodes, occurrences }; const occurrenceByPath = new Map(); const registry = new ArtifactPathRegistry(); let totalBytes = 0; + const digestEntry = async ( + currentReader: ArtifactReader, + entry: ArtifactEntry, + logicalPath: string, + ): Promise => { + if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted) + return undefined; + const remainingBytes = limits.maxTotalBytes - totalBytes; + if (remainingBytes <= 0) + throw new ArtifactReaderFailure( + "limit", + "Artifact total byte limit exceeded", + ); + if (entry.declaredSize !== null && entry.declaredSize > remainingBytes) + throw new ArtifactReaderFailure( + "limit", + "Declared artifact bytes exceed remaining cumulative limit", + ); + const digest = await hashReadable( + await currentReader.open(entry, signal), + Math.min(limits.maxEntryBytes, remainingBytes), + signal, + ); + totalBytes += digest.bytes; + if (entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256) + throw new ArtifactReaderFailure( + "integrity", + `Artifact integrity metadata disagrees with content: ${logicalPath}`, + undefined, + { + logicalPath, + declaredSha256: entry.declaredSha256, + calculatedSha256: digest.sha256, + unpacked: entry.unpacked, + }, + ); + const classified = + entry.kind === "slice" + ? ({ kind: "universal-slice", format: "mach-o" } as const) + : classifyArtifactContent(logicalPath, digest.prefix); + return createArtifactNode({ + sha256: digest.sha256, + size: digest.bytes, + kind: classified.kind, + format: classified.format, + executable: entry.executable, + contentState: "embedded", + }); + }; const visit = async ( currentReader: ArtifactReader, prefix: string, @@ -278,10 +351,7 @@ const scanReader = async ( prefix.length === 0 ? entry.path : `${prefix}/${entry.path}`, limits, ); - const expandableAsar = - entry.kind === "file" && - logicalPath.toLowerCase().endsWith(".asar") && - entry.adapterKey.startsWith("/"); + const expandableAsar = isExpandableAsar(entry, logicalPath); registry.add(logicalPath, expandableAsar ? "directory" : entry.kind); preflightEntry(entry, limits); const parent = nearestParent(logicalPath, occurrenceByPath); @@ -290,73 +360,16 @@ const scanReader = async ( logicalPath, parent?.occurrence_id ?? null, ); - if ( - (entry.kind === "file" || entry.kind === "slice") && - !entry.encrypted - ) { - const remainingBytes = limits.maxTotalBytes - totalBytes; - if (remainingBytes <= 0) - throw new ArtifactReaderFailure( - "limit", - "Artifact total byte limit exceeded", - ); - if (entry.declaredSize !== null && entry.declaredSize > remainingBytes) - throw new ArtifactReaderFailure( - "limit", - "Declared artifact bytes exceed remaining cumulative limit", - ); - const digest = await hashReadable( - await currentReader.open(entry, signal), - Math.min(limits.maxEntryBytes, remainingBytes), - signal, - ); - totalBytes += digest.bytes; - if ( - entry.declaredSha256 !== null && - entry.declaredSha256 !== digest.sha256 - ) - throw new ArtifactReaderFailure( - "integrity", - `Artifact integrity metadata disagrees with content: ${logicalPath}`, - undefined, - { - logicalPath, - declaredSha256: entry.declaredSha256, - calculatedSha256: digest.sha256, - unpacked: entry.unpacked, - }, - ); - if (totalBytes > limits.maxTotalBytes) - throw new ArtifactReaderFailure( - "limit", - "Artifact total byte limit exceeded", - ); - const classified = - entry.kind === "slice" - ? ({ kind: "universal-slice", format: "mach-o" } as const) - : classifyArtifactContent(logicalPath, digest.prefix); - const node = createArtifactNode({ - sha256: digest.sha256, - size: digest.bytes, - kind: classified.kind, - format: classified.format, - executable: entry.executable, - contentState: "embedded", - }); + const node = await digestEntry(currentReader, entry, logicalPath); + if (node !== undefined) { nodes.set(node.artifact_id, node); occurrence.artifact_id = node.artifact_id; occurrence.hash_status = "verified"; } occurrences.push(occurrence); occurrenceByPath.set(logicalPath, occurrence); - if (expandableAsar) { - const nested = new AsarArtifactReader(entry.adapterKey); - try { - await visit(nested, logicalPath); - } finally { - await nested.close(); - } - } + if (expandableAsar) + await visitNestedAsar(entry.adapterKey, logicalPath, visit); } }; await visit(reader, ""); diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index c9d7cc76b..e07d93c7d 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -148,10 +148,12 @@ class ArtifactClient implements AnalysisClient { edgeOffset: parsed.edge_offset, edgeLimit: parsed.edge_limit, }, - options?.signal, { - nativeMountApproved: parsed.native_mount_approved === true, - nativeMountEnabled: this.nativeMountEnabled, + ...(options?.signal === undefined ? {} : { signal: options.signal }), + nativeMount: { + nativeMountApproved: parsed.native_mount_approved === true, + nativeMountEnabled: this.nativeMountEnabled, + }, }, ); return ok( From 07e1b5ac5f6a1724b1064c47ea05f04ccffcb4f3 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:20:40 +0800 Subject: [PATCH 6/8] refactor(setup): split setup and CLI registration --- src/application/Setup.ts | 167 +++++++++++++++------------------- src/application/SetupSkill.ts | 55 +++++++++++ src/cli.ts | 103 +++++++++++---------- 3 files changed, 181 insertions(+), 144 deletions(-) create mode 100644 src/application/SetupSkill.ts diff --git a/src/application/Setup.ts b/src/application/Setup.ts index e60052b7b..f6cf299d9 100644 --- a/src/application/Setup.ts +++ b/src/application/Setup.ts @@ -22,6 +22,9 @@ import { type LinuxDistribution, } from "./LinuxHopper.js"; import { installMacHopper } from "./MacHopper.js"; +import { installCanonicalSkill } from "./SetupSkill.js"; + +export { installCanonicalSkill } from "./SetupSkill.js"; const registrationCommand = (): readonly string[] => process.env.npm_command === "exec" @@ -100,6 +103,65 @@ export type SetupConfirmation = ( actions: readonly SetupAction[], ) => Promise; +const setupPlan = ( + platform: NodeJS.Platform, + hopperPath: string | undefined, + clients: readonly SetupClient[], +): readonly SetupAction[] => [ + ...(hopperPath === undefined + ? [ + { + kind: "install_hopper" as const, + target: + platform === "darwin" + ? "~/Applications/Hopper Disassembler.app" + : "system package manager", + detail: + "Download the official Hopper package, verify it, install it, and open Hopper for activation.", + external: true, + }, + ] + : []), + ...clients + .filter(({ format }) => format !== "unsupported") + .map( + (client): SetupAction => ({ + kind: "configure_client", + target: client.configPath, + detail: `Add the REA MCP registration for ${client.name}; preserve unrelated configuration.`, + external: false, + }), + ), + { + kind: "install_skill", + target: "~/.agents/skills/rea-analysis/SKILL.md", + detail: "Install or update the bundled REA analysis skill.", + external: false, + }, +]; + +const configureDetectedClients = async (options: { + readonly host: SetupHost; + readonly detectedClients: readonly SetupClient[]; + readonly hopperPath: string | undefined; + readonly clients: Record; + readonly appliedActions: string[]; +}): Promise => { + for (const client of options.detectedClients) { + const result = await options.host.configureClient( + client, + options.hopperPath, + registrationCommand(), + ); + options.clients[client.name] = result; + if (result.status === "failed") + return `${client.name} configuration ${result.reason} verification failed; no successful configuration was reported.`; + if (result.status === "configured") + options.appliedActions.push(`configured_${client.name}`); + } + return undefined; +}; + /** * Install prerequisites and configure detected clients idempotently. * Discovery always precedes mutation. Interactive confirmation or explicit @@ -125,38 +187,7 @@ export const runSetup = async ( if (unsupported !== undefined) return fail(unsupported); let hopperPath = await host.hopperPath(); const detectedClients = await host.detectedClients(); - plannedActions = [ - ...(hopperPath === undefined - ? [ - { - kind: "install_hopper" as const, - target: - host.platform === "darwin" - ? "~/Applications/Hopper Disassembler.app" - : "system package manager", - detail: - "Download the official Hopper package, verify it, install it, and open Hopper for activation.", - external: true, - }, - ] - : []), - ...detectedClients - .filter(({ format }) => format !== "unsupported") - .map( - (client): SetupAction => ({ - kind: "configure_client", - target: client.configPath, - detail: `Add the REA MCP registration for ${client.name}; preserve unrelated configuration.`, - external: false, - }), - ), - { - kind: "install_skill", - target: "~/.agents/skills/rea-analysis/SKILL.md", - detail: "Install or update the bundled REA analysis skill.", - external: false, - }, - ]; + plannedActions = setupPlan(host.platform, hopperPath, detectedClients); let approved = options.approved; let interactiveApproval = false; if (!approved && confirm !== undefined && !options.structured) { @@ -191,20 +222,14 @@ export const runSetup = async ( }; appliedActions.push("installed_hopper"); } - for (const client of detectedClients) { - const result = await host.configureClient( - client, - hopperPath, - registrationCommand(), - ); - clients[client.name] = result; - if (result.status === "failed") - return fail( - `${client.name} configuration ${result.reason} verification failed; no successful configuration was reported.`, - ); - if (result.status === "configured") - appliedActions.push(`configured_${client.name}`); - } + const clientFailure = await configureDetectedClients({ + host, + detectedClients, + hopperPath, + clients, + appliedActions, + }); + if (clientFailure !== undefined) return fail(clientFailure); const skill = await host.installSkill(); if (skill === "failed") return fail("Agent skill installation or readback failed."); @@ -485,56 +510,6 @@ export const configureTomlClient = async ( }; }; -/** Transactionally install or upgrade the versioned canonical REA skill. */ -export const installCanonicalSkill = async ( - home: string, -): Promise<"installed" | "unchanged" | "failed"> => { - const destination = join( - home, - ".agents/skills", - PRODUCT_IDENTITY.skillName, - "SKILL.md", - ); - const backup = `${destination}.rea.backup`; - let original: string | undefined; - try { - const content = await readFile( - new URL( - `../../skills/${PRODUCT_IDENTITY.skillName}/SKILL.md`, - import.meta.url, - ), - "utf8", - ); - original = await readFile(destination, "utf8").catch(() => undefined); - if (original === content) return "unchanged"; - await mkdir(dirname(destination), { recursive: true }); - if (original !== undefined) - await writeFileAtomic(backup, original, { - encoding: "utf8", - mode: 0o600, - }); - await writeFileAtomic(destination, content, { - encoding: "utf8", - mode: 0o600, - }); - if ((await readFile(destination, "utf8")) !== content) - throw new Error("skill readback mismatch"); - return "installed"; - } catch { - try { - if (original === undefined) await rm(destination, { force: true }); - else - await writeFileAtomic(destination, original, { - encoding: "utf8", - mode: 0o600, - }); - } catch { - // The backup remains beside the skill for explicit operator recovery. - } - return "failed"; - } -}; - const major = (version: string): number => Number.parseInt(version.split(".")[0] ?? "0", 10); const exists = async (path: string): Promise => { diff --git a/src/application/SetupSkill.ts b/src/application/SetupSkill.ts new file mode 100644 index 000000000..5df937780 --- /dev/null +++ b/src/application/SetupSkill.ts @@ -0,0 +1,55 @@ +import { mkdir, readFile, rm } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import writeFileAtomic from "write-file-atomic"; + +import { PRODUCT_IDENTITY } from "../identity.js"; + +/** Transactionally install or upgrade the versioned canonical REA skill. */ +export const installCanonicalSkill = async ( + home: string, +): Promise<"installed" | "unchanged" | "failed"> => { + const destination = join( + home, + ".agents/skills", + PRODUCT_IDENTITY.skillName, + "SKILL.md", + ); + const backup = `${destination}.rea.backup`; + let original: string | undefined; + try { + const content = await readFile( + new URL( + `../../skills/${PRODUCT_IDENTITY.skillName}/SKILL.md`, + import.meta.url, + ), + "utf8", + ); + original = await readFile(destination, "utf8").catch(() => undefined); + if (original === content) return "unchanged"; + await mkdir(dirname(destination), { recursive: true }); + if (original !== undefined) + await writeFileAtomic(backup, original, { + encoding: "utf8", + mode: 0o600, + }); + await writeFileAtomic(destination, content, { + encoding: "utf8", + mode: 0o600, + }); + if ((await readFile(destination, "utf8")) !== content) + throw new Error("skill readback mismatch"); + return "installed"; + } catch { + try { + if (original === undefined) await rm(destination, { force: true }); + else + await writeFileAtomic(destination, original, { + encoding: "utf8", + mode: 0o600, + }); + } catch { + // The backup remains beside the skill for explicit operator recovery. + } + return "failed"; + } +}; diff --git a/src/cli.ts b/src/cli.ts index 209ddef1c..2b027e9aa 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -51,6 +51,7 @@ export const createCli = (): ReturnType => { }, }); + registerSetupCommands(cli, logger); registerCoreCommands(cli, logger); registerFunctionCommand(cli, logger); registerSearchCommand(cli, logger); @@ -73,6 +74,60 @@ const registerCoreCommands = ( detail: z.enum(["concise", "detailed"]).default("concise"), limit: z.number().int().min(1).max(50).default(10), }); + cli.command("analyze", { + description: "Get an overview of an app", + args: z.object({ + path: z.string().describe("App, program, or Hopper database path"), + }), + options: overviewOptions, + run: ({ args, options }) => + logCliCommand(logger, "analyze", () => + runDirectAnalysis( + args.path, + "binary_overview", + { detail: options.detail, limit: options.limit }, + logger, + ), + ), + }); + cli.command("inspect", { + description: "Inspect an app overview with evidence", + args: z.object({ + path: z.string().describe("App, program, or Hopper database path"), + }), + options: overviewOptions, + run: ({ args, options }) => + logCliCommand(logger, "inspect", () => + runDirectAnalysis( + args.path, + "binary_overview", + { detail: options.detail, limit: options.limit }, + logger, + ), + ), + }); + cli.command("decompile", { + description: "Read one part of an app as code", + args: z.object({ + path: z.string().describe("App or program path"), + address: z.string().describe("Procedure address"), + }), + run: ({ args }) => + logCliCommand(logger, "decompile", () => + runDirectAnalysis( + args.path, + "procedure_pseudo_code", + { procedure: args.address }, + logger, + ), + ), + }); +}; + +const registerSetupCommands = ( + cli: ReturnType, + logger: Logger, +): void => { cli.command("setup", { description: "Install requirements and configure coding agents", options: z.object({ @@ -132,54 +187,6 @@ const registerCoreCommands = ( ), ), }); - cli.command("analyze", { - description: "Get an overview of an app", - args: z.object({ - path: z.string().describe("App, program, or Hopper database path"), - }), - options: overviewOptions, - run: ({ args, options }) => - logCliCommand(logger, "analyze", () => - runDirectAnalysis( - args.path, - "binary_overview", - { detail: options.detail, limit: options.limit }, - logger, - ), - ), - }); - cli.command("inspect", { - description: "Inspect an app overview with evidence", - args: z.object({ - path: z.string().describe("App, program, or Hopper database path"), - }), - options: overviewOptions, - run: ({ args, options }) => - logCliCommand(logger, "inspect", () => - runDirectAnalysis( - args.path, - "binary_overview", - { detail: options.detail, limit: options.limit }, - logger, - ), - ), - }); - cli.command("decompile", { - description: "Read one part of an app as code", - args: z.object({ - path: z.string().describe("App or program path"), - address: z.string().describe("Procedure address"), - }), - run: ({ args }) => - logCliCommand(logger, "decompile", () => - runDirectAnalysis( - args.path, - "procedure_pseudo_code", - { procedure: args.address }, - logger, - ), - ), - }); }; const confirmSetup = async ( From f0a9a3cd258d60046dcd74feea243956489c60dc Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:23:54 +0800 Subject: [PATCH 7/8] docs(skill): document capture v4 and DMG mounts --- skills/rea-analysis/SKILL.md | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/skills/rea-analysis/SKILL.md b/skills/rea-analysis/SKILL.md index 45e0dae15..d397aa660 100644 --- a/skills/rea-analysis/SKILL.md +++ b/skills/rea-analysis/SKILL.md @@ -2,7 +2,7 @@ name: rea-analysis description: Reverse engineer apps with REA. Explore how features work, then build a version tailored to your project. metadata: - version: "10" + version: "11" tool_count: 68 --- @@ -36,13 +36,19 @@ REA accepts a `.app` bundle directly. Do not expose its internal `Contents/MacOS Briefly tell the user what you will investigate. Open the app with `open_binary`, begin with `binary_overview`, and narrow the investigation around the requested feature. Use decompilation, strings, names, callers, callees, and cross-references as needed. -For applications, ZIP/APK/IPA packages, or Electron ASAR archives, call +For applications, ZIP/APK/IPA packages, Electron ASAR archives, or DMGs, call `inventory_artifact` before extraction. Follow its deterministic occurrence pages and cite graph manifest IDs. `extract_artifact` requires explicit user approval, an absent absolute output root, and selected occurrence IDs; never extract every entry implicitly. Symlinks and encrypted entries are inventory -facts, not extractable files. Nested containers require a separate deliberate -inventory step. +facts, not extractable files. Inventory traverses discovered ASARs within the +same graph and shared limits, including ASARs inside an approved mounted DMG. + +Native DMG traversal is macOS-only, read-only, and disabled by default. Use it +only when the user approves that inventory call with +`native_mount_approved: true` and the operator has separately enabled +`REA_ARTIFACT_NATIVE_MOUNT_ENABLED=true`. Without both gates, retain the DMG +root-hash-only result. Never imply that approval changes extraction authority. Explain conclusions in plain language. Point to the relevant decompiled code, strings, names, and connections so the user can see how the explanation was reached. Do not claim to recover original source code or automatically clone an application. @@ -57,6 +63,15 @@ requires per-call approval plus operator policy, and uses host networking only when the operator explicitly permits it. It is behavioral evidence, not a security sandbox. +`capture_process_scenario` produces Process Capture v4 only. If a boundary +reports Process Capture v3, tell the user or calling agent to rerun the original +scenario; v3 cannot be upgraded because it lacks required manifest and +settlement evidence. Treat the v4 manifest commitments as compatibility and +provenance evidence, and distinguish root exit from descendant settlement. +When comparing stored captures, set `max_capture_age_ms` when the task requires +fresh behavioral evidence. Different scenario and executable digests are +allowed, but schema and comparison-contract digests must be compatible. + Do not let unanswered questions disappear. Use `record_unknown` only with explicit approval, attach supporting and contradicting evidence IDs, and record the authority/environment still required. Use `update_unknown` with the current From 19bb2317d1bd6773e5fa3f61e32cbf3d6d257830 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:26:36 +0800 Subject: [PATCH 8/8] fix(ci): remove redundant process exports --- src/application/ProcessHarness.ts | 1 - src/domain/processCapture.ts | 1 - src/domain/processCaptureValidation.ts | 4 ++-- src/domain/processScenario.ts | 6 ++---- tests/setupSkill.test.ts | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/application/ProcessHarness.ts b/src/application/ProcessHarness.ts index 986867a53..67f540e80 100644 --- a/src/application/ProcessHarness.ts +++ b/src/application/ProcessHarness.ts @@ -35,7 +35,6 @@ import { import { assertNotCancelled } from "./ProcessCaptureAuthority.js"; export { probeProcessCaptureCapability } from "./ProcessCaptureCapability.js"; -export type { ProcessCaptureCapability } from "./ProcessCaptureCapability.js"; interface EnvironmentOptions { readonly scenario: ProcessScenario; diff --git a/src/domain/processCapture.ts b/src/domain/processCapture.ts index 03d451408..945fcd879 100644 --- a/src/domain/processCapture.ts +++ b/src/domain/processCapture.ts @@ -325,7 +325,6 @@ export const processCaptureSchema: z.ZodType = z.object({ }); export { validateProcessCapture } from "./processCaptureValidation.js"; -export type { ProcessCaptureValidationIssue } from "./processCaptureValidation.js"; import { validateProcessCapture } from "./processCaptureValidation.js"; /** Parse unknown input as v4 and reject invalid commitments or semantics. */ diff --git a/src/domain/processCaptureValidation.ts b/src/domain/processCaptureValidation.ts index 571617746..8c7b62d4e 100644 --- a/src/domain/processCaptureValidation.ts +++ b/src/domain/processCaptureValidation.ts @@ -1,5 +1,5 @@ import type { ProcessCapture } from "./processCapture.js"; -import { canonicalProcessSha256 } from "./processScenario.js"; +import { digestProcessCommitment } from "./processScenario.js"; /** One pure semantic validation failure in an otherwise shaped v4 capture. */ export interface ProcessCaptureValidationIssue { @@ -35,7 +35,7 @@ const validateCommitments = ( ["replay_plan_sha256", manifest.replay_plan], ] as const) require(manifest[field] === - canonicalProcessSha256( + digestProcessCommitment( value, ), `manifest.${field}`, "commitment does not match its canonical value"); require(Date.parse(manifest.started_at) <= diff --git a/src/domain/processScenario.ts b/src/domain/processScenario.ts index 5d7d5cea1..a3ec96704 100644 --- a/src/domain/processScenario.ts +++ b/src/domain/processScenario.ts @@ -42,7 +42,8 @@ const outputChunkSchema = z.object({ export const LEGACY_PROCESS_CAPTURE_MESSAGE = "Process Capture v3 is unsupported. Re-run the scenario with this REA version using capture_process_scenario to produce Process Capture v4; captures cannot be upgraded because v4 requires new manifest and settlement evidence."; -export const canonicalProcessSha256 = (value: unknown): string => { +/** Compute a canonical SHA-256 commitment independent of object key order. */ +export const digestProcessCommitment = (value: unknown): string => { const serialized = canonicalize(value); if (serialized === undefined) throw new TypeError("Process capture commitment is not canonical JSON"); @@ -86,9 +87,6 @@ export const processComparisonContract = ( replay: scenario.replay, }); -/** Compute a canonical SHA-256 commitment independent of object key order. */ -export const digestProcessCommitment = canonicalProcessSha256; - /** * Boundary schema for one explicitly approved, bounded process experiment. * Defaults are part of the evidence contract and must remain deterministic. diff --git a/tests/setupSkill.test.ts b/tests/setupSkill.test.ts index 97c981272..537a8e5bf 100644 --- a/tests/setupSkill.test.ts +++ b/tests/setupSkill.test.ts @@ -27,7 +27,7 @@ describe("canonical skill transaction", () => { expect(await readFile(`${destination}.rea.backup`, "utf8")).toBe( "stale managed skill\n", ); - expect(await readFile(destination, "utf8")).toContain('version: "10"'); + expect(await readFile(destination, "utf8")).toContain('version: "11"'); expect(await readFile(destination, "utf8")).toContain( `tool_count: ${String(TOOL_CONTRACTS.length)}`, );