import { createHash } from "node:crypto"; import { readFile, writeFile, mkdir } from "node:fs/promises"; import { resolve, dirname } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; import { getEncoding } from "js-tiktoken"; import { CodeModeBridge, connectUpstream, json } from "./bridge.mjs"; // Replay isolates transport/discovery costs. It does not simulate model reasoning. const workflows = { "sharex.dll": { tools: ["inspect_managed_artifact", "inspect_managed_native_boundaries"], question: "Report assembly/module identity, target framework and declared P/Invoke libraries with their uncertainty.", code: ` const artifact = await call("inspect_managed_artifact", {path: input.path}); const boundaries = await call("inspect_managed_native_boundaries", {}); const n = artifact.normalized_result; return {artifact_sha256: artifact.subject?.digest.sha256, classification: n.classification, assembly: n.assembly, module: n.module, target_frameworks: n.target_frameworks, coverage: n.coverage, native_boundaries: boundaries.normalized_result, evidence_ids: [artifact.evidence_id, boundaries.evidence_id], limitations: [...artifact.limitations, ...boundaries.limitations]};`, }, "sqlite.wasm": { tools: ["inspect_wasm_artifact"], question: "Report exact WASM validation, all exported names and runtime/network observation states without executing it.", code: ` const wasm = await call("inspect_wasm_artifact", {path: input.path}); const n = wasm.normalized_result; return {artifact_sha256: wasm.subject?.digest.sha256, validation: n.validation, sections: n.sections, exports: n.exports, runtime_execution: n.runtime_execution, network_fetch: n.network_fetch, evidence_ids: [wasm.evidence_id], limitations: [...wasm.limitations, ...n.limitations]};`, }, "obsidian.asar": { tools: ["analyze_javascript_application", "inspect_analysis_view"], question: "Report application identity, analysis coverage/counts and the first eight module identities.", code: ` const summary = await call("analyze_javascript_application", { input_path: input.path, detail: "summary", max_heap_mb: 1024, analysis_timeout_ms: 300000, integrity_policy: "fail" }); const parent = summary.normalized_result.parent_evidence_id; const page = await call("inspect_analysis_view", { source: {kind: "retained-evidence", evidence_id: parent}, view: {kind: "page", collection: "modules", offset: 0, limit: 8} }); return {artifact_sha256: summary.subject?.digest.sha256, summary: summary.normalized_result, modules: page.normalized_result, evidence_ids: [summary.evidence_id, parent, page.evidence_id], limitations: [...summary.limitations, ...page.limitations]};`, }, "obsidian-inventory": { artifact: "obsidian.asar", tools: ["open_binary", "inspect_artifact", "close_binary"], question: "Report ASAR identity, complete inventory manifest, entry-kind counts, root package/main/preload occurrences, integrity contradictions and inventory coverage; do not claim AST or runtime analysis.", code: ` await call("open_binary", {path: input.path}); try { const inspected = await call("inspect_artifact", {integrity_policy: "fail"}); const n = inspected.normalized_result; const inventory = n.substeps.find(x => x.operation === "inventory_artifact").evidence; const graph = inventory.normalized_result; const counts = {}; for (const row of graph.occurrences) counts[row.entry_kind] = (counts[row.entry_kind] ?? 0) + 1; return {artifact_sha256: inspected.subject.digest.sha256, manifest: graph.manifest, entry_kind_counts: counts, requested_paths: graph.occurrences.filter(x => ["package.json", "main.js", "preload.js"].includes(x.logical_path)), integrity_contradictions: graph.integrity_contradictions, coverage: n.coverage, unexplored_branches: n.unexplored_branches, evidence_ids: [inspected.evidence_id, inventory.evidence_id], limitations: [...inspected.limitations, ...n.limitations, ...inventory.limitations]}; } finally { await call("close_binary", {}); }`, }, "newpipe.apk": { tools: [ "inspect_android_package", "search_android_classes", "inspect_android_class", ], question: "Report package/version, exact MainActivity matches and its onCreate method descriptors.", code: ` const pkg = await call("inspect_android_package", {path: input.path}); const found = await call("search_android_classes", {path: input.path, query: "MainActivity"}); if (!found.normalized_result.matches.includes("org.schabi.newpipe.MainActivity")) throw new Error("Requested shipped MainActivity was not found"); const cls = await call("inspect_android_class", { path: input.path, class_name: "org.schabi.newpipe.MainActivity" }); return {artifact_sha256: pkg.subject?.digest.sha256, package_name: pkg.normalized_result.package_name, version_name: pkg.normalized_result.version_name, version_code: pkg.normalized_result.version_code, matches: found.normalized_result.matches, coverage: found.normalized_result.coverage, onCreate: cls.normalized_result.methods.filter(x => x.name === "onCreate"), evidence_ids: [pkg.evidence_id, found.evidence_id, cls.evidence_id], limitations: [...pkg.limitations, ...found.limitations, ...cls.limitations]};`, }, "openwrt.bin": { tools: ["inspect_firmware_regions"], question: "Report firmware identity, every reported region offset/signature and examined coverage.", code: ` const inspected = await call("inspect_firmware_regions", {path: input.path}); return {artifact_sha256: inspected.subject?.digest.sha256, result: inspected.normalized_result, evidence_ids: [inspected.evidence_id], limitations: inspected.limitations};`, }, jq: { tools: [ "open_binary", "resolve_containing_procedure", "analyze_function", "close_binary", ], question: "Resolve the observed ELF entry address and report the recovered function's first 512 pseudocode units and caller/callee counts.", code: ` await call("open_binary", {path: input.path, provider_id: "ghidra"}); try { const found = await call("resolve_containing_procedure", {address: input.entry_address}); const fn = await call("analyze_function", {procedure: input.entry_address}); const n = fn.normalized_result; return {artifact_sha256: fn.subject?.digest.sha256, entry_address: input.entry_address, containing_procedure: found.normalized_result, procedure: n.procedure, pseudocode_prefix: n.pseudocode.slice(0, 512), caller_count: n.callers.length, callee_count: n.callees.length, evidence_ids: [found.evidence_id, fn.evidence_id], limitations: [...found.limitations, ...fn.limitations, ...n.limitations]}; } finally { await call("close_binary", {}); }`, }, }; const { values } = parseArgs({ options: { artifact: { type: "string" }, workflow: { type: "string" }, mode: { type: "string", default: "direct" }, "artifact-root": { type: "string" }, providers: { type: "string" }, output: { type: "string" }, }, }); if ( !workflows[values.workflow ?? values.artifact] || !["direct", "discovery", "bridge"].includes(values.mode) || !values["artifact-root"] || !values.output ) throw new Error( "Use --artifact ID [--workflow NAME] --mode direct|discovery|bridge --artifact-root ABS --output NEW_DIR [--providers JSON]", ); const experiment = dirname(fileURLToPath(import.meta.url)); const repository = resolve(experiment, "../.."); const corpus = JSON.parse( await readFile(resolve(experiment, "corpus.json"), "utf8"), ); const artifact = corpus.artifacts.find((x) => x.id === values.artifact); const workflowName = values.workflow ?? values.artifact; const workflow = workflows[workflowName]; if (!artifact || artifact.id !== (workflow.artifact ?? workflowName)) throw new Error( "Selected workflow does not match the frozen corpus artifact", ); const artifactPath = resolve(values["artifact-root"], artifact.file); const bytes = await readFile(artifactPath); const sha256 = createHash("sha256").update(bytes).digest("hex"); if (sha256 !== artifact.sha256 || bytes.length !== artifact.bytes) throw new Error("Artifact identity differs from frozen corpus"); await mkdir(values.output, { recursive: false, mode: 0o700 }); const encoding = getEncoding("cl100k_base"); const measure = (value) => { const text = typeof value === "string" ? value : JSON.stringify(value); return { bytes: Buffer.byteLength(text), cl100k_estimated_tokens: encoding.encode(text).length, }; }; const modelText = (reply) => reply.content .filter((item) => item.type === "text") .map((item) => item.text) .join("\n"); const environment = values.providers ? JSON.parse(await readFile(values.providers, "utf8")) : {}; const started = performance.now(); const upstream = await connectUpstream({ command: process.execPath, args: [resolve(repository, "scripts/rea.mjs"), "mcp"], env: environment, }); const connected = performance.now(); const records = []; const realCall = upstream.call; upstream.call = async (name, args, options) => { const began = performance.now(); try { const reply = await realCall(name, args, { timeout: 600_000, ...options }); const parsed = json(reply); records.push({ name, arguments: args, elapsed_ms: performance.now() - began, is_error: reply.isError === true, input: measure(args), output: measure(reply), text_output: measure(modelText(reply)), evidence_id: parsed.evidence_id, reply, }); return reply; } catch (error) { records.push({ name, arguments: args, elapsed_ms: performance.now() - began, failure: String(error), input: measure(args), }); throw error; } }; const bridge = new CodeModeBridge(upstream); const input = { path: artifactPath }; if (artifact.id === "jq") { if (bytes.subarray(0, 6).toString("hex") !== "7f454c460201") throw new Error("Expected ELF64 little-endian header for entry seed"); input.entry_address = `0x${bytes.readBigUInt64LE(24).toString(16)}`; } const code = `const input = ${JSON.stringify(input)}; async function call(name, args) { const reply = await tools[name](args); const parsed = json(reply); if (reply.isError) throw new Error(JSON.stringify(parsed)); return parsed; } ${workflow.code}`; let discovery; let value; let failure; let bridgeReply; const recovery = []; const operationStarted = performance.now(); try { if (values.mode !== "direct") discovery = await bridge.search({ names: workflow.tools }); if (values.mode === "bridge") { bridgeReply = await bridge.execute({ code, timeout_ms: 900_000 }); value = bridgeReply.value; failure = bridgeReply.bridge_error; } else { const tools = Object.fromEntries( upstream.catalog.map((tool) => [ tool.name, (args) => upstream.call(tool.name, args), ]), ); const AsyncFunction = Object.getPrototypeOf( async function () {}, ).constructor; value = await new AsyncFunction("tools", "json", code)(tools, json); } } catch (error) { failure = String(error); } finally { const partials = new Set( records.flatMap((record) => { if (!record.reply?.isError) return []; try { const partial = json(record.reply).error?.details?.partial_observation; return partial?.kind === "retained-evidence" ? [partial.evidence_id] : []; } catch { return []; // Preserve non-JSON failures in calls.json, without inventing IDs. } }), ); for (const evidence_id of partials) { // Capture already retained partial facts before disposing the replay session. // Recovery is not a successful completion of the failed analyst workflow. try { const reply = await realCall( "inspect_analysis_view", { source: { kind: "retained-evidence", evidence_id }, view: { kind: "summary" }, }, { timeout: 60_000 }, ); recovery.push({ parent_evidence_id: evidence_id, reply }); } catch (error) { recovery.push({ parent_evidence_id: evidence_id, failure: String(error), }); } } await upstream.close(); } const catalogExposure = values.mode === "direct" ? upstream.catalog : discovery; const summary = { scope: "deterministic_real_provider_replay_not_model_accuracy", baseline_sha: "fb94d9dde0ccda83ae520ac6fb22b1d283562cf3", artifact: artifact.id, workflow: workflowName, artifact_sha256: sha256, mode: values.mode, question: workflow.question, startup_ms: connected - started, operation_ms: performance.now() - operationStarted, original_calls: records.map(({ reply: _reply, ...metadata }) => metadata), exposed_catalog: catalogExposure ? measure(catalogExposure) : null, direct_input_definitions: measure( upstream.catalog.map((tool) => ({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema, })), ), execution_code: values.mode === "bridge" ? measure(code) : null, exposed_tool_results: values.mode === "bridge" ? [measure(bridgeReply)] : records.filter((x) => x.output).map((x) => x.output), exposed_text_results: values.mode === "bridge" ? [measure(bridgeReply)] : records.filter((x) => x.text_output).map((x) => x.text_output), projection: value, failure: failure ?? null, partial_recovery: recovery, correctness: failure ? "failed_workflow" : "requires_cross_profile_fact_review", estimator: "cl100k_base_compact_MCP_JSON_including_envelope_not_actual_model_tokens", }; await writeFile( resolve(values.output, "summary.json"), JSON.stringify(summary, null, 2), ); await writeFile(resolve(values.output, "calls.json"), JSON.stringify(records)); await writeFile(resolve(values.output, "code.js"), code); console.log( JSON.stringify({ artifact: artifact.id, mode: values.mode, calls: records.length, startup_ms: summary.startup_ms, operation_ms: summary.operation_ms, failure: summary.failure ? String( typeof summary.failure === "object" ? summary.failure.message : summary.failure, ).slice(0, 1000) : null, }), ); if (failure) process.exitCode = 1;