From be6863e9a16bcc3a09d87c50981e69292795a512 Mon Sep 17 00:00:00 2001 From: kvalls <95490470+kvalls@users.noreply.github.com> Date: Fri, 9 Oct 2026 16:11:06 +0100 Subject: [PATCH 01/27] feat(ghidra): support native PE DLLs on Windows --- .github/workflows/real-ghidra-windows.yml | 20 +++ AGENTS.md | 2 +- docs/installation.md | 2 +- docs/managed-code-analysis.md | 5 +- docs/provider-evaluation.md | 9 +- docs/roadmap.md | 2 +- docs/windows-ghidra-p0.md | 15 +- scripts/create-ghidra-windows-fixture.mjs | 155 +++++++++++++-------- scripts/verify-packaged-ghidra-windows.mjs | 114 ++++++++++++--- scripts/verify-real-ghidra-windows.mjs | 27 +++- src/ghidra/GhidraAnalysisProfile.ts | 6 + src/ghidra/GhidraProvider.behavior.test.ts | 84 ++++++----- src/ghidra/GhidraProvider.ts | 9 +- src/ghidra/GhidraProviderCapabilities.ts | 2 +- 14 files changed, 316 insertions(+), 136 deletions(-) diff --git a/.github/workflows/real-ghidra-windows.yml b/.github/workflows/real-ghidra-windows.yml index 42dc38fa4..d6de7469f 100644 --- a/.github/workflows/real-ghidra-windows.yml +++ b/.github/workflows/real-ghidra-windows.yml @@ -64,6 +64,14 @@ jobs: env: REA_ANALYSIS_PROVIDER: ghidra run: npm run verify:ghidra:windows -- --x86 | Tee-Object -FilePath windows-ghidra-x86-proof.log + - name: Verify native x64 DLL Windows Ghidra operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows -- --dll | Tee-Object -FilePath windows-ghidra-dll-proof.log + - name: Verify native x86 DLL Windows Ghidra operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows -- --x86 --dll | Tee-Object -FilePath windows-ghidra-x86-dll-proof.log - name: Verify ordinary-user CLI and MCP operations env: REA_ANALYSIS_PROVIDER: ghidra @@ -72,6 +80,14 @@ jobs: env: REA_ANALYSIS_PROVIDER: ghidra run: npm run verify:ghidra:windows:package -- --x86 | Tee-Object -FilePath windows-ghidra-x86-package-proof.log + - name: Verify ordinary-user x64 DLL CLI and MCP operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows:package -- --dll | Tee-Object -FilePath windows-ghidra-dll-package-proof.log + - name: Verify ordinary-user x86 DLL CLI and MCP operations + env: + REA_ANALYSIS_PROVIDER: ghidra + run: npm run verify:ghidra:windows:package -- --x86 --dll | Tee-Object -FilePath windows-ghidra-x86-dll-package-proof.log - uses: actions/upload-artifact@v4 if: always() with: @@ -81,4 +97,8 @@ jobs: windows-ghidra-package-proof.log windows-ghidra-x86-proof.log windows-ghidra-x86-package-proof.log + windows-ghidra-dll-proof.log + windows-ghidra-x86-dll-proof.log + windows-ghidra-dll-package-proof.log + windows-ghidra-x86-dll-package-proof.log retention-days: 30 diff --git a/AGENTS.md b/AGENTS.md index 1acb82552..01ecf3d91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Product Direction -REA exposes reverse-engineering tools through a CLI and MCP server. Hopper, the bring-your-own Ghidra adapter, and the bring-your-own IDA MCP adapter are operation-capable deep binary-analysis providers. Ghidra is supported on Linux x64/arm64 and macOS x64/arm64 with matching native decompiler tools, and has an experimental Windows x64 P0 boundary for approved native x86 and x86-64 PE applications; on Linux and macOS it supplies inventory, function analysis, and atomic function annotation edits in an ephemeral database, without modifying executable bytes or controlling a GUI. Windows P0 has no mutation authority. IDA adapts upstream legacy attached GUI and modern headless database-supervisor profiles for live read-only analysis; initial real verification covers Windows, and an attached GUI database is never saved or closed. Keep provider-specific code out of the domain and application layers. +REA exposes reverse-engineering tools through a CLI and MCP server. Hopper, the bring-your-own Ghidra adapter, and the bring-your-own IDA MCP adapter are operation-capable deep binary-analysis providers. Ghidra is supported on Linux x64/arm64 and macOS x64/arm64 with matching native decompiler tools, and has an experimental Windows x64 P0 boundary for approved native x86 and x86-64 PE applications and DLLs; on Linux and macOS it supplies inventory, function analysis, and atomic function annotation edits in an ephemeral database, without modifying executable bytes or controlling a GUI. Windows P0 has no mutation authority. IDA adapts upstream legacy attached GUI and modern headless database-supervisor profiles for live read-only analysis; initial real verification covers Windows, and an attached GUI database is never saved or closed. Keep provider-specific code out of the domain and application layers. Prioritize: diff --git a/docs/installation.md b/docs/installation.md index a0be32920..dfa3bc84b 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -452,7 +452,7 @@ entry comments atomically and returns refreshed analysis. These session metadata edits leave executable bytes unchanged and are discarded on close. GUI controls require Hopper; Windows P0 remains read-only. -Windows P0 admits native x86 and x86-64 PE applications on fixed local NTFS volumes. +Windows P0 admits native x86 and x86-64 PE applications and DLLs on fixed local NTFS volumes. The npm package bundles native Job Object ownership, protected private runtime DACLs, and handle-based path admission; no separate addon installation is needed. See the [Windows Ghidra P0 guide](windows-ghidra-p0.md) for verified scope and diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index 92bc23060..bb87a7a63 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -125,8 +125,9 @@ The upstream analyzer currently gates its own analysis to x86-64. NativeAOT's broader platform matrix does not establish REA coverage: PE/COFF, ELF, Mach-O, shared libraries, target architectures, and runtime metadata versions each need their own provider verification. REA's experimental Windows Ghidra boundary is -limited to admitted native x86 and x86-64 PE applications, so it does not currently -establish coverage for PE DLLs. Mobile NativeAOT targets should remain +limited to admitted native x86 and x86-64 PE applications and DLLs for static +analysis. This does not establish Windows NativeAOT metadata-recovery coverage. +Mobile NativeAOT targets should remain experimental until REA has a matching provider and package-level verification. For REA's native analysis, preserve the same artifact path and digest used by diff --git a/docs/provider-evaluation.md b/docs/provider-evaluation.md index 8aa655e52..a18303a92 100644 --- a/docs/provider-evaluation.md +++ b/docs/provider-evaluation.md @@ -18,7 +18,7 @@ comments atomically, verifies readback, and returns refreshed function analysis. Changes live only in the session database and leave executable bytes unchanged. Ghidra does not expose GUI controls; Windows P0 remains read-only. -Windows x64 P0 supports native x86 and x86-64 PE applications on local NTFS with +Windows x64 P0 supports native x86 and x86-64 PE applications and DLLs on local NTFS with bundled Job Object ownership, protected runtime DACLs, and handle-based path admission. Real ordinary-user verification covers all 25 read-only operations through the packaged CLI and MCP, target integrity, and cleanup. This remains @@ -62,7 +62,7 @@ maximum, when the release leaves those at the 12.1 defaults). Otherwise doctor probes `java`/`javac` or `java.exe`/`javac.exe` from `PATH`. Supported Linux and macOS hosts accept compatible ELF, PE, and Mach-O executable targets. Host admission and target compatibility are separate checks. Windows x64 P0 admits only native, -non-managed, non-DLL x86 and x86-64 PE applications on fixed local NTFS, with the packaged +non-managed x86 and x86-64 PE applications and DLLs on fixed local NTFS, with the packaged native authority available before launch. The launcher creates one ephemeral runtime root with project, @@ -136,8 +136,9 @@ Mach-O target coverage. It requires `clang`, LLD, and `lld-link`; set these tools before compilation. Keeping this matrix separate lets Linux host/provider acceptance run with only the host compiler. -`npm run verify:ghidra:windows` and its `-- --x86` variant check the source-owned native x86 and x86-64 PE -fixture, all 25 read-only operations, target/snapshot/import SHA-256 linkage, +`npm run verify:ghidra:windows` and its `-- --x86` variant check source-owned native x86 and x86-64 PE +applications. Adding `-- --dll` or `-- --x86 --dll` selects entry-point-free DLLs +with exported caller/callee functions. These lanes check all 25 read-only operations, target/snapshot/import SHA-256 linkage, and project, endpoint, process, and runtime cleanup. The independent native lane checks DACLs, handle admission, cancellation, and Job Object lifecycle. `npm run verify:ghidra:windows:package` packs and installs REA into an isolated diff --git a/docs/roadmap.md b/docs/roadmap.md index b4ac0ba54..077efa632 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -87,7 +87,7 @@ and exact commands. Experimental Windows x64 P0 now supplies bundled Job Object ownership, protected runtime DACLs, and handle-based admission for native x86 and x86-64 PE -applications on local NTFS. Real ordinary-user CLI/MCP verification covers +applications and DLLs on local NTFS. Real ordinary-user CLI/MCP verification covers all 25 read-only operations and cleanup. The [Windows P0 guide](windows-ghidra-p0.md) describes this boundary and unverified broader coverage. diff --git a/docs/windows-ghidra-p0.md b/docs/windows-ghidra-p0.md index ec54f5908..5f50d832c 100644 --- a/docs/windows-ghidra-p0.md +++ b/docs/windows-ghidra-p0.md @@ -17,7 +17,7 @@ additional permission flag or degraded mode is required. - Node.js 22.x (>=22.19), 24.x (>=24.11), or 26+; - an operator-installed Ghidra 12.1.x distribution (verified with 12.1.4); - a 64-bit full JDK inside that installation's declared Java range (JDK 21 or newer, with no maximum, for current 12.1 releases); -- an explicit native, non-managed, non-DLL x86 or x86-64 PE application; and +- an explicit native, non-managed x86 or x86-64 PE application or DLL; and - the 25 read-only Ghidra inventory, memory, and function-analysis operations. Loaded memory reads and file offsets preserve Ghidra's observed source mappings. @@ -38,6 +38,13 @@ REA selects its packaged bridge scripts by explicit path. Same-named files or directories in the caller's working directory do not select a different script or require users to clean that directory before analysis. +Native DLL analysis preserves the PE shared-library role in the analysis profile. +REA imports an immutable copy for static analysis; it does not load the library +into the analyst process or execute its entry routine. The source-owned DLL +fixtures have no entry routine or imports and expose two exported functions. +The same native snapshot, path-admission and cleanup requirements apply to EXEs +and DLLs. + ## Configuration ```powershell @@ -100,7 +107,7 @@ and artifact SHA-256 before loading the package-owned addon. Native failure reasons remain distinct from a valid installation and an unsupported target. PE header classification and SHA-256 come from the same open file. The Windows -provider admits native x86 and x86-64 PE applications and rejects unsupported roles, +provider admits native x86 and x86-64 PE applications and DLLs and rejects unsupported roles, architectures, managed images, and malformed headers. The original selected source coordinate is preserved for native admission. @@ -159,7 +166,11 @@ artifact and manifest. npm run verify:windows-native npm run verify:ghidra:windows npm run verify:ghidra:windows -- --x86 +npm run verify:ghidra:windows -- --dll +npm run verify:ghidra:windows -- --x86 --dll npm run verify:ghidra:windows:package -- --x86 +npm run verify:ghidra:windows:package -- --dll +npm run verify:ghidra:windows:package -- --x86 --dll npm run verify:ghidra:windows:package -- C:\fixtures\installed-rea C:\fixtures\sample.exe ``` diff --git a/scripts/create-ghidra-windows-fixture.mjs b/scripts/create-ghidra-windows-fixture.mjs index 6538ec57f..e610b4688 100644 --- a/scripts/create-ghidra-windows-fixture.mjs +++ b/scripts/create-ghidra-windows-fixture.mjs @@ -3,68 +3,103 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +// Source-owned static fixtures: no compiler, target loading or execution. for (const architecture of ["x86_64", "x86"]) { - const x86 = architecture === "x86"; - const output = resolve( - root, - "build", - "fixtures", - x86 ? "rea-ghidra-windows-x86.exe" : "rea-ghidra-windows.exe", - ); - const bytes = Buffer.alloc(1024); - const peOffset = 0x80; - const optionalHeader = peOffset + 24; - const optionalHeaderSize = x86 ? 0xe0 : 0xf0; - const sectionTable = optionalHeader + optionalHeaderSize; + for (const dll of [false, true]) { + const x86 = architecture === "x86"; + const output = resolve( + root, + "build", + "fixtures", + `rea-ghidra-windows${x86 ? "-x86" : ""}.${dll ? "dll" : "exe"}`, + ); + const bytes = Buffer.alloc(1024); + const peOffset = 0x80; + const optionalHeader = peOffset + 24; + const optionalHeaderSize = x86 ? 0xe0 : 0xf0; + const sectionTable = optionalHeader + optionalHeaderSize; + const directories = optionalHeader + (x86 ? 96 : 112); - bytes.write("MZ", 0, "ascii"); - bytes.writeUInt32LE(peOffset, 0x3c); - bytes.write("PE\0\0", peOffset, "binary"); - bytes.writeUInt16LE(x86 ? 0x14c : 0x8664, peOffset + 4); - bytes.writeUInt16LE(1, peOffset + 6); - bytes.writeUInt16LE(optionalHeaderSize, peOffset + 20); - bytes.writeUInt16LE(x86 ? 0x102 : 0x22, peOffset + 22); + bytes.write("MZ", 0, "ascii"); + bytes.writeUInt32LE(peOffset, 0x3c); + bytes.write("PE\0\0", peOffset, "binary"); + bytes.writeUInt16LE(x86 ? 0x14c : 0x8664, peOffset + 4); + bytes.writeUInt16LE(1, peOffset + 6); + bytes.writeUInt16LE(optionalHeaderSize, peOffset + 20); + bytes.writeUInt16LE( + (x86 ? 0x102 : 0x22) | (dll ? 0x2000 : 0), + peOffset + 22, + ); - bytes.writeUInt16LE(x86 ? 0x10b : 0x20b, optionalHeader); - bytes.writeUInt32LE(0x200, optionalHeader + 4); - bytes.writeUInt32LE(0x1000, optionalHeader + 16); - bytes.writeUInt32LE(0x1000, optionalHeader + 20); - if (x86) bytes.writeUInt32LE(0x400000, optionalHeader + 28); - else bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24); - bytes.writeUInt32LE(0x1000, optionalHeader + 32); - bytes.writeUInt32LE(0x200, optionalHeader + 36); - bytes.writeUInt16LE(6, optionalHeader + 40); - bytes.writeUInt16LE(6, optionalHeader + 48); - bytes.writeUInt32LE(0x2000, optionalHeader + 56); - bytes.writeUInt32LE(0x200, optionalHeader + 60); - bytes.writeUInt16LE(3, optionalHeader + 68); - bytes.writeUInt16LE(x86 ? 0x8140 : 0x8160, optionalHeader + 70); - if (x86) { - bytes.writeUInt32LE(0x10_0000, optionalHeader + 72); - bytes.writeUInt32LE(0x1000, optionalHeader + 76); - bytes.writeUInt32LE(0x10_0000, optionalHeader + 80); - bytes.writeUInt32LE(0x1000, optionalHeader + 84); - bytes.writeUInt32LE(16, optionalHeader + 92); - } else { - bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72); - bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80); - bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88); - bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96); - bytes.writeUInt32LE(16, optionalHeader + 108); + bytes.writeUInt16LE(x86 ? 0x10b : 0x20b, optionalHeader); + bytes.writeUInt32LE(0x200, optionalHeader + 4); + bytes.writeUInt32LE(dll ? 0 : 0x1000, optionalHeader + 16); + bytes.writeUInt32LE(0x1000, optionalHeader + 20); + if (x86) bytes.writeUInt32LE(0x400000, optionalHeader + 28); + else bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24); + bytes.writeUInt32LE(0x1000, optionalHeader + 32); + bytes.writeUInt32LE(0x200, optionalHeader + 36); + bytes.writeUInt16LE(6, optionalHeader + 40); + bytes.writeUInt16LE(6, optionalHeader + 48); + bytes.writeUInt32LE(0x2000, optionalHeader + 56); + bytes.writeUInt32LE(0x200, optionalHeader + 60); + bytes.writeUInt16LE(3, optionalHeader + 68); + bytes.writeUInt16LE(x86 ? 0x8140 : 0x8160, optionalHeader + 70); + if (x86) { + bytes.writeUInt32LE(0x10_0000, optionalHeader + 72); + bytes.writeUInt32LE(0x1000, optionalHeader + 76); + bytes.writeUInt32LE(0x10_0000, optionalHeader + 80); + bytes.writeUInt32LE(0x1000, optionalHeader + 84); + bytes.writeUInt32LE(16, optionalHeader + 92); + } else { + bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72); + bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80); + bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88); + bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96); + bytes.writeUInt32LE(16, optionalHeader + 108); + } + + bytes.write(".text", sectionTable, "ascii"); + bytes.writeUInt32LE(dll ? 0x180 : 0x16, sectionTable + 8); + bytes.writeUInt32LE(0x1000, sectionTable + 12); + bytes.writeUInt32LE(0x200, sectionTable + 16); + bytes.writeUInt32LE(0x200, sectionTable + 20); + bytes.writeUInt32LE(0x6000_0020, sectionTable + 36); + + // EXE entry returns zero. DLL caller adds one to the callee's 42. + bytes.set( + dll + ? [0xe8, 0x0b, 0, 0, 0, 0x83, 0xc0, 1, 0xc3] + : [0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], + 0x200, + ); + bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210); + + if (dll) { + // Export both functions; an entry-point-free library is discovered by exports. + bytes.writeUInt32LE(0x1040, directories); + bytes.writeUInt32LE(0x130, directories + 4); + const exports = 0x240; + bytes.writeUInt32LE(0x1100, exports + 12); + bytes.writeUInt32LE(1, exports + 16); + bytes.writeUInt32LE(2, exports + 20); + bytes.writeUInt32LE(2, exports + 24); + bytes.writeUInt32LE(0x1080, exports + 28); + bytes.writeUInt32LE(0x1088, exports + 32); + bytes.writeUInt32LE(0x1090, exports + 36); + bytes.writeUInt32LE(0x1000, 0x280); + bytes.writeUInt32LE(0x1010, 0x284); + bytes.writeUInt32LE(0x1120, 0x288); + bytes.writeUInt32LE(0x1140, 0x28c); + bytes.writeUInt16LE(0, 0x290); + bytes.writeUInt16LE(1, 0x292); + bytes.write("rea-fixture.dll\0", 0x300, "binary"); + bytes.write("fixture_caller\0", 0x320, "binary"); + bytes.write("fixture_return42\0", 0x340, "binary"); + } + + await mkdir(dirname(output), { recursive: true }); + await writeFile(output, bytes); + if (!x86 && !dll) process.stdout.write(`${output}\n`); } - - bytes.write(".text", sectionTable, "ascii"); - bytes.writeUInt32LE(0x16, sectionTable + 8); - bytes.writeUInt32LE(0x1000, sectionTable + 12); - bytes.writeUInt32LE(0x200, sectionTable + 16); - bytes.writeUInt32LE(0x200, sectionTable + 20); - bytes.writeUInt32LE(0x6000_0020, sectionTable + 36); - - // entry calls the second function, returns zero; second returns 42. - bytes.set([0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], 0x200); - bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210); - - await mkdir(dirname(output), { recursive: true }); - await writeFile(output, bytes); - if (!x86) process.stdout.write(`${output}\n`); } diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index 17d29c4ad..268d949d9 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -15,7 +15,7 @@ import { import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { promisify } from "node:util"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; @@ -24,10 +24,43 @@ if (process.platform !== "win32" || process.arch !== "x64") throw new Error( "Packaged Windows Ghidra acceptance requires a Windows x64 host.", ); -const workspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-")); const exec = promisify(execFile); -const x86Fixture = process.argv[2] === "--x86"; -const packageRootArgument = x86Fixture ? undefined : process.argv[2]; +const arguments_ = process.argv.slice(2); +// Windows locks the loaded package-owned addon until the verifier exits. +// Keep installation cleanup in a parent that never loads that addon. +if (arguments_[0] !== "--internal-workspace") { + const ownedWorkspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-")); + try { + const result = await exec( + process.execPath, + [ + fileURLToPath(import.meta.url), + "--internal-workspace", + ownedWorkspace, + ...arguments_, + ], + { timeout: 900_000, maxBuffer: 8 * 1024 * 1024 }, + ); + process.stdout.write(result.stdout); + process.stderr.write(result.stderr); + } catch (cause) { + if (typeof cause.stdout === "string") process.stdout.write(cause.stdout); + if (typeof cause.stderr === "string") process.stderr.write(cause.stderr); + throw cause; + } finally { + await rm(ownedWorkspace, { recursive: true, force: true }); + } + process.exit(0); +} +assert.ok(arguments_[1] !== undefined, "Missing verifier workspace."); +const workspace = resolve(arguments_[1]); +const fixtureArguments = arguments_.slice(2); +const x86Fixture = fixtureArguments.includes("--x86"); +const dllFixture = fixtureArguments.includes("--dll"); +const [packageRootArgument, targetArgument, reportArgument] = + fixtureArguments.filter( + (argument) => argument !== "--x86" && argument !== "--dll", + ); let packageRoot = resolve(packageRootArgument ?? "."); // The default lane verifies the npm artifact in an isolated prefix. Explicit // package roots support an already-installed artifact without a second install. @@ -75,17 +108,17 @@ if (packageRootArgument === undefined) { ); packageRoot = join(prefix, "node_modules", "rea-agents"); } catch (cause) { - await rm(workspace, { recursive: true, force: true }); - throw cause; + throw new Error("Packaged Windows verifier installation failed.", { + cause, + }); } } const target = resolve( - process.argv[3] ?? - (x86Fixture - ? "build/fixtures/rea-ghidra-windows-x86.exe" - : "build/fixtures/rea-ghidra-windows.exe"), + targetArgument ?? + `build/fixtures/rea-ghidra-windows${x86Fixture ? "-x86" : ""}.${dllFixture ? "dll" : "exe"}`, ); let TOOL_CONTRACTS, native, token, sha256; +let parsedTarget; try { ({ TOOL_CONTRACTS } = await import( pathToFileURL(join(packageRoot, "dist/contracts/toolContracts.js")) @@ -107,9 +140,25 @@ try { ); const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); sha256 = digest(await readFile(target)); + const { parseBinaryTarget } = await import( + pathToFileURL(join(packageRoot, "dist/application/BinaryTargetResolver.js")) + ); + parsedTarget = await parseBinaryTarget(target); + assert.equal(parsedTarget.ok, true); + assert.equal(parsedTarget.value.format, "pe"); + assert.equal(parsedTarget.value.managed, false); + if (targetArgument === undefined) { + assert.equal( + parsedTarget.value.architecture, + x86Fixture ? "x86" : "x86_64", + ); + assert.equal( + parsedTarget.value.executableRole, + dllFixture ? "shared-library" : "application", + ); + } } catch (cause) { - await rm(workspace, { recursive: true, force: true }); - throw cause; + throw new Error("Packaged Windows verifier prerequisites failed.", { cause }); } const digest = (bytes) => createHash("sha256").update(bytes).digest("hex"); const runtimeParent = join(workspace, "runtime with spaces"); @@ -145,6 +194,8 @@ const report = { architecture: "x64", nonAdmin: true, targetSha256: sha256, + targetArchitecture: parsedTarget.value.architecture, + targetRole: parsedTarget.value.executableRole, cli: {}, mcpOperations: [], nativeControls: "available", @@ -208,8 +259,8 @@ try { }); const call = async (name, arguments_) => { report.lastOperation = name; - if (process.argv[4] !== undefined) - await writeFile(process.argv[4], `${JSON.stringify(report)}\n`); + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); const contract = TOOL_CONTRACTS.find((value) => value.name === name); assert.ok(contract !== undefined, `Missing canonical contract: ${name}`); const argumentsParsed = contract.inputSchema.parse(arguments_); @@ -230,6 +281,25 @@ try { await call("list_names", {}); const procedures = await call("list_procedures", {}); assert.ok(procedures.length > 0); + if (targetArgument === undefined) { + const calleeAddress = x86Fixture ? "0x401010" : "0x140001010"; + const calleeCode = await call("procedure_pseudo_code", { + procedure: calleeAddress, + }); + assert.match(calleeCode, /\b(?:42|0x2a)\b/iu); + if (dllFixture) { + const callerAddress = x86Fixture ? "0x401000" : "0x140001000"; + assert.ok( + ( + await call("procedure_callees", { procedure: callerAddress }) + ).includes(calleeAddress), + ); + assert.match( + await call("procedure_pseudo_code", { procedure: callerAddress }), + /\+\s*1\b/u, + ); + } + } assert.ok((await call("list_segments", {})).length > 0); await call("list_strings", {}); const procedure = @@ -296,6 +366,12 @@ try { "Packaged verification omitted an admitted operation.", ); const session = await call("binary_session", {}); + assert.equal( + session.analysis_provider_binding?.analysis_profile?.parameters + ?.executable_role, + parsedTarget.value.executableRole, + "MCP profile must preserve the selected PE role.", + ); assert.equal( session.analysis_provider_binding?.selection_source, "auto-single-candidate", @@ -327,8 +403,8 @@ try { report.callerScriptCollisionsPreserved = true; report.runtimeCleanup = true; report.ok = true; - if (process.argv[4] !== undefined) - await writeFile(process.argv[4], `${JSON.stringify(report)}\n`); + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); process.stdout.write(`${JSON.stringify(report)}\n`); } catch (cause) { report.failure = { @@ -338,9 +414,7 @@ try { .replaceAll(target, "") .replaceAll(workspace, ""), }; - if (process.argv[4] !== undefined) - await writeFile(process.argv[4], `${JSON.stringify(report)}\n`); + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); throw cause; -} finally { - await rm(workspace, { recursive: true, force: true }); } diff --git a/scripts/verify-real-ghidra-windows.mjs b/scripts/verify-real-ghidra-windows.mjs index 1c5b3b829..3332ea4e2 100644 --- a/scripts/verify-real-ghidra-windows.mjs +++ b/scripts/verify-real-ghidra-windows.mjs @@ -62,21 +62,21 @@ if ( ); const root = resolve(fileURLToPath(new URL("..", import.meta.url))); -const architecture = process.argv[2] === "--x86" ? "x86" : "x86_64"; +const architecture = process.argv.includes("--x86") ? "x86" : "x86_64"; +const dllFixture = process.argv.includes("--dll"); +const role = dllFixture ? "shared-library" : "application"; const targetPath = resolve( root, "build", "fixtures", - architecture === "x86" - ? "rea-ghidra-windows-x86.exe" - : "rea-ghidra-windows.exe", + `rea-ghidra-windows${architecture === "x86" ? "-x86" : ""}.${dllFixture ? "dll" : "exe"}`, ); const target = await parseBinaryTarget(targetPath); if ( !target.ok || target.value.format !== "pe" || target.value.architecture !== architecture || - target.value.executableRole !== "application" || + target.value.executableRole !== role || target.value.managed !== false ) throw new Error( @@ -89,6 +89,8 @@ const profile = await resolveGhidraAnalysisProfile( ); if (!profile.ok || profile.value.profile === null) throw new Error("Windows Ghidra profile could not be committed"); +if (profile.value.profile.parameters.executable_role !== role) + throw new Error("Windows Ghidra profile lost the observed PE role"); const client = new GhidraClient({ platform: installation.platform, @@ -162,6 +164,19 @@ try { throw new Error( "Windows fixture decompilation lost its known return value", ); + if (dllFixture) { + const callerAddress = architecture === "x86" ? "0x401000" : "0x140001000"; + const callees = await functionOperation("procedure_callees", { + procedure: callerAddress, + }); + if (!callees.includes(callee.address)) + throw new Error("DLL export caller lost its controlled call reference"); + const callerCode = await functionOperation("procedure_pseudo_code", { + procedure: callerAddress, + }); + if (typeof callerCode !== "string" || !/\+\s*1\b/u.test(callerCode)) + throw new Error("DLL export caller lost its addition of one"); + } await inventory("inspect_native_load_image", {}); const memory = await inventory("read_bytes", { @@ -254,7 +269,7 @@ try { limitations: [ "approved-non-sensitive-fixtures-only", "windows-x64-local-ntfs-only", - "native-x86-and-x86-64-pe-applications-only", + "native-x86-and-x86-64-pe-applications-and-dlls-only", "no-gui-or-mutation-authority", ], }; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 1af7469fa..eeb8646a4 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -34,6 +34,12 @@ export const resolveGhidraAnalysisProfile = ( profile: createAnalysisProfile(provider, { target_kind: target.kind, target_format: target.format, + ...(target.format === "pe" + ? { + executable_role: target.executableRole ?? null, + managed: target.managed ?? null, + } + : {}), architecture: target.architecture ?? null, available_architectures: [ ...(target.availableArchitectures ?? []), diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 7bbbc8c89..88e1eb07f 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -270,7 +270,7 @@ describe("Ghidra platform support", () => { expect( ghidra.inspectTargetSupport({ ...nativeApplication, - executableRole: "shared-library", + executableRole: "non-executable", }), ).toMatchObject({ status: "unsupported", @@ -295,45 +295,59 @@ describe("Ghidra platform support", () => { code: "architecture_unsupported", }); }); - it("admits generated PE32 applications while rejecting unsupported x86 roles", async () => { +}); + +describe("Ghidra Windows PE fixture admission", () => { + it("admits generated PE32 and PE32+ DLLs and applications while preserving native-only Windows admission", async () => { const root = fileURLToPath(new URL("../../", import.meta.url)); await promisify(execFile)(process.execPath, [ join(root, "scripts/create-ghidra-windows-fixture.mjs"), ]); - const path = join(root, "build/fixtures/rea-ghidra-windows-x86.exe"); - const bytes = await readFile(path); - const metadata = parseExecutableHeader(bytes, "x64"); - expect(metadata).toMatchObject({ - ok: true, - value: { - format: "pe", - architecture: "x86", - executableRole: "application", - managed: false, - }, - }); - if (!metadata.ok) throw new Error(metadata.error); - if (metadata.value.format !== "pe") - throw new Error("Expected a PE fixture"); - const target: BinaryTarget = { - path, - sha256: createHash("sha256").update(bytes).digest("hex"), - kind: "executable", - ...metadata.value, - }; const ghidra = provider({ ...installationHost(), platform: "win32" }); - expect(ghidra.inspectTargetSupport(target).status).toBe("supported"); - for (const [candidate, code] of [ - [ - { ...target, executableRole: "shared-library" }, - "target_role_unsupported", - ], - [{ ...target, managed: true }, "managed_target_unsupported"], - ] as const) - expect(ghidra.inspectTargetSupport(candidate)).toMatchObject({ - status: "unsupported", - code, - }); + for (const architecture of ["x86", "x86_64"] as const) { + for (const dll of [false, true]) { + const path = join( + root, + "build/fixtures", + `rea-ghidra-windows${architecture === "x86" ? "-x86" : ""}.${dll ? "dll" : "exe"}`, + ); + const bytes = await readFile(path); + const metadata = parseExecutableHeader(bytes, "x64"); + const role = dll ? "shared-library" : "application"; + expect(metadata).toMatchObject({ + ok: true, + value: { + format: "pe", + architecture, + executableRole: role, + managed: false, + }, + }); + if (!metadata.ok) throw new Error(metadata.error); + if (metadata.value.format !== "pe") + throw new Error("Expected a PE fixture"); + const target: BinaryTarget = { + path, + sha256: createHash("sha256").update(bytes).digest("hex"), + kind: "executable", + ...metadata.value, + }; + expect(ghidra.inspectTargetSupport(target).status).toBe("supported"); + for (const [candidate, code] of [ + [ + { ...target, executableRole: "non-executable" }, + "target_role_unsupported", + ], + [{ ...target, managed: true }, "managed_target_unsupported"], + [{ ...target, architecture: "arm64" }, "architecture_unsupported"], + ] as const) { + expect(ghidra.inspectTargetSupport(candidate)).toMatchObject({ + status: "unsupported", + code, + }); + } + } + } }); }); diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index d657a156e..d782827d9 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -247,12 +247,15 @@ const inspectWindowsP0TargetSupport = ( reason: "Windows Ghidra P0 accepts x86 and x86-64 PE targets only.", diagnostics, }; - if (target.executableRole !== "application") + if ( + target.executableRole !== "application" && + target.executableRole !== "shared-library" + ) return { status: "unsupported", code: "target_role_unsupported", reason: - "Windows Ghidra P0 accepts PE applications, not DLL or non-executable images.", + "Windows Ghidra P0 accepts PE applications and DLLs, not non-executable or unclassified images.", diagnostics, }; if (target.managed !== false) @@ -260,7 +263,7 @@ const inspectWindowsP0TargetSupport = ( status: "unsupported", code: "managed_target_unsupported", reason: - "Windows Ghidra P0 accepts native PE applications; managed or unclassified PE targets are unsupported.", + "Windows Ghidra P0 accepts native PE applications and DLLs; managed or unclassified PE targets are unsupported.", diagnostics, }; return { diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index edbbb11c3..5b9a6ca8b 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -31,7 +31,7 @@ export const healthLimitations = Object.freeze([ /** Additional limitations applied to the experimental Windows x64 P0 boundary. */ export const windowsP0Limitations = Object.freeze([ - "Windows Ghidra P0 accepts approved native x86 and x86-64 PE applications only; DLL, managed, hostile, sensitive, and mutable-path targets are unsupported.", + "Windows Ghidra P0 accepts approved native x86 and x86-64 PE applications and DLLs; managed, hostile, sensitive, and mutable-path targets are unsupported.", "The Windows bridge uses authenticated IPv4 loopback because Node path-based IPC does not expose Java AF_UNIX sockets; the endpoint file contains no bearer token.", "Windows sessions require the matching packaged Windows x64 native addon, local NTFS targets and runtimes, and Windows 10 or later. Native handles enforce path admission, private DACLs, and Job Object ownership automatically.", ]); From e10c934d5a756c1a4c7b547ebd02c403ddfde7b8 Mon Sep 17 00:00:00 2001 From: GRAB Racem Date: Sat, 10 Oct 2026 01:36:11 +0200 Subject: [PATCH 02/27] fix(ghidra): keep the cross-format branch callee multi-block The cross-format oracle requires a callee whose recovered control flow has more than one basic block, and rea_cross_branch provided it through a symmetric if/else of tail calls. Clang 21 folds that shape into a conditional negate plus one tail branch, a single basic block, so the lane rejected a correct Ghidra analysis of the fixture. Give the branch a loop whose trip count depends on a volatile global: no compiler can eliminate the backedge, and the callee stays multi-block at every optimization level on every toolchain. --- tests/conformance/ghidra/cross-format.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/conformance/ghidra/cross-format.c b/tests/conformance/ghidra/cross-format.c index f718b9b54..22efa6159 100644 --- a/tests/conformance/ghidra/cross-format.c +++ b/tests/conformance/ghidra/cross-format.c @@ -8,10 +8,15 @@ __attribute__((noinline, used)) int rea_cross_leaf(int value) { } __attribute__((noinline, used)) int rea_cross_branch(int value) { - if (value > 10) { - return rea_cross_leaf(value); + /* A volatile-dependent loop keeps this callee multi-block at every + optimization level: newer clang folds a symmetric if/else of tail calls + into a conditional negate, and cross-format verification requires a + recovered multi-block callee. */ + int adjusted = value > 10 ? value + 1 : -value; + while (adjusted > 100) { + adjusted -= rea_cross_global; } - return rea_cross_leaf(-value); + return rea_cross_leaf(adjusted); } __attribute__((noinline, used)) int rea_cross_indirect( From 40f632a155bff2defdea09e467229c703bc0b0e9 Mon Sep 17 00:00:00 2001 From: GRAB Racem Date: Sat, 10 Oct 2026 02:06:15 +0200 Subject: [PATCH 03/27] ci: re-run the suite after the main-branch shard flake The failed shard is the JavaScriptSemanticGraphBuilder short-circuit mutation test, which also failed on main run 38003459598 at the same base commit (08ec2bf8) before this branch started CI, and passed on main run 38003459600 minutes later at the same commit. From e17290cd2a17931d7a2af70f74b76b94276bd788 Mon Sep 17 00:00:00 2001 From: GRAB Racem Date: Sat, 10 Oct 2026 02:34:52 +0200 Subject: [PATCH 04/27] ci: re-run the suite on the corrected main checkpoint b8a8190b restored the guarded-mutation uncertainty the failing test encodes, so the shard should pass against the new merge base. From e4a8e0b728154fe0bf181684ccd0d73df45cf4bf Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 10 Oct 2026 00:44:32 +0000 Subject: [PATCH 05/27] fix(javascript): recognize Electron APIs through renamed bindings Electron recognition matched callee spellings, so a bundler rename such as esbuild's `import { ipcMain as ipcMain2 } from "electron"`, or a destructured `const { contextBridge: bridge } = require("electron")`, hid every IPC handler, BrowserWindow, contextBridge and utility process behind it. Resolve each call and construction root lexically to its Electron import or require origin, and classify the callee under that export's name. Shadowed locals keep their own meaning. On Hermes Desktop v0.21.6, whose main process is one esbuild ESM bundle, recognized main IPC handlers rise from 1 to 204, BrowserWindow constructions from 0 to 11, and paired renderer transmissions from 1 to 201. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01XPY2GmufMNGQKoEgRUx1pd --- .../electronStaticAnalysisBrowser.ts | 8 +- .../javascript/electronStaticAnalysisIpc.ts | 3 +- .../electronStaticAnalysisValues.ts | 21 ++++- .../javascript/javascriptSemanticAnalysis.ts | 79 +++++++++++++++++++ .../javascript/javascriptStaticAnalysis.ts | 27 +++---- .../javascriptStaticAnalysisState.ts | 2 + .../electronApplicationAnalysis.test.ts | 66 ++++++++++++++++ 7 files changed, 186 insertions(+), 20 deletions(-) diff --git a/src/domain/javascript/electronStaticAnalysisBrowser.ts b/src/domain/javascript/electronStaticAnalysisBrowser.ts index f29e7cf8f..a44baf89a 100644 --- a/src/domain/javascript/electronStaticAnalysisBrowser.ts +++ b/src/domain/javascript/electronStaticAnalysisBrowser.ts @@ -12,6 +12,7 @@ import type { import { boundedExpression, collectContextBridgeMembers, + electronCalleeName, electronStaticValue, objectProperty, } from "./electronStaticAnalysisValues.js"; @@ -19,7 +20,6 @@ import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; import { addLocatedFinding } from "./javascriptStaticAnalysisFindings.js"; import { argumentNode, - calleeName, range, staticPath, staticPathResolutionContext, @@ -42,7 +42,7 @@ const inspectBrowserWindow = ( node: t.NewExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); if (name !== "BrowserWindow" && !name.endsWith(".BrowserWindow")) return; const options = argumentNode(node.arguments[0]); const collected = collectWindowOptions(context.source, options); @@ -165,7 +165,7 @@ const inspectContextBridge = ( node: t.CallExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); const main = name === "contextBridge.exposeInMainWorld" || name.endsWith(".contextBridge.exposeInMainWorld"); @@ -213,7 +213,7 @@ const inspectUtilityProcess = ( node: t.CallExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); if (name !== "utilityProcess.fork" && !name.endsWith(".utilityProcess.fork")) return; const moduleNode = argumentNode(node.arguments[0]); diff --git a/src/domain/javascript/electronStaticAnalysisIpc.ts b/src/domain/javascript/electronStaticAnalysisIpc.ts index 9efc2670b..a560ea184 100644 --- a/src/domain/javascript/electronStaticAnalysisIpc.ts +++ b/src/domain/javascript/electronStaticAnalysisIpc.ts @@ -7,6 +7,7 @@ import type { } from "./electronStaticAnalysisTypes.js"; import { boundedExpression, + electronCalleeName, electronStaticValue, handlerKind, } from "./electronStaticAnalysisValues.js"; @@ -36,7 +37,7 @@ const inspectIpcCall = ( node: t.CallExpression, context: JavaScriptFindingContext, ): void => { - const name = calleeName(node.callee); + const name = electronCalleeName(node.callee, context); const descriptor = ipcDescriptor(name); if (descriptor === undefined) return; const channelNode = argumentNode(node.arguments[0]); diff --git a/src/domain/javascript/electronStaticAnalysisValues.ts b/src/domain/javascript/electronStaticAnalysisValues.ts index 2a800e6bb..c2bb6e76a 100644 --- a/src/domain/javascript/electronStaticAnalysisValues.ts +++ b/src/domain/javascript/electronStaticAnalysisValues.ts @@ -6,7 +6,26 @@ import { readExactJavaScriptLiteral, semanticStaticPropertyName, } from "./javascriptAstValues.js"; -import { sourceSlice } from "./javascriptStaticAnalysisHelpers.js"; +import { calleeName, sourceSlice } from "./javascriptStaticAnalysisHelpers.js"; +import type { JavaScriptFindingContext } from "./javascriptStaticAnalysisState.js"; + +/** Name a callee, spelling a root renamed from Electron as its export. */ +export const electronCalleeName = ( + callee: t.Node, + context: JavaScriptFindingContext, +): string => { + const name = calleeName(callee); + let root = callee; + while (t.isMemberExpression(root) || t.isOptionalMemberExpression(root)) { + if (!t.isNode(root.object)) return name; + root = root.object; + } + if (!t.isIdentifier(root)) return name; + const exported = context.electronBindings?.get(root.start ?? -1); + return exported === undefined + ? name + : `${exported}${name.slice(root.name.length)}`; +}; /** Preserve one literal value or the exact inert expression. */ export const electronStaticValue = ( diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 5a873429e..b5c1c7f5d 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -29,6 +29,7 @@ import { semanticVariableScope, } from "./javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { semanticRequireOrigin } from "./javascriptSemanticRequireOrigin.js"; import { collectSemanticMemberMutations } from "./javascriptSemanticMemberMutations.js"; import { collectSemanticReturns, @@ -282,6 +283,84 @@ const resolveGlobalAliasFact = ( return openGlobalFact(name); }; +const ELECTRON_MODULE = /^electron(?:\/(?:common|main|renderer|utility))?$/u; + +/** + * Electron export paths for call and construction roots bound to Electron + * under another name, such as esbuild's `import { ipcMain as ipcMain2 }`. + * Keys are root identifier offsets; roots spelled as their export are omitted. + */ +export const classifyParsedJavaScriptElectronBindings = ( + file: ParsedJavaScriptSource, +): ReadonlyMap => { + const facts = new Map(); + const state = createState(file.program); + collectDefinitions(file.program, state); + const exportsByBinding = new Map(); + traverseJavaScriptAst(file.program, { + enter: (node) => { + if ( + !t.isCallExpression(node) && + !t.isOptionalCallExpression(node) && + !t.isNewExpression(node) + ) + return; + const root = calleeRoot(node.callee); + if (root === undefined) return; + const binding = resolveSemanticBindingState(state, root, root.name); + if (binding === undefined) return; + let exported = exportsByBinding.get(binding.bindingId); + if (exported === undefined) { + exported = electronExport(binding, state); + exportsByBinding.set(binding.bindingId, exported); + } + if (exported !== null && exported !== root.name) + facts.set(root.start ?? -1, exported); + }, + }); + return facts; +}; + +const calleeRoot = (node: t.Node): t.Identifier | undefined => { + while (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { + if (!t.isNode(node.object)) return undefined; + node = node.object; + } + return t.isIdentifier(node) ? node : undefined; +}; + +// Read import and require origins directly; general value evaluation is +// unnecessary here and costly across large vendor bundles. +const electronExport = ( + binding: JavaScriptSemanticBindingState, + state: JavaScriptSemanticAnalysisState, +): string | null => { + const origin = bindingOrigin(binding, state); + return origin !== undefined && + ELECTRON_MODULE.test(origin.specifier) && + origin.importedPath.length > 0 + ? origin.importedPath.join(".") + : null; +}; + +const bindingOrigin = ( + binding: JavaScriptSemanticBindingState, + state: JavaScriptSemanticAnalysisState, +): JavaScriptModuleOrigin | undefined => { + if (binding.directOrigins.length > 0) + return binding.directOrigins.length === 1 + ? binding.directOrigins[0] + : undefined; + const [initializer] = binding.initializers; + if (binding.mutable || binding.initializers.length !== 1) return undefined; + const origin = semanticRequireOrigin(initializer?.node, state); + const projection = initializer?.projection ?? []; + return origin === undefined || + !projection.every((key): key is string => typeof key === "string") + ? undefined + : { ...origin, importedPath: [...origin.importedPath, ...projection] }; +}; + const createState = (program: t.Program): JavaScriptSemanticAnalysisState => { const root: JavaScriptSemanticScopeState = { scopeId: semanticScopeId("program", program), diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index af20eaa92..524af811e 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -25,9 +25,11 @@ import { finalizeLocatedFindings } from "./javascriptStaticAnalysisFindings.js"; import { createJavaScriptAnalysisAccumulator, type JavaScriptAnalysisAccumulator as AnalysisAccumulator, + type JavaScriptFindingContext, } from "./javascriptStaticAnalysisState.js"; import type { JavaScriptStaticAnalysis } from "./javascriptStaticAnalysisTypes.js"; import { + classifyParsedJavaScriptElectronBindings, classifyParsedJavaScriptOpenReceivers, type JavaScriptOpenReceiverFact, } from "./javascriptSemanticAnalysis.js"; @@ -57,21 +59,24 @@ export const analyzeParsedJavaScriptStaticSource = ( openReceiverFacts?: ReadonlyMap, ): JavaScriptStaticAnalysis => { const accumulator = createJavaScriptAnalysisAccumulator(source.length); - traverseStaticSource(source, file, accumulator, openReceiverFacts); + traverseStaticSource(file, { + source, + accumulator, + ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), + electronBindings: classifyParsedJavaScriptElectronBindings(file), + }); addSourceMapDirectives(source, file.comments ?? [], accumulator); return finalizeStaticAnalysis(source, file, accumulator); }; const traverseStaticSource = ( - source: string, file: ParsedJavaScriptSource, - accumulator: AnalysisAccumulator, - openReceiverFacts?: ReadonlyMap, + findings: JavaScriptFindingContext, ): void => { traverseJavaScriptAst(file, { enter: (node) => { - accumulator.visitedNodes += 1; - inspectNode(source, node, accumulator, openReceiverFacts); + findings.accumulator.visitedNodes += 1; + inspectNode(node, findings); return undefined; }, }); @@ -156,16 +161,10 @@ const finalizeStaticAnalysis = ( }; const inspectNode = ( - source: string, node: t.Node, - accumulator: AnalysisAccumulator, - openReceiverFacts?: ReadonlyMap, + findings: JavaScriptFindingContext, ): void => { - const findings = { - source, - accumulator, - ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), - }; + const { source, accumulator } = findings; inspectElectronStaticNode(node, findings); if (t.isCallExpression(node)) { inspectBundlerRegistration(source, node, accumulator); diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 23de6952b..47e25bccd 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -65,6 +65,8 @@ export interface JavaScriptFindingContext { | "local-indexed-db" | "local-cache-storage" >; + /** Electron export paths for renamed call roots, by identifier offset. */ + readonly electronBindings?: ReadonlyMap; } /** Candidate import, require, worker, or service-worker reference. */ diff --git a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts index 3484f98a2..4180e1818 100644 --- a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts +++ b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts @@ -89,6 +89,33 @@ describe("static Electron application analysis", () => { expect(requestedMembers).toEqual(["*"]); }); + it("maps Electron boundaries through bundler-renamed bindings", async () => { + const root = await renamedBindingFixtureDirectory(); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const graph = parseJavaScriptApplicationGraph(result.graph); + const handledChannels = graph.nodes.flatMap(({ kind, observations }) => + kind === "ipc-handler" + ? observations.flatMap(({ properties }) => + properties.side === "main" ? [properties.channel] : [], + ) + : [], + ); + + expect(result.electron_summary).toMatchObject({ + browser_windows: 1, + context_bridge_apis: 1, + exposed_api_members: 2, + ipc: { + main_handlers: 2, + paired_renderer_transmissions: 2, + unpaired_literal_renderer_transmissions: 0, + }, + utility_processes: 1, + }); + expect(handledChannels.sort()).toEqual(["rea:read", "rea:write"]); + }); + it("returns a tagged cancellation without executing application code", async () => { const root = await fixtureDirectory(); const controller = new AbortController(); @@ -299,6 +326,45 @@ const fixtureDirectory = async (): Promise => { return root; }; +const renamedBindingFixtureDirectory = async (): Promise => { + const root = await createTestTempDirectory("rea-electron-renamed-"); + await Promise.all([ + writeFile( + join(root, "package.json"), + JSON.stringify({ name: "rea-electron-renamed", main: "main.mjs" }), + ), + // Bundlers rename colliding imports, e.g. esbuild's ipcMain2. + writeFile( + join(root, "main.mjs"), + String.raw` +import { BrowserWindow as BrowserWindow2, ipcMain as ipcMain2 } from "electron"; +import { ipcMain as ipcMain3, utilityProcess as utility } from "electron/main"; + +new BrowserWindow2({ webPreferences: { preload: "./preload.cjs" } }); +ipcMain2.handle("rea:read", async () => "value"); +ipcMain3.on("rea:write", () => undefined); +utility.fork("./worker.js"); +function register(ipcMain2) { + ipcMain2.handle("rea:shadowed", async () => "local"); +} +export { register }; +`, + ), + writeFile( + join(root, "preload.cjs"), + String.raw` +const { contextBridge: bridge, ipcRenderer: ipc } = require("electron"); +bridge.exposeInMainWorld("reaApi", { + read: () => ipc.invoke("rea:read"), + write: (value) => ipc.send("rea:write", value), +}); +`, + ), + writeFile(join(root, "worker.js"), "module.exports = {};"), + ]); + return root; +}; + const computedMemberFixtureDirectory = async (): Promise => { const root = await createTestTempDirectory("rea-electron-computed-members-"); await mkdir(join(root, "native")); From c29931ce81dc530d2623ec992714a70b1f3e50e4 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:25:00 -0400 Subject: [PATCH 06/27] fix(javascript): scan direct returns without recursive stack growth Use the shared AST child enumeration with an explicit stack, preserving return order and nested callable boundaries. Refs #1065. Co-authored-by: rudycelekli <47457359+rudycelekli@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- .../javascript/javascriptSemanticReturns.ts | 14 +++--- .../javascriptApplicationDeepReturns.test.ts | 44 +++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) create mode 100644 tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts diff --git a/src/domain/javascript/javascriptSemanticReturns.ts b/src/domain/javascript/javascriptSemanticReturns.ts index bb931fba5..44cbbea5b 100644 --- a/src/domain/javascript/javascriptSemanticReturns.ts +++ b/src/domain/javascript/javascriptSemanticReturns.ts @@ -131,15 +131,17 @@ const directReturnExpressions = (callable: t.Node): ReturnExpression[] => { return [{ node: callable.body, location: range(callable.body) }]; if (!t.isFunction(callable)) return []; const output: ReturnExpression[] = []; - const visit = (node: t.Node): void => { - if (t.isFunction(node) || t.isClass(node)) return; + const pending = childNodes(callable.body).reverse(); + while (pending.length > 0) { + const node = pending.pop(); + if (node === undefined || t.isFunction(node) || t.isClass(node)) continue; if (t.isReturnStatement(node)) { output.push({ node: node.argument ?? null, location: range(node) }); - return; + continue; } - for (const child of childNodes(node)) visit(child); - }; - for (const child of childNodes(callable.body)) visit(child); + // Reverse children so the explicit stack retains source-tree order. + for (const child of childNodes(node).reverse()) pending.push(child); + } return output; }; diff --git a/tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts b/tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts new file mode 100644 index 000000000..32a2d99af --- /dev/null +++ b/tests/boundary/filesystem/javascriptApplicationDeepReturns.test.ts @@ -0,0 +1,44 @@ +import { execFile } from "node:child_process"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("analyzes direct returns after deep source without exhausting the default process stack", async () => { + const root = await createTestTempDirectory("rea-deep-direct-returns-"); + await writeFile( + join(root, "app.js"), + `export function render() { object${".property".repeat(20_000)}; function nested() { return "nested"; } class Inner { method() { return "method"; } } return "ready"; }`, + ); + const service = new URL( + "../../../dist/application/javascript/JavaScriptApplicationService.js", + import.meta.url, + ).href; + const script = ` + import { analyzeJavaScriptApplication } from ${JSON.stringify(service)}; + const result = await analyzeJavaScriptApplication({ input_path: ${JSON.stringify(root)}, format: "directory" }); + if (!result.ok) throw result.error; + const nodes = result.value.normalized_result.semantic_graph.nodes; + console.log(JSON.stringify(nodes.filter(node => node.kind === "return-site").map(node => node.function_node_id))); + `; + const { stdout, stderr } = await promisify(execFile)( + process.execPath, + ["--input-type=module", "-e", script], + { + timeout: 10_000, + // Bound the child heap while retaining Node's default stack size. + env: { + HOME: root, + TMPDIR: root, + NODE_OPTIONS: "--max-old-space-size=768", + }, + }, + ); + expect(stderr).toBe(""); + const owners: unknown = JSON.parse(stdout); + expect(owners).toHaveLength(3); + expect(new Set(Array.isArray(owners) ? owners : []).size).toBe(3); +}); From 6375ccaf04b4835cc67080ee1ecd91d1fc46645c Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:35:50 -0700 Subject: [PATCH 07/27] perf(native): record macOS command provenance once in its Evidence (#1430) inspect_macho, inspect_signature, inspect_plist, list_architectures and demangle_swift already return their command provenance inside the normalized result. The provider also copied the same array into raw_result, so every response carried it twice. raw_result now stays null, as docs/mcp-contracts.md describes for results without a distinct provider representation. Co-authored-by: Claude Opus 5.5 --- src/native/NativeMacOSProvider.ts | 6 --- src/native/NativeMachoInspection.ts | 2 - .../providers/native/nativeProvider.test.ts | 37 +++++++++++++++++++ 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 12c94814f..d63db2506 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -233,7 +233,6 @@ class NativeMacOSClient implements AnalysisClient { return observation.ok ? ok( createAnalysisExecution(observation.value.result, IDENTITY, { - rawResult: { provenance: observation.value.provenance }, limitations: observation.value.limitations, locations: observation.value.locations, }), @@ -317,7 +316,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: [], locations: architectureLocations(result.architectures.items), }); @@ -351,7 +349,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: [], locations: [], }); @@ -448,7 +445,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: result.limitations, locations: [], }); @@ -611,7 +607,6 @@ class NativeMacOSClient implements AnalysisClient { }); return ok({ result: jsonValueSchema.parse(result), - provenance, limitations: parsed.value.limitations, locations: [{ kind: "artifact-path", path: plist.value }], }); @@ -659,7 +654,6 @@ class NativeMacOSClient implements AnalysisClient { interface NativeObservation { readonly result: JsonValue; - readonly provenance: readonly NativeCommandInvocation[]; readonly limitations: readonly string[]; readonly locations: readonly EvidenceLocation[]; } diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index 417b43fef..2ef2c95ae 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -15,7 +15,6 @@ import { parseOtoolLoadCommands } from "./parsers/otool.js"; interface NativeMachoObservation { readonly result: JsonValue; - readonly provenance: readonly NativeCommandInvocation[]; readonly limitations: readonly string[]; readonly locations: readonly EvidenceLocation[]; } @@ -116,7 +115,6 @@ export const inspectNativeMacho = async ( : normalized; return ok({ result: jsonValueSchema.parse(result), - provenance: result.provenance, limitations: result.limitations, locations: fileOffsetLocations(result, segmentOffset), }); diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index d5be920ba..70a3b06ad 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -552,6 +552,43 @@ describe("native macOS provider inspection", () => { }); }); +describe("native macOS provider raw results", () => { + it("records command provenance once, in the normalized result", async () => { + directory = await createTestTempDirectory("rea-native-"); + const app = join(directory, "Fixture.app"); + const executable = join(app, "Contents/MacOS/Fixture"); + await mkdir(join(app, "Contents/MacOS"), { recursive: true }); + await writeFile(executable, "fixture"); + await writeFile(join(app, "Contents/Info.plist"), "fixture"); + const client = new NativeMacOSProvider( + {}, + new FixtureRunner(), + "darwin", + ).createClient(await nativeMachoTargetForFile(executable, app)); + const requests: readonly (readonly [ + Parameters[0], + Parameters[1], + ])[] = [ + ["inspect_macho", {}], + ["list_architectures", {}], + ["inspect_signature", {}], + ["inspect_plist", {}], + ["demangle_swift", { symbols: ["$s4Test3fooyyF", "plain_symbol"] }], + ]; + + for (const [operation, parameters] of requests) { + const execution = await client.execute(operation, parameters); + if (!execution.ok) throw execution.error; + expect(execution.value.rawResult, operation).toBeNull(); + expect(execution.value.result, operation).toMatchObject({ + provenance: expect.arrayContaining([ + expect.objectContaining({ tool: expect.any(String) }), + ]), + }); + } + }); +}); + describe("native plist defaults for iOS-style bundles", () => { it("defaults to the Info.plist the bundle program was resolved from", async () => { directory = await createTestTempDirectory("rea-native-flat-"); From 0b9815b6e7080081d8fd426ad9710aee36761afa Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 21:38:52 -0700 Subject: [PATCH 08/27] docs(skill): save CLI application Evidence and read a summary view (#1429) The skill's CLI fallback printed the complete application Evidence to stdout. A real ASAR produces hundreds of megabytes (334 MB for Obsidian), which an agent then reads into its context. Save the Evidence to a file and project a summary or module page with inspect-analysis-view instead. Co-authored-by: Claude Opus 5.5 --- docs/cli.md | 17 +++++++++++++++++ skill-src/reverse-engineer-anything/SKILL.md | 19 +++++++++++++------ src/generatedPackageMetadata.ts | 2 +- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/cli.md b/docs/cli.md index 2a9863c1e..245251a28 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -20,6 +20,23 @@ files when neither `--provider` nor `--snapshot` is supplied. See [JavaScript artifact reconstruction](javascript-artifact-reconstruction.md) for results, integrity checks and coverage. +The complete application Evidence can be hundreds of megabytes for a real +application. When an agent or a person reads the result, save it to a file and +project a summary, a module page or one module from the saved Evidence: + +```bash +rea analyze-javascript-application /absolute/path/to/app --json > app-evidence.json +jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json +rea inspect-analysis-view app-view.json +``` + +For example, a 334 MB Obsidian application Evidence file yields a summary view +of about 10 KB. Replace the view with +`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}` for a +module page. See +[JavaScript application workflows](javascript-application-workflows.md) for +views, feature traces and comparisons that take the same saved Evidence. + ## Native analysis Configure [Hopper or Ghidra](installation.md#hopper), or the diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/skill-src/reverse-engineer-anything/SKILL.md index d1650f26e..3eaa239b8 100644 --- a/skill-src/reverse-engineer-anything/SKILL.md +++ b/skill-src/reverse-engineer-anything/SKILL.md @@ -2,7 +2,7 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: - version: "34" + version: "35" --- # REA @@ -62,13 +62,20 @@ with a version-pinned `rea-agents` package or continue through the CLI. A concrete CLI fallback for an operator-supplied JavaScript tree or ASAR is: ```bash -npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json +npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json > app-evidence.json +jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json +npx -y rea-agents@latest inspect-analysis-view app-view.json ``` -No MCP registration or native engine is required. Read the returned Evidence, -graph, limitations, and unknowns with the same care as an MCP result; the CLI -returns the Evidence record directly. This fallback does not establish that -MCP is configured. Native CLI tasks still require their selected engine. +No MCP registration or native engine is required. The CLI returns the complete +Evidence record directly, and a real application can produce hundreds of +megabytes, so save it to a file rather than reading it into context. Start from +the summary view, then request a module page +(`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}`) or one +module the same way, or pass the saved Evidence inline to +`trace-application-feature`. Read graph, limitations, and unknowns with the +same care as an MCP result. This fallback does not establish that MCP is +configured. Native CLI tasks still require their selected engine. ## Route the target first diff --git a/src/generatedPackageMetadata.ts b/src/generatedPackageMetadata.ts index 29177bf02..468117b05 100644 --- a/src/generatedPackageMetadata.ts +++ b/src/generatedPackageMetadata.ts @@ -5,5 +5,5 @@ export const PACKAGE_METADATA = { serverSdkVersion: "2.3.1", clientSdkVersion: "2.3.1", coreSdkVersion: "2.3.1", - skillVersion: "34", + skillVersion: "35", } as const; From dc5af6d68f74155d32a0be957c88afb0ee4be699 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:49:09 +0800 Subject: [PATCH 09/27] perf(mcp): reuse output validators across catalog refreshes (#1433) Advertise content- and dialect-bound identities for self-contained output schemas so the SDK can reuse unchanged Ajv validators after complete tools/list refreshes. Preserve explicit IDs and relative reference bases. Verify changed schemas, availability transitions, reconnects, numeric and selector constraints, explicit IDs, relative references, and advertised dialects through the pinned SDK boundaries. Refs #1065, #723 --- docs/mcp-contracts.md | 7 ++ src/contracts/toolOutputReferences.test.ts | 58 ++++++++++ src/contracts/toolSchemaMetadata.ts | 44 ++++++- .../boundary/mcp/outputValidatorReuse.test.ts | 108 ++++++++++++++++++ 4 files changed, 213 insertions(+), 4 deletions(-) create mode 100644 tests/boundary/mcp/outputValidatorReuse.test.ts diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 7bf2fb76c..70d843335 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -36,6 +36,13 @@ after SDK conversion and check advertised validation and actual calls. Individual model APIs can impose additional nesting limits; complete producer captures can exceed ten structural levels. +Self-contained output schemas advertise a content-bound `$id`, including their +declared dialect. SDK validators can reuse compiled schemas across complete +catalog refreshes and equivalent tool outputs; changing the schema changes its +identity. Explicit schema IDs and relative external reference bases are +preserved. This keeps every tool and validation rule in discovery and does not +change the SDK's catalog invalidation or availability checks. + `compare_web_captures` accepts exactly one of two input shapes: - Passive: `before` and `after` each contain `inspection`, the complete diff --git a/src/contracts/toolOutputReferences.test.ts b/src/contracts/toolOutputReferences.test.ts index 70c1f5ea5..cda49f1d5 100644 --- a/src/contracts/toolOutputReferences.test.ts +++ b/src/contracts/toolOutputReferences.test.ts @@ -1,11 +1,69 @@ import { Ajv } from "ajv"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { AjvJsonSchemaValidator } from "@modelcontextprotocol/client/validators/ajv"; import { expect, it } from "vitest"; import { z } from "zod"; import { toolContract } from "./toolContracts.js"; import { toolOutputSchemaWithMetadata } from "./toolSchemaMetadata.js"; +it("keeps dialect-specific validators distinct after JSON schema serialization", () => { + const contract = { + ...toolContract("binary_session"), + outputSchema: z.strictObject({ value: z.number().int().min(1).max(8) }), + }; + const output = toolOutputSchemaWithMetadata(contract); + const provider = new AjvJsonSchemaValidator(); + for (const target of ["draft-2020-12", "draft-07"] as const) { + const projected = output["~standard"].jsonSchema.output({ target }); + const parsed = z + .record(z.string(), z.unknown()) + .parse(JSON.parse(JSON.stringify(projected))); + const validate = provider.getValidator(parsed); + expect(validate({ value: 8 }).valid).toBe(true); + expect(validate({ value: 9 }).valid).toBe(false); + expect(validate({ value: 1.5 }).valid).toBe(false); + } +}); + +it("preserves explicit schema identity and relative reference resolution", () => { + const value = z.string(); + const canonical = z.strictObject({ value }); + const contract = { + ...toolContract("binary_session"), + outputSchema: canonical, + }; + const projected = toolOutputSchemaWithMetadata(contract)[ + "~standard" + ].jsonSchema.output({ + target: "draft-2020-12", + libraryOptions: { + override: (context: { + readonly zodSchema: unknown; + readonly jsonSchema: Record; + }) => { + if (context.zodSchema !== value) return; + delete context.jsonSchema.type; + context.jsonSchema.$ref = "child.json"; + }, + }, + }); + const ajv = new Ajv2020({ strict: false }); + ajv.addSchema({ $id: "child.json", type: "string", minLength: 3 }); + const validate = ajv.compile(projected); + expect(validate({ value: "abc" })).toBe(true); + expect(validate({ value: "ab" })).toBe(false); + + const identified = toolOutputSchemaWithMetadata({ + ...contract, + outputSchema: canonical.meta({ $id: "https://example.test/result" }), + })["~standard"].jsonSchema.output({ target: "draft-2020-12" }); + expect(identified.$id).toBe("https://example.test/result"); + const explicit = ajv.compile(identified); + expect(explicit({ value: "abc" })).toBe(true); + expect(explicit({ value: 1 })).toBe(false); +}); + it.each(["draft-2020-12", "draft-07"] as const)( "preserves refined numeric bounds and strict repeated objects through %s references", (target) => { diff --git a/src/contracts/toolSchemaMetadata.ts b/src/contracts/toolSchemaMetadata.ts index f2d395ece..8f43f1708 100644 --- a/src/contracts/toolSchemaMetadata.ts +++ b/src/contracts/toolSchemaMetadata.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import type { ToolContract } from "./toolContractTypes.js"; import { presentInputJsonSchema } from "./inputSchemaPresentation.js"; +import { digestCanonicalValue } from "../domain/canonicalDigest.js"; const PROPERTY_DESCRIPTIONS: Readonly> = { addresses: "Ordered provider-normalized procedure addresses to analyze.", @@ -69,20 +70,55 @@ export const toolInputSchemaWithMetadata = ( }, ); -/** Share repeated output definitions without changing canonical validation. */ +/** Share output definitions and content-bound validator identities on the wire. */ export const toolOutputSchemaWithMetadata = ( contract: Contract, ): Contract["outputSchema"] => withAdvertisedJsonSchema( contract.outputSchema, "output", - (project) => (options) => - project({ + (project) => (options) => { + const shared = project({ ...options, libraryOptions: { reused: "ref", ...options.libraryOptions }, - }), + }); + // An explicit ID owns the reference base. Relative external references + // also depend on that base, so leave those projections unchanged. + if ("$id" in shared || hasRelativeSchemaReference(shared)) return shared; + return { + ...shared, + $id: `urn:rea:tool-output:sha256:${digestCanonicalValue( + { target: options.target, schema: shared }, + "Tool output schema", + )}`, + }; + }, ); +const hasRelativeSchemaReference = ( + schema: Record, +): boolean => { + const pending: unknown[] = [schema]; + const visited = new WeakSet(); + while (pending.length > 0) { + const current = pending.pop(); + if (current === null || typeof current !== "object" || visited.has(current)) + continue; + visited.add(current); + for (const [key, value] of Object.entries(current)) { + if ( + ["$id", "$ref", "$dynamicRef", "$recursiveRef"].includes(key) && + typeof value === "string" && + !value.startsWith("#") && + !URL.canParse(value) + ) + return true; + if (value !== null && typeof value === "object") pending.push(value); + } + } + return false; +}; + const withAdvertisedJsonSchema = ( canonical: Schema, io: "input" | "output", diff --git a/tests/boundary/mcp/outputValidatorReuse.test.ts b/tests/boundary/mcp/outputValidatorReuse.test.ts new file mode 100644 index 000000000..256c29d58 --- /dev/null +++ b/tests/boundary/mcp/outputValidatorReuse.test.ts @@ -0,0 +1,108 @@ +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import { McpServer } from "@modelcontextprotocol/server"; +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { toolOutputSchemaWithMetadata } from "../../../src/contracts/toolSchemaMetadata.js"; + +const advertised = < + Schema extends z.ZodType>>, +>( + schema: Schema, +): Schema => + toolOutputSchemaWithMetadata({ + ...toolContract("binary_session"), + outputSchema: schema, + }); + +const connect = async (server: McpServer, client: Client): Promise => { + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); +}; + +it("validates changed schemas after catalog refreshes and availability changes", async () => { + const client = new Client({ name: "output-reuse-test", version: "1" }); + const server = new McpServer({ name: "output-reuse-fixture", version: "1" }); + const schema = advertised(z.strictObject({ state: z.literal("ready") })); + let state = "ready"; + const selected = server.registerTool( + "selected", + { inputSchema: z.strictObject({}), outputSchema: schema }, + () => ({ content: [], structuredContent: { state } }), + ); + server.registerTool( + "equivalent", + { inputSchema: z.strictObject({}), outputSchema: schema }, + () => ({ content: [], structuredContent: { state: "ready" } }), + ); + try { + await connect(server, client); + await client.listTools(); + expect( + (await client.callTool({ name: "selected" })).structuredContent, + ).toEqual({ + state: "ready", + }); + await client.listTools(); + await client.callTool({ name: "equivalent" }); + + selected.update({ + outputSchema: advertised(z.strictObject({ state: z.literal("updated") })), + }); + state = "updated"; + await client.listTools(); + expect( + (await client.callTool({ name: "selected" })).structuredContent, + ).toEqual({ + state: "updated", + }); + selected.disable(); + const unavailable = await client.listTools(); + expect(unavailable.tools.some((tool) => tool.name === "selected")).toBe( + false, + ); + await expect(client.callTool({ name: "selected" })).rejects.toThrow( + /not found|disabled/iu, + ); + await client.callTool({ name: "equivalent" }); + selected.enable(); + await client.listTools(); + await client.callTool({ name: "selected" }); + } finally { + await Promise.allSettled([client.close(), server.close()]); + } +}); + +it("uses the new server's schema after reconnecting the client", async () => { + const client = new Client({ name: "output-reconnect-test", version: "1" }); + const servers: McpServer[] = []; + try { + for (const state of ["first", "second"]) { + const server = new McpServer({ name: `fixture-${state}`, version: "1" }); + servers.push(server); + server.registerTool( + "selected", + { + inputSchema: z.strictObject({}), + outputSchema: advertised(z.strictObject({ state: z.literal(state) })), + }, + () => ({ content: [], structuredContent: { state } }), + ); + await connect(server, client); + await client.listTools(); + expect( + (await client.callTool({ name: "selected" })).structuredContent, + ).toEqual({ state }); + await client.close(); + await server.close(); + } + } finally { + await Promise.allSettled([ + client.close(), + ...servers.map((server) => server.close()), + ]); + } +}); From 0a14ca41b46e42d36d46ba07384ff13a7a97456d Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:00:11 +0800 Subject: [PATCH 10/27] fix(javascript): preserve effects through yielded array references (#1437) --- .../javascript/javascriptSemanticAnalysis.ts | 20 ++- .../javascriptSemanticIterationSources.ts | 12 +- .../javascriptSemanticMemberMutations.ts | 15 +- .../javascriptIterationAliases.test.ts | 145 ++++++++++++++++++ 4 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 tests/acceptance/applications/javascriptIterationAliases.test.ts diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 5a873429e..1cf53fb31 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -565,9 +565,21 @@ const bindAssignment = ( else if (t.isForOfStatement(node) || t.isForInStatement(node)) for (const pattern of t.isVariableDeclaration(node.left) ? node.left.declarations.map(({ id }) => id) - : [node.left]) + : [node.left]) { for (const identifier of assignedPatternIdentifiers(pattern)) addAssignment(identifier, node, scope, state); + if (t.isForOfStatement(node)) + // Keep the loop value unknown, but preserve references yielded into + // each bound slot so writes/escapes can reach their iterable origins. + bindPattern({ + pattern, + initializer: node, + scope, + state, + mutable: true, + referenceOnly: true, + }); + } }; const assignedPatternIdentifiers = ( @@ -780,7 +792,11 @@ const bindPattern = (input: BindPatternInput): void => { } if (t.isIdentifier(pattern)) { if (referenceOnly) { - const binding = scope.bindings.get(pattern.name); + const binding = resolveSemanticBindingFromScope( + scope, + pattern.name, + state, + ); if (binding !== undefined && initializer !== null) binding.referenceInitializers.push({ node: initializer, diff --git a/src/domain/javascript/javascriptSemanticIterationSources.ts b/src/domain/javascript/javascriptSemanticIterationSources.ts index 1b9e7d714..c83eb94ad 100644 --- a/src/domain/javascript/javascriptSemanticIterationSources.ts +++ b/src/domain/javascript/javascriptSemanticIterationSources.ts @@ -8,8 +8,9 @@ interface IterationSourceValue { readonly projection: readonly (string | number | null)[]; } -/** An implicit iterator receiver, with guards for destructuring defaults. */ +/** An iterator receiver or yielded reference store, with default guards. */ export interface SemanticIterationSource extends IterationSourceValue { + readonly kind: "iteration" | "reference-store"; readonly mutation: t.Node; readonly fallbackSources?: readonly IterationSourceValue[]; readonly requiredSources?: readonly IterationSourceValue[]; @@ -30,8 +31,10 @@ const add = ( source: IterationSourceValue, mutation: t.Node, guards: SourceGuards, + kind: SemanticIterationSource["kind"] = "iteration", ): void => { sources.push({ + kind, node: source.node, projection: source.projection, mutation, @@ -66,6 +69,13 @@ const visit = ( guards: SourceGuards = {}, ): void => { const { sources, mutation } = context; + if (t.isForOfStatement(mutation) && t.isMemberExpression(pattern)) { + // Storing a yielded reference in a property has no lexical loop binding. + // Preserve its escape rather than assuming later writes cannot reach it. + if (source !== undefined) + add(sources, source, mutation, guards, "reference-store"); + return; + } if (t.isTSParameterProperty(pattern)) { visit(context, pattern.parameter, source, guards); return; diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 21a6aded0..4e4ea7b8b 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -324,7 +324,7 @@ export const collectSemanticMemberMutations = ( bindings: new Set(), effect: "escape", originAt: source.mutation, - iterationOnly: true, + iterationOnly: source.kind === "iteration", }); if (t.isAssignmentExpression(node)) markTarget(node.left, node); else if (t.isUpdateExpression(node)) markTarget(node.argument, node); @@ -369,6 +369,19 @@ export const collectSemanticMemberMutations = ( const { initializer } = reference; if (referenceSourcesInactive(initializer, state)) pendingReferences.push(reference); + else if (t.isForOfStatement(initializer.node)) + deferIterable({ + node: initializer.node.right, + projection: [], + path: [null, ...reference.path], + fallbackPath: [], + bindings: reference.bindings, + effect: reference.effect, + originAt: initializer.node, + ...(reference.mutation === undefined + ? {} + : { mutation: reference.mutation }), + }); else if ( initializer.copyKind === "array-rest" && initializer.copyProjectionOffset !== undefined diff --git a/tests/acceptance/applications/javascriptIterationAliases.test.ts b/tests/acceptance/applications/javascriptIterationAliases.test.ts new file mode 100644 index 000000000..103995480 --- /dev/null +++ b/tests/acceptance/applications/javascriptIterationAliases.test.ts @@ -0,0 +1,145 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { projectedExportReturnShapesSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const source = [ + "export function inline() { const shared = { x: 1, keep: 7 }; for (const t of [shared]) t.x = 2; return { x: shared.x, keep: shared.keep }; }", + "export function named() { const shared = { x: 1 }; const arr = [shared]; for (const t of arr) t.x = 2; return shared.x; }", + "export function declaredVar() { const shared = { x: 1 }; for (var t of [shared]) t.x = 2; return shared.x; }", + "export function assigned() { const shared = { x: 1 }; let t; for (t of [shared]) t.x = 2; return shared.x; }", + "export function assignedDestructured() { const shared = { x: 1 }; let t; for ({child: t} of [{child: shared}]) t.x = 2; return shared.x; }", + "export function declaredVarDestructured() { const shared = { x: 1 }; for (var {child: t} of [{child: shared}]) t.x = 2; return shared.x; }", + "export function memberTarget() { const shared = { x: 1 }; const holder = {}; for (holder.item of [shared]) holder.item.x = 2; return shared.x; }", + "export function memberPattern() { const shared = { x: 1 }; const holder = {}; for ({child: holder.item} of [{child: shared}]) holder.item.x = 2; return shared.x; }", + "export function destructured() { const shared = { x: 1 }; for (const {child: t} of [{child: shared}]) t.x = 2; return shared.x; }", + "export function arrayDestructured() { const shared = { x: 1 }; for (const [t] of [[shared]]) t.x = 2; return shared.x; }", + "export function spread() { const shared = { x: 1 }; const arr = [shared]; for (const t of [...arr]) t.x = 2; return shared.x; }", + "export function objectRestChild() { const shared = { x: 1 }; for (const {...t} of [{child: shared}]) t.child.x = 2; return shared.x; }", + "export function arrayRestChild() { const shared = { x: 1 }; for (const [...t] of [[shared]]) t[0].x = 2; return shared.x; }", + "export function defaultChild() { const shared = { x: 1 }; for (const {child: t = shared} of [{}]) t.x = 2; return shared.x; }", + "export function updated() { const shared = { x: 1 }; for (const t of [shared]) t.x++; return shared.x; }", + "export function deleted() { const shared = { x: 1 }; for (const t of [shared]) delete t.x; return shared.x; }", + "export function escaped() { const shared = { x: 1 }; for (const t of [shared]) consume(t); return shared.x; }", + "export async function awaited() { const shared = { x: 1 }; for await (const t of [shared]) t.x = 2; return shared.x; }", + "export function indexed() { const shared = { x: 1 }; const arr = [shared]; for (let i = 0; i < arr.length; i++) { const t = arr[i]; t.x = 2; } return shared.x; }", + "export function copyWrite() { const shared = { x: 1 }; for (const {...t} of [shared]) t.x = 2; return shared.x; }", + "export function noWrite() { const shared = { x: 1 }; for (const t of [shared]) { const read = t.x; } return shared.x; }", + "export function empty() { const shared = { x: 1 }; for (const t of []) t.x = 2; return shared.x; }", + "export function keys() { const shared = { x: 1 }; for (const key in [shared]) { const read = key; } return shared.x; }", + "export function shadowed() { const t = { x: 1 }; const shared = { x: 3 }; for (const t of [shared]) t.x = 2; return t.x; }", +].join("\n"); + +const assertReturns = (value: unknown): void => { + const analysis = javascriptApplicationAnalysisResultSchema.parse(value); + const projections = analysis.graph.nodes + .flatMap(({ observations }) => observations) + .filter( + ({ properties }) => properties.semantic_role === "export-return-shapes", + ) + .map(({ properties }) => + projectedExportReturnShapesSchema.parse(properties), + ); + const fields = (name: string) => { + const projection = projections.find( + ({ exported_name }) => exported_name === name, + ); + if (projection === undefined) throw new Error(`Missing export ${name}`); + expect(projection.return_shape_coverage).toMatchObject({ + status: "complete", + projection_complete: true, + }); + return projection.static_return_shapes.flatMap(({ fields }) => fields); + }; + expect(fields("inline")).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "/x", state: "unknown" }), + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ]), + ); + for (const name of [ + "named", + "declaredVar", + "assigned", + "assignedDestructured", + "declaredVarDestructured", + "memberTarget", + "memberPattern", + "destructured", + "arrayDestructured", + "spread", + "objectRestChild", + "arrayRestChild", + "defaultChild", + "updated", + "deleted", + "escaped", + "awaited", + "indexed", + ]) + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "", state: "unknown" }), + ); + for (const name of ["copyWrite", "noWrite", "empty", "keys", "shadowed"]) + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "", state: "literal", value: 1 }), + ); +}; + +cliTest( + "preserves yielded-object mutation uncertainty through CLI and stdio MCP", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-iteration-aliases-"); + await writeFile(join(root, "app.js"), source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + }); + expect(cliResponse.exitCode).toBe(0); + const contract = toolContract("analyze_javascript_application"); + const cliEvidence = contract.outputSchema.parse(cliResponse.json); + assertReturns(cliEvidence.normalized_result); + const client = new Client({ name: "iteration-aliases-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(response.isError).not.toBe(true); + const evidence = contract.outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + assertReturns(evidence.normalized_result); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + assertReturns( + contract.outputSchema.parse(JSON.parse(text.text)).normalized_result, + ); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } + }, + 120_000, +); From ce16f7bc951180b68707fc8cec7e20a0511bb5b3 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:05:12 +0800 Subject: [PATCH 11/27] test(native): verify macOS provenance through real CLI and MCP Verify five native operations through the public CLI and real SDK stdio MCP using a source-built macOS fixture. Retain normalized provenance, response parity and unchanged input bytes. Refs #1430, #1065, #723. --- scripts/verify/apple/macos-bundle.mjs | 3 + scripts/verify/apple/native-provenance.mjs | 71 ++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 scripts/verify/apple/native-provenance.mjs diff --git a/scripts/verify/apple/macos-bundle.mjs b/scripts/verify/apple/macos-bundle.mjs index c10abcb2e..e2a130842 100644 --- a/scripts/verify/apple/macos-bundle.mjs +++ b/scripts/verify/apple/macos-bundle.mjs @@ -17,6 +17,7 @@ import { import { verifyDyldEnvironment } from "./dyld-environment.mjs"; import { verifyDylibResolution } from "./macos-dylib-e2e.mjs"; import { verifyMachOOccurrences } from "./macho-occurrences.mjs"; +import { verifyNativeProvenance } from "./native-provenance.mjs"; const exec = promisify(execFile); @@ -69,6 +70,7 @@ const EXPECTED_BUNDLES = [ let report; try { const { app, executable } = await buildMacosBundleFixture(root); + const nativeProvenance = await verifyNativeProvenance(app, executable); const machoOccurrences = await verifyMachOOccurrences(app, executable); const zip = join(root, "MacFixture.zip"); await exec("/usr/bin/ditto", ["-c", "-k", "--keepParent", app, zip]); @@ -166,6 +168,7 @@ try { containers: containers.map(({ format }) => format), bundles: EXPECTED_BUNDLES.length, dmg_detached: true, + native_provenance: nativeProvenance, macho_occurrences: machoOccurrences, dylib_resolution: dylibs, dyld_environment: dyldEnvironment, diff --git a/scripts/verify/apple/native-provenance.mjs b/scripts/verify/apple/native-provenance.mjs new file mode 100644 index 000000000..ad7bf6cc1 --- /dev/null +++ b/scripts/verify/apple/native-provenance.mjs @@ -0,0 +1,71 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; + +import { toolContract } from "../../../dist/contracts/toolContracts.js"; +import { + artifactCliEvidence, + withArtifactMcp, +} from "../../lib/artifact-e2e.mjs"; + +/** Inspect the compiled bundle with real native utilities through both adapters. */ +export async function verifyNativeProvenance(app, executable) { + const original = await readFile(executable); + const sha256 = createHash("sha256").update(original).digest("hex"); + const symbols = ["$s4main3FooV", "plain_symbol"]; + const operations = [ + ["inspect_macho", []], + ["inspect_signature", []], + ["inspect_plist", []], + ["list_architectures", []], + ["demangle_swift", symbols], + ]; + const observations = []; + await withArtifactMcp(app, async (client) => { + const definitions = new Map( + (await client.listTools()).tools.map((tool) => [tool.name, tool]), + ); + for (const [operation, arguments_] of operations) { + const cli = toolContract(operation).outputSchema.parse( + await artifactCliEvidence( + operation.replaceAll("_", "-"), + app, + arguments_, + ), + ); + const response = await client.callTool( + { + name: operation, + arguments: operation === "demangle_swift" ? { symbols } : {}, + }, + { toolDefinition: definitions.get(operation) }, + ); + assert.notEqual(response.isError, true, JSON.stringify(response)); + const mcp = toolContract(operation).outputSchema.parse( + response.structuredContent, + ); + assert.equal(cli.raw_result, null); + assert.equal(mcp.raw_result, null); + assert.ok(cli.normalized_result.provenance.length > 0); + assert.deepEqual(mcp.normalized_result, cli.normalized_result); + assert.deepEqual( + JSON.parse(response.content.find((item) => item.type === "text").text), + response.structuredContent, + ); + await client.ping(); + observations.push({ + operation, + commands: cli.normalized_result.provenance.length, + cli_bytes: Buffer.byteLength(JSON.stringify(cli)), + mcp_bytes: Buffer.byteLength(JSON.stringify(response)), + }); + } + }); + assert.deepEqual(await readFile(executable), original); + return { + real_native_utilities: true, + target_sha256: sha256, + bytes_unchanged: true, + observations, + }; +} From bb6fa1c6f500c6b6af42f1e3f72bd42c1dce91f6 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:13:05 +0800 Subject: [PATCH 12/27] docs(process): verify independent full-report hashing budgets (#1440) --- docs/process-capture.md | 46 +++++++- .../processFilesystemCoverageAdapters.test.ts | 104 ++++++++++++++++++ 2 files changed, 148 insertions(+), 2 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index 21f1f5343..1d3d23e7f 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -159,8 +159,11 @@ Trace assertions use coverage for the sources they select, so an assertion about complete raw terminal output need not fail because rendered snapshots were omitted. Captures missing `truncation_details` are rejected on import; preserve them as historical files and recapture for current comparisons. -These diagnostics use the existing scenario -budgets; they do not introduce a separate file-hashing budget. +`limits.file_bytes` is already the cumulative whole-file hashing budget, +independent of `limits.output_bytes`, which bounds retained terminal content. +Each filesystem checkpoint starts with a fresh `file_bytes` budget across all +selected roots; it is not a per-file allowance. File contents are read in +64 KiB chunks for hashing and are not embedded in capture Evidence. When the host withholds an unrelated process’s ownership token, REA leaves that process untouched and records its PID and reason in `cleanup.unverified_processes` @@ -176,6 +179,45 @@ as well. A newly started unreadable process can prevent verified cleanup even after the selected command exits. REA preserves this uncertainty and leaves that process untouched. +## Hash a report independently of terminal output + +Select the producer's report directory and give `file_bytes` enough room for +the complete files you need, including other files in those roots. For example, +save this scenario and run `rea capture-process ./scenario.json --json > capture.json`: + +```json +{ + "executable": "node", + "arguments": [ + "-e", + "const fs=require('node:fs');fs.mkdirSync('reports',{recursive:true});fs.writeFileSync('reports/final.json',JSON.stringify({status:'complete',data:'X'.repeat(131072)}));console.log('x'.repeat(8192));" + ], + "working_directory": ".", + "filesystem_observation_paths": ["./reports"], + "limits": { "output_bytes": 1024, "file_bytes": 262144 } +} +``` + +The report exceeds the terminal budget but fits the independent 256 KiB hash +budget. Its complete SHA-256 appears in `normalized_result.files_after` at +`root_0:final.json`; check `truncation_details.filesystem_after` for omissions +and actual bytes hashed. The verbose terminal output is truncated without +preventing the report digest. The same scenario object is the MCP +`capture_process_scenario` input. REA never returns a prefix hash as a full-file +digest: an insufficient remaining budget produces a null digest and an +explicit omission reason. + +A digest binds the complete bytes observed at that checkpoint, not the +producer's notion of a finished report. A stable periodic or running report +can also have a complete digest. Choose the final file and verify the +producer's documented completion marker, exit outcome and settlement +observations before calling it final. `after_settlement` names a filesystem +checkpoint, not a guarantee that an application finished successfully or +that an unobserved writer will never change the file. REA checks opened-file +identity/state around hashing and reports observed changes or unavailable +files; cancellation stops the hash. If an external sealer produces a manifest, +keep its completion claims and digest separate from REA's checkpoint facts. + ## Compare two captures Compare saved capture Evidence with: diff --git a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts index ec7ce5b4c..ff3b118a7 100644 --- a/tests/boundary/process/processFilesystemCoverageAdapters.test.ts +++ b/tests/boundary/process/processFilesystemCoverageAdapters.test.ts @@ -1,4 +1,5 @@ import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { promisify } from "node:util"; @@ -7,6 +8,7 @@ import { expect, it, onTestFinished } from "vitest"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { compareProcessCaptures } from "../../../src/domain/process/processComparison.js"; import { parseProcessCapture } from "../../../src/domain/process/processCaptureParsing.js"; +import { parseProcessScenario } from "../../../src/domain/process/processScenario.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -58,6 +60,108 @@ const captureViaMcp = async (scenario: Record) => { return evidence; }; +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "hashes the documented complete report despite omitted terminal output through %s", + async (adapter) => { + const document = await readFile("docs/process-capture.md", "utf8"); + const example = document + .split("## Hash a report independently of terminal output")[1] + ?.match(/```json\n([\s\S]*?)\n```/u)?.[1]; + if (example === undefined) + throw new Error("Missing report capture example"); + const scenario = parseProcessScenario(JSON.parse(example)); + const root = await createTestTempDirectory("rea-full-report-adapter-"); + const evidence = await (adapter === "cli" ? captureViaCli : captureViaMcp)({ + ...scenario, + executable: process.execPath, + working_directory: root, + filesystem_observation_paths: [join(root, "reports")], + }); + const capture = parseProcessCapture(evidence.normalized_result); + const bytes = await readFile(join(root, "reports", "final.json")); + expect(JSON.parse(bytes.toString())).toEqual({ + status: "complete", + data: "X".repeat(131072), + }); + expect(bytes.length).toBeGreaterThan(scenario.limits.output_bytes); + expect(capture.files_after).toContainEqual( + expect.objectContaining({ + path: "root_0:final.json", + size: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), + }), + ); + expect(capture.truncation_details.filesystem_after).toMatchObject({ + enumeration_truncated: false, + hash_budget_bytes: scenario.limits.file_bytes, + hashed_bytes: bytes.length, + hash_omissions: [], + }); + expect(capture.truncation_details.raw_terminal.retained_bytes).toBeLessThan( + capture.truncation_details.raw_terminal.observed_bytes, + ); + }, + 20_000, +); + +itWithCaptureCapability.each(["cli", "mcp"] as const)( + "accounts for exact cumulative hash budgets independently at each %s checkpoint", + async (adapter) => { + const runCapture = adapter === "cli" ? captureViaCli : captureViaMcp; + for (const shortfall of [0, 1]) { + const root = await createTestTempDirectory("rea-report-cumulative-"); + const periodic = Buffer.from('{"status":"running"}'); + const final = Buffer.from('{"status":"complete"}'); + await writeFile(join(root, "periodic.json"), periodic); + const budget = periodic.length + final.length - shortfall; + const evidence = await runCapture({ + executable: process.execPath, + arguments: [ + "-e", + "require('node:fs').writeFileSync('final.json','{\"status\":\"complete\"}')", + ], + working_directory: root, + filesystem_observation_paths: [root], + limits: { output_bytes: 1, file_bytes: budget }, + }); + const capture = parseProcessCapture(evidence.normalized_result); + expect(capture.truncation_details.filesystem_before).toMatchObject({ + hashed_bytes: periodic.length, + hash_omissions: [], + }); + expect(capture.files_after).toContainEqual( + expect.objectContaining({ + path: "root_0:final.json", + sha256: createHash("sha256").update(final).digest("hex"), + }), + ); + expect(capture.truncation_details.filesystem_after).toMatchObject({ + enumeration_truncated: false, + hashed_bytes: final.length + (shortfall === 0 ? periodic.length : 0), + hash_omissions: + shortfall === 0 + ? [] + : [ + { + path: "root_0:periodic.json", + reason: "file_bytes_budget", + remaining_budget_bytes: periodic.length - 1, + }, + ], + }); + expect( + capture.files_after.find(({ path }) => path === "root_0:periodic.json") + ?.sha256, + ).toBe( + shortfall === 0 + ? createHash("sha256").update(periodic).digest("hex") + : null, + ); + } + }, + 20_000, +); + itWithCaptureCapability.each(["cli", "mcp"] as const)( "reports unknown absence instead of deletion in truncated %s capture Evidence", async (adapter) => { From eeb83f65817819644af8e6fbf8a120227dfb88d4 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 22:16:44 -0700 Subject: [PATCH 13/27] perf(cli): defer Playwright and isomorphic-git until a command needs them Load browser composition after CLI scenario validation and load Git support only for local reference trees with Git metadata. Preserve default outputs and actual command behavior. Refs #1065. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- src/application/ReferenceSourceVcsAdapter.ts | 4 +- src/cliBrowserScenarioCommands.ts | 4 +- tests/boundary/cli/cliStartupImports.test.ts | 76 ++++++++++++++++++++ 3 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 tests/boundary/cli/cliStartupImports.test.ts diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts index 42ab4e738..3ddad25d7 100644 --- a/src/application/ReferenceSourceVcsAdapter.ts +++ b/src/application/ReferenceSourceVcsAdapter.ts @@ -2,8 +2,6 @@ import fs from "node:fs"; import { lstat, readFile } from "node:fs/promises"; import { join, resolve } from "node:path"; -import { resolveRef } from "isomorphic-git"; - export type ReferenceSourceVcsInfo = | { readonly kind: "git"; @@ -42,6 +40,8 @@ export const readReferenceSourceVcs = async ( }; const resolveSourceHead = async (root: string): Promise => { + // isomorphic-git is loaded on first use so CLI and MCP startup skip it. + const { resolveRef } = await import("isomorphic-git"); const dotgit = join(root, ".git"); if (!(await lstat(dotgit)).isFile()) return resolveRef({ fs, dir: root, ref: "HEAD" }); diff --git a/src/cliBrowserScenarioCommands.ts b/src/cliBrowserScenarioCommands.ts index f98e415cb..a1a1f86e1 100644 --- a/src/cliBrowserScenarioCommands.ts +++ b/src/cliBrowserScenarioCommands.ts @@ -2,7 +2,6 @@ import { Cli } from "incur"; import { z } from "zod"; import { captureBrowserScenario } from "./application/BrowserScenarioCaptureService.js"; -import { createBrowserScenarioProvider } from "./composition/browserScenario.js"; import { CLI_COMMANDS } from "./cliCommandNames.js"; import { parseCliJsonInput, resolveCliJsonPaths } from "./cliJsonInput.js"; import { logCliCommand } from "./cliLogging.js"; @@ -44,6 +43,9 @@ export const registerBrowserScenarioCommands = ( projectInputIssues(scenario.error.issues, input.value), ), ); + // Load Playwright only for this command, not during CLI startup. + const { createBrowserScenarioProvider } = + await import("./composition/browserScenario.js"); const result = await captureBrowserScenario( createBrowserScenarioProvider(environment), scenario.data, diff --git a/tests/boundary/cli/cliStartupImports.test.ts b/tests/boundary/cli/cliStartupImports.test.ts new file mode 100644 index 000000000..c084b6212 --- /dev/null +++ b/tests/boundary/cli/cliStartupImports.test.ts @@ -0,0 +1,76 @@ +import { execFile } from "node:child_process"; +import { mkdir } from "node:fs/promises"; +import { join } from "node:path"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const MARKER = "REA_TEST_RESOLVED "; +// Dependencies that only one command uses; startup must not resolve them. +const DEFERRED_PACKAGES = ["playwright-core", "isomorphic-git"] as const; +// Report each resolved deferred package on stderr without changing resolution. +const RESOLUTION_HOOK = `data:text/javascript,${encodeURIComponent(` +import { registerHooks } from "node:module"; +const packages = ${JSON.stringify(DEFERRED_PACKAGES)}; +registerHooks({ + resolve(specifier, context, next) { + const resolved = next(specifier, context); + const name = packages.find((item) => + resolved.url.includes("/node_modules/" + item + "/"), + ); + if (name !== undefined) + process.stderr.write(${JSON.stringify(MARKER)} + name + "\\n"); + return resolved; + }, +}); +`)}`; + +const resolvedPackages = async ( + arguments_: readonly string[], +): Promise> => { + const { stderr } = await execute( + process.execPath, + [`--import=${RESOLUTION_HOOK}`, ...arguments_], + { cwd: process.cwd(), maxBuffer: 16 * 1_024 * 1_024 }, + ); + return new Set( + stderr + .split("\n") + .filter((line) => line.startsWith(MARKER)) + .map((line) => line.slice(MARKER.length)), + ); +}; + +const evaluate = (source: string): readonly string[] => [ + "--input-type=module", + "--eval", + source, +]; + +describe("CLI startup imports", () => { + it("detects each deferred package when its command code loads", async () => { + await expect( + resolvedPackages( + evaluate('await import("./dist/composition/browserScenario.js");'), + ), + ).resolves.toContain("playwright-core"); + const source = await createTestTempDirectory("rea-startup-imports-"); + await mkdir(join(source, ".git")); + await expect( + resolvedPackages( + evaluate( + `const { readReferenceSourceVcs } = await import("./dist/application/ReferenceSourceVcsAdapter.js"); await readReferenceSourceVcs(${JSON.stringify(source)});`, + ), + ), + ).resolves.toContain("isomorphic-git"); + }); + + it("does not load command-specific packages to register commands", async () => { + await expect( + resolvedPackages(["scripts/rea.mjs", "--help"]), + ).resolves.toEqual(new Set()); + }); +}); From 33b9b31a6fb50cc3e6f4591d0395dbba15b51f13 Mon Sep 17 00:00:00 2001 From: keemsisi Date: Sat, 10 Oct 2026 06:21:13 +0100 Subject: [PATCH 14/27] perf(javascript): index change-graph nodes once per comparison Resolve each compared graph node through a per-side index while preserving right-hand evidence selection and complete comparison results. Refs #1065. Co-authored-by: keemsisi <7439375+keemsisi@users.noreply.github.com> --- .../javascriptApplicationChangeGraph.test.ts | 107 ++++++++++++++++++ .../javascriptApplicationChangeGraph.ts | 25 ++-- 2 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 src/domain/javascript/javascriptApplicationChangeGraph.test.ts diff --git a/src/domain/javascript/javascriptApplicationChangeGraph.test.ts b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts new file mode 100644 index 000000000..2bbd653af --- /dev/null +++ b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; + +import { + createJavaScriptApplicationEdge, + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, + type JavaScriptApplicationGraph, +} from "./javascriptApplicationGraph.js"; +import { + artifactEvidence, + completeCoverage, +} from "./javascriptApplicationGraph.fixture.js"; +import { compareJavaScriptApplicationVersions } from "./javascriptApplicationVersionComparison.js"; + +const LEFT_SHA = "a".repeat(64); +const RIGHT_SHA = "b".repeat(64); + +const digest = (value: number): string => value.toString(16).padStart(64, "0"); + +const buildGraph = ( + artifactSha: string, + changed: number, +): JavaScriptApplicationGraph => { + const nodes = ["a", "b", "c", "d"].map((name, index) => + createJavaScriptApplicationNode({ + kind: "javascript-module", + identity: { + strategy: "canonical-path", + stability: "artifact-version", + artifact_sha256: artifactSha, + path: `${name}.js`, + }, + observations: [ + { + label: `${name}.js`, + properties: { + structural_fingerprint_sha256: digest(index + 1), + structural_fingerprint_algorithm: "fixture", + source_sha256: digest(index === 1 || index === 3 ? changed : 0), + }, + evidence: artifactEvidence(artifactSha, `${name}.js`), + }, + ], + }), + ); + const [a, b] = nodes; + if (a === undefined || b === undefined) + throw new Error("Missing fixture node"); + return createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + root_node_ids: [a.node_id], + nodes, + edges: [ + createJavaScriptApplicationEdge({ + source_node_id: a.node_id, + target_node_id: b.node_id, + relation: "imports", + properties: {}, + evidence: artifactEvidence(artifactSha, "a.js"), + }), + ], + coverage: completeCoverage, + limitations: [], + }); +}; + +describe("application change graph", () => { + it("anchors each changed_from edge to its own right-hand observation", () => { + const left = buildGraph(LEFT_SHA, 1); + const right = buildGraph(RIGHT_SHA, 2); + const result = compareJavaScriptApplicationVersions({ + left: { + evidenceId: `ev_${"1".repeat(64)}`, + rootArtifactSha256: LEFT_SHA, + graph: left, + }, + right: { + evidenceId: `ev_${"2".repeat(64)}`, + rootArtifactSha256: RIGHT_SHA, + graph: right, + }, + leftNativeEvidence: [], + rightNativeEvidence: [], + }); + const changedFrom = result.graph.edges.filter( + ({ relation }) => relation === "changed_from", + ); + expect(changedFrom).toHaveLength(2); + for (const edge of changedFrom) { + const source = right.nodes.find( + ({ node_id: id }) => id === edge.source_node_id, + ); + const target = left.nodes.find( + ({ node_id: id }) => id === edge.target_node_id, + ); + expect(source?.observations[0]?.label).toBe( + target?.observations[0]?.label, + ); + expect(edge.evidence.artifact).toEqual( + source?.observations[0]?.evidence.artifact, + ); + expect(edge.evidence.location).toEqual( + source?.observations[0]?.evidence.location, + ); + } + }); +}); diff --git a/src/domain/javascript/javascriptApplicationChangeGraph.ts b/src/domain/javascript/javascriptApplicationChangeGraph.ts index 4c80b3204..134fff520 100644 --- a/src/domain/javascript/javascriptApplicationChangeGraph.ts +++ b/src/domain/javascript/javascriptApplicationChangeGraph.ts @@ -53,8 +53,14 @@ export const buildJavaScriptApplicationChangeGraph = ( ({ source_node_id: source, target_node_id: target }) => retained.has(source) && retained.has(target), ); + const leftById = new Map( + input.left.nodes.map((node) => [node.node_id, node]), + ); + const rightById = new Map( + input.right.nodes.map((node) => [node.node_id, node]), + ); const comparisonEdges = input.items.flatMap((item) => - changedFromEdge(item, input, retained), + changedFromEdge(item, input, retained, leftById, rightById), ); const candidateEdges = uniqueEdges([...sourceEdges, ...comparisonEdges]); const rootNodeIds = preferredRoots.filter((nodeId) => retained.has(nodeId)); @@ -104,6 +110,8 @@ const changedFromEdge = ( item: ApplicationVersionComparisonItem, input: ChangeGraphInput, retained: ReadonlySet, + leftById: ReadonlyMap, + rightById: ReadonlyMap, ): ApplicationEdge[] => { const left = item.left_node_id; const right = item.right_node_id; @@ -117,7 +125,12 @@ const changedFromEdge = ( !retained.has(right) ) return []; - const evidence = comparisonEvidence(item, input); + const evidence = comparisonEvidence( + item, + input, + leftById.get(left), + rightById.get(right), + ); return [ createJavaScriptApplicationEdge({ source_node_id: right, @@ -138,13 +151,9 @@ const changedFromEdge = ( const comparisonEvidence = ( item: ApplicationVersionComparisonItem, input: ChangeGraphInput, + left: ApplicationNode | undefined, + right: ApplicationNode | undefined, ): ApplicationGraphEvidence => { - const right = input.right.nodes.find( - ({ node_id: id }) => id === item.right_node_id, - ); - const left = input.left.nodes.find( - ({ node_id: id }) => id === item.left_node_id, - ); const source = right?.observations[0]?.evidence ?? left?.observations[0]?.evidence; const complete = From 7633dd8d5dcb94eb5c2fca8b69858949c51b0eaf Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:43:01 +0800 Subject: [PATCH 15/27] fix(ghidra): publish Windows verification success after cleanup --- docs/native-investigation.md | 2 +- scripts/verify-packaged-ghidra-windows.mjs | 31 +++++++++++++--------- src/ghidra/GhidraProvider.behavior.test.ts | 26 ++++++++++++++++++ 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 4dad23b3b..b3051cc64 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -379,7 +379,7 @@ the tool fails with a `debugger-attach-denied` reason. Install a Ghidra 12.1.x release and the 64-bit full JDK it declares, then configure REA to use them. Ghidra analysis supports Linux x64/arm64 and macOS x64/arm64; each installation requires the matching native decompiler. Experimental Windows x64 P0 admits native -x86-64 PE applications on local NTFS using bundled Job Object ownership, +x86 and x86-64 PE applications and DLLs on local NTFS using bundled Job Object ownership, protected runtime DACLs, and handle-based path admission. See [Windows Ghidra P0](windows-ghidra-p0.md) and [issue #527](https://github.com/morluto/rea/issues/527). On Linux and macOS, `annotate_native_function` atomically edits a function name diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index 268d949d9..be21fde12 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -26,12 +26,21 @@ if (process.platform !== "win32" || process.arch !== "x64") ); const exec = promisify(execFile); const arguments_ = process.argv.slice(2); +const internalWorkspace = arguments_[0] === "--internal-workspace"; +const fixtureArguments = internalWorkspace ? arguments_.slice(2) : arguments_; +const x86Fixture = fixtureArguments.includes("--x86"); +const dllFixture = fixtureArguments.includes("--dll"); +const [packageRootArgument, targetArgument, reportArgument] = + fixtureArguments.filter( + (argument) => argument !== "--x86" && argument !== "--dll", + ); // Windows locks the loaded package-owned addon until the verifier exits. // Keep installation cleanup in a parent that never loads that addon. -if (arguments_[0] !== "--internal-workspace") { +if (!internalWorkspace) { const ownedWorkspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-")); + let result; try { - const result = await exec( + result = await exec( process.execPath, [ fileURLToPath(import.meta.url), @@ -41,8 +50,6 @@ if (arguments_[0] !== "--internal-workspace") { ], { timeout: 900_000, maxBuffer: 8 * 1024 * 1024 }, ); - process.stdout.write(result.stdout); - process.stderr.write(result.stderr); } catch (cause) { if (typeof cause.stdout === "string") process.stdout.write(cause.stdout); if (typeof cause.stderr === "string") process.stderr.write(cause.stderr); @@ -50,17 +57,17 @@ if (arguments_[0] !== "--internal-workspace") { } finally { await rm(ownedWorkspace, { recursive: true, force: true }); } + const report = JSON.parse(result.stdout); + report.workspaceCleanup = true; + report.ok = true; + if (reportArgument !== undefined) + await writeFile(reportArgument, `${JSON.stringify(report)}\n`); + process.stdout.write(`${JSON.stringify(report)}\n`); + process.stderr.write(result.stderr); process.exit(0); } assert.ok(arguments_[1] !== undefined, "Missing verifier workspace."); const workspace = resolve(arguments_[1]); -const fixtureArguments = arguments_.slice(2); -const x86Fixture = fixtureArguments.includes("--x86"); -const dllFixture = fixtureArguments.includes("--dll"); -const [packageRootArgument, targetArgument, reportArgument] = - fixtureArguments.filter( - (argument) => argument !== "--x86" && argument !== "--dll", - ); let packageRoot = resolve(packageRootArgument ?? "."); // The default lane verifies the npm artifact in an isolated prefix. Explicit // package roots support an already-installed artifact without a second install. @@ -402,7 +409,7 @@ try { assert.deepEqual(await readdir(join(callerDirectory, name)), []); report.callerScriptCollisionsPreserved = true; report.runtimeCleanup = true; - report.ok = true; + // Only the parent can attest success after removing the installation. if (reportArgument !== undefined) await writeFile(reportArgument, `${JSON.stringify(report)}\n`); process.stdout.write(`${JSON.stringify(report)}\n`); diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 88e1eb07f..c607b9950 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -198,6 +198,32 @@ describe("Ghidra Mach-O slice support", () => { }); describe("Ghidra platform support", () => { + it("commits the PE role and managed classification to profile identity", async () => { + const ghidra = provider({ ...installationHost(), platform: "win32" }); + const application = peTarget("x86_64"); + const library = { + ...application, + executableRole: "shared-library", + } as const; + const digests = new Set(); + for (const target of [ + application, + library, + { ...library, managed: true }, + ]) { + const resolved = await ghidra.resolveAnalysisProfile(target); + if (!resolved.ok) throw resolved.error; + const profile = resolved.value.profile; + if (profile === null) throw new Error("Expected a PE analysis profile"); + expect(profile.parameters).toMatchObject({ + executable_role: target.executableRole, + managed: target.managed, + }); + digests.add(profile.digest); + } + expect(digests.size).toBe(3); + }); + it("keeps Windows annotation mutation unavailable independently of native controls", () => { const ghidra = provider({ ...installationHost(), platform: "win32" }); expect( From ea1dbda1c747400fe2228c1802a3721995216277 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:47:06 +0800 Subject: [PATCH 16/27] test(ghidra): share result projection client setup --- src/ghidra/GhidraProvider.behavior.test.ts | 74 ++++++++-------------- 1 file changed, 26 insertions(+), 48 deletions(-) diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index c607b9950..c0518297b 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -540,24 +540,23 @@ describe("Ghidra client projection", () => { }); describe("Ghidra result projection", () => { + const createElfClient = async (ghidra: GhidraProvider) => { + const target = executableTarget("elf", "x86_64"); + const resolved = await ghidra.resolveAnalysisProfile(target); + if (!resolved.ok) throw resolved.error; + if (resolved.value.profile === null) + throw new Error("Expected a bound Ghidra profile"); + return ghidra.createClient(target, resolved.value.profile); + }; + it("rejects malformed inventory output before Evidence creation", async () => { const ghidra = provider(installationHost(), () => ({ start: () => Promise.resolve(ok(sessionInfo())), callTool: () => Promise.resolve(ok({ items: "not-an-inventory" })), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - - await expect( - ghidra - .createClient(executableTarget("elf", "x86_64"), resolved.value.profile) - .execute("list_procedures", {}), - ).resolves.toMatchObject({ + const client = await createElfClient(ghidra); + await expect(client.execute("list_procedures", {})).resolves.toMatchObject({ ok: false, error: { _tag: "AnalysisOutputError" }, }); @@ -586,21 +585,13 @@ describe("Ghidra result projection", () => { ), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - + const client = await createElfClient(ghidra); await expect( - ghidra - .createClient( - executableTarget("elf", "x86_64"), - resolved.value.profile, - ) - .execute("list_procedures", {}), - ).resolves.toMatchObject({ ok: false, error: { _tag: tag } }); + client.execute("list_procedures", {}), + ).resolves.toMatchObject({ + ok: false, + error: { _tag: tag }, + }); }, ); @@ -622,19 +613,12 @@ describe("Ghidra result projection", () => { ), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound profile"); + const client = await createElfClient(ghidra); await expect( - ghidra - .createClient(executableTarget("elf", "x86_64"), resolved.value.profile) - .execute("annotate_native_function", { - procedure: "0x10100", - name: "bad name", - }), + client.execute("annotate_native_function", { + procedure: "0x10100", + name: "bad name", + }), ).resolves.toMatchObject({ ok: false, error: { @@ -662,16 +646,10 @@ describe("Ghidra result projection", () => { ), close: () => Promise.resolve(ok(null)), })); - const resolved = await ghidra.resolveAnalysisProfile( - executableTarget("elf", "x86_64"), - ); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - - const result = await ghidra - .createClient(executableTarget("elf", "x86_64"), resolved.value.profile) - .execute("procedure_pseudo_code", { procedure: "main" }); + const client = await createElfClient(ghidra); + const result = await client.execute("procedure_pseudo_code", { + procedure: "main", + }); expect(result).toMatchObject({ ok: false, error: { _tag: tag }, From e09cfe9b5970d75da478c9dd741451cc2be5c91a Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:47:15 +0800 Subject: [PATCH 17/27] fix(javascript): invalidate references stored through property targets (#1445) * fix(javascript): invalidate references stored through property targets * refactor(javascript): share rest keys and return-shape fixtures --- docs/javascript-application-workflows.md | 6 +- src/domain/javascript/javascriptAstValues.ts | 10 ++ .../javascript/javascriptSemanticAnalysis.ts | 14 +-- .../javascriptSemanticIterationSources.ts | 69 ++++++++++--- .../javascriptIterationAliases.test.ts | 81 ++------------- .../applications/javascriptSlotStores.test.ts | 98 +++++++++++++++++++ tests/fixtures/javascriptReturnShapes.ts | 91 +++++++++++++++++ 7 files changed, 271 insertions(+), 98 deletions(-) create mode 100644 tests/acceptance/applications/javascriptSlotStores.test.ts create mode 100644 tests/fixtures/javascriptReturnShapes.ts diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index f26c16d1f..bdc0dcf21 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -183,8 +183,10 @@ Direct return expressions, including expression-bodied arrows, are evaluated through an execution-free value lattice. Literal object fields and direct return sites are represented in the result. Calls, dynamic spreads, computed keys, and parser recovery remain partial or unknown. -Objects and arrays passed to calls, constructors, or tagged templates, or used -as method receivers, are not assumed unchanged after the invocation. Aliases +Objects and arrays passed to calls, constructors, or tagged templates, used +as method receivers, or stored through property targets are not assumed +unchanged afterward. Property stores conservatively retain reference escape +uncertainty rather than pretending to execute later writes. Aliases and shared children in spread and rest copies retain that uncertainty; copied primitive slots and unrelated containing properties remain known. Object rest excludes consumed keys, array rest excludes the consumed prefix for known diff --git a/src/domain/javascript/javascriptAstValues.ts b/src/domain/javascript/javascriptAstValues.ts index f0a0528c0..5479c8d1f 100644 --- a/src/domain/javascript/javascriptAstValues.ts +++ b/src/domain/javascript/javascriptAstValues.ts @@ -76,6 +76,16 @@ export const semanticStaticPropertyKey = ( return !computed && t.isIdentifier(property) ? property.name : null; }; +/** Read exact consumed object-pattern keys, leaving dynamic keys unknown. */ +export const semanticObjectPatternKeys = ( + pattern: t.ObjectPattern, +): readonly string[] => + pattern.properties.flatMap((property) => { + if (t.isRestElement(property)) return []; + const key = semanticStaticPropertyKey(property.key, property.computed); + return key === null ? [] : [key]; + }); + /** * Display an exact JavaScript string as nonempty label text. Graph labels and * artifact-local keys are nonempty, so the legal empty value is shown as `""`; diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 1cf53fb31..452d29074 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -14,7 +14,10 @@ import { immutableSemanticBindings, immutableSemanticScopes, } from "./javascriptSemanticProjection.js"; -import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; +import { + semanticObjectPatternKeys, + semanticStaticPropertyKey, +} from "./javascriptAstValues.js"; import type { JavaScriptSemanticAnalysisState, JavaScriptSemanticBindingState, @@ -880,14 +883,7 @@ const bindPattern = (input: BindPatternInput): void => { referenceOnly: true, copyKind: "object-rest", copyProjectionOffset: projection.length, - copyExcludedKeys: pattern.properties.flatMap((property) => { - if (t.isRestElement(property)) return []; - const key = semanticStaticPropertyKey( - property.key, - property.computed, - ); - return key === null ? [] : [key]; - }), + copyExcludedKeys: semanticObjectPatternKeys(pattern), }); } else { const name = semanticStaticPropertyKey(property.key, property.computed); diff --git a/src/domain/javascript/javascriptSemanticIterationSources.ts b/src/domain/javascript/javascriptSemanticIterationSources.ts index c83eb94ad..ea3708e51 100644 --- a/src/domain/javascript/javascriptSemanticIterationSources.ts +++ b/src/domain/javascript/javascriptSemanticIterationSources.ts @@ -1,14 +1,21 @@ import * as t from "@babel/types"; -import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; -import { semanticArrayIndex } from "./javascriptSemanticPropertyPaths.js"; +import { + semanticObjectPatternKeys, + semanticStaticPropertyKey, +} from "./javascriptAstValues.js"; +import { + semanticArrayIndex, + type JavaScriptSemanticPropertyPath, +} from "./javascriptSemanticPropertyPaths.js"; +import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; interface IterationSourceValue { readonly node: t.Node; - readonly projection: readonly (string | number | null)[]; + readonly projection: JavaScriptSemanticPropertyPath; } -/** An iterator receiver or yielded reference store, with default guards. */ +/** An iterator receiver or property reference store, with default guards. */ export interface SemanticIterationSource extends IterationSourceValue { readonly kind: "iteration" | "reference-store"; readonly mutation: t.Node; @@ -69,11 +76,25 @@ const visit = ( guards: SourceGuards = {}, ): void => { const { sources, mutation } = context; - if (t.isForOfStatement(mutation) && t.isMemberExpression(pattern)) { - // Storing a yielded reference in a property has no lexical loop binding. - // Preserve its escape rather than assuming later writes cannot reach it. + if (t.isMemberExpression(pattern)) { + // Property stores have no lexical alias binding. Preserve the selected + // reference's escape rather than claiming later writes cannot reach it. if (source !== undefined) - add(sources, source, mutation, guards, "reference-store"); + add( + sources, + source.restStartIndex === undefined + ? source + : { + node: source.node, + projection: [ + ...source.projection, + { excludedKeys: [], startIndex: source.restStartIndex }, + ], + }, + mutation, + guards, + "reference-store", + ); return; } if (t.isTSParameterProperty(pattern)) { @@ -104,8 +125,29 @@ const visit = ( } if (t.isObjectPattern(pattern)) { for (const property of pattern.properties) { - // Object rest creates a fresh object and has no iterable binding pattern. - if (t.isRestElement(property)) continue; + if (t.isRestElement(property)) { + // A rest copy owns its scalar slots; only selected child references + // escape when that fresh copy is stored through a member target. + visit( + context, + property.argument, + source === undefined + ? undefined + : { + node: source.node, + projection: [ + ...source.projection, + { + excludedKeys: [ + ...new Set(semanticObjectPatternKeys(pattern)), + ].sort(compareUnicodeCodePoints), + }, + ], + }, + guards, + ); + continue; + } visit( context, property.value, @@ -144,7 +186,7 @@ const visit = ( visit(context, pattern.argument, undefined, guards); }; -/** Describe implicit iteration sources without deciding their runtime values. */ +/** Describe iterator receivers and property reference stores without evaluation. */ export const semanticIterationSources = ( node: t.Node, parent: t.Node | null, @@ -153,7 +195,10 @@ export const semanticIterationSources = ( const context = { sources, mutation: node }; if (t.isVariableDeclarator(node) && node.init != null) visit(context, node.id, { node: node.init, projection: [] }); - else if (t.isAssignmentExpression(node) && node.operator === "=") + else if ( + t.isAssignmentExpression(node) && + ["=", "||=", "&&=", "??="].includes(node.operator) + ) visit(context, node.left, { node: node.right, projection: [] }); else if (t.isForOfStatement(node)) { add(sources, { node: node.right, projection: [] }, node, {}); diff --git a/tests/acceptance/applications/javascriptIterationAliases.test.ts b/tests/acceptance/applications/javascriptIterationAliases.test.ts index 103995480..a7f039eb8 100644 --- a/tests/acceptance/applications/javascriptIterationAliases.test.ts +++ b/tests/acceptance/applications/javascriptIterationAliases.test.ts @@ -1,13 +1,9 @@ -import { Client } from "@modelcontextprotocol/client"; -import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; -import { writeFile } from "node:fs/promises"; -import { join, resolve } from "node:path"; import { expect } from "vitest"; -import { toolContract } from "../../../src/contracts/toolContracts.js"; -import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; -import { projectedExportReturnShapesSchema } from "../../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; -import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + verifyJavaScriptReturnShapes, + type JavaScriptReturnFields, +} from "../../fixtures/javascriptReturnShapes.js"; import { cliTest } from "../../support/cli/cliFixture.js"; const source = [ @@ -37,27 +33,7 @@ const source = [ "export function shadowed() { const t = { x: 1 }; const shared = { x: 3 }; for (const t of [shared]) t.x = 2; return t.x; }", ].join("\n"); -const assertReturns = (value: unknown): void => { - const analysis = javascriptApplicationAnalysisResultSchema.parse(value); - const projections = analysis.graph.nodes - .flatMap(({ observations }) => observations) - .filter( - ({ properties }) => properties.semantic_role === "export-return-shapes", - ) - .map(({ properties }) => - projectedExportReturnShapesSchema.parse(properties), - ); - const fields = (name: string) => { - const projection = projections.find( - ({ exported_name }) => exported_name === name, - ); - if (projection === undefined) throw new Error(`Missing export ${name}`); - expect(projection.return_shape_coverage).toMatchObject({ - status: "complete", - projection_complete: true, - }); - return projection.static_return_shapes.flatMap(({ fields }) => fields); - }; +const assertReturns = (fields: JavaScriptReturnFields): void => { expect(fields("inline")).toEqual( expect.arrayContaining([ expect.objectContaining({ path: "/x", state: "unknown" }), @@ -95,51 +71,6 @@ const assertReturns = (value: unknown): void => { cliTest( "preserves yielded-object mutation uncertainty through CLI and stdio MCP", - async ({ cli }) => { - const root = await createTestTempDirectory("rea-iteration-aliases-"); - await writeFile(join(root, "app.js"), source); - const cliResponse = await cli.run({ - arguments: [ - "analyze-javascript-application", - root, - "--artifact-format", - "directory", - "--json", - ], - }); - expect(cliResponse.exitCode).toBe(0); - const contract = toolContract("analyze_javascript_application"); - const cliEvidence = contract.outputSchema.parse(cliResponse.json); - assertReturns(cliEvidence.normalized_result); - const client = new Client({ name: "iteration-aliases-e2e", version: "1" }); - const transport = new StdioClientTransport({ - command: process.execPath, - args: [resolve("scripts/rea.mjs"), "mcp"], - cwd: process.cwd(), - env: { PATH: process.env.PATH ?? "" }, - stderr: "pipe", - }); - transport.stderr?.on("data", () => undefined); - try { - await client.connect(transport); - const response = await client.callTool({ - name: "analyze_javascript_application", - arguments: { input_path: root, format: "directory" }, - }); - expect(response.isError).not.toBe(true); - const evidence = contract.outputSchema.parse(response.structuredContent); - expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); - assertReturns(evidence.normalized_result); - const text = response.content.find((item) => item.type === "text"); - if (text?.type !== "text") throw new Error("Missing MCP text result"); - assertReturns( - contract.outputSchema.parse(JSON.parse(text.text)).normalized_result, - ); - await client.ping(); - } finally { - await client.close(); - await transport.close(); - } - }, + ({ cli }) => verifyJavaScriptReturnShapes(cli, source, assertReturns), 120_000, ); diff --git a/tests/acceptance/applications/javascriptSlotStores.test.ts b/tests/acceptance/applications/javascriptSlotStores.test.ts new file mode 100644 index 000000000..3af7e3eee --- /dev/null +++ b/tests/acceptance/applications/javascriptSlotStores.test.ts @@ -0,0 +1,98 @@ +import { expect } from "vitest"; + +import { + verifyJavaScriptReturnShapes, + type JavaScriptReturnFields, +} from "../../fixtures/javascriptReturnShapes.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +const stores = { + object: "const o = {a: {x: 0}}; o.a = shared; o.a.x = 2;", + array: "const arr = [{x: 0}]; arr[0] = shared; arr[0].x = 2;", + captured: "const o = {a: {x: 0}}; o.a = shared; const t = o.a; t.x = 2;", + nested: "const o = {a: {b: {x: 0}}}; o.a.b = shared; o.a.b.x = 2;", + quoted: 'const o = {}; o["a/b~c"] = shared; o["a/b~c"].x = 2;', + dynamic: 'const o = {}; const key = "entry"; o[key] = shared; o[key].x = 2;', + receiverAlias: "const o = {}; const t = o; t.a = shared; o.a.x = 2;", + transitive: "const p = {}; const o = {}; o.a = p; p.b = shared; o.a.b.x = 2;", + overwritten: "const o = {}; o.a = {x: 0}; o.a = shared; o.a.x = 2;", + replacedReference: + "const o = {}; const other = {x: 0}; o.a = other; o.a = shared; o.a.x = 2;", + copiedDestination: + "const o = {}; o.a = shared; const copy = {...o}; copy.a.x = 2;", + iteration: "const arr = []; arr[0] = shared; for (const t of arr) t.x = 2;", + nestedArray: "const o = {}; o.a = [shared]; o.a[0].x = 2;", + orAssignment: "const o = {a: null}; o.a ||= shared; o.a.x = 2;", + nullishAssignment: "const o = {a: null}; o.a ??= shared; o.a.x = 2;", + andAssignment: "const o = {a: {x: 0}}; o.a &&= shared; o.a.x = 2;", + objectPattern: "const o = {}; ({child: o.a} = {child: shared}); o.a.x = 2;", + arrayPattern: "const o = {}; [o.a] = [shared]; o.a.x = 2;", + defaultPattern: "const o = {}; ({child: o.a = shared} = {}); o.a.x = 2;", + objectRestMember: + "const o = {}; ({...o.a} = {child: shared}); o.a.child.x = 2;", + arrayRestMember: "const o = {}; [...o.a] = [shared]; o.a[0].x = 2;", + iteratedObjectRest: + "const o = {}; for ({...o.a} of [{child: shared}]) o.a.child.x = 2;", + iteratedArrayRest: "const o = {}; for ([...o.a] of [[shared]]) o.a[0].x = 2;", + classField: + "class Holder {} const o = new Holder(); o.a = shared; o.a.x = 2;", +}; +const source = [ + ...Object.entries(stores).map( + ([name, body]) => + `export function ${name}() { const shared = {x: 1}; ${body} return shared.x; }`, + ), + "export function primitive() { const n = 1; const o = {}; o.a = n; o.a = 2; return n; }", + "export function nestedChild() { const shared = {inner: {x: 1}}; const o = {}; o.a = shared; o.a.inner.x = 2; return shared.inner.x; }", + "export function readOnlyStore() { const shared = {x: 1}; const o = {}; o.a = shared; const captured = o.a.x; return shared.x; }", + "export function copiedScalar() { const shared = {x: 1}; const o = {}; o.a = {...shared}; o.a.x = 2; return shared.x; }", + "export function copiedArrayScalar() { const shared = {x: 1}; const o = {}; o.a = [shared.x]; o.a[0] = 2; return shared.x; }", + "export function unrelated() { const shared = {x: 1}; const other = {x: 9}; const o = {}; o.a = shared; o.a.x = 2; return other.x; }", + "export function copiedChild() { const child = {x: 1}; const parent = {child, keep: 7}; const o = {}; o.a = {...parent}; o.a.child.x = 2; return {child: child.x, keep: parent.keep}; }", + "export function objectRestExclusion() { const child = {x: 1}; const skip = {x: 3}; const parent = {child, skip, keep: 7}; const o = {}; let ignored; ({skip: ignored, ...o.a} = parent); o.a.child.x = 2; return {child: child.x, skip: skip.x, keep: parent.keep}; }", + "export function arrayRestOffset() { const child = {x: 1}; const skip = {x: 3}; const items = [skip, child]; const o = {}; let ignored; [ignored, ...o.a] = items; o.a[0].x = 2; return {child: child.x, skip: items[0].x}; }", +].join("\n"); + +const assertReturns = (fields: JavaScriptReturnFields): void => { + for (const name of [...Object.keys(stores), "nestedChild"]) { + const field = fields(name).find(({ path }) => path === ""); + expect(field, name).toBeDefined(); + expect(field?.state, name).toMatch(/^(unknown|literal)$/); + if (field?.state === "literal") expect(field.value, name).toBe(2); + } + const readOnly = fields("readOnlyStore").find(({ path }) => path === ""); + expect(readOnly?.state).toMatch(/^(unknown|literal)$/); + if (readOnly?.state === "literal") expect(readOnly.value).toBe(1); + for (const [name, value] of Object.entries({ + primitive: 1, + copiedScalar: 1, + copiedArrayScalar: 1, + unrelated: 9, + })) + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "", state: "literal", value }), + ); + const child = fields("copiedChild").find(({ path }) => path === "/child"); + expect(child?.state).toMatch(/^(unknown|literal)$/); + if (child?.state === "literal") expect(child.value).toBe(2); + expect(fields("copiedChild")).toContainEqual( + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ); + for (const name of ["objectRestExclusion", "arrayRestOffset"]) { + const child = fields(name).find(({ path }) => path === "/child"); + expect(child?.state, name).toMatch(/^(unknown|literal)$/); + if (child?.state === "literal") expect(child.value, name).toBe(2); + expect(fields(name), name).toContainEqual( + expect.objectContaining({ path: "/skip", state: "literal", value: 3 }), + ); + } + expect(fields("objectRestExclusion")).toContainEqual( + expect.objectContaining({ path: "/keep", state: "literal", value: 7 }), + ); +}; + +cliTest( + "does not report stale literals after storing references in member slots", + ({ cli }) => verifyJavaScriptReturnShapes(cli, source, assertReturns), + 120_000, +); diff --git a/tests/fixtures/javascriptReturnShapes.ts b/tests/fixtures/javascriptReturnShapes.ts new file mode 100644 index 000000000..02c66bc84 --- /dev/null +++ b/tests/fixtures/javascriptReturnShapes.ts @@ -0,0 +1,91 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { expect } from "vitest"; + +import { toolContract } from "../../src/contracts/toolContracts.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { projectedExportReturnShapesSchema } from "../../src/domain/javascript/javascriptExportShapeComparisonSchemas.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; +import type { TestCli } from "../support/cli/cliFixture.js"; + +const exportedReturnFields = (value: unknown) => { + const analysis = javascriptApplicationAnalysisResultSchema.parse(value); + const projections = analysis.graph.nodes + .flatMap(({ observations }) => observations) + .filter( + ({ properties }) => properties.semantic_role === "export-return-shapes", + ) + .map(({ properties }) => + projectedExportReturnShapesSchema.parse(properties), + ); + return (name: string) => { + const projection = projections.find( + ({ exported_name }) => exported_name === name, + ); + if (projection === undefined) throw new Error(`Missing export ${name}`); + expect(projection.return_shape_coverage).toMatchObject({ + status: "complete", + projection_complete: true, + }); + return projection.static_return_shapes.flatMap(({ fields }) => fields); + }; +}; + +/** Select complete return fields from a real JavaScript application result. */ +export type JavaScriptReturnFields = ReturnType; + +/** Exercise identical return assertions through the compiled CLI and stdio MCP. */ +export const verifyJavaScriptReturnShapes = async ( + cli: TestCli, + source: string, + check: (fields: JavaScriptReturnFields) => void, +): Promise => { + const root = await createTestTempDirectory("rea-return-shapes-"); + await writeFile(join(root, "app.js"), source); + const cliResponse = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--artifact-format", + "directory", + "--json", + ], + }); + expect(cliResponse.exitCode).toBe(0); + const contract = toolContract("analyze_javascript_application"); + const cliEvidence = contract.outputSchema.parse(cliResponse.json); + check(exportedReturnFields(cliEvidence.normalized_result)); + const client = new Client({ name: "return-shapes-e2e", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: { PATH: process.env.PATH ?? "" }, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const response = await client.callTool({ + name: "analyze_javascript_application", + arguments: { input_path: root, format: "directory" }, + }); + expect(response.isError).not.toBe(true); + const evidence = contract.outputSchema.parse(response.structuredContent); + expect(evidence.evidence_id).toBe(cliEvidence.evidence_id); + check(exportedReturnFields(evidence.normalized_result)); + const text = response.content.find((item) => item.type === "text"); + if (text?.type !== "text") throw new Error("Missing MCP text result"); + check( + exportedReturnFields( + contract.outputSchema.parse(JSON.parse(text.text)).normalized_result, + ), + ); + await client.ping(); + } finally { + await client.close(); + await transport.close(); + } +}; From 0d2082d36b8bed3cda5daa22adee2163652fc511 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:56:40 +0800 Subject: [PATCH 18/27] fix(javascript): recognize unchanged var and let Electron aliases --- .../javascript/javascriptSemanticAnalysis.ts | 14 ++++- .../electronApplicationAnalysis.test.ts | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index b5c1c7f5d..d490adf65 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -352,9 +352,17 @@ const bindingOrigin = ( ? binding.directOrigins[0] : undefined; const [initializer] = binding.initializers; - if (binding.mutable || binding.initializers.length !== 1) return undefined; - const origin = semanticRequireOrigin(initializer?.node, state); - const projection = initializer?.projection ?? []; + // A var/let declaration permits writes without proving one occurred. + // Require one unconditional initializer and no actual binding writes. + if ( + initializer === undefined || + binding.initializers.length !== 1 || + binding.definitions.some(({ kind }) => kind === "assignment") || + state.conditionalInitializers.has(initializer.node) + ) + return undefined; + const origin = semanticRequireOrigin(initializer.node, state); + const projection = initializer.projection; return origin === undefined || !projection.every((key): key is string => typeof key === "string") ? undefined diff --git a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts index 4180e1818..dd7bd8809 100644 --- a/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts +++ b/tests/composition/javascript-applications/electronApplicationAnalysis.test.ts @@ -116,6 +116,18 @@ describe("static Electron application analysis", () => { expect(handledChannels.sort()).toEqual(["rea:read", "rea:write"]); }); + it("maps unchanged var and let require aliases while excluding reassigned bindings", async () => { + const root = await requireAliasFixtureDirectory(); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + + expect(result.electron_summary.ipc).toMatchObject({ + main_handlers: 2, + paired_renderer_transmissions: 2, + unpaired_literal_renderer_transmissions: 0, + }); + }); + it("returns a tagged cancellation without executing application code", async () => { const root = await fixtureDirectory(); const controller = new AbortController(); @@ -326,6 +338,50 @@ const fixtureDirectory = async (): Promise => { return root; }; +const requireAliasFixtureDirectory = async (): Promise => { + const root = await createTestTempDirectory("rea-electron-require-aliases-"); + await Promise.all([ + writeFile( + join(root, "package.json"), + JSON.stringify({ + name: "rea-electron-require-aliases", + main: "main.cjs", + }), + ), + writeFile( + join(root, "main.cjs"), + String.raw` +var { ipcMain: main } = require("electron"); +let listener = require("electron/main").ipcMain; +main.handle("rea:read", async () => "value"); +listener.on("rea:write", () => undefined); + +let replaced = require("electron").ipcMain; +replaced = { handle() {} }; +replaced.handle("rea:reassigned", () => undefined); + +var assignedLater; +assignedLater.handle("rea:before-assignment", () => undefined); +assignedLater = require("electron").ipcMain; + +if (globalThis.registerIpc) { + var conditional = require("electron").ipcMain; +} +conditional.handle("rea:conditional", () => undefined); +`, + ), + writeFile( + join(root, "preload.cjs"), + String.raw` +const { ipcRenderer } = require("electron"); +ipcRenderer.invoke("rea:read"); +ipcRenderer.send("rea:write", "value"); +`, + ), + ]); + return root; +}; + const renamedBindingFixtureDirectory = async (): Promise => { const root = await createTestTempDirectory("rea-electron-renamed-"); await Promise.all([ From 66863b609fe5bdbb46b8634399ea08439a94a9f4 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:02:02 +0800 Subject: [PATCH 19/27] test(javascript): reuse comparison node fixtures Use the established typed graph fixture constructor while retaining right-hand comparison provenance assertions. Restore the unchanged duplicate-code gate on combined main. Closes #1446. Refs #1434, #1443, #1065. --- .../javascriptApplicationChangeGraph.test.ts | 28 ++++++++----------- .../javascriptApplicationGraph.fixture.ts | 2 +- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/domain/javascript/javascriptApplicationChangeGraph.test.ts b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts index 2bbd653af..e3c8e1310 100644 --- a/src/domain/javascript/javascriptApplicationChangeGraph.test.ts +++ b/src/domain/javascript/javascriptApplicationChangeGraph.test.ts @@ -3,12 +3,12 @@ import { describe, expect, it } from "vitest"; import { createJavaScriptApplicationEdge, createJavaScriptApplicationGraph, - createJavaScriptApplicationNode, type JavaScriptApplicationGraph, } from "./javascriptApplicationGraph.js"; import { artifactEvidence, completeCoverage, + fixtureNode, } from "./javascriptApplicationGraph.fixture.js"; import { compareJavaScriptApplicationVersions } from "./javascriptApplicationVersionComparison.js"; @@ -22,26 +22,22 @@ const buildGraph = ( changed: number, ): JavaScriptApplicationGraph => { const nodes = ["a", "b", "c", "d"].map((name, index) => - createJavaScriptApplicationNode({ - kind: "javascript-module", - identity: { + fixtureNode( + "javascript-module", + { strategy: "canonical-path", stability: "artifact-version", artifact_sha256: artifactSha, path: `${name}.js`, }, - observations: [ - { - label: `${name}.js`, - properties: { - structural_fingerprint_sha256: digest(index + 1), - structural_fingerprint_algorithm: "fixture", - source_sha256: digest(index === 1 || index === 3 ? changed : 0), - }, - evidence: artifactEvidence(artifactSha, `${name}.js`), - }, - ], - }), + `${name}.js`, + { + structural_fingerprint_sha256: digest(index + 1), + structural_fingerprint_algorithm: "fixture", + source_sha256: digest(index === 1 || index === 3 ? changed : 0), + }, + artifactEvidence(artifactSha, `${name}.js`), + ), ); const [a, b] = nodes; if (a === undefined || b === undefined) diff --git a/src/domain/javascript/javascriptApplicationGraph.fixture.ts b/src/domain/javascript/javascriptApplicationGraph.fixture.ts index 00f7c229a..3002b822f 100644 --- a/src/domain/javascript/javascriptApplicationGraph.fixture.ts +++ b/src/domain/javascript/javascriptApplicationGraph.fixture.ts @@ -109,7 +109,7 @@ type FixtureNodeArguments = [ evidence: ApplicationGraphEvidence, ]; -const fixtureNode = ( +export const fixtureNode = ( ...[kind, identity, label, properties, evidence]: FixtureNodeArguments ): FixtureNode => createJavaScriptApplicationNode({ From 1ec490f36180cf6d3a8f353e9e43920dbd81ece0 Mon Sep 17 00:00:00 2001 From: Zeynel <137368801+zeynelacikgoez@users.noreply.github.com> Date: Sat, 10 Oct 2026 08:04:31 +0200 Subject: [PATCH 20/27] fix(javascript): stop replaying earlier writes for each object property write Repeated property writes to one object made member-mutation collection cubic. Each write evaluated its target binding to rule out a primitive, and every evaluation folded all earlier mutation paths, copying every slot once per path. 2,000 writes to one object took 27 s and 4,000 took 203 s. - A recorded write always leaves a non-primitive binding value, so only the first write to a binding needs the primitive check. - Apply mutation paths per slot in one pass with the same ordering semantics instead of one full slot copy per path. Co-authored-by: Claude Opus 5.5 Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> --- ...tSemanticAnalysis.member_mutations.test.ts | 21 ++++ .../javascriptSemanticMemberMutations.ts | 5 +- .../javascriptSemanticMutationValues.ts | 118 ++++++++++++------ .../javascript/javascriptSemanticValues.ts | 10 +- 4 files changed, 106 insertions(+), 48 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts index f28436eb1..3450060cb 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.member_mutations.test.ts @@ -590,3 +590,24 @@ describe("property mutations through TypeScript satisfies aliases", () => { ).toEqual({ status: "literal", value: "TOKEN" }); }); }); + +describe("property mutation collection on minified bundles", () => { + it("records each write to one object without replaying earlier writes", () => { + const writes = Array.from( + { length: 2000 }, + (_, index) => `source.p${index} = ${index};`, + ).join(" "); + const start = performance.now(); + const value = resultValue( + `const source = { token: "TOKEN" }; ${writes} return [source.token, source.p1999];`, + ); + expect(performance.now() - start).toBeLessThan(2000); + expect(value).toMatchObject({ + status: "array", + items: [ + { value: { status: "literal", value: "TOKEN" } }, + { value: { status: "unknown" } }, + ], + }); + }, 30000); +}); diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 4e4ea7b8b..1414d407f 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -194,8 +194,11 @@ export const collectSemanticMemberMutations = ( current.originAt?.end, ]); if (recordedEffects.has(identity)) continue; + // A recorded write invalidates the binding to a non-primitive value, so + // only its first write can target a primitive. Re-evaluating it would + // replay every earlier write and make repeated writes superlinear. const value = - effect === "write" + effect === "write" && binding.mutatedPaths.length === 0 ? evaluateSemanticBinding(binding, state) : undefined; const primitiveWrite = diff --git a/src/domain/javascript/javascriptSemanticMutationValues.ts b/src/domain/javascript/javascriptSemanticMutationValues.ts index dbbc9d457..bc2213da7 100644 --- a/src/domain/javascript/javascriptSemanticMutationValues.ts +++ b/src/domain/javascript/javascriptSemanticMutationValues.ts @@ -33,49 +33,89 @@ export const invalidateSemanticEscapedPath = ( : { ...value, items: value.items.map(invalidateChild) }; }; -/** Invalidate only slots that an explicit mutation can affect. */ -export const invalidateSemanticMutationPath = ( +/** + * Invalidate only slots that explicit mutations can affect, as if applying each + * path in order. Paths are grouped per slot so that many writes to one object + * cost one pass over its slots rather than one pass per write. + */ +export const invalidateSemanticMutationPaths = ( value: JavaScriptSemanticValue, - path: JavaScriptSemanticPropertyPath, + paths: readonly JavaScriptSemanticPropertyPath[], ): JavaScriptSemanticValue => { - const [key, ...remaining] = path; - const unknown: JavaScriptSemanticValue = { - status: "unknown", - reason: "This property may have been mutated.", + if (paths.length === 0) return value; + // An unknown key or a replaced value leaves nothing that a later path can + // restore, wherever it occurs in the sequence. + if ( + (value.status !== "object" && value.status !== "array") || + paths.some(([key]) => key === undefined || key === null) + ) + return { + status: "unknown", + reason: "This property may have been mutated.", + }; + // Writing an array length discards every item known before it. + const reset = + value.status === "array" + ? paths.findLastIndex(([key]) => key === "length") + : -1; + const base: typeof value = + reset < 0 + ? value + : { status: "array", items: [], unknownItems: true, omittedItems: null }; + const slots = base.status === "object" ? base.properties : base.items; + const slotsByName = new Map(); + slots.forEach((property, index) => { + const indexes = slotsByName.get(property.name) ?? []; + indexes.push(index); + slotsByName.set(property.name, indexes); + }); + const slotPaths = new Map(); + const replacedSlots = new Set(); + const select = (index: number, remaining: JavaScriptSemanticPropertyPath) => { + const selected = slotPaths.get(index) ?? []; + selected.push(remaining); + slotPaths.set(index, selected); + if (remaining.length === 0) replacedSlots.add(index); }; - if (key === undefined || key === null) return unknown; - if (value.status === "object" || value.status === "array") { - if (value.status === "array" && key === "length") - return { - status: "array", - items: [], - unknownItems: true, - omittedItems: null, - }; - const slots = value.status === "object" ? value.properties : value.items; - const observed = slots.some((property) => - semanticPropertyPathKeyMatches(key, property.name), - ); - const properties = slots.map((property) => - semanticPropertyPathKeyMatches(key, property.name) - ? { + // An added slot is already unknown, so later paths that select it are inert. + const added = new Set(); + for (const [key, ...remaining] of paths.slice(reset + 1)) { + if (key === undefined || key === null) continue; + if (typeof key === "object") { + slots.forEach((property, index) => { + if (semanticPropertyPathKeyMatches(key, property.name)) + select(index, remaining); + }); + continue; + } + const indexes = slotsByName.get(String(key)); + if (indexes === undefined) added.add(String(key)); + else for (const index of indexes) select(index, remaining); + } + const properties: JavaScriptSemanticProperty[] = slots.map( + (property, index) => { + const selected = slotPaths.get(index); + return selected === undefined + ? property + : { ...property, - ...(remaining.length === 0 + ...(replacedSlots.has(index) ? { presence: "unknown-coverage" as const } : {}), - value: invalidateSemanticMutationPath(property.value, remaining), - } - : property, - ); - if (!observed && typeof key !== "object") - properties.push({ - name: String(key), - value: unknown, - presence: "unknown-coverage", - }); - return value.status === "object" - ? { ...value, properties } - : { ...value, items: properties }; - } - return unknown; + value: invalidateSemanticMutationPaths(property.value, selected), + }; + }, + ); + for (const name of added) + properties.push({ + name, + value: { + status: "unknown", + reason: "This property may have been mutated.", + }, + presence: "unknown-coverage", + }); + return base.status === "object" + ? { ...base, properties } + : { ...base, items: properties }; }; diff --git a/src/domain/javascript/javascriptSemanticValues.ts b/src/domain/javascript/javascriptSemanticValues.ts index 884915948..87f6ad540 100644 --- a/src/domain/javascript/javascriptSemanticValues.ts +++ b/src/domain/javascript/javascriptSemanticValues.ts @@ -10,7 +10,7 @@ import { semanticSlotAtPath } from "./javascriptSemanticSlots.js"; import { invalidateSemanticEscapedPath, - invalidateSemanticMutationPath, + invalidateSemanticMutationPaths, } from "./javascriptSemanticMutationValues.js"; import type { JavaScriptBindingProvenance } from "./javascriptSemanticIr.js"; @@ -211,13 +211,7 @@ const evaluateBinding = ( if (isPrimitive(captured)) value = captured; } } - let projected = value; - for (const path of binding.mutatedPaths) { - projected = invalidateSemanticMutationPath(projected, path); - // Every subsequent write returns this same root unknown. Replaying those - // paths at every newly collected effect otherwise becomes quadratic. - if (projected.status === "unknown") break; - } + let projected = invalidateSemanticMutationPaths(value, binding.mutatedPaths); for (const escape of binding.escapedPaths) { // Escapes only invalidate object references; other lattice values survive. if (projected.status !== "object" && projected.status !== "array") break; From c89c8f53187aac02e87f0ca11e3dd120b8c7efef Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:05:56 +0800 Subject: [PATCH 21/27] docs(mcp): record real Windows first-query recovery --- docs/mcp-contracts.md | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index 70d843335..fd801cc0a 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -193,10 +193,27 @@ the current client, so it is not a fresh-start recovery. On the same connection: This close/reopen flow was exercised on one real Linux stdio connection after a controlled startup timeout, followed by a successful overview and function -analysis. It is not a Windows/macOS coverage claim. REA cleans only resources it -owns and never switches to another provider automatically. A provider timeout, -installation failure, or host permission denial needs its own reported recovery; -increasing a client deadline alone does not fix those failures. +analysis. A separate Windows x64 stdio run with Ghidra 12.1.4, JDK 21 and Node +22.19.0 exercised the same connection and caller-selected native PE: the default +60-second client request timed out, close succeeded, and reopening before a +240-second request completed the cold overview in about 163 seconds. The next +overview took about 19 ms. The cold and warm overviews reported two procedures +and two segments; the input digest was unchanged and no owned runtime root or +observed process-family member remained after final close. This was a small +1,024-byte fixture, so the longer request setting remains an example, not a +guarantee for larger targets. No macOS cold-start recovery was exercised. + +With a supplied progress token, the Windows run received an operation-start +notification before timeout and start/completion notifications for the +successful retry. It received no intermediate import or auto-analysis progress. +These operation markers do not measure analysis work completed or extend the +client's deadline. Each new provider session still imports the target into a +fresh temporary project. + +REA cleans only resources it owns and never switches to another provider +automatically. A provider timeout, installation failure, or host permission +denial needs its own reported recovery; increasing a client deadline alone does +not fix those failures. ## Tool results From 2781cad20125ff3a210b795f60620b048e1bbe9b Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:18:13 -0700 Subject: [PATCH 22/27] perf(process): stop padding rendered terminal lines to full width (#1450) Rendered frames padded every visible row with trailing spaces to the terminal width. Blank rows cost a full row each and every frame repeats them, so the padding made up 14-39% of capture Evidence (2 KB of sparse output produced 228 KB). The padding carried no information: columns is recorded and serialized_state keeps the exact terminal state. Lines are now trimmed after normalization, so line.padEnd(columns) restores the previous value. The comparison contract commits rendered_line_format so older padded captures do not compare as the same contract. Co-authored-by: Claude Opus 5.5 --- docs/process-capture.md | 6 +++ src/domain/process/processScenario.ts | 1 + src/process/capture/TerminalRenderer.test.ts | 47 +++++++++++++++++++ src/process/capture/TerminalRenderer.ts | 17 ++++--- .../processCaptureNormalization.test.ts | 1 + 5 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 src/process/capture/TerminalRenderer.test.ts diff --git a/docs/process-capture.md b/docs/process-capture.md index 1d3d23e7f..c315e3e9e 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -110,6 +110,12 @@ while comparisons and trace assertions use the normalized text. Retaining the original adds at most `limits.output_bytes` of text, because it comes from the same admitted PTY chunks rather than a second unbounded stream. +Rendered frame `lines` hold the visible rows after normalization, without +trailing spaces; pad a line to the frame's `columns` to restore the fixed-width +row. `serialized_state` keeps the exact terminal state. Captures commit this +line format in their comparison contract, so a capture whose lines were padded +to full width does not compare as the same contract. + Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and `listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with ports from 0 through 65535. A port may end a sentence, as in `port: 8080.`; diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index 8312e1d2a..0ecf2c259 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -108,6 +108,7 @@ export const processComparisonContract = ( ...(scenario.normalization.ports ? { port_normalization_version: "contextual-endpoints-v2" } : {}), + rendered_line_format: "trailing-spaces-trimmed-v1", }); /** diff --git a/src/process/capture/TerminalRenderer.test.ts b/src/process/capture/TerminalRenderer.test.ts new file mode 100644 index 000000000..e5a19996b --- /dev/null +++ b/src/process/capture/TerminalRenderer.test.ts @@ -0,0 +1,47 @@ +import { expect, it } from "vitest"; + +import { TerminalRenderer } from "./TerminalRenderer.js"; + +const renderFrames = async ( + data: string, + normalize: (value: string) => string = (value) => value, +) => { + const renderer = new TerminalRenderer({ + columns: 20, + rows: 4, + scrollback: 10, + maxBytes: 100_000, + normalize, + }); + renderer.write(data, 0); + const frames = await renderer.frames(); + const retention = renderer.retention(); + await renderer.dispose(); + return { frames, retention }; +}; + +it("omits trailing blank cells from visible lines without losing rows", async () => { + const { frames, retention } = await renderFrames( + "left right \r\n\r\n indented", + ); + const [frame] = frames; + expect(frame?.lines).toEqual(["left right", "", " indented", ""]); + // Fixed-width rows remain recoverable from the recorded column count. + expect(frame?.lines.map((line) => line.padEnd(frame.columns, " "))).toEqual([ + "left right ", + " ".repeat(20), + " indented ", + " ".repeat(20), + ]); + expect(retention.retained_bytes).toBe( + Buffer.byteLength(frame?.serialized_state ?? "") + + Buffer.byteLength("left right indented"), + ); +}); + +it("trims only after normalization sees the full-width row", async () => { + const { frames } = await renderFrames("pid 4242", (value) => + value.replace(/4242 +$/u, ""), + ); + expect(frames[0]?.lines[0]).toBe("pid "); +}); diff --git a/src/process/capture/TerminalRenderer.ts b/src/process/capture/TerminalRenderer.ts index e7f1a0fc8..4322e2781 100644 --- a/src/process/capture/TerminalRenderer.ts +++ b/src/process/capture/TerminalRenderer.ts @@ -14,6 +14,8 @@ const HeadlessPackage = const SerializePackage = require("@xterm/addon-serialize") as typeof import("@xterm/addon-serialize"); +const TRAILING_SPACES = / +$/u; + interface TerminalRendererOptions { readonly columns: number; readonly rows: number; @@ -106,13 +108,16 @@ export class TerminalRenderer { const lines: string[] = []; for (let row = 0; row < this.#terminal.rows; row += 1) { const line = buffer.getLine(buffer.viewportY + row); + // Trailing blank cells carry no information beyond `columns`; dropping + // them keeps sparse screens from costing a full row per blank line. lines.push( - this.options.normalize( - (line?.translateToString(false, 0, this.#terminal.cols) ?? "").padEnd( - this.#terminal.cols, - " ", - ), - ), + this.options + .normalize( + ( + line?.translateToString(false, 0, this.#terminal.cols) ?? "" + ).padEnd(this.#terminal.cols, " "), + ) + .replace(TRAILING_SPACES, ""), ); } const serializedState = this.options.normalize( diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 58cf22c0f..8156aff97 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -64,6 +64,7 @@ itWithCaptureCapability( ).toBe(true); expect(capture.manifest.comparison_contract).toMatchObject({ port_normalization_version: "contextual-endpoints-v2", + rendered_line_format: "trailing-spaces-trimmed-v1", }); expect(capture.exit).toMatchObject({ code: 0, reason: "exited" }); expect(capture.cleanup).toMatchObject({ From 696fb834b8873ac9f6df0c4d45377bdd7c3c5762 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:20:22 +0800 Subject: [PATCH 23/27] fix(skills): expose authored skill to repository discovery --- .../reverse-engineer-anything/SKILL.md | 0 .../references/android-applications.md | 0 .../references/evidence-workflows.md | 0 .../references/javascript-applications.md | 0 .../references/native-and-artifacts.md | 0 .../references/runtime-observation.md | 0 .github/pull_request_template.md | 2 +- AGENTS.md | 2 +- CONTRIBUTING.md | 8 +- docs/installation.md | 2 +- scripts/generate-package-metadata.mjs | 2 +- scripts/generate-skill-metadata.mjs | 4 +- scripts/verify-package-pack.mjs | 2 +- scripts/verify-package.mjs | 4 + scripts/verify-repository-skill.mjs | 97 +++++++++++++++++++ .../filesystem/dependencyInstall.test.ts | 2 +- .../filesystem/generatedSkill.test.ts | 2 +- turbo.json | 6 +- 18 files changed, 119 insertions(+), 14 deletions(-) rename {skill-src => .agents/skills}/reverse-engineer-anything/SKILL.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/android-applications.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/evidence-workflows.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/javascript-applications.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/native-and-artifacts.md (100%) rename {skill-src => .agents/skills}/reverse-engineer-anything/references/runtime-observation.md (100%) create mode 100644 scripts/verify-repository-skill.mjs diff --git a/skill-src/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md similarity index 100% rename from skill-src/reverse-engineer-anything/SKILL.md rename to .agents/skills/reverse-engineer-anything/SKILL.md diff --git a/skill-src/reverse-engineer-anything/references/android-applications.md b/.agents/skills/reverse-engineer-anything/references/android-applications.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/android-applications.md rename to .agents/skills/reverse-engineer-anything/references/android-applications.md diff --git a/skill-src/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/evidence-workflows.md rename to .agents/skills/reverse-engineer-anything/references/evidence-workflows.md diff --git a/skill-src/reverse-engineer-anything/references/javascript-applications.md b/.agents/skills/reverse-engineer-anything/references/javascript-applications.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/javascript-applications.md rename to .agents/skills/reverse-engineer-anything/references/javascript-applications.md diff --git a/skill-src/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/native-and-artifacts.md rename to .agents/skills/reverse-engineer-anything/references/native-and-artifacts.md diff --git a/skill-src/reverse-engineer-anything/references/runtime-observation.md b/.agents/skills/reverse-engineer-anything/references/runtime-observation.md similarity index 100% rename from skill-src/reverse-engineer-anything/references/runtime-observation.md rename to .agents/skills/reverse-engineer-anything/references/runtime-observation.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ffbeb73b..e33ce01f4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -28,7 +28,7 @@ intentionally not included. --> - Provider, bridge, target-format, or platform compatibility: - Evidence, artifact, provenance, or reconstruction contract: - Process execution, authorization, cleanup, or containment impact: -- Generated metadata (`docs/public/product-catalog.json`), package, or installation impact (CI generates and validates ignored catalogs and `skills/`; edit authored instructions in `skill-src/` and never commit build outputs): +- Generated metadata (`docs/public/product-catalog.json`), package, or installation impact (CI generates and validates ignored catalogs and `skills/`; edit authored instructions in `.agents/skills/reverse-engineer-anything/` and never commit build outputs): ## Evidence and regression coverage diff --git a/AGENTS.md b/AGENTS.md index 15d3984ff..6e5a2df44 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,7 @@ Use [.nvmrc](.nvmrc) and `package.json#packageManager` for the pinned toolchain. Use relevant checks for the change. The complete local gate, `npm run check:pr`, is optional for broad changes or CI diagnosis. See [CONTRIBUTING.md](CONTRIBUTING.md) for development, generated-file ownership, and release conventions. -`docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/` are ignored outputs. Edit source contracts and `skill-src/`, then regenerate as needed. `.cache/mcp-tool-catalog.json` is test metadata; runtime builds and source checking do not consume it. Never commit binaries, provider project documents, credentials, `dist/`, `node_modules/`, or local planning artifacts. +`docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/` are ignored outputs. Edit source contracts and `.agents/skills/reverse-engineer-anything/`, then regenerate as needed. `.cache/mcp-tool-catalog.json` is test metadata; runtime builds and source checking do not consume it. Never commit binaries, provider project documents, credentials, `dist/`, `node_modules/`, or local planning artifacts. ## Tool and Test Changes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07c71c4c5..9f4e6969b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,8 +95,12 @@ Pre-commit formats and lints staged files; pre-push runs `check:fast`. `evidence:generate` prepares only the managed commitment and its runtime/skill dependencies. The ignored outputs are `docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/`. Edit authored -skill instructions in `skill-src/`. CI validates generated outputs and retains -them as artifacts; do not commit them. Reviewed source metadata, including +skill instructions in `.agents/skills/reverse-engineer-anything/`, where Skills +can discover the public bundle directly from a clean checkout. `verify:package` +installs that bundle with Skills 1.7.2 using default repository-root discovery +and checks every installed file, alongside the packaged setup checks. +CI validates generated outputs and retains them as artifacts; do not commit +them. Reviewed source metadata, including `src/generatedPackageMetadata.ts` and `docs/error-contract.schema.json`, remains tracked and checked for freshness. diff --git a/docs/installation.md b/docs/installation.md index e54cc73ae..38ea09f5a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -95,7 +95,7 @@ npx skills add morluto/rea --skill reverse-engineer-anything This installs agent instructions and bundled references, not REA MCP registration or analysis engines. Follow the skill's -[conditional connection guide](https://github.com/morluto/rea/blob/main/skill-src/reverse-engineer-anything/SKILL.md#connect-only-when-needed). +[conditional connection guide](https://github.com/morluto/rea/blob/main/.agents/skills/reverse-engineer-anything/SKILL.md#connect-only-when-needed). Working tools can be used immediately. If tools are missing, inspect the current client's registration with `doctor --client codex --json` (substitute its client ID), then plan repairs with `setup --client codex --dry-run --json`. Show and diff --git a/scripts/generate-package-metadata.mjs b/scripts/generate-package-metadata.mjs index 8247dac1c..0130401b1 100644 --- a/scripts/generate-package-metadata.mjs +++ b/scripts/generate-package-metadata.mjs @@ -17,7 +17,7 @@ const lock = JSON.parse( await readFile(join(root, "package-lock.json"), "utf8"), ); const skill = await readFile( - join(root, "skill-src/reverse-engineer-anything/SKILL.md"), + join(root, ".agents/skills/reverse-engineer-anything/SKILL.md"), "utf8", ); const skillVersion = /^\s{2}version:\s*"([^"]+)"\s*$/mu.exec(skill)?.[1]; diff --git a/scripts/generate-skill-metadata.mjs b/scripts/generate-skill-metadata.mjs index 5bb2e3ccf..3ca0a467d 100644 --- a/scripts/generate-skill-metadata.mjs +++ b/scripts/generate-skill-metadata.mjs @@ -10,7 +10,7 @@ for (const argument of arguments_) throw new Error(`Unknown skill metadata option: ${argument}`); const root = join(dirname(fileURLToPath(import.meta.url)), ".."); -const sourceRoot = join(root, "skill-src/reverse-engineer-anything"); +const sourceRoot = join(root, ".agents/skills/reverse-engineer-anything"); const outputRoot = join(root, "skills/reverse-engineer-anything"); const check = arguments_.has("--check"); const current = ( @@ -25,7 +25,7 @@ const { PRODUCT_IDENTITY } = await import( ); if (/^\s{2}(?:tool_count|catalog_digest):/mu.test(current)) throw new Error( - "Catalog metadata belongs in the generated skill, not skill-src", + "Catalog metadata belongs in the generated skill, not the authored source", ); const versionLine = /^ {2}version: "[^"\r\n]+"$/mu; if (!versionLine.test(current)) diff --git a/scripts/verify-package-pack.mjs b/scripts/verify-package-pack.mjs index 96f05417b..fc9676745 100644 --- a/scripts/verify-package-pack.mjs +++ b/scripts/verify-package-pack.mjs @@ -38,7 +38,7 @@ export async function verifyPackagePack({ root, workspace }) { throw new Error("package did not expose both rea command entry points"); if (packedManifest.mcpName !== "io.github.morluto/rea") throw new Error("package did not retain its MCP Registry ownership marker"); - if (packedFiles.some((path) => path.startsWith("package/skill-src/"))) + if (packedFiles.some((path) => path.startsWith("package/.agents/"))) throw new Error( "package included authored skill sources instead of only the generated bundle", ); diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index 24923f7f6..b70b06064 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -17,6 +17,7 @@ import { verifyPackageMcp } from "./verify-package-mcp.mjs"; import { verifyPackagePack } from "./verify-package-pack.mjs"; import { verifyPackagePlatform } from "./verify-package-platform.mjs"; import { verifyPackageSetup } from "./verify-package-setup.mjs"; +import { verifyRepositorySkill } from "./verify-repository-skill.mjs"; import { verifyManaged } from "./verify-package-managed.mjs"; import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; @@ -34,9 +35,11 @@ let tarball; let mcpStartup; let mcpModuleLoading; let update; +let repositorySkill; let report; try { + repositorySkill = await verifyRepositorySkill({ root, workspace }); ({ tarball } = await verifyPackagePack({ root, workspace })); const environmentData = await verifyPackageEnvironment({ root, @@ -156,6 +159,7 @@ try { configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, + repositorySkill, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, diff --git a/scripts/verify-repository-skill.mjs b/scripts/verify-repository-skill.mjs new file mode 100644 index 000000000..8340bce7b --- /dev/null +++ b/scripts/verify-repository-skill.mjs @@ -0,0 +1,97 @@ +import { cp, mkdir, readFile, readdir } from "node:fs/promises"; +import { dirname, join, relative } from "node:path"; + +import { exec, pathExists } from "./lib/verify-package-core.mjs"; + +/** Exercise Skills' default root discovery without ignored build outputs. */ +export async function verifyRepositorySkill({ root, workspace }) { + const snapshot = join(workspace, "repository"); + const consumer = join(workspace, "skill-consumer"); + const home = join(workspace, "skill-consumer-home"); + const tracked = ( + await exec( + "git", + ["ls-files", "--cached", "--format=%(objectmode) %(path)", "-z"], + { cwd: root }, + ) + ).stdout + .split("\0") + .filter(Boolean); + for (const entry of tracked) { + const mode = entry.slice(0, 6); + const path = entry.slice(7); + const destination = join(snapshot, path); + // A GitHub clone without submodule initialization has only these directories. + if (mode === "160000") { + await mkdir(destination, { recursive: true }); + continue; + } + await mkdir(dirname(destination), { recursive: true }); + await cp(join(root, path), destination, { verbatimSymlinks: true }); + } + if (await pathExists(join(snapshot, "skills"))) + throw new Error("repository skill check must run without generated skills"); + await mkdir(consumer, { recursive: true }); + await mkdir(home, { recursive: true }); + const skillsVersion = "1.7.2"; + await exec( + "npm", + [ + "exec", + "--yes", + "--package", + `skills@${skillsVersion}`, + "--", + "skills", + "add", + snapshot, + "--skill", + "reverse-engineer-anything", + "--agent", + "codex", + "--yes", + ], + { + cwd: consumer, + env: { + ...process.env, + HOME: home, + USERPROFILE: home, + XDG_CONFIG_HOME: join(home, ".config"), + CODEX_HOME: join(home, ".codex"), + DO_NOT_TRACK: "1", + SKILLS_NO_TELEMETRY: "1", + CI: "1", + }, + timeout: 120_000, + }, + ); + const authoredRoot = join( + snapshot, + ".agents/skills/reverse-engineer-anything", + ); + const installedSkills = join(consumer, ".agents/skills"); + const installedRoot = join(installedSkills, "reverse-engineer-anything"); + const files = await filePaths(authoredRoot); + if ( + JSON.stringify(await readdir(installedSkills)) !== + JSON.stringify(["reverse-engineer-anything"]) || + JSON.stringify(await filePaths(installedRoot)) !== JSON.stringify(files) + ) + throw new Error( + "Skills did not install exactly the selected public bundle", + ); + for (const path of files) { + const authored = await readFile(join(authoredRoot, path)); + if (!authored.equals(await readFile(join(installedRoot, path)))) + throw new Error(`Skills changed authored skill content: ${path}`); + } + return { skillsVersion, files }; +} + +async function filePaths(root) { + return (await readdir(root, { recursive: true, withFileTypes: true })) + .filter((entry) => entry.isFile()) + .map((entry) => relative(root, join(entry.parentPath, entry.name))) + .sort(); +} diff --git a/tests/boundary/filesystem/dependencyInstall.test.ts b/tests/boundary/filesystem/dependencyInstall.test.ts index 66378360f..806d91b32 100644 --- a/tests/boundary/filesystem/dependencyInstall.test.ts +++ b/tests/boundary/filesystem/dependencyInstall.test.ts @@ -169,7 +169,7 @@ const createBuildCacheFixture = async (): Promise => { "scripts/clean-build-output.mjs", "scripts/generate-package-metadata.mjs", "scripts/lib/generated-file.mjs", - "skill-src/reverse-engineer-anything/SKILL.md", + ".agents/skills/reverse-engineer-anything/SKILL.md", "src/generatedPackageMetadata.ts", ]; await Promise.all( diff --git a/tests/boundary/filesystem/generatedSkill.test.ts b/tests/boundary/filesystem/generatedSkill.test.ts index 5df6e3146..2b7ebf754 100644 --- a/tests/boundary/filesystem/generatedSkill.test.ts +++ b/tests/boundary/filesystem/generatedSkill.test.ts @@ -21,7 +21,7 @@ async function fixture(eol = "\n") { await mkdir(dirname(join(root, path)), { recursive: true }); await copyFile(join(repositoryRoot, path), join(root, path)); } - const sourceRoot = join(root, "skill-src/reverse-engineer-anything"); + const sourceRoot = join(root, ".agents/skills/reverse-engineer-anything"); const outputRoot = join(root, "skills/reverse-engineer-anything"); await mkdir(join(sourceRoot, "references"), { recursive: true }); await mkdir(join(root, "dist")); diff --git a/turbo.json b/turbo.json index 392523708..5b93503d7 100644 --- a/turbo.json +++ b/turbo.json @@ -39,7 +39,7 @@ "package-lock.json", "scripts/generate-package-metadata.mjs", "scripts/lib/generated-file.mjs", - "skill-src/reverse-engineer-anything/SKILL.md", + ".agents/skills/reverse-engineer-anything/SKILL.md", "src/generatedPackageMetadata.ts" ] }, @@ -138,7 +138,7 @@ "docs/**", "!docs/public/product-catalog.json", "!docs/verification/managed-conformance-*.json", - "skill-src/**", + ".agents/skills/reverse-engineer-anything/**", "README.md", { "mode": "dependencyOutputs", @@ -170,7 +170,7 @@ "package-lock.json", "scripts/generate-skill-metadata.mjs", "scripts/lib/generated-file.mjs", - "skill-src/**" + ".agents/skills/reverse-engineer-anything/**" ], "outputs": ["skills/**"] }, From ef96657d6ba0c1f22ebdb1e6bd4d9a6ac9335002 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:21:07 -0700 Subject: [PATCH 24/27] perf(javascript): hash each semantic evidence context once (#1457) The semantic graph's evidence-context registry canonicalized the full provenance context three times for every node, relation, unknown and fingerprint: once to hash its ID and twice more to compare against the stored row. A file has only a few distinct contexts, so nearly all of that work repeated a known answer. Interning now keys contexts by their canonical text. Each fact canonicalizes once, and only a new context is hashed and checked for a digest collision. IDs and output are unchanged; Obsidian's 334 MB Evidence is byte-identical and analysis drops from about 47.0 s to 44.4 s. Co-authored-by: Claude Opus 5.5 --- .../javascript/javascriptSemanticGraph.ts | 23 +++++++---- .../javascriptSemanticGraphQuery.test.ts | 40 +++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/domain/javascript/javascriptSemanticGraph.ts b/src/domain/javascript/javascriptSemanticGraph.ts index e6cf1d530..69b3bb967 100644 --- a/src/domain/javascript/javascriptSemanticGraph.ts +++ b/src/domain/javascript/javascriptSemanticGraph.ts @@ -56,6 +56,9 @@ const evidenceContextId = (context: ApplicationGraphEvidenceContext): string => /** Graph-scoped owner for canonical provenance shared by semantic facts. */ export class JavaScriptSemanticEvidenceContextRegistry { readonly #contexts = new Map(); + // Facts share a few contexts, so key them by canonical text and hash each + // distinct context once instead of on every fact. + readonly #idsByCanonicalContext = new Map(); /** Intern normalized non-location provenance at fact construction. */ intern( @@ -63,18 +66,21 @@ export class JavaScriptSemanticEvidenceContextRegistry { ): JavaScriptSemanticEvidenceReference { const normalized = normalizeEvidence(evidence); const { location, ...context } = normalized; + const canonicalContext = canonicalJson( + context, + "JavaScript semantic graph", + ); + const internedId = this.#idsByCanonicalContext.get(canonicalContext); + if (internedId !== undefined) return { context_id: internedId, location }; const contextId = evidenceContextId(context); - const row = { ...context, context_id: contextId }; - const existing = this.#contexts.get(contextId); - if ( - existing !== undefined && - canonicalJson(existing, "JavaScript semantic graph") !== - canonicalJson(row, "JavaScript semantic graph") - ) + // This canonical context is new, so a row already stored under its + // digest holds different provenance. + if (this.#contexts.has(contextId)) throw new TypeError( "JavaScript semantic evidence context digest collision", ); - this.#contexts.set(contextId, row); + this.#contexts.set(contextId, { ...context, context_id: contextId }); + this.#idsByCanonicalContext.set(canonicalContext, contextId); return { context_id: contextId, location }; } @@ -101,6 +107,7 @@ export class JavaScriptSemanticEvidenceContextRegistry { /** Release interned contexts after the owning graph is finalized. */ clear(): void { this.#contexts.clear(); + this.#idsByCanonicalContext.clear(); } } diff --git a/src/domain/javascript/javascriptSemanticGraphQuery.test.ts b/src/domain/javascript/javascriptSemanticGraphQuery.test.ts index ddf3015f0..363cf29ad 100644 --- a/src/domain/javascript/javascriptSemanticGraphQuery.test.ts +++ b/src/domain/javascript/javascriptSemanticGraphQuery.test.ts @@ -1,6 +1,7 @@ import { expect, it } from "vitest"; import { z } from "zod"; +import { digestCanonicalValue } from "../canonicalDigest.js"; import type { ApplicationGraphEvidence } from "./javascriptApplicationEvidenceSchemas.js"; import { JAVASCRIPT_SEMANTIC_NODE_KINDS, @@ -615,3 +616,42 @@ it.each(invalidGraphCases)( expect(issues(() => complete(steps))).toEqual(expected); }, ); + +it("interns each distinct evidence context once under its content digest", () => { + const registry = new JavaScriptSemanticEvidenceContextRegistry(); + const elsewhere: ApplicationGraphEvidence = { + ...evidence(), + location: { + available: true, + value: { + kind: "source-range", + source: "other.js", + start: { line: 7, column: 2 }, + end: { line: 7, column: 9 }, + }, + }, + }; + const first = registry.intern(evidence()); + const repeated = registry.intern(elsewhere); + const reordered = registry.intern({ + ...evidence("inferred"), + limitations: [ + "Static reachability does not prove runtime execution.", + "Static reachability does not prove runtime execution.", + ], + }); + const inferred = registry.intern(evidence("inferred")); + const { location: _location, ...context } = evidence(); + + expect(repeated.context_id).toBe(first.context_id); + expect(repeated.location).toEqual(elsewhere.location); + expect(reordered.context_id).toBe(inferred.context_id); + expect(inferred.context_id).not.toBe(first.context_id); + expect(first.context_id).toBe( + `jsrg_evidence_${digestCanonicalValue(context, "test")}`, + ); + expect(registry.contexts.map(({ context_id }) => context_id)).toEqual( + [first.context_id, inferred.context_id].toSorted(), + ); + expect(registry.resolve(repeated)).toEqual(elsewhere); +}); From b118f044926ae6b4880c4214621ae6c5b2a71f17 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Fri, 9 Oct 2026 23:21:24 -0700 Subject: [PATCH 25/27] perf(apple): stop repeating the archive path in Interface Builder evidence (#1460) Every Interface Builder node and edge evidence record prefixed its description with the archive's relative path, which artifact_path in the same record already states. Bundles carry thousands of objects across localized nibs, so the repeated path made up 6-10% of decode_interface_builder output (XLD: 7.38 MB to 6.62 MB). Descriptions now name only the decoded item, such as "object 81" or "outlet o_quality". Co-authored-by: Claude Opus 5.5 --- .../apple/interfaceBuilderGraph.test.ts | 43 +++++++++++++++++++ src/domain/apple/interfaceBuilderGraph.ts | 4 +- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/domain/apple/interfaceBuilderGraph.test.ts b/src/domain/apple/interfaceBuilderGraph.test.ts index 13d6f79fb..fb1c0b9ad 100644 --- a/src/domain/apple/interfaceBuilderGraph.test.ts +++ b/src/domain/apple/interfaceBuilderGraph.test.ts @@ -509,3 +509,46 @@ describe("AppKit keyed archive connectors", () => { expect(to).toEqual(expect.arrayContaining(["objc_selector", "unknown"])); }); }); + +it("names the archive once per evidence record", () => { + const relativePath = "Views/Main.storyboardc/scene.nib/objects.nib"; + const result = buildInterfaceBuilderAnalysis({ + targetSha256: hash, + toolVersion: "test", + documents: [ + { + relativePath, + archiveSha256: hash, + documentKind: "storyboard_scene", + raw: { + "com.apple.ibtool.document.objects": { + controller: { customClass: "StoreViewController" }, + next: { customClass: "BuildViewController" }, + }, + "com.apple.ibtool.document.connections": { + controller: [ + { type: "segue", identifier: "showBuild", destinationId: "next" }, + ], + }, + }, + }, + ], + limits: interfaceBuilderLimitsSchema.parse({}), + }); + const evidence = [...result.graph.nodes, ...result.graph.edges].flatMap( + (item) => item.evidence, + ); + expect(new Set(evidence.map(({ artifact_path }) => artifact_path))).toEqual( + new Set([relativePath]), + ); + expect(evidence.map(({ description }) => description)).toEqual( + expect.arrayContaining([ + "compiled document", + "object controller", + "segue showBuild", + ]), + ); + expect( + evidence.filter(({ description }) => description.includes(relativePath)), + ).toEqual([]); +}); diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index 17e94d584..bd35130a8 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -72,7 +72,9 @@ export const buildInterfaceBuilderAnalysis = (input: { const evidenceFor = (description: string) => [ { kind: "interface_builder_resource" as const, - description: `${document.relativePath}: ${description}`, + // artifact_path names the archive; repeating it here cost a path per + // node and edge. + description, location: { address: null, file_offset: null }, artifact_path: document.relativePath, artifact_sha256: document.archiveSha256, From ea923287751043c7acfaa35a0acd10fe32f785d6 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:33:57 +0800 Subject: [PATCH 26/27] fix(mcp): validate doctor probe against its discovered output contract (#1461) Validate the target-free probe against the named definition discovered from the complete production catalog. Preserve initialized server and inventory observations when the SDK rejects the probe, and retain the schema failure reason, absolute deadline and owned-child cleanup. The negative stdio replay regression fails on original main and passes the fix; actual production-boundary fault injection confirms rejection. Nine doctor/help tests, serial full static checks and real small-target CLI/MCP parity pass. Successful doctor reports remain byte-identical. No performance or model-token reduction is claimed. Closes #1454. Refs #1065, #723, #1433. --- src/mcpDoctor.ts | 26 +++++++--- tests/boundary/mcp/mcpDoctor.test.ts | 77 +++++++++++++++++++++++++++- tests/fixtures/mcpDoctorReply.mjs | 48 +++++++++++++++++ 3 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 tests/fixtures/mcpDoctorReply.mjs diff --git a/src/mcpDoctor.ts b/src/mcpDoctor.ts index 0f5de39fc..62a588280 100644 --- a/src/mcpDoctor.ts +++ b/src/mcpDoctor.ts @@ -6,6 +6,7 @@ import { resolve } from "node:path"; import { CATALOG_IDENTITY } from "./catalogIdentity.js"; import { PROMPT_CONTRACTS } from "./contracts/promptContracts.js"; import { TOOL_CONTRACTS } from "./contracts/toolContracts.js"; +import { err, ok } from "./domain/result.js"; import { PRODUCT_IDENTITY } from "./identity.js"; import { MCP_STARTUP_POLICY } from "./mcpStartupPolicy.js"; @@ -131,11 +132,22 @@ const inspectProductionMcpSession = async ( signal: AbortSignal, ) => { const request = requestOptions(deadline, signal); - const [tools, prompts, requestFlow] = await Promise.all([ + const [tools, prompts] = await Promise.all([ client.listTools(undefined, request), client.listPrompts(undefined, request), - client.callTool({ name: "binary_session", arguments: {} }, request), ]); + const sessionTool = tools.tools.find(({ name }) => name === "binary_session"); + const requestFlow = await client + .callTool( + { name: "binary_session", arguments: {} }, + { + ...requestOptions(deadline, signal), + ...(sessionTool === undefined ? {} : { toolDefinition: sessionTool }), + }, + ) + .then(ok, (cause: unknown) => + err({ kind: "request-flow" as const, detail: errorMessage(cause) }), + ); const inventory = { tools: compareInventory( TOOL_CONTRACTS.map(({ name }) => name), @@ -147,7 +159,7 @@ const inspectProductionMcpSession = async ( ), }; const observedIdentity = parseIdentityToolResult( - requestFlow.structuredContent, + requestFlow.ok ? requestFlow.value.structuredContent : undefined, ); const serverVersion = client.getServerVersion(); const protocolVersion = client.getNegotiatedProtocolVersion(); @@ -173,8 +185,10 @@ const inspectProductionMcpSession = async ( }, { name: "request-flow", - ok: requestFlow.isError !== true, - detail: "binary_session target-free request", + ok: requestFlow.ok && requestFlow.value.isError !== true, + detail: requestFlow.ok + ? "binary_session target-free request" + : requestFlow.error.detail, }, ]; return { @@ -194,7 +208,7 @@ const inspectProductionMcpSession = async ( }, request_flow: { tool: "binary_session", - ok: requestFlow.isError !== true, + ok: requestFlow.ok && requestFlow.value.isError !== true, }, checks, }; diff --git a/tests/boundary/mcp/mcpDoctor.test.ts b/tests/boundary/mcp/mcpDoctor.test.ts index 9858e4ff2..e9b68b36f 100644 --- a/tests/boundary/mcp/mcpDoctor.test.ts +++ b/tests/boundary/mcp/mcpDoctor.test.ts @@ -1,4 +1,6 @@ -import { readFile } from "node:fs/promises"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -43,6 +45,79 @@ describe("production MCP doctor", () => { ); }, 30_000); + it("rejects a schema-invalid probe while retaining the discovered inventory", async () => { + const root = await createTestTempDirectory("rea-mcp-doctor-reply-"); + const snapshotPath = join(root, "reply.json"); + const environment = { PATH: process.env.PATH ?? "" }; + const client = new Client({ name: "doctor-reply-capture", version: "1" }); + const transport = new StdioClientTransport({ + command: process.execPath, + args: [resolve("scripts/rea.mjs"), "mcp"], + cwd: process.cwd(), + env: environment, + stderr: "pipe", + }); + transport.stderr?.on("data", () => undefined); + try { + await client.connect(transport); + const tools = await client.listTools(); + const prompts = await client.listPrompts(); + const toolDefinition = tools.tools.find( + ({ name }) => name === "binary_session", + ); + if (toolDefinition === undefined) + throw new Error("Production binary_session contract is missing"); + const reply = await client.callTool( + { name: "binary_session", arguments: {} }, + { toolDefinition }, + ); + await writeFile( + snapshotPath, + JSON.stringify({ + serverInfo: client.getServerVersion(), + protocolVersion: client.getNegotiatedProtocolVersion(), + tools, + prompts, + reply, + }), + ); + } finally { + await client.close(); + await transport.close(); + } + // Replay real production metadata through a synthetic stdio producer. + for (const mode of ["valid", "invalid"]) { + const result = await runProductionMcpDoctor({ + command: process.execPath, + args: [ + resolve("tests/fixtures/mcpDoctorReply.mjs"), + snapshotPath, + mode, + ], + cwd: process.cwd(), + environment, + }); + expect(result.healthy).toBe(mode === "valid"); + expect(result.inventory?.tools.observed).toBe( + CATALOG_IDENTITY.counts.mcp_tools, + ); + expect(result.inventory?.prompts.observed).toBe( + CATALOG_IDENTITY.counts.mcp_prompts, + ); + expect(result.checks).toContainEqual({ + name: "initialize", + ok: true, + detail: expect.any(String), + }); + if (mode === "invalid") + expect(result.checks).toContainEqual({ + name: "request-flow", + ok: false, + detail: expect.stringMatching(/output schema/iu), + }); + } + }, 30_000); + it("reports exact missing, unexpected, and duplicate inventory names", () => { expect(compareInventory(["a", "b"], ["b", "c", "c"])).toEqual({ expected: 2, diff --git a/tests/fixtures/mcpDoctorReply.mjs b/tests/fixtures/mcpDoctorReply.mjs new file mode 100644 index 000000000..14dad1240 --- /dev/null +++ b/tests/fixtures/mcpDoctorReply.mjs @@ -0,0 +1,48 @@ +import { readFile } from "node:fs/promises"; +import { createInterface } from "node:readline"; + +const snapshot = JSON.parse(await readFile(process.argv[2], "utf8")); +const reply = structuredClone(snapshot.reply); +if (process.argv[3] === "invalid") { + reply.structuredContent.schema_violation_control = true; + reply.content = reply.content.map((item) => + item.type === "text" + ? { ...item, text: JSON.stringify(reply.structuredContent) } + : item, + ); +} + +const input = createInterface({ input: process.stdin }); +input.on("line", (line) => { + const request = JSON.parse(line); + if (request.id === undefined) return; + let result; + switch (request.method) { + case "initialize": + result = { + protocolVersion: snapshot.protocolVersion, + capabilities: { tools: {}, prompts: {} }, + serverInfo: snapshot.serverInfo, + }; + break; + case "tools/list": + result = snapshot.tools; + break; + case "prompts/list": + result = snapshot.prompts; + break; + case "tools/call": + if (request.params.name === "binary_session") result = reply; + break; + case "ping": + result = {}; + break; + } + const response = + result === undefined + ? { error: { code: -32601, message: "Unknown fixture method or tool" } } + : { result }; + process.stdout.write( + `${JSON.stringify({ jsonrpc: "2.0", id: request.id, ...response })}\n`, + ); +}); From 4e3d3354397374116577555200319975fc2ff86e Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:37:49 +0800 Subject: [PATCH 27/27] test(android): retain incomplete cleanup failure reasons (#1443) Include the full stop disposition in the fixture assertion message, so incomplete owned teardown retains its typed reason in Vitest failure reporters. Preserve the existing cleanup requirement without retries or relaxed ownership checks. Seven process-boundary tests, full serial typecheck/lint/duplication/format checks, and 19 final-head CI checks pass after integrating the combined-main fixture fix. A controlled failure-rendering probe verifies that the explicit message preserves the reason. The original CI cleanup failure remains unreproduced; this is diagnostic work with a synthetic JADX producer, not a root-cause or real-engine fix. Refs #1441, #1446, #1448, #1065. --- tests/fixtures/android/jadx.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/fixtures/android/jadx.ts b/tests/fixtures/android/jadx.ts index 3488647a0..c82469039 100644 --- a/tests/fixtures/android/jadx.ts +++ b/tests/fixtures/android/jadx.ts @@ -74,7 +74,10 @@ export const createJadxProtocolFixture = async ( ownsProcessLifetime: true, cleanup: () => cleanupOwnedProcessGroup(spawned.ownership), }); - expect((await supervisor.stop()).status).not.toBe("incomplete"); + const stopped = await supervisor.stop(); + expect(stopped, JSON.stringify(stopped)).not.toMatchObject({ + status: "incomplete", + }); if (options.cwd !== undefined) await rm(options.cwd, { recursive: true, force: true }); });