diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 45240351d..1960c6d1c 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -115,8 +115,14 @@ ask when matches are ambiguous. Never choose an example app on the user's behalf In a target-free session, use `open_binary` to bind any archive/package or native target whose analysis tool operates on the active target. Do not call an unadvertised tool; inspect `binary_session` with `{}` and its -`result.tool_availability` for availability reasons and remediation. The tool -list stays complete; availability depends on the operation, target, and host. +`result.tool_availability` for availability reasons and remediation. Expand each +group's `names` into entries, inheriting the inventory-level client requirements +unless the group overrides them. In its closed effect and annotation vocabularies, +listed flags are true and omitted known flags are false. The grouped name union +is the complete tool inventory; use `tools/list` for canonical tool schemas. +Provider candidate facts describe possible providers, while top-level session +capabilities describe the current route and `analysis_provider_binding` names the +selected provider. Do not treat those scopes as interchangeable. ## Work summary-first diff --git a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md index 5e5f6251d..9b06a93fc 100644 --- a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md +++ b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md @@ -11,14 +11,14 @@ main and an older release expose the same arguments. ## Choose the remedy from the observed failure -| Observation | Meaning and next step | -| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. | -| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. | -| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. | -| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. | -| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Resolve only the prerequisite needed for the task; a complete tool catalog does not imply every operation is available. | -| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. | +| Observation | Meaning and next step | +| --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. | +| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. | +| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. | +| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. | +| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Expand each group's tool names and inherit the shared client requirements unless overridden. Use the operation's availability, reason, and remediation; a complete tool catalog does not imply every operation is available. | +| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. | If the client cannot launch its shell/sandbox helper or read the installed skill, inspect its launch diagnostic. For example, Codex can refuse helper aliases when @@ -46,7 +46,7 @@ rea setup --client codex --dry-run --json Use the current supported client's ID: `claude_code`, `claude_desktop`, `codex`, `cursor`, `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`, `commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, -`hermes`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a +`hermes`, `dsh`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a scope. `grok_bot` uses a chat registration step rather than local `mcp.json`. For unsupported clients, use version-pinned manual stdio registration or CLI. diff --git a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md index 97e873a04..5ccd021b9 100644 --- a/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md +++ b/.agents/skills/reverse-engineer-anything/references/evidence-workflows.md @@ -46,3 +46,9 @@ observed exit signal stays in `exit.signal`, `exit.code` is null for every deadline exit, and the code of a finalizer that exited on its own is in `exit.finalization.exit_code` (null on a signal exit or when no exit was observed). + +Use `inspect_analysis_view` with `view: {kind: "api-surface"}` on retained +`inspect_web_network_capture` HAR Evidence to group observed request lines by +method, origin and path template. Format matches are inferences, not handler +identity or schemas. Group pages cite parent record ordinals; consult record +coverage and exclusions before claiming completeness. No requests are replayed. diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 9f09ad82f..a045e5a62 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -50,10 +50,19 @@ runs and may change files or use the network. Hardened-runtime targets need the `get-task-allow` entitlement. Use `binary_session` with no arguments to check the open target, selected -provider, and alignment. Its default `result.tool_availability` includes the -complete tool inventory with availability reasons and remediation. When choosing -a tool, use its entry in that result; `tools/list` retains the complete catalog -when the target or provider state changes. +provider, and alignment. Its default `result.tool_availability` carries a +complete inventory grouped by shared facts. Expand each group's `names` and +inherit the inventory-level client requirements unless a group overrides them; +availability, reasons, remediation, modes, and flags remain inline. The +`effects` and `annotations` arrays name true flags from fixed vocabularies, so an +omitted known flag is false. `tools/list` remains the canonical source for full +tool schemas when target or provider state changes. + +Provider capability groups carry provider identity and ordered limitations. +Prepend each set's `shared_limitations` to a group's `limitations` to recover the +operation's full ordered list. Candidate capabilities describe possible +providers. Top-level session capabilities describe the current route, and +`analysis_provider_binding` identifies the selected provider. ## DOS MZ with Ghidra diff --git a/AGENTS.md b/AGENTS.md index c46340ab0..6ee0229f2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,9 @@ Preserve caller-selected inputs and local evidence, including output, URLs, path Dependencies flow inward: domain semantics and contracts, provider adapters, shared application workflows, then CLI/MCP adapters. Keep engine protocols and provider-specific code out of domain/application layers. See [docs/architecture.mermaid](docs/architecture.mermaid) when changing composition. -Give each interpretation and resource a clear owner. Preserve identity, omission, uncertainty, and source evidence across boundaries. Retain ownership of resources whose cleanup failed. Reuse `src/process/` supervision and identity primitives; a PID and executable path alone do not establish ownership after exit or reuse. +Give each interpretation and resource a clear owner. Consumers must use that owner's parsed facts rather than reinterpret raw source. Preserve identity, omission, uncertainty, and source evidence across boundaries. Retain ownership of resources whose cleanup failed. Reuse `src/process/` supervision and identity primitives; a PID and executable path alone do not establish ownership after exit or reuse. + +Keep proof of ownership separate from validity, editability, and readiness. Malformed owned data may still be safely removable. Unresolved identity or applicability must remain unknown rather than be treated as absent, foreign, or aligned. Assess brittleness in touched code during implementation, debugging, review, and maintenance. Fix the producing representation or ownership rule before adding downstream repairs. Normalize once at the owning boundary using format-aware parsers. Remove superseded representations, aliases, wrappers, and redundant tests after verifying affected consumers. Introduce abstractions only when they remove observed complexity. diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 350b6011f..e45f97172 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -908,6 +908,19 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("procedure", function == null ? JsonNull.INSTANCE : GSON.toJsonTree(canonicalAddress(function.getEntryPoint()))); result.addProperty("architecture", currentProgram.getLanguageID().getIdAsString()); result.addProperty("mode", currentProgram.getLanguage().getLanguageDescription().getVariant()); + if (currentProgram.getLanguageID().getIdAsString().startsWith("MIPS:")) { + JsonArray decoderContext = new JsonArray(); + var modeRegister = currentProgram.getRegister("ISA_MODE"); + if (modeRegister != null) { + var modeValue = currentProgram.getProgramContext().getValue(modeRegister, address, false); + JsonObject context = new JsonObject(); + context.addProperty("register", modeRegister.getName()); + context.add("value", modeValue == null ? JsonNull.INSTANCE : GSON.toJsonTree(modeValue.toString())); + context.addProperty("source", "ghidra-program-context"); + decoderContext.add(context); + } + result.add("decoder_context", decoderContext); + } JsonArray operands = new JsonArray(); JsonArray references = new JsonArray(); JsonArray destinations = new JsonArray(); diff --git a/bridge/pwntools/layout_impl.py b/bridge/pwntools/layout_impl.py index 0cae2b46f..57ba9880e 100644 --- a/bridge/pwntools/layout_impl.py +++ b/bridge/pwntools/layout_impl.py @@ -206,6 +206,11 @@ def inspect_elf(path, cache): "visibility": raw.st_other.visibility, "section_index": raw.st_shndx, }) if h.sh_type in ("SHT_REL", "SHT_RELA"): + if h.sh_size % h.sh_entsize != 0: + raise LayoutFailure( + "format", + f"ELF relocation section {index} size is not a multiple of its {h.sh_entsize}-byte entries.", + ) if image.header.e_type == "ET_REL" and h.sh_info >= len(all_sections): raise LayoutFailure("format", f"ELF relocation section {index} references missing target section {h.sh_info}.") symbol_table = None diff --git a/docs/adb-device-analysis.md b/docs/adb-device-analysis.md index 59ab98017..416e78161 100644 --- a/docs/adb-device-analysis.md +++ b/docs/adb-device-analysis.md @@ -109,12 +109,12 @@ existing analysis families directly, for example - **Device paths are not host paths.** `base_apk_device_path` values live on the device filesystem. Only pulls and captures produce host-local paths, each paired with byte size and SHA-256 digest. A pulled file's host name is - the device path's last segment, accepted only when every host can store it: - a `pull_adb_file` whose device path ends in a separator, `.`/`..`, a - control or reserved character, a Windows reserved device name, a trailing - dot or space, or a `:` alternate-data-stream marker is refused as - `invalid_request` before any `adb pull` runs, and REA never renames a - device-reported file. + the device path's final POSIX segment. REA preserves the original device + path and filename; a missing final segment or `.`/`..` is refused before + output creation. Windows hosts follow [Windows filename rules](https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file), + while POSIX hosts retain valid POSIX filenames such as names with spaces or + colons. A package pull checks reported names against the destination + filesystem before pulling, and refuses names that would share a local path. - **Splits are part of the answer.** `pm path` enumerates the base APK and every installed split; pulls cover the whole set. System packages whose APKs keep other names report the `unknown` role rather than guessing. diff --git a/docs/android-analysis.md b/docs/android-analysis.md index 03fd39850..5a4a2f879 100644 --- a/docs/android-analysis.md +++ b/docs/android-analysis.md @@ -107,7 +107,11 @@ the same source byte limit and explicit truncation reporting. upstream engine and should be checked against the retained XML when needed. Signature verification is explicitly `not_performed`. - Class search consumes all pages and checks their counts. An empty `query` - inventories all class names; class names are decompiler representations. + inventories all class-name rows; class names are decompiler representations. + Repeated names remain repeated in `matches` so the inventory row count and + producer evidence stay visible. `duplicate_names` identifies matching names + with multiple rows and their occurrence counts; a name alone does not + distinguish which provider class node a later lookup selects. - Class inventories read parsed metadata without generating source and return full reported display types, fields and inner class names. Synthetic members that code generation hides can appear in the inventory; it stays consistent diff --git a/docs/ghidra-mips.md b/docs/ghidra-mips.md index 248a5c0d1..59d04a8b0 100644 --- a/docs/ghidra-mips.md +++ b/docs/ghidra-mips.md @@ -16,9 +16,10 @@ PE, managed and Mach-O architecture sets are unchanged. The Ghidra lane admits ELF32 `ET_EXEC` and `ET_REL`, little or big endian, with explicit o32 and `EF_MIPS_ARCH_32R2` flags and no machine-specific or ASE encoding flags. These are header declarations, not proof that every instruction -conforms to that ISA. ELF64, n32/n64, unspecified ABI, MIPS16/microMIPS, -machine-specific variants, shared objects and PSP PRX are not covered by this -lane. Their family may be identified, but provider selection returns the +conforms to that ISA. ELF64, n32/n64, unspecified ABI, microMIPS, +machine-specific variants and PSP PRX are not covered by this standard-instruction +lane. MIPS16 shared images have the separate bounded profile below; other shared +or compressed images remain unsupported. Their family may be identified, but provider selection returns the specific unsupported constraint rather than guessing a compatible profile. Explicit `EF_MIPS_FP64` and `EF_MIPS_NAN2008` declarations are refused. Their @@ -49,6 +50,63 @@ values are retained for diagnosis but do not gain provider support. These checks bind the declared interpretation; the fixtures do not establish full floating-point behavioral conformance. +### MIPS16 shared images + +A separate profile admits ELF32 `ET_DYN`, either byte order, declaring explicit +o32, MIPS32r2 and exactly the MIPS16 ASE. Machine variants, n32, FP64 and NAN2008 +remain refused. Source inspection must examine the complete dynamic relocation +declarations through uniquely mapped file-backed `PT_LOAD` ranges. This profile +verifies `R_MIPS_NONE` and symbol-index-zero `R_MIPS_REL32`; section declarations +must describe that same dynamic table. Unavailable section headers and additional +section relocation tables (including `--emit-relocs` output) remain unsupported. +Symbol-bearing REL32, +other relocation types, RELA and PLT declarations remain unsupported. A real +negative control shows that Ghidra can normalize a symbol-bearing MIPS16 function +pointer to an even address, losing the source mode tag. REA refuses that shape +rather than rewriting the source or silently repairing the analysis result. + +The profile pins the endian-matched `MIPS::32:default` processor and default +compiler specification, and checks the bridge-reported language after import. +The generic processor preserves mixed MIPS32/MIPS16 code through per-address +`ISA_MODE` context; a global `16e` processor would impose one mode across the +image. A caller-selected incompatible processor or compiler specification is +explicitly refused without replacing the requested choice. + +An observed ABI flags record must agree with the header and MIPS16 ASE, and +must declare the same verified floating-point combinations as the standard +lane. With no observed record, every ABI declaration table must have been +completely examined. The header still declares o32, but floating-point ABI +semantics remain unknown and are stated in Evidence; selecting the default +compiler specification does not establish floating-point storage. Partial +section tables do not prove record absence. The existing soft-float prototype +policy applies when the source explicitly declares soft-float. + +Returned addresses are analysis coordinates, not runtime load addresses. +`inspect_native_load_image` preserves original relocation bytes, current loaded +memory bytes and provider status. External GOT bindings remain unverified +without their defining images. The provider's `modified_sha256` describes its +ephemeral relocated image and can differ from `original_sha256`; it does not +mean that the caller's artifact was written. `inspect_native_instruction` retains the program +language variant in `mode` and reports actual per-address `ISA_MODE` separately +in `decoder_context`, including null when unavailable. Artifact bytes and digests +remain unchanged; this workflow does not execute the shared library. + +The optional real-provider check builds eight source-owned GNU controls, covering +both byte orders, mixed MIPS32/MIPS16 code, declared and absent ABI records, and +the rejected symbol-bearing and additional section relocations. It compares source instruction bytes, +function symbols and the local function-pointer tag with actual CLI/MCP results, +checks context through a snapshot round trip, and verifies process cleanup. +It requires caller-supplied GNU MIPS cross-binutils and Ghidra; no fixture target +is executed or patched after linking. Floating-point behavior of compressed +functions is not established by these integer controls. + +```bash +REA_MIPS_AS=/absolute/path/to/mips-linux-gnu-as \ +REA_MIPS_LD=/absolute/path/to/mips-linux-gnu-ld \ +GHIDRA_INSTALL_DIR=/absolute/path/to/ghidra \ +npm run verify:ghidra:mips-shared +``` + MIPS header and ABI record facts are committed in the analysis profile so interpretation changes invalidate profile-bound snapshots. The support-lane identity is versioned independently of file identity. Evidence, lifecycle @@ -186,8 +244,10 @@ from the original file bytes. HI16/LO16 are checked as the effective address each LO16 forms with each HI16 it resolves, including several HI16s before one LO16, so a missing signed-low carry fails and a HI16 without a following LO16 fails. The lane also requires matching Ghidra relocation entries and statuses, -section file offsets and sizes, coverage of a shared HI16, a LO16 carry, -external code and data and `R_MIPS_32` words, direct calls from the entry +section file offsets and sizes, coverage of a shared HI16, a LO16 resolving +several pending HI16s (from an assembly block, since Clang does not emit that +from C), a LO16 carry, external code and data and `R_MIPS_32` words, direct +calls from the entry function, the relocatable support lane in the snapshot profile, the relocatable limitation in Evidence and an unchanged source object. The fixture also calls `powf`, an external whose float prototype comes from Ghidra's C diff --git a/docs/installation.md b/docs/installation.md index 36f307c31..71533a6fe 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -89,8 +89,11 @@ running server or its registration. The npm release checked on **2026-10-11** was **6.4.0**, published from [`a56f7bb6`](https://github.com/morluto/rea/commit/a56f7bb668ab23fec8d33f7af353ac7ed2fe1895). -The actual registry package accepted dry-run setup plans for all 17 client IDs -listed below, including Qwen Code, Pi and Hermes. OpenCode's plan selected +The actual registry package accepted dry-run setup plans for the 17 client IDs +in that release, including Qwen Code, Pi and Hermes. DeepSeek Harness (`dsh`) +is on main and is not in npm 6.4.0; `rea setup --client dsh` against `@latest` +reports an unsupported client until a release containing it is published. +OpenCode's plan selected `OPENCODE_CONFIG_DIR` even when that directory did not yet exist. These checks used an isolated home on macOS with Node 24; they verify planning and argument admission, without writing client configuration or establishing live integration @@ -162,6 +165,10 @@ listed after the table because its connector is not one of these files: | OMP | `omp` | | Pi | `pi` | | Hermes | `hermes` | +| DeepSeek Harness | `dsh` | + +`dsh` is main-only until a release after 6.4.0. The registry check above does +not cover it. For GitHub Copilot CLI 1.0.95, `gpt-4.1` model metadata can block ordinary chat with REA's full input-schema profile before sending a model request @@ -259,11 +266,15 @@ configuration and skill destinations; doctor and uninstall use those same paths. ### DeepSeek Harness -DeepSeek Harness uses a Cordis YAML patch sequence rather than a `mcpServers` -object. REA's `setup --client` does not edit this format. For Harness -`0.2.0-rc.2`, append the following entry to `$DSH_HOME/cordis.patch.yml` -(`~/.dsh/cordis.patch.yml` when `DSH_HOME` is unset). Preserve existing patches -and replace the executable paths with absolute paths from your installation: +DeepSeek Harness registers MCP clients through a Cordis YAML patch sequence +rather than a `mcpServers` object. `rea setup --client dsh` appends (or +replaces) the REA entry in the home patch `$DSH_HOME/cordis.patch.yml` +(`~/.dsh/cordis.patch.yml` when `DSH_HOME` is unset), preserving unrelated +patches and comments. `rea doctor --client dsh` checks that same entry id. +`rea uninstall` removes it by scanning supported clients; uninstall takes no +`--client` flag. The home patch applies to every Harness profile, so the +registration is written once and no per-profile edit is needed. The written +entry is: ```yaml - insert: @@ -274,17 +285,20 @@ and replace the executable paths with absolute paths from your installation: transport: stdio command: /absolute/path/to/node args: ["/absolute/path/to/rea/scripts/rea.mjs", mcp] - cwd: !!js process.cwd() ``` +Setup preserves caller-owned Cordis expressions and replaces launcher fields +with literal values that doctor can verify. It refuses to edit an expression +that produces the entire REA config, or a group-targeted REA insertion. Use a +literal config and a root-level insertion as shown above. + For a global npm installation, the launcher is under -`$(npm root -g)/rea-agents/scripts/rea.mjs`. Check the Node executable with -`command -v node`. Register `serverName: rea` only once in each active profile, -then restart Harness. It exposes tools as `mcp__rea__`. -Run `rea setup --skill --dry-run`, review the destination, then -`rea setup --skill --yes` to install the workflow in `~/.agents/skills`. -Harness discovers that directory, or `$DSH_AGENTS_HOME/skills` when configured; -its native `skill` tool can load `reverse-engineer-anything`. +`$(npm root -g)/rea-agents/scripts/rea.mjs`. Restart Harness after setup; it +exposes tools as `mcp__rea__`. Run `rea setup --skill --dry-run`, +review the destination, then `rea setup --skill --yes` to install the workflow +in `~/.agents/skills` (`$DSH_AGENTS_HOME/skills` when configured); Harness +discovers that directory, and its native `skill` tool can load +`reverse-engineer-anything`. REA's optional [native client verification](testing.md#end-to-end-integration-and-golden-evidence) checks headless chat, skill loading, all tool input schemas and actual analysis @@ -779,6 +793,32 @@ is no fixed per-operation or response-size ceiling. Unresolved computed calls remain unknown, reference-kind provenance is preserved, and provider-specific pseudocode is never treated as original source or Hopper-equivalent text. +### Missing macOS native decompiler + +If `rea doctor --json` reports `native_decompiler` as missing on macOS x64, +the extracted Ghidra archive may include the decompiler source without the +macOS executable. For Ghidra 12.1.4, you can build it manually with an existing +Xcode Command Line Tools installation, `make`, and `bison`: + +```bash +cd "$GHIDRA_INSTALL_DIR/Ghidra/Features/Decompiler/src/decompile/cpp" +make ghidra_opt +decompiler_dir="$GHIDRA_INSTALL_DIR/Ghidra/Features/Decompiler/os/mac_x86_64" +mkdir -p "$decompiler_dir" +test ! -e "$decompiler_dir/decompile" && test ! -L "$decompiler_dir/decompile" && + install -m 755 ghidra_opt "$decompiler_dir/decompile" +rea doctor --json +``` + +The copy step requires the destination to be absent and preserves an existing +decompiler. This recipe is specific to macOS x64: the +[Ghidra 12.1.4 Makefile](https://github.com/NationalSecurityAgency/ghidra/blob/Ghidra_12.1.4_build/Ghidra/Features/Decompiler/src/decompile/cpp/Makefile) +explicitly selects `-arch x86_64` on Darwin. For other architectures or a Gradle +build, follow the matching release's +[native component instructions](https://github.com/NationalSecurityAgency/ghidra/blob/Ghidra_12.1.4_build/GhidraDocs/GettingStarted.md#building-native-components). +REA accepts the matching executable under either `os//` or +`build/os//`; it does not build the component during setup or diagnosis. + ### Ghidra import language and analysis seeds Ghidra detects the language from the executable header. To set it (for example, Borland C++ PE images): @@ -882,8 +922,12 @@ verification workflows. Unexpected request failures retain their original internal cause. CLI and MCP errors expose Error names, messages, and available codes under -`details.diagnostics.failure_cause`; other rejection values retain their -primitive value or an explicit type. Shutdown warnings include the failure +`details.diagnostics.failure_cause`, including bounded nested causes and explicit +truncation markers. Darwin ownership-helper compilation failures also retain +the compiler outcome, exit code or signal, and bounded stderr; a timed-out +compiler remains distinguishable from a compiler that exited with an error. +Other rejection values retain their primitive value or an explicit type. +Shutdown warnings include the failure kind, message, and diagnostics, while process and temporary-project cleanup continues. These messages redact known bridge authentication tokens and preserve local paths and other analysis context. diff --git a/docs/javascript-artifact-reconstruction.md b/docs/javascript-artifact-reconstruction.md index eb37c6a2b..b721d829d 100644 --- a/docs/javascript-artifact-reconstruction.md +++ b/docs/javascript-artifact-reconstruction.md @@ -174,6 +174,24 @@ default). Reaching it fails the current source with the same memory resource failure and reports `worker_process_memory_limit_bytes`, also a coverage limit; the next source starts a new analyzer. Bun elsewhere enforces no analyzer limit. +The default semantic capacities are 100000 nodes across the application and +20000 per source. Large bundles may explicitly select `max_semantic_nodes` and +`max_semantic_nodes_per_file` (CLI: `--max-semantic-nodes` and +`--max-semantic-nodes-per-file`), each from 1 to 1000000. The per-file offer is +also bounded by the remaining application capacity. Raising these controls does +not bypass worker heap, execution deadline or parent result admission. Truncated +coverage reports both effective node ceilings; omitted relationships remain +unknown. + +While sources remain, the sequential worker reserves one next compact static +checkpoint at its 8 MiB bound with the same 6x retention expansion used at +admission. Earlier checkpoints are already charged to retained bytes; the +remaining inventory does not multiply the reserve. Streamed file transfers keep the default 512 MiB +allowance, and can grow with the parent's offered retention budget (8x byte +expansion), up to 2 GiB. Individual records remain bounded to 8 MiB. The parent +still independently checks actual bytes, record overhead, digests and source +identities before adopting results. + A source that exhausts the analyzer heap does not terminate the CLI/MCP process. Completed static facts are retained, other files can still be analyzed, and unexamined semantic relationships remain explicit unknowns. The parent @@ -366,9 +384,30 @@ Source-map contents are parsed as local input. The graph stores original source Only explicitly present BrowserWindow values are observations. REA does not substitute version-dependent Electron defaults for omitted `webPreferences`. -Dynamic option objects, bridge keys, API objects, and IPC channel expressions +Unresolved option objects, bridge keys, API objects, and IPC channel expressions remain unknown and make coverage partial. +Recognized literal module-table bootstraps also support bounded lexical +substitution through module exports, import helper bodies, local wrapper calls, +returned closures, object spreads, and callback parameters. Recovered Electron +paths retain source ranges, module identities, literal channels and branch +conditions as static candidates. Unknown spreads, reassigned bindings, accessors, +dynamic channels and unsupported syntax do not become exact implementations. +This pass neither executes the bundle nor proves that runtime guards pass. +An ambiguous or unresolved handler retains its channel registration but does +not acquire a path to one arbitrarily selected callback. Unknown factory +conditions retain both return branches, including fallthrough. Short-circuit +selection distinguishes truthiness from nullishness; uncertain `??` paths +retain a nullish guard rather than a falsy guard. + +These paths appear as contextual `javascript-expression` nodes. An exact `api` +seed such as `desktop.copy` can trace an exposed member through a unique IPC +pairing to an external API call. `addListener` is normalized to `on`. Semantic +projection retains these boundary paths before bulk function definitions, within +the existing per-file node budget; omitted general semantics remain explicitly +partial. Wrapper expansion is bounded by one million visits/substitutions and +128 nested substitutions, with exhaustion reported as unknown coverage. + Electron API identity follows proven lexical module aliases. Namespace interop also preserves identity when the complete Rolldown helper operations, lexical roles, live getters, and captured intrinsic origins match, or when a stable @@ -415,7 +454,7 @@ do not establish that an application has no Electron boundary usage. IPC pairing is an inference, not an observation. A renderer `invoke` pairs only with one unique `ipcMain.handle`/`handleOnce` candidate on the same exact literal -channel; a renderer send pairs only with one unique `ipcMain.on`/`once` +channel; a renderer send pairs only with one unique `ipcMain.on`/`once`/`addListener` candidate. Dynamic channels are never paired, and multiple compatible handlers are reported as ambiguous without adding a caller-to-handler edge. A matching channel still does not prove registration order, reachability, or runtime use. diff --git a/docs/managed-code-analysis.md b/docs/managed-code-analysis.md index ad642b7a5..18360ec3d 100644 --- a/docs/managed-code-analysis.md +++ b/docs/managed-code-analysis.md @@ -58,6 +58,34 @@ reports `resource-limit` with unavailable or partial coverage; omitted facts do not become empty observations. Ambiguous PE section mappings are rejected rather than interpreted according to section order. +### Selected member inspection + +When whole-assembly member decoding reaches the representation budget, select +known declaring types before their signatures and CIL bodies are decoded: + +```bash +rea inspect-managed-members /absolute/path/Assembly.dll --type Example.Parser +``` + +MCP uses `inspect_managed_members` with +`{"path":"/absolute/path/Assembly.dll","type_names":["Example.Parser"]}`. +Repeat `--type` or supply multiple unique `type_names` for several types. +Names match the reported `full_name` exactly. Top-level names are namespace +qualified; nested types currently report bare names, which can select several +types with that name. + +The result preserves the requested names in `selection.type_names` and reports +unmatched names separately. If metadata names are unreadable, limitations retain +that uncertainty. With no selection, `selection` is null and whole-assembly +inspection keeps its existing resource policy. Selection restricts types, +fields, methods, call edges and field accesses; member references remain an +assembly-wide inventory, so output size is not guaranteed to be small. + +Coverage describes the selected members. Comparisons leave members outside the +opposite observation's selection unknown, and application graphs report their +selected scope as partial. Neither makes an absence claim about the whole +assembly. Older member Evidence without `selection` remains readable. + ## Analysis objective REA's managed-code track is intended to answer five different questions without diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index cd1a75316..9a8d35d7f 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -113,17 +113,48 @@ Schema does not express those cross-field invariants. Call `binary_session` with `{}` and read `result.tool_availability` to choose a callable operation for the current target, provider, host, and negotiated client -capabilities. The default result includes the complete inventory with each -tool's availability, reason, and remediation. Each entry also reports required -and optional negotiated client features plus the currently missing features. -The optional inputs `expected_package_version`, `expected_catalog_digest`, and +capabilities. The default result contains the complete inventory in groups that +share status facts. `tool_availability.client_requirements` is the baseline for +each group; a group's optional `client_requirements` replaces that baseline for +the group's tools. Expand each group's `names` into one tool entry, carrying its +surface, availability, reason, remediation, default-mode status, modes, and +client requirements. Compare the union of group names with `tools/list` when +checking that a client received the full catalog. An operation absent from all +groups is missing from the complete inventory; it is not an implicit +unavailable entry. + +The `effects` and `annotations` arrays list the true members of their fixed +vocabularies. A known member omitted from either array is false. This applies +only to these closed flag sets; it does not make an omitted tool, provider, mode, +or limitation false. Keep availability reasons, remediation, client-feature +requirements, and mode facts inline with the grouped tool names. The optional +inputs `expected_package_version`, `expected_catalog_digest`, and `expected_server_path` compare the live session with the caller's expectations. +`server_identity.catalog` contains catalog counts and schema digests. Grouped +tool availability carries the name set and current per-tool facts; `tools/list` +is the canonical source for complete tool definitions, including input and +output schemas. Clients that previously read `server_identity.catalog.tools` +should expand `tool_availability.groups` and use `tools/list` for schemas. The +same catalog identity summary is returned by CLI `capabilities` and `providers`; +MCP tool definitions remain available through `tools/list`. + +Provider capabilities are grouped by provider identity and shared facts. In each +set, `shared_limitations` is the ordered common prefix; append a group's ordered +`limitations` suffix to recover each operation's exact limitations. Group +`effects` arrays use the same true-only closed-vocabulary rule as tool +availability. Expand every group operation while retaining its provider +identity, availability, reason, rejection code, effects, and ordered +limitations. + `binary_session.analysis_provider_candidates` is authoritative for deep-engine -discovery. Target-free discovery is sorted by provider ID, reports host -availability and `unknown` target support, and does not create an analysis -client. `open_binary.provider_id` accepts a concrete provider ID or `auto`; it -uses the same parser and selection policy as CLI `--provider` and +discovery. Each candidate's capability sets describe that possible provider; +they do not select it. The top-level `capabilities` describe the current route, +while `analysis_provider_binding` identifies the concrete selected route. These +fields are not interchangeable. Target-free discovery is sorted by provider ID, +reports host availability and `unknown` target support, and does not create an +analysis client. `open_binary.provider_id` accepts a concrete provider ID or +`auto`; it uses the same parser and selection policy as CLI `--provider` and `REA_ANALYSIS_PROVIDER`. A successful deep open exposes one immutable provider, concrete version, selection source, and complete analysis profile through `analysis_provider_binding`. Ambiguity and unknown, unavailable, or unsupported @@ -165,6 +196,11 @@ Provider calls receive the request cancellation signal. Artifact traversal, hashing, version comparisons, Hopper requests, and process capture check the same signal. Cancellation is distinct from timeout. A cleanup failure uses `cleanup_incomplete` and lists only the owned resource kinds that remain. +An atomic Evidence bundle or snapshot write can publish its complete output +before staging cleanup fails. In that case, `details.published: true`, +`details.published_path`, and `details.bytes` retain the publication receipt. +Use that receipt to distinguish completed publication from a write that never +committed. Staging resources remain owned until cleanup is verified. Native call tracing and process capture retain available observations in `details.partial_observation` on failure, including when cleanup succeeds. The observation reports its partial coverage; cleanup details describe host @@ -310,9 +346,12 @@ directly to `compare_functions`, and `inspect_artifact` Evidence to `compare_artifacts`. Use `get_evidence_bundle` when the task needs broader retained session history or an explicit bundle for transfer. -`get_call_graph` keeps provider-limited relationship reads on their node as -`limitations`, alongside any observed `calls`. An empty call list with such a -limitation is incomplete coverage, not proof that the function has no calls. +`get_call_graph` returns a `graph` keyed by traversal depth and a top-level +`limitations` array with deduplicated execution caveats. Each node keeps its +relationship coverage and read-specific `limitations`, alongside observed +`calls`. When a provider does not classify its caveats, they remain on that +node and coverage is `unknown`. An empty call list with partial or unknown +coverage is not proof that the function has no calls. `trace_call_path` reports each limited expansion in `residual_unknowns` and sets `truncated: true`; any path already supported by observed edges remains visible. Tool errors remain in their existing error or failure fields. @@ -380,6 +419,13 @@ analysis, so later views and application workflows read it without repeating analysis or moving the complete record through the response. A server without session retention refuses summary detail with `capability_unavailable`. +`inspect_analysis_view` also accepts `api-surface` for retained HAR capture Evidence, +grouping request lines by method, origin and format-rule path template. Templated +segments are inferences. Because paging follows the full grouping, every page is +inferred Evidence whenever any group is templated, and derived Evidence only when +no group was templated. +See [derived API surfaces](web-network-captures.md#derived-api-surface). + `inspect_analysis_view` projects a caller-selected view of already completed `inspect_binary_layout`, `analyze_javascript_application`, or `analyze_function` Evidence. Native views select procedure, pseudocode, assembly, basic blocks, diff --git a/docs/native-investigation.md b/docs/native-investigation.md index 1d2b6c01a..b878b17b1 100644 --- a/docs/native-investigation.md +++ b/docs/native-investigation.md @@ -6,11 +6,20 @@ coverage and unknowns. The CLI and MCP use the same application workflows. ## Static inspection -`inspect_macho` retains otool segment `file_offset` values relative to the -selected Mach-O slice. Its evidence file-offset ranges address the original -input file and include the observed lipo slice offset for universal binaries. -If that offset is unavailable, segment evidence locations are omitted with an -explicit limitation; architecture inventory locations remain available. +`inspect_macho.selected_architecture` identifies the exact selected lipo slice +and its `container_file_offset` in the input file. A thin Mach-O has a known +offset of zero; an unknown universal-slice offset stays null. The Evidence +subject format describes the input container: `mach-o-universal` for FAT files, +including those with one slice. The normalized result describes the selected +Mach-O image. Architecture +`file_offset` values are container-relative. Entrypoint, segment and section +`file_offset` values retain their source meaning and are relative to the +selected Mach-O slice. Raw load-command fields retain their original values. +Evidence locations map the LC_MAIN entrypoint and nonempty file-backed segment +ranges to container offsets by adding the observed slice offset. Zero-fill +segments do not produce file ranges. When the selected slice's container offset +is unknown, those mapped locations are omitted with an explicit limitation; +architecture inventory locations remain available. Hopper's `address_to_file_offset` combines the native image-relative mapping with original-file coordinates only after checking the source against the session's @@ -53,6 +62,10 @@ complete observation. classes are preserved. Select `root`, `offset` and `limit` explicitly. UID 0 cannot distinguish conditional nil from ordinary nil. Limits are 64 MiB of input, 200,000 objects/references and 128 nesting levels. + XML dictionary entries decoded as `__proto__` are omitted with an explicit + count because REA results cannot represent that key. XML tag whitespace and + comments do not change that handling. Other values and the original archive + digest are preserved; key interpretation follows the pinned plist decoder. - `decode_interface_builder` / `rea decode-interface-builder ` reads compiled storyboard and nib resources into objects, outlets, actions, connections and controller/class names. `verify:interface-builder` compiles @@ -192,8 +205,14 @@ rea inspect-native-data-type --address --provider Instruction facts include bytes, length, decoder text, ordered register/scalar/ address tokens, typed references and flow destinations. Mid-instruction, data, outside-memory and undecodable addresses have separate outcomes. Effective -memory base/index/displacement roles and per-instruction context mode remain -unavailable; `mode` is the program language variant. +memory base/index/displacement roles remain unavailable. Ghidra reports its program +language variant in `mode`; other providers retain their own mode label. Ghidra +MIPS results additionally expose `decoder_context` +with the observed `ISA_MODE` register at the queried address and its provider +source. A null value is unavailable, not mode zero; an empty array reports no +available context register, and omission preserves older captures or providers +that do not expose this facet. The context is an analysis observation, not proof +of a runtime processor state. `inspect_native_instruction` also returns `incoming_references` to the exact queried address, including data, interior-instruction and undecodable addresses. diff --git a/docs/testing.md b/docs/testing.md index 09b01e79c..92cf8dc44 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -112,8 +112,12 @@ Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of a named sticky profile with `HERMES_HOME` still pointing to its root. `verify:gemini-client` requires an installed Gemini CLI (verified with -`@google/gemini-cli@0.63.0`); select it with `REA_VERIFY_GEMINI_COMMAND`. -The optional POSIX lane uses native `GEMINI_CLI_HOME` discovery in an isolated +`@google/gemini-cli@0.63.0` on POSIX and native Windows); select it with +`REA_VERIFY_GEMINI_COMMAND`. A selected `.js`, `.mjs` or `.cjs` entry point runs +under the verifier's Node. On Windows, select the npm install's +`node_modules/@google/gemini-cli/bundle/gemini.js`, because only a `gemini.cmd` +shim reaches `PATH`. +The optional lane uses native `GEMINI_CLI_HOME` discovery in an isolated Git project, checks setup plans, backups and idempotence, activates the installed personal skill, validates all forwarded input JSON Schemas against their declared dialect, forwards the complete REA catalog and checks full JavaScript @@ -121,7 +125,7 @@ Evidence for a Unicode path. Use `-- chat` for ordinary chat with REA enabled, or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed REA package. The local fixture exercises the real Gemini API adapter, including its `untrusted_context` tool-result envelope. Token counts are synthetic; live -Google API and native Windows compatibility remain unverified. The lane disables +Google API compatibility remains unverified. The lane disables the client's memory-based relaunch to preserve the caller's Node heap budget. `verify:opencode-client` requires an installed OpenCode (verified with @@ -268,9 +272,13 @@ unverified; skill discovery uses an explicit isolated custom directory. `verify:deepseek-client` requires an installed DeepSeek Harness (`dsh`; verified with `@deepseek-ai/dsh@0.2.0-rc.2`) and Git. Set `REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane -installs the skill through `rea setup --skill`, writes the native Cordis MCP -patch into an isolated `DSH_HOME`, and binds `HOME` and `USERPROFILE` to the -disposable account. The native default `~/.agents/skills` search path discovers +seeds caller model settings and a stale Cordis registration in an isolated +`DSH_HOME`, then runs public `setup --client dsh` dry-run, setup, repeated setup, +and scoped doctor. It checks the backup and unrelated patches before booting +Harness with the generated registration. After native execution it uninstalls +through the public CLI and checks that only caller patches remain. `HOME` and +`USERPROFILE` point to the disposable account. The native default +`~/.agents/skills` search path discovers the installed skill without a `DSH_AGENTS_HOME` override. A separate Git root prevents inherited project skills from masking a missing installation. The default `call` mode checks skill loading, complete catalog discovery, the actual forwarded input schemas and regexes, @@ -281,6 +289,12 @@ context capacity declared by its default DeepSeek Flash model; it does not establish live DeepSeek API or native Windows coverage. Receipts include request artifacts, result digests and owned-process lineage. +The DSH acceptance journey also exercises public setup, approval, doctor, +generated-launcher MCP startup and uninstall without an installed Harness. +That journey proves REA workflow behavior, not native Cordis expression +evaluation. Filesystem regressions retain tagged values and unrelated rows +across alias dependencies and check refusal of unsafe YAML without mutation. + Full E2E tests invoke the production command dispatcher and real providers, without fake launchers, runners or responses. `verify:keyed-archive` writes Foundation binary, XML, and integer archives using one run-owned Swift module diff --git a/docs/tool-design.md b/docs/tool-design.md index 934722850..a4dd155f0 100644 --- a/docs/tool-design.md +++ b/docs/tool-design.md @@ -55,6 +55,11 @@ satisfy. A plan without execution does not establish runtime behavior. - Keep observed, derived, and inferred relationships distinct. Preserve unresolved edges and partial provider facts instead of requiring every provider to supply the same metadata. +- For caller/callee reads, `relationship_coverage` describes only the + provider-observed direct relationship set. `complete` means that set was + returned without a known read limit; it does not claim every possible runtime + edge. Provider and execution caveats stay in `limitations` and do not imply + incomplete relationship coverage. Missing coverage remains `unknown`. - Declare process, filesystem, network, UI, and mutation effects truthfully; enforce the requested target and lifecycle. - Add limits only for real format, protocol, authority, or measured resource diff --git a/docs/wasm-artifacts.md b/docs/wasm-artifacts.md index 5289976a7..21e2735a0 100644 --- a/docs/wasm-artifacts.md +++ b/docs/wasm-artifacts.md @@ -4,6 +4,15 @@ local WASM bytes using caller-supplied WABT 1.0.42. No active binary target is required. REA does not install tools or fetch artifacts and does not execute WASM. +`open_binary` is not the WASM inspection route. A recognized WASM v1 header +returns `target_unavailable` with category `unsupported_target` and constraint +`wasm_requires_static_inspection`, including the selected path in a valid +`inspect_wasm_artifact` call. Use that target-free tool or +`rea inspect-wasm-artifact --format json` with the toolchain below. +Check `binary_session` for configured capability availability. A recognized +header is not validation of the complete module; missing WABT is reported +separately as an unavailable capability, not as missing native engine support. + Set `REA_WABT_BIN_DIRECTORY` to an absolute directory containing executable `wasm-validate`, `wasm-objdump` and `wasm2wat` (with `.exe` on Windows). Missing executables, unsupported banners and inaccessible tools produce actionable diff --git a/docs/web-network-captures.md b/docs/web-network-captures.md index f82b36109..1edb4993a 100644 --- a/docs/web-network-captures.md +++ b/docs/web-network-captures.md @@ -100,6 +100,11 @@ resources follow the same explicit declarations without changing error types; sensitive issue pointers identify a real ancestor instead of a fabricated coordinate. Header/cookie authentication values and known transport URL userinfo are excluded structurally, including native backups. +HAR redaction and API-surface projection share the same recorded-URL parser. +Canonical userinfo spans are removed without rewriting the remaining source +text. Credential-bearing spellings without a safe raw span, such as slashless +or extra-slash HTTP URLs, are excluded as a whole transport field. Ordinary +extension text and noncredential path/query values remain unchanged. Generated free-text limitations follow the same literal declarations, including global, per-record and Evidence explanations added during projection. Contract @@ -112,6 +117,74 @@ wire fragments. Native dictionary duplicate keys have the unchanged upstream decoder's last-value semantics; this adapter does not claim to recover discarded duplicates or the original numeric spelling of parsed native floats. +## Derived API surface + +Pass retained HAR Evidence to `inspect_analysis_view` with +`view: {"kind":"api-surface","offset":0,"limit":50}`. Omitted offset and limit +are 0 and 50. The view groups request lines by method, WHATWG URL origin and +path template, sorted by origin, template and method. Query strings, fragments, +credentials, headers and bodies are not copied into groups. + +| Rule | Exact segment format | Placeholder | +| ---------- | ---------------------------------------------------- | ------------ | +| `numeric` | ASCII digits only | `{numeric}` | +| `uuid` | RFC 9562 textual UUID, any version, case-insensitive | `{uuid}` | +| `hex` | At least 32 hexadecimal digits | `{hex}` | +| `iso-date` | Valid calendar date `YYYY-MM-DD` | `{iso-date}` | + +Rules apply in table order to the recorded, still-encoded request path, read +before WHATWG dot-segment removal, so `/a/%2e%2e/admin` and `/admin` stay distinct. Segment indexes start +at zero after the leading slash. Each matched segment lists its observed literals +and `basis: "inferred"`. `templated_groups` counts templated groups across every +page. Because paging follows the full grouping, every page is inferred Evidence +when that count is nonzero, and derived Evidence only when no group was templated. Version prefixes, short opaque IDs +and digit-bearing route names stay literal. Samples merge only when method, +origin, and exact segment rules match. The view does not infer further +cross-sample patterns. + +Groups cite parent record ordinals and distinct statuses and MIME types (lowercase +type/subtype without parameters). A malformed media type is unknown, not a content +type. Flags report single samples, mixed statuses or mixed content types. +`coverage` pages groups; `records` separately reports examined, total and excluded +parent records. Ordinal subsets carry a limitation. Null URLs are excluded as +`url-redacted`; unparseable URLs, URLs without an origin, spellings without +`scheme://authority` (such as `https:host/path`), and authorities containing a +tab or newline as `url-unparseable`. +Unavailable request objects or methods are explicitly excluded. Unavailable +response fields, including HAR's `status: 0` and an empty `mimeType`, remain +unknown, not invented statuses or MIME types. + +The view makes no handler-identity, schema or causality claim. It performs no +network activity. Native mitmproxy captures return a typed unsupported error. +Parents inspected with `sensitive_values`, or whose Evidence does not record +`sensitive_value_count`, return a typed unsupported error whose reason is the +canonical token `sensitive_values_unretained`. The view would write path +templates and limitation text that it cannot check against declarations the +parent does not retain, so the refusal does not repeat declaration text. + +One accumulator owns group identity, observed-value/status/MIME sets, ordinals, +exclusions and deduplicated limitations. It scans path spans without splitting +literal paths into arrays. Before schema parsing, it reserves the capture clone +and validation scratch by walking the parent without cloning it. Inline requests +borrow the Evidence value through CLI/MCP envelope validation; the application +admits the entire portable envelope before authenticated parsing or hashing. +The envelope clone, hash scratch and capture clone stay charged during grouping. +Temporary URL/path/MIME work and retained state must +fit together within a 96 MiB projection representation budget. A whole group or +merge is admitted before allocating its retained sets, map entries or array +values. Fixed group storage, distinct text, exclusions and limitations count as +well as matched segments. Per-record reservations are released after each +record; repeated values do not accumulate another retained charge. Only the +requested page is materialized as output groups. + +The representation budget uses conservative storage and worst-case JSON-escaping +allowances; it is not an enforced V8 heap or process-memory limit. It is separate +from the decoder's actual reply-byte limit and MCP transport admission. Exceeding +it returns `resource_constraint` without a partial view. Limitations on the +parent envelope, capture and examined records are deduplicated onto the view. +Portable captures must retain unique ordinals within their declared total; +caller-selected subsets can preserve any requested ordinal order. + ## Resource and lifecycle boundaries Complete evidence has a 32 MiB input, 96 MiB decoder reply and 64-level nesting diff --git a/package-lock.json b/package-lock.json index aaed5cd61..de9c141f5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6707,7 +6707,7 @@ }, "node_modules/rolldown": { "version": "1.2.12", - "resolved": "https://mirrors.cloud.tencent.com/npm/rolldown/-/rolldown-1.2.12.tgz", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.12.tgz", "integrity": "sha512-8wafseiaG80xmXSfqidUNqZcylTlhmPZZt+za2m+js2sFZ8dTNlhIOV2WcbIPx2hgwPBJpEUGFAMZ9bgBBLTSQ==", "dev": true, "license": "MIT", diff --git a/package.json b/package.json index 316809a20..4e4c6a7c4 100644 --- a/package.json +++ b/package.json @@ -165,6 +165,7 @@ "verify:ghidra": "npm run build:cached && node scripts/verify-real-ghidra.mjs", "verify:ghidra:seeds": "npm run build:cached && node scripts/verify/ghidra/seeds.mjs", "verify:ghidra:mips": "npm run build:cached && node scripts/verify-real-ghidra-mips.mjs", + "verify:ghidra:mips-shared": "npm run build:cached && node scripts/verify-real-ghidra-mips-shared.mjs", "verify:ghidra:mips-relocatable": "npm run build:cached && node scripts/verify-real-ghidra-mips-relocatable.mjs", "verify:ghidra:swift": "npm run build:cached && node scripts/verify-real-ghidra-swift.mjs", "verify:ida": "npm run build:cached && node scripts/verify-real-ida.mjs", @@ -255,8 +256,8 @@ "verify:evm:interface": "npm run build:cached && node scripts/verify-evm-interface.mjs", "verify:recorded:crash": "npm run build:cached && node scripts/verify-recorded-crash.mjs", "mcp-catalog:check:built": "node scripts/generate-mcp-tool-catalog.mjs --check", - "compile": "node scripts/clean-build-output.mjs && tsc -p tsconfig.build.json && node scripts/bundle-mcp-runtime.mjs", - "compile:termux": "node scripts/clean-build-output.mjs && node --max-old-space-size=3072 node_modules/typescript-termux/lib/tsc.js -p tsconfig.build.json && node scripts/bundle-mcp-runtime.mjs", + "compile": "node scripts/clean-build-output.mjs && tsc -p tsconfig.build.json && node scripts/bundle-runtime.mjs", + "compile:termux": "node scripts/clean-build-output.mjs && node --max-old-space-size=3072 node_modules/typescript-termux/lib/tsc.js -p tsconfig.build.json && node scripts/bundle-runtime.mjs", "artifacts:mcp-catalog": "node scripts/generate-mcp-tool-catalog.mjs", "artifacts:skills": "node scripts/generate-skill-metadata.mjs", "artifacts:product-catalog": "node scripts/generate-product-catalog.mjs", diff --git a/scripts/bundle-mcp-runtime.d.mts b/scripts/bundle-mcp-runtime.d.mts deleted file mode 100644 index 7fff6f125..000000000 --- a/scripts/bundle-mcp-runtime.d.mts +++ /dev/null @@ -1 +0,0 @@ -export const bundleMcpRuntime: (root: string) => Promise; diff --git a/scripts/bundle-mcp-runtime.mjs b/scripts/bundle-mcp-runtime.mjs deleted file mode 100644 index 2750883f5..000000000 --- a/scripts/bundle-mcp-runtime.mjs +++ /dev/null @@ -1,104 +0,0 @@ -import { parse } from "@babel/parser"; -import { readFile } from "node:fs/promises"; -import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -/** - * Reduce filesystem resolution during MCP startup. Keep dependencies external, - * dynamic imports lazy, and the original compiled files for CLI/worker entrypoints. - */ -export const bundleMcpRuntime = async (root) => { - const compiledRoot = join(root, "dist"); - const { rolldown, RUNTIME_MODULE_ID } = await import("rolldown"); - const { imports = {} } = JSON.parse( - await readFile(join(root, "package.json"), "utf8"), - ); - const bundle = await rolldown({ - input: { main: join(compiledRoot, "main.js") }, - platform: "node", - treeshake: false, - external: (source) => !source.startsWith(".") && !isAbsolute(source), - plugins: [ - { - name: "preserve-compiled-module-locations", - transform(code, id) { - if (id === RUNTIME_MODULE_ID) return null; - if (!code.includes("import.meta")) return null; - const sourcePath = relative(compiledRoot, id); - if ( - isAbsolute(sourcePath) || - sourcePath === ".." || - sourcePath.startsWith(`..${sep}`) - ) - throw new Error(`MCP bundle input is outside dist: ${id}`); - // All chunks are directly under dist/mcp. Preserve the original - // module URL, including URL escaping, for bridges and createRequire. - const moduleUrl = `new URL(${JSON.stringify(`../${sourcePath.split(sep).map(encodeURIComponent).join("/")}`)}, import.meta.url).href`; - const edits = []; - const visit = (node) => { - if (node === null || typeof node !== "object") return; - const isMeta = (value) => - value?.type === "MetaProperty" && - value.meta.name === "import" && - value.property.name === "meta"; - if ( - node.type === "CallExpression" && - node.callee.type === "MemberExpression" && - !node.callee.computed && - isMeta(node.callee.object) && - node.callee.property.name === "resolve" && - node.arguments.length === 1 && - node.arguments[0].type === "StringLiteral" && - Object.hasOwn(imports, node.arguments[0].value) - ) - // Package imports resolve in the same package scope after bundling. - return; - if ( - node.type === "MemberExpression" && - !node.computed && - isMeta(node.object) && - node.property.name === "url" - ) { - edits.push({ start: node.start, end: node.end }); - return; - } - if (isMeta(node)) - throw new Error( - `Unsupported import.meta use in MCP bundle: ${id}`, - ); - for (const value of Object.values(node)) { - if (Array.isArray(value)) value.forEach(visit); - else visit(value); - } - }; - visit(parse(code, { sourceType: "module" }).program); - for (const { start, end } of edits.sort((a, b) => b.start - a.start)) - code = code.slice(0, start) + moduleUrl + code.slice(end); - return { code, map: null }; - }, - }, - ], - }); - try { - await bundle.write({ - dir: join(compiledRoot, "mcp"), - format: "esm", - entryFileNames: "[name].js", - chunkFileNames: "[name]-[hash].js", - codeSplitting: true, - strictExecutionOrder: true, - minify: false, - sourcemap: false, - }); - } finally { - await bundle.close(); - } -}; - -if ( - process.argv[1] !== undefined && - pathToFileURL(resolve(process.argv[1])).href === import.meta.url -) - await bundleMcpRuntime( - resolve(dirname(fileURLToPath(import.meta.url)), ".."), - ); diff --git a/scripts/bundle-runtime.d.mts b/scripts/bundle-runtime.d.mts new file mode 100644 index 000000000..1f9637971 --- /dev/null +++ b/scripts/bundle-runtime.d.mts @@ -0,0 +1 @@ +export const bundleRuntime: (root: string) => Promise; diff --git a/scripts/bundle-runtime.mjs b/scripts/bundle-runtime.mjs new file mode 100644 index 000000000..bac18ed2c --- /dev/null +++ b/scripts/bundle-runtime.mjs @@ -0,0 +1,140 @@ +import { parse } from "@babel/parser"; +import { readFile } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +/** + * Reduce filesystem resolution during CLI and MCP startup. Keep dependencies + * external, dynamic imports lazy, and original compiled worker entrypoints. + */ +export const bundleRuntime = async (root) => { + const compiledRoot = resolve(root, "dist"); + // CLI, MCP, workers and updater fixtures share one release-identity owner. + const metadataModule = join(compiledRoot, "generatedPackageMetadata.js"); + const isMetadataModule = (id) => + isAbsolute(id) && resolve(id) === metadataModule; + const { rolldown, RUNTIME_MODULE_ID } = await import("rolldown"); + const { imports = {} } = JSON.parse( + await readFile(join(root, "package.json"), "utf8"), + ); + for (const [directory, entries] of [ + ["mcp", { main: "main.js" }], + [ + "cli-runtime", + { + cli: "cli.js", + cliOutput: "cliOutput.js", + streamedJsonOutput: "cli/streamedJsonOutput.js", + mcpDoctor: "mcpDoctor.js", + }, + ], + ]) { + const bundle = await rolldown({ + input: Object.fromEntries( + Object.entries(entries).map(([name, path]) => [ + name, + join(compiledRoot, path), + ]), + ), + platform: "node", + treeshake: false, + external: (source) => + isMetadataModule(source) || + (!source.startsWith(".") && !isAbsolute(source)), + plugins: [ + { + name: "preserve-compiled-module-locations", + transform(code, id) { + if (id === RUNTIME_MODULE_ID) return null; + if (!code.includes("import.meta")) return null; + const sourcePath = relative(compiledRoot, id); + if ( + isAbsolute(sourcePath) || + sourcePath === ".." || + sourcePath.startsWith(`..${sep}`) + ) + throw new Error(`Runtime bundle input is outside dist: ${id}`); + // Each output directory is directly under dist. Preserve the original + // module URL, including URL escaping, for bridges and createRequire. + const moduleUrl = `new URL(${JSON.stringify(`../${sourcePath.split(sep).map(encodeURIComponent).join("/")}`)}, import.meta.url).href`; + const edits = []; + const visit = (node) => { + if (node === null || typeof node !== "object") return; + const isMeta = (value) => + value?.type === "MetaProperty" && + value.meta.name === "import" && + value.property.name === "meta"; + if ( + node.type === "CallExpression" && + node.callee.type === "MemberExpression" && + !node.callee.computed && + isMeta(node.callee.object) && + node.callee.property.name === "resolve" && + node.arguments.length === 1 && + node.arguments[0].type === "StringLiteral" && + Object.hasOwn(imports, node.arguments[0].value) + ) + // Package imports resolve in the same package scope after bundling. + return; + if ( + node.type === "MemberExpression" && + !node.computed && + isMeta(node.object) && + node.property.name === "url" + ) { + edits.push({ start: node.start, end: node.end }); + return; + } + if (isMeta(node)) + throw new Error( + `Unsupported import.meta use in runtime bundle: ${id}`, + ); + for (const value of Object.values(node)) { + if (Array.isArray(value)) value.forEach(visit); + else visit(value); + } + }; + visit(parse(code, { sourceType: "module" }).program); + for (const { start, end } of edits.sort( + (a, b) => b.start - a.start, + )) + code = code.slice(0, start) + moduleUrl + code.slice(end); + return { code, map: null }; + }, + }, + ], + }); + try { + await bundle.write({ + dir: join(compiledRoot, directory), + format: "esm", + entryFileNames: "[name].js", + chunkFileNames: + directory === "mcp" + ? "[name]-[hash].js" + : (chunk) => { + // Keep lazy facades distinguishable when source basenames repeat. + if (typeof chunk.facadeModuleId !== "string") + return "[name]-[hash].js"; + const path = relative(compiledRoot, chunk.facadeModuleId); + return `${path.replace(/\.js$/u, "").split(sep).join("__")}-[hash].js`; + }, + paths: (id) => + isMetadataModule(id) ? "../generatedPackageMetadata.js" : id, + minifyInternalExports: directory === "mcp", + codeSplitting: true, + strictExecutionOrder: true, + minify: false, + sourcemap: false, + }); + } finally { + await bundle.close(); + } + } +}; + +if ( + process.argv[1] !== undefined && + pathToFileURL(resolve(process.argv[1])).href === import.meta.url +) + await bundleRuntime(resolve(dirname(fileURLToPath(import.meta.url)), "..")); diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index 7ac038a0a..def0effbb 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -1,5 +1,11 @@ import { analysisErrorProjectionSchema } from "../../dist/contracts/errorSchemas.js"; import { toolContract } from "../../dist/contracts/toolContracts.js"; +import { + expandProviderCapabilities, + expandToolAvailability, + providerCapabilitySet, +} from "../../dist/contracts/toolOutputSchemaPrimitives.js"; +import { compareUnicodeCodePoints } from "../../dist/domain/unicodeCodePointOrder.js"; /** Read the textual JSON projection from one MCP tool call. */ export const mcpTextValue = (result) => { @@ -10,6 +16,10 @@ export const mcpTextValue = (result) => { const jsonValue = (result) => JSON.parse(mcpTextValue(result)); +/** Sort canonical contract names using REA's code-point ordering. */ +export const sortVerifierNames = (names) => + [...names].sort(compareUnicodeCodePoints); + /** Read the complete canonical diagnostic from an MCP tool failure. */ export const requireMcpToolError = (result) => { if (result.isError !== true) throw new Error("Expected an MCP tool failure"); @@ -35,6 +45,44 @@ const operationResultKey = (contract) => { return evidence ? "normalized_result" : "result"; }; +/** Expand compact session metadata once at the verifier boundary. */ +export const expandMcpSessionStatus = (status) => ({ + ...status, + capabilities: expandProviderCapabilities(status.capabilities), + analysis_provider_candidates: expandProviderCandidates( + status.analysis_provider_candidates, + ), + tool_availability: expandToolAvailability(status.tool_availability), +}); + +/** Expand provider capability groups while retaining candidate identity/state. */ +export const expandProviderCandidates = (candidates) => + candidates.map((candidate) => ({ + ...candidate, + capabilities: expandProviderCapabilities( + providerCapabilitySet.array().parse(candidate.capabilities), + ), + })); + +/** Expand CLI provider capability groups at the same owning boundary. */ +export const expandProviderOutput = (providers) => ({ + ...providers, + capabilities: expandProviderCapabilities( + providerCapabilitySet.array().parse(providers.capabilities), + ), + analysis_provider_candidates: expandProviderCandidates( + providers.analysis_provider_candidates, + ), +}); + +/** Expand the CLI capabilities command's compact provider descriptor groups. */ +export const expandCapabilitiesOutput = (capabilities) => ({ + ...capabilities, + capabilities: expandProviderCapabilities( + providerCapabilitySet.array().parse(capabilities.capabilities), + ), +}); + /** Validate an Evidence-producing tool output and return its normalized result. */ export const requireMcpEvidenceResult = (result, operation) => { const contract = toolContract(operation); @@ -48,7 +96,10 @@ export const requireMcpLifecycleResult = (result, operation) => { const contract = toolContract(operation); if (operationResultKey(contract) !== "result") throw new Error(`${operation} requires the Evidence result extractor`); - return requireOutput(result, operation, contract).result; + const projected = requireOutput(result, operation, contract).result; + return operation === "binary_session" + ? expandMcpSessionStatus(projected) + : projected; }; /** Project a mixed verifier call using its exact named canonical output schema. */ @@ -58,7 +109,10 @@ export const requireMcpOperationResult = ( contract = toolContract(operation), ) => { const key = operationResultKey(contract); - return requireOutput(result, operation, contract)[key]; + const projected = requireOutput(result, operation, contract)[key]; + return operation === "binary_session" + ? expandMcpSessionStatus(projected) + : projected; }; /** Verify provider identity directly from the tool result. */ diff --git a/scripts/lib/native-value-e2e.mjs b/scripts/lib/native-value-e2e.mjs index e65700b06..513259b30 100644 --- a/scripts/lib/native-value-e2e.mjs +++ b/scripts/lib/native-value-e2e.mjs @@ -11,6 +11,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../../dist/domain/evidence.js"; import { analysisProfileSchema } from "../../dist/domain/analysisProfile.js"; import { nativeValueTraceSchema } from "../../dist/domain/native/nativeValueTrace.js"; +import { requireMcpLifecycleResult } from "./mcp-verifier-results.mjs"; import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs"; const defaultEntrypoint = fileURLToPath(new URL("../rea.mjs", import.meta.url)); @@ -172,13 +173,16 @@ async function verifyNativeValueAdapters( { timeout: 180000 }, ); assert.notEqual(opened.isError, true, JSON.stringify(opened)); - const inventory = await client.callTool({ + const inventoryReply = await client.callTool({ name: "binary_session", arguments: {}, }); - assert.notEqual(inventory.isError, true, JSON.stringify(inventory)); + const inventory = requireMcpLifecycleResult( + inventoryReply, + "binary_session", + ); assert.ok( - inventory.structuredContent?.result?.tool_availability.some( + inventory.tool_availability.some( (tool) => tool.name === "trace_native_values" && tool.available, ), "Native value tracing is unavailable in real provider discovery", @@ -267,11 +271,14 @@ async function verifyNativeValueAdapters( arguments: {}, }); assert.notEqual(closed.isError, true, JSON.stringify(closed)); - const status = await client.callTool({ + const statusReply = await client.callTool({ name: "binary_session", arguments: {}, }); - assert.equal(status.structuredContent?.result?.open, false); + assert.equal( + requireMcpLifecycleResult(statusReply, "binary_session").open, + false, + ); assert.deepEqual( (await readdir(env.TMPDIR)).filter((name) => name.startsWith("rea-ghidra-"), diff --git a/scripts/lib/real-ghidra-boundaries.mjs b/scripts/lib/real-ghidra-boundaries.mjs index 83cc74b22..f6ee1527a 100644 --- a/scripts/lib/real-ghidra-boundaries.mjs +++ b/scripts/lib/real-ghidra-boundaries.mjs @@ -13,8 +13,6 @@ import { promisify } from "node:util"; import Ajv from "ajv/dist/2020.js"; import addFormats from "ajv-formats"; -import { analysisErrorProjectionSchema } from "../../dist/contracts/errorSchemas.js"; - import { verifyLegacyGhidraReferenceSnapshot } from "./ghidra-reference-snapshot-e2e.mjs"; import { verifyGhidraSnapshotLifecycle } from "./real-ghidra-snapshot-lifecycle.mjs"; import { verifyGhidraTargetAdmission } from "./real-ghidra-target-admission.mjs"; diff --git a/scripts/lib/real-hopper-workflows.mjs b/scripts/lib/real-hopper-workflows.mjs index c0b6595ed..9c71353ad 100644 --- a/scripts/lib/real-hopper-workflows.mjs +++ b/scripts/lib/real-hopper-workflows.mjs @@ -37,7 +37,7 @@ export async function verifyHopperWorkflows(call, invalid, procedure) { ); assert.equal(cycle.length, 2, "fixture omitted its source-owned call cycle"); const cycleGraph = await call("get_call_graph", { address: cycle[0].value }); - const cycleNodes = Object.values(cycleGraph).flat(); + const cycleNodes = Object.values(cycleGraph.graph).flat(); assert.equal(cycleNodes.length, 2); assert.deepEqual( cycleNodes.map((node) => node.address).sort(), diff --git a/scripts/lib/verify-package-core.mjs b/scripts/lib/verify-package-core.mjs index 5371edffc..a78b516da 100644 --- a/scripts/lib/verify-package-core.mjs +++ b/scripts/lib/verify-package-core.mjs @@ -2,6 +2,10 @@ import { execFile } from "node:child_process"; import { lstat } from "node:fs/promises"; import { promisify } from "node:util"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { + requireMcpLifecycleResult, + sortVerifierNames, +} from "./mcp-verifier-results.mjs"; /** Promisified child_process execution helper. */ export const exec = promisify(execFile); @@ -46,11 +50,10 @@ export const verifyCompleteToolCatalog = async (client, options) => { { name: "binary_session", arguments: {} }, options, ); - const availability = status.structuredContent?.result?.tool_availability; - if (!Array.isArray(availability)) - throw new Error("packaged MCP omitted tool availability"); - const expected = availability.map(({ name }) => name).sort(); - const observed = listed.tools.map(({ name }) => name).sort(); + const expanded = requireMcpLifecycleResult(status, "binary_session"); + const availability = expanded.tool_availability; + const expected = sortVerifierNames(availability.map(({ name }) => name)); + const observed = sortVerifierNames(listed.tools.map(({ name }) => name)); if (JSON.stringify(observed) !== JSON.stringify(expected)) throw new Error( "packaged MCP tool inventory diverged from session metadata", diff --git a/scripts/prepack.mjs b/scripts/prepack.mjs index 3b9f9adf1..9d0958d8e 100644 --- a/scripts/prepack.mjs +++ b/scripts/prepack.mjs @@ -47,6 +47,10 @@ if (await pathExists(join(root, "src"))) { for (const path of [ "dist/main.js", "dist/mcp/main.js", + "dist/cli-runtime/cli.js", + "dist/cli-runtime/cliOutput.js", + "dist/cli-runtime/streamedJsonOutput.js", + "dist/cli-runtime/mcpDoctor.js", "dist/mcpDoctor.js", "dist/cli.js", "dist/cliOutput.js", diff --git a/scripts/rea.mjs b/scripts/rea.mjs index 57272d987..3b1ef6c4a 100755 --- a/scripts/rea.mjs +++ b/scripts/rea.mjs @@ -33,8 +33,12 @@ if ( const runtimeFiles = isMcpMode ? ["dist/mcp/main.js"] : isMcpDoctorMode - ? ["dist/mcp/main.js", "dist/mcpDoctor.js"] - : ["dist/cli.js", "dist/cliOutput.js", "dist/cli/streamedJsonOutput.js"]; + ? ["dist/mcp/main.js", "dist/cli-runtime/mcpDoctor.js"] + : [ + "dist/cli-runtime/cli.js", + "dist/cli-runtime/cliOutput.js", + "dist/cli-runtime/streamedJsonOutput.js", + ]; if (!(await compiledRuntimeExists(runtimeFiles))) { process.stderr.write( @@ -45,7 +49,8 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { const { runEntrypoint } = await import("../dist/mcp/main.js"); await runEntrypoint(); } else if (isMcpDoctorMode) { - const { runProductionMcpDoctorCli } = await import("../dist/mcpDoctor.js"); + const { runProductionMcpDoctorCli } = + await import("../dist/cli-runtime/mcpDoctor.js"); const result = await runProductionMcpDoctorCli(args.slice(2), { dispatcherPath: fileURLToPath(import.meta.url), packageRoot, @@ -53,14 +58,14 @@ if (!(await compiledRuntimeExists(runtimeFiles))) { process.stdout.write(result.output); process.exitCode = result.exitCode; } else { - const { createCli } = await import("../dist/cli.js"); + const { createCli } = await import("../dist/cli-runtime/cli.js"); const { createStreamedCliJsonOutput } = - await import("../dist/cli/streamedJsonOutput.js"); + await import("../dist/cli-runtime/streamedJsonOutput.js"); const { renderCliOutputArgumentError, renderEmptyFilteredCliOutput, validateCliOutputArguments, - } = await import("../dist/cliOutput.js"); + } = await import("../dist/cli-runtime/cliOutput.js"); const outputArguments = validateCliOutputArguments(args); if (!outputArguments.ok) { process.stdout.write(renderCliOutputArgumentError(outputArguments)); diff --git a/scripts/verify-apktool.mjs b/scripts/verify-apktool.mjs index 8117038bc..aaec8a7ac 100644 --- a/scripts/verify-apktool.mjs +++ b/scripts/verify-apktool.mjs @@ -1,7 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { readFile, stat } from "node:fs/promises"; import { ApktoolResourceAnalysisService } from "../dist/application/apktool/ApktoolResourceAnalysisService.js"; import { createApktoolResourceAnalysisProvider } from "../dist/composition/apktool.js"; @@ -11,8 +9,8 @@ if (options.help) { process.stdout.write( "Usage: npm run verify:apktool [-- --apk PATH]\n" + "Requires a caller-selected apktool launcher (REA_APKTOOL_COMMAND or\n" + - "PATH) and a Java runtime. Without --apk, a small signed probe APK is\n" + - "expected at the path given by --apk; nothing is downloaded or built.\n", + "PATH) and a Java runtime. Without --apk, only client availability is\n" + + "checked. Supply a local APK to verify decoding; nothing is downloaded or built.\n", ); process.exit(0); } @@ -38,70 +36,80 @@ const run = async (operation, input) => { return result.value.normalized_result; }; -const client = await run("inspect_apktool_client", {}); -if (client !== null) { - check( - "apktool launcher reports a version", - typeof client.client.apktool_version === "string" && - client.client.apktool_version !== "", - `command=${client.client.command} version=${client.client.apktool_version}`, - ); -} +try { + const client = await run("inspect_apktool_client", {}); + if (client !== null) { + check( + "apktool launcher reports a version", + typeof client.client.apktool_version === "string" && + client.client.apktool_version !== "", + `command=${client.client.command} version=${client.client.apktool_version}`, + ); + } -if (options.apk !== null) { - const decoded = await run("decode_android_resources", { - path: options.apk, - include_strings: true, - }); - if (decoded !== null) { - const bytes = await stat(options.apk).then((info) => info.size); - const digest = createHash("sha256") - .update(await readFile(options.apk)) - .digest("hex"); - check( - "decoded target identity matches the file on disk", - decoded.target.bytes === bytes && decoded.target.sha256 === digest, - `${bytes} bytes`, - ); - check( - "metadata carries version and SDK facts", - typeof decoded.metadata.version_name === "string" && - decoded.metadata.min_sdk_version !== null && - decoded.metadata.target_sdk_version !== null, - `version=${String(decoded.metadata.version_name)} sdk=${String(decoded.metadata.min_sdk_version)}..${String(decoded.metadata.target_sdk_version)}`, - ); - check( - "decoded manifest names the package", - typeof decoded.metadata.package_name === "string" && - decoded.manifest !== null && - decoded.manifest.includes(decoded.metadata.package_name), - String(decoded.metadata.package_name), - ); - check( - "string resources projected", - decoded.strings.length > 0, - `${decoded.strings.length} string(s)`, - ); - if (decoded.locales.length > 0) { - const locale = await run("decode_android_resources", { - path: options.apk, - include_strings: true, - locale: decoded.locales[0], - }); - if (locale !== null) - check( - `locale table projected for ${String(decoded.locales[0])}`, - Array.isArray(locale.strings), - `${locale.strings.length} string(s)`, - ); + if (options.apk !== null) { + const decoded = await run("decode_android_resources", { + path: options.apk, + include_strings: true, + }); + if (decoded !== null) { + const bytes = await stat(options.apk).then((info) => info.size); + const digest = createHash("sha256") + .update(await readFile(options.apk)) + .digest("hex"); + check( + "decoded target identity matches the file on disk", + decoded.target.bytes === bytes && decoded.target.sha256 === digest, + `${bytes} bytes`, + ); + check( + "metadata carries version and SDK facts", + typeof decoded.metadata.version_name === "string" && + decoded.metadata.min_sdk_version !== null && + decoded.metadata.target_sdk_version !== null, + `version=${String(decoded.metadata.version_name)} sdk=${String(decoded.metadata.min_sdk_version)}..${String(decoded.metadata.target_sdk_version)}`, + ); + check( + "decoded manifest names the package", + typeof decoded.metadata.package_name === "string" && + decoded.manifest !== null && + decoded.manifest.includes(decoded.metadata.package_name), + String(decoded.metadata.package_name), + ); + check( + "string resources projected", + decoded.strings.length > 0, + `${decoded.strings.length} string(s)`, + ); + if (decoded.locales.length > 0) { + const locale = await run("decode_android_resources", { + path: options.apk, + include_strings: true, + locale: decoded.locales[0], + }); + if (locale !== null) + check( + `locale table projected for ${String(decoded.locales[0])}`, + Array.isArray(locale.strings), + `${locale.strings.length} string(s)`, + ); + } } } -} - -try { - await provider.close(); -} catch { - // The provider owns per-call workspaces only; closing is best-effort. +} catch (cause) { + failures += 1; + process.stdout.write( + `[FAIL] verifier execution — ${cause instanceof Error ? cause.message : String(cause)}\n`, + ); +} finally { + try { + await provider.close(); + } catch (cause) { + failures += 1; + process.stdout.write( + `[FAIL] provider cleanup — ${cause instanceof Error ? cause.message : String(cause)}\n`, + ); + } } if (failures > 0) { @@ -116,7 +124,12 @@ function parseArguments(argv) { const argument = argv[index]; if (argument === "--help" || argument === "-h") parsed.help = true; else if (argument === "--apk") { - parsed.apk = argv[index + 1] ?? null; + const path = argv[index + 1]; + if (path === undefined) { + process.stderr.write("--apk requires a local APK path\n"); + process.exit(2); + } + parsed.apk = path; index += 1; } else { process.stderr.write(`unknown argument: ${argument}\n`); diff --git a/scripts/verify-binary-layout.mjs b/scripts/verify-binary-layout.mjs index 40e7e798d..e4718e1df 100644 --- a/scripts/verify-binary-layout.mjs +++ b/scripts/verify-binary-layout.mjs @@ -662,6 +662,16 @@ try { bytes.writeBigUInt64LE(8n, header + 56); return [`undersized-entry-${type}`, bytes, "invalid_input"]; }); + const nonintegralSymbolEntries = ["SHT_SYMTAB", "SHT_DYNSYM"].map((type) => { + const section = protectedReport.sections.find((item) => item.type === type); + assert.notEqual(section, undefined); + const bytes = Buffer.from(sectionBearingBytes); + const header = Number(BigInt(section.header_location.offset)); + const size = bytes.readBigUInt64LE(header + 32); + assert.ok(size + 1n <= BigInt(bytes.length) - BigInt(section.offset)); + bytes.writeBigUInt64LE(size + 1n, header + 32); + return [`nonintegral-entry-count-${type}`, bytes, "invalid_input"]; + }); const relocationSection = relocatable.sections.find( (item) => item.type === "SHT_RELA", ); @@ -677,6 +687,17 @@ try { bytes.writeBigUInt64LE(8n, header + 56); return [`undersized-entry-${type}`, bytes, "invalid_input"]; }); + const nonintegralRelocationEntries = [ + ["RELA", 4, 24], + ["REL", 9, 16], + ].map(([type, sectionType, entrySize]) => { + const bytes = Buffer.from(object); + const header = Number(BigInt(relocationSection.header_location.offset)); + bytes.writeUInt32LE(sectionType, header + 4); + bytes.writeBigUInt64LE(BigInt(entrySize + 1), header + 32); + bytes.writeBigUInt64LE(BigInt(entrySize), header + 56); + return [`nonintegral-entry-count-${type}`, bytes, "invalid_input"]; + }); const invalidRelocationReferences = [ ["missing-symbol-table", relocatable.sections.length + 100], ["wrong-symbol-table-kind", 1], @@ -1066,7 +1087,9 @@ try { ], ...zeroSymbolEntries, ...undersizedSymbolEntries, + ...nonintegralSymbolEntries, ...undersizedRelocationEntries, + ...nonintegralRelocationEntries, ...invalidRelocationReferences, ["unterminated-dynamic-segment", shortDynamic, "invalid_input"], ["unterminated-interpreter", unterminatedInterpreter, "invalid_input"], diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index 30bc66ba7..2c0d79bc0 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -5,11 +5,16 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; +import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; import { exec } from "./lib/verify-package-core.mjs"; +import { + expandMcpSessionStatus, + sortVerifierNames, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createAnthropicModelFixture } from "./verify/clients/anthropic-model-fixture.mjs"; @@ -58,6 +63,8 @@ await writeFile( 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); let skillRequested = false; +let sessionRequested = false; +let sessionStatusSeen = false; let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; @@ -73,7 +80,7 @@ const containsText = (value, text) => { return Object.values(value).some((nested) => containsText(nested, text)); return false; }; -const extractEvidence = (value) => { +const extractInlineResult = (value, matches) => { if (typeof value === "string") { try { // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. @@ -81,20 +88,16 @@ const extractEvidence = (value) => { /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, "", ); - return extractEvidence(JSON.parse(json)); + return extractInlineResult(JSON.parse(json), matches); } catch { return undefined; } } if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; + if (matches(value)) return value; for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; + const result = extractInlineResult(nested, matches); + if (result) return result; } } return undefined; @@ -125,10 +128,12 @@ const fixture = await createAnthropicModelFixture({ 0, -"analyze_javascript_application".length, ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const expected = sortVerifierNames( + TOOL_CONTRACTS.map((t) => prefix + t.name), + ); const reaTools = tools.filter((t) => t.name.startsWith(prefix)); assert.deepEqual( - reaTools.map((t) => t.name).sort(), + sortVerifierNames(reaTools.map((t) => t.name)), expected, "Claude must forward the complete REA catalog after connection", ); @@ -146,11 +151,61 @@ const fixture = await createAnthropicModelFixture({ schemasChecked++; } } + if (mode === "call" && !sessionStatusSeen) { + const observedStatus = extractInlineResult( + body.messages, + (value) => + Boolean(value.result?.server_identity) && + Array.isArray(value.result?.tool_availability?.groups), + ); + if (observedStatus) { + const parsedStatus = + toolContract("binary_session").outputSchema.parse(observedStatus); + const status = { + ...parsedStatus, + result: expandMcpSessionStatus(parsedStatus.result), + }; + assert.deepEqual( + status.result.server_identity.catalog, + CATALOG_IDENTITY, + "binary_session must return the exact compact catalog counts and digests", + ); + assert.equal( + Object.hasOwn(status.result.server_identity.catalog, "tools"), + false, + "binary_session must not inline the duplicated canonical tool catalog", + ); + assert.deepEqual( + sortVerifierNames( + status.result.tool_availability.map((entry) => entry.name), + ), + sortVerifierNames(TOOL_CONTRACTS.map((contract) => contract.name)), + "binary_session must retain complete canonical tool availability", + ); + sessionStatusSeen = true; + } else { + assert( + !sessionRequested, + "Claude must return the inline binary_session result before continuing", + ); + const session = tools.find( + (tool) => tool.name === `${prefix}binary_session`, + ); + assert(session, "Claude must expose the REA binary_session tool"); + sessionRequested = true; + return call(session.name, {}); + } + } skillLoaded ||= expectedSkillBody !== undefined && containsText(body.messages, expectedSkillBody) && containsText(body.messages, skillDirectory); - const observed = extractEvidence(body.messages); + const observed = extractInlineResult( + body.messages, + (value) => + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0, + ); if (observed) { const evidence = toolContract( "analyze_javascript_application", @@ -184,7 +239,7 @@ const fixture = await createAnthropicModelFixture({ await writeFile(join(lab, "claude-evidence.json"), bytes); evidenceSeen = true; } - if (mode === "chat" || evidenceSeen) { + if (mode === "chat" || (sessionStatusSeen && evidenceSeen)) { finalSeen = true; return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; } @@ -294,6 +349,7 @@ try { "Skill", "WaitForMcpServers", "mcp__rea__analyze_javascript_application", + ...(mode === "call" ? ["mcp__rea__binary_session"] : []), ], }, }), @@ -324,7 +380,7 @@ try { assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); assert.deepEqual(parsed.permission_denials, []); assert(finalSeen); - if (mode === "call") assert(evidenceSeen && skillLoaded); + if (mode === "call") assert(evidenceSeen && skillLoaded && sessionStatusSeen); } catch (cause) { failure = cause; if (typeof cause?.stdout === "string") @@ -355,6 +411,7 @@ try { schemasChecked, pendingServerWaits, evidenceSeen, + sessionStatusSeen, skillLoaded, artifactDigest, finalSeen, diff --git a/scripts/verify-deepseek-client.mjs b/scripts/verify-deepseek-client.mjs index 12aed2a1e..42aae272b 100644 --- a/scripts/verify-deepseek-client.mjs +++ b/scripts/verify-deepseek-client.mjs @@ -5,11 +5,12 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { parse as parseYaml } from "yaml"; import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; -import { exec } from "./lib/verify-package-core.mjs"; +import { exec, pathExists } from "./lib/verify-package-core.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs"; @@ -32,6 +33,7 @@ const skillPath = join( "SKILL.md", ); const cli = join(runtimeRoot, "scripts", "rea.mjs"); +const configPath = join(profile, "cordis.patch.yml"); const verifier = createVerifierRun(); await Promise.all([ mkdir(account), @@ -66,32 +68,32 @@ const clientVersion = ( timeout: 30_000, }) ).stdout.trim(); -for (const args of [ - ["setup", "--skill", "--dry-run"], - ["setup", "--skill", "--yes"], -]) { +const lifecycle = []; +const runRea = async (args, artifact) => { const result = await exec(process.execPath, [cli, ...args, "--json"], { env: environment, cwd: workspace, timeout: 60_000, }); - const parsed = JSON.parse(result.stdout); - assert.equal(parsed.status, args.includes("--dry-run") ? "planned" : "ready"); - if (args.includes("--dry-run")) { - assert.equal(parsed.plannedActions.length, 1); - assert.equal(parsed.plannedActions[0].kind, "install_skill"); - assert.equal(parsed.plannedActions[0].target, dirname(skillPath)); - } - await writeFile( - join(lab, args.includes("--dry-run") ? "setup-plan.json" : "setup.json"), - result.stdout, - ); -} -assert( - (await readFile(skillPath, "utf8")).includes( - "name: reverse-engineer-anything", - ), -); + await writeFile(join(lab, `${artifact}.json`), result.stdout); + return JSON.parse(result.stdout); +}; +const patches = (text) => + parseYaml(text, { + customTags: [ + { + tag: "tag:yaml.org,2002:js", + resolve: (source) => ({ __jsExpr: source }), + }, + ], + }); +const withoutRea = (text) => + patches(text).flatMap((op) => { + if (op.id === "rea-mcp") return []; + if (!Array.isArray(op.insert)) return [op]; + const insert = op.insert.filter((entry) => entry.id !== "rea-mcp"); + return insert.length === 0 ? [] : [{ ...op, insert }]; + }); let catalog; let patternCount = 0; @@ -247,10 +249,9 @@ const fixture = await createOpenAiModelFixture({ }); let failure; try { - // Use the client's native Cordis patch DSL; generic `rea setup --client` does not edit it. - await writeFile( - join(profile, "cordis.patch.yml"), - ` + // Seed model settings and a stale registration; the public setup workflow + // must produce the registration this native client actually boots. + const original = ` - id: llm-pi-ai config: providers: @@ -275,13 +276,59 @@ try { config: serverName: rea transport: stdio - command: ${JSON.stringify(process.execPath)} - args: ${JSON.stringify([cli, "mcp"])} + command: stale-launcher + args: [mcp] cwd: !!js process.cwd() env: REA_PROCESS_RUN_ID: ${JSON.stringify(verifier.run_id)} -`, +`; + await writeFile(configPath, original); + const planned = await runRea( + ["setup", "--client", "dsh", "--dry-run"], + "setup-plan", ); + assert.equal(planned.status, "planned"); + assert(planned.plannedActions.some((action) => action.target === configPath)); + assert( + planned.plannedActions.some( + (action) => action.target === dirname(skillPath), + ), + ); + assert( + planned.plannedActions.every( + (action) => + action.target === configPath || action.target === dirname(skillPath), + ), + ); + assert.equal(await readFile(configPath, "utf8"), original); + assert.equal(await pathExists(`${configPath}.rea.backup`), false); + lifecycle.push("dry-run"); + const setup = await runRea(["setup", "--client", "dsh", "--yes"], "setup"); + assert.equal(setup.status, "ready"); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + const configured = await readFile(configPath, "utf8"); + assert.deepEqual(withoutRea(configured), withoutRea(original)); + assert(configured.includes("!!js process.cwd()")); + assert( + (await readFile(skillPath, "utf8")).includes( + "name: reverse-engineer-anything", + ), + ); + lifecycle.push("setup"); + const repeated = await runRea( + ["setup", "--client", "dsh", "--yes"], + "setup-repeat", + ); + assert.deepEqual(repeated.appliedActions, []); + assert.equal(await readFile(configPath, "utf8"), configured); + assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original); + lifecycle.push("idempotent-setup"); + const doctor = await runRea( + ["doctor", "--client", "dsh", "--skill"], + "doctor", + ); + assert.equal(doctor.healthy, true); + lifecycle.push("doctor"); const execution = exec( command, [ @@ -304,6 +351,20 @@ try { assert.equal(fixture.failure, undefined); assert(finalSeen && result.stdout.includes("REA_CLIENT_COMPATIBILITY_OK")); if (mode === "call") assert(evidenceSeen && skillLoaded); + lifecycle.push(mode === "call" ? "native-tool-invocation" : "native-chat"); + const uninstall = await runRea(["uninstall"], "uninstall"); + assert.equal(uninstall.status, "complete"); + assert( + uninstall.items.some( + (item) => item.name === "dsh" && item.status === "removed", + ), + ); + assert.deepEqual( + patches(await readFile(configPath, "utf8")), + withoutRea(original), + ); + assert.equal(await pathExists(skillPath), false); + lifecycle.push("uninstall"); } catch (cause) { failure = cause; if (typeof cause?.stdout === "string") @@ -316,6 +377,7 @@ try { client: "deepseek-harness", clientVersion, mode, + lifecycle, passed: failure === undefined, artifacts: lab, runtimeRoot, diff --git a/scripts/verify-gemini-client.mjs b/scripts/verify-gemini-client.mjs index c2d940077..8d1195f74 100644 --- a/scripts/verify-gemini-client.mjs +++ b/scripts/verify-gemini-client.mjs @@ -2,7 +2,7 @@ import assert from "node:assert/strict"; import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; import { @@ -13,14 +13,18 @@ import { exec } from "./lib/verify-package-core.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createGeminiModelFixture } from "./verify/clients/gemini-model-fixture.mjs"; -assert( - process.platform !== "win32", - "Native Gemini verifier requires POSIX; Windows is unverified", -); const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; const command = process.env.REA_VERIFY_GEMINI_COMMAND ?? "gemini"; const mode = process.argv[2] ?? "call"; +// npm installs of Gemini CLI provide a JavaScript entry point; on Windows only a gemini.cmd shim reaches +// PATH, which Node does not spawn without a shell, so a selected .js entry runs under this Node. It is +// resolved once, because the preflight runs from the caller's directory and the client from the workspace. +const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined; +const launch = (args) => + entryPoint === undefined + ? [command, args] + : [process.execPath, [entryPoint, ...args]]; assert( ["chat", "call"].includes(mode), "Usage: verify-gemini-client.mjs [chat|call]", @@ -60,11 +64,15 @@ const environment = { let clientVersion; try { clientVersion = ( - await exec(command, ["--version"], { env: environment, timeout: 30_000 }) + await exec(...launch(["--version"]), { env: environment, timeout: 30_000 }) ).stdout.trim(); } catch (cause) { + const windowsHint = + process.platform === "win32" + ? " On Windows, select the JavaScript entry point; npm installs provide it as node_modules/@google/gemini-cli/bundle/gemini.js." + : ""; throw new Error( - `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND. ${String(cause)}`, + `Gemini preflight failed for ${command}; install Gemini CLI or set REA_VERIFY_GEMINI_COMMAND.${windowsHint} ${String(cause)}`, ); } const original = JSON.stringify( @@ -103,11 +111,11 @@ for (const args of [ assert(parsed.plannedActions.some((a) => a.target === configPath)); assert( parsed.plannedActions.some((a) => - a.target.startsWith(join(profile, ".agents", "skills") + "/"), + a.target.startsWith(join(profile, ".agents", "skills") + sep), ), ); assert( - parsed.plannedActions.every((a) => a.target.startsWith(profile + "/")), + parsed.plannedActions.every((a) => a.target.startsWith(profile + sep)), ); } } @@ -300,8 +308,7 @@ const fixture = await createGeminiModelFixture({ let failure; try { const response = await exec( - command, - [ + ...launch([ "--model", "gemini-2.5-flash", "--approval-mode", @@ -314,7 +321,7 @@ try { mode === "chat" ? "Say hello briefly." : `Use REA to analyze the JavaScript directory ${target}. Report the result.`, - ], + ]), { env: { ...environment, diff --git a/scripts/verify-package-capabilities.mjs b/scripts/verify-package-capabilities.mjs index 2137a4c5a..acf848974 100644 --- a/scripts/verify-package-capabilities.mjs +++ b/scripts/verify-package-capabilities.mjs @@ -1,13 +1,19 @@ import { json, run } from "./lib/verify-package-core.mjs"; +import { + expandCapabilitiesOutput, + expandProviderOutput, +} from "./lib/mcp-verifier-results.mjs"; /** Verify packaged capabilities, providers, and non-Linux search results. */ export async function verifyPackageCapabilitiesAndSearch({ cli, environment }) { - const capabilities = json( - await run(cli, ["capabilities", "--json"], environment), + const capabilities = expandCapabilitiesOutput( + json(await run(cli, ["capabilities", "--json"], environment)), ); if (!Array.isArray(capabilities.capabilities)) throw new Error("packaged capabilities CLI failed"); - const providers = json(await run(cli, ["providers", "--json"], environment)); + const providers = expandProviderOutput( + json(await run(cli, ["providers", "--json"], environment)), + ); const candidates = providers.analysis_provider_candidates; if ( !Array.isArray(candidates) || diff --git a/scripts/verify-package-mcp.mjs b/scripts/verify-package-mcp.mjs index 7bb22715d..0d03abab1 100644 --- a/scripts/verify-package-mcp.mjs +++ b/scripts/verify-package-mcp.mjs @@ -1,4 +1,7 @@ -import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { join } from "node:path"; import { execFile } from "node:child_process"; @@ -21,14 +24,14 @@ const verifyMcpToolsAndPrompts = async (client, mcpOptions) => { }; const verifyMcpTargetFree = async (client, mcpOptions) => { - const status = await client.callTool( + const result = await client.callTool( { name: "binary_session", arguments: {} }, mcpOptions, ); - const currentDocument = - status.structuredContent?.result?.tool_availability?.find( - ({ name }) => name === "current_document", - ); + const status = requireMcpLifecycleResult(result, "binary_session"); + const currentDocument = status.tool_availability.find( + ({ name }) => name === "current_document", + ); if ( currentDocument?.available !== false || currentDocument.reason !== "target_required" @@ -67,15 +70,13 @@ const verifyMcpOpenAndBind = async (client, mcpOptions) => { ); if (opened.isError === true) throw new Error("packaged MCP could not open a binary"); - const providerStatusEnvelope = json( - prompts.mcpText( - await client.callTool( - { name: "binary_session", arguments: {} }, - mcpOptions, - ), + const providerStatus = requireMcpLifecycleResult( + await client.callTool( + { name: "binary_session", arguments: {} }, + mcpOptions, ), + "binary_session", ); - const providerStatus = providerStatusEnvelope.result; if ( providerStatus.analysis_provider_binding?.provider?.id !== "hopper" || providerStatus.analysis_provider_binding?.selection_source !== "request" diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index b38072c8c..2c5f15e62 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -1,7 +1,10 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + expandProviderOutput, + requireMcpOperationResult, +} from "./lib/mcp-verifier-results.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -230,14 +233,16 @@ const assertCleanup = async () => { assert.equal(entries.length, 0, "Provider runtime cleanup left files."); }; try { - const providers = JSON.parse( - ( - await exec(process.execPath, [entry, "providers", "--json"], { - env: environment, - cwd: callerDirectory, - timeout: 30_000, - }) - ).stdout, + const providers = expandProviderOutput( + JSON.parse( + ( + await exec(process.execPath, [entry, "providers", "--json"], { + env: environment, + cwd: callerDirectory, + timeout: 30_000, + }) + ).stdout, + ), ); assert.equal(providers.analysis_provider_binding, null); assert.ok(Array.isArray(providers.analysis_provider_candidates)); diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs index 12859d4bb..8c0281d56 100644 --- a/scripts/verify-published-package.mjs +++ b/scripts/verify-published-package.mjs @@ -10,6 +10,11 @@ import { pathToFileURL } from "node:url"; import { lt, rcompare } from "semver"; import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; +import { + requireMcpLifecycleResult, + sortVerifierNames, +} from "./lib/mcp-verifier-results.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; const verifierRun = createVerifierRun(); @@ -54,27 +59,39 @@ try { await client.connect(transport); const tools = await client.listTools(); const toolNames = tools.tools.map(({ name }) => name); - const canonicalToolNames = new Set( - CATALOG_IDENTITY.tools.map(({ name }) => name), - ); + const canonicalToolNames = new Set(TOOL_CONTRACTS.map(({ name }) => name)); const unknownToolNames = toolNames.filter( (name) => !canonicalToolNames.has(name), ); + const publishedToolNames = new Set(toolNames); + const missingToolNames = [...canonicalToolNames].filter( + (name) => !publishedToolNames.has(name), + ); publishedToolCount = toolNames.length; if ( publishedToolCount === 0 || - new Set(toolNames).size !== publishedToolCount || - unknownToolNames.length > 0 + publishedToolNames.size !== publishedToolCount || + unknownToolNames.length > 0 || + missingToolNames.length > 0 ) throw new Error( - `Published MCP exposed an invalid capability-scoped tool projection (${String(publishedToolCount)} tools, unknown: ${unknownToolNames.join(", ") || "none"})`, + `Published MCP exposed an invalid canonical tool catalog (${String(publishedToolCount)} tools, unknown: ${unknownToolNames.join(", ") || "none"}, missing: ${missingToolNames.join(", ") || "none"})`, ); - const status = await client.callTool({ + const statusReply = await client.callTool({ name: "binary_session", arguments: {}, }); - if (status.isError === true) - throw new Error("Published target-free MCP session check failed"); + const status = requireMcpLifecycleResult(statusReply, "binary_session"); + const availabilityNames = sortVerifierNames( + status.tool_availability.map(({ name }) => name), + ); + if ( + JSON.stringify(availabilityNames) !== + JSON.stringify(sortVerifierNames(canonicalToolNames)) + ) + throw new Error( + "Published binary_session omitted canonical tool availability", + ); update = await verifyPublishedUpdate({ version, canaryRoot }); } catch (cause) { throw new Error(`Published package verification failed: ${stderr}`, { diff --git a/scripts/verify-real-ghidra-com.mjs b/scripts/verify-real-ghidra-com.mjs index aaba1871b..5fabc1b8b 100644 --- a/scripts/verify-real-ghidra-com.mjs +++ b/scripts/verify-real-ghidra-com.mjs @@ -158,7 +158,7 @@ try { observations: { ...x.observations, source_files: x.observations.source_files.map( - ({ name, ...file }) => file, + ({ name: _name, ...file }) => file, ), }, }); diff --git a/scripts/verify-real-ghidra-mips-relocatable.mjs b/scripts/verify-real-ghidra-mips-relocatable.mjs index ab26f0699..da8a821b1 100755 --- a/scripts/verify-real-ghidra-mips-relocatable.mjs +++ b/scripts/verify-real-ghidra-mips-relocatable.mjs @@ -163,6 +163,7 @@ async function verifyFixture(byteOrder, kernelModule) { }); assert.equal(coverage.unplaced, 0, "Fixture references an empty section"); assert.ok(coverage.shared_hi16, "No HI16 shared by several LO16s"); + assert.ok(coverage.multiple_hi16, "No LO16 resolved several pending HI16s"); assert.ok(coverage.lo16_carry, "No LO16 >= 0x8000 forced a HI16 carry"); assert.ok(coverage.external_code, "No external code relocation"); assert.ok(coverage.external_data, "No external data relocation"); diff --git a/scripts/verify-real-ghidra-mips-shared.mjs b/scripts/verify-real-ghidra-mips-shared.mjs new file mode 100644 index 000000000..da52495ce --- /dev/null +++ b/scripts/verify-real-ghidra-mips-shared.mjs @@ -0,0 +1,338 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { promisify } from "node:util"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js"; +import { parseEvidence } from "../dist/domain/evidence.js"; +import { parseAnalysisSnapshot } from "../dist/domain/analysisSnapshot.js"; +import { buildMips16SharedFixture } from "../tests/conformance/ghidra/mips16-shared-fixture.mjs"; +import { + requireMcpOperationResult, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +if ( + process.env.GHIDRA_INSTALL_DIR === undefined || + !["linux", "darwin"].includes(process.platform) +) + throw new Error( + "This optional lane requires BYO Ghidra on Linux/macOS and GNU MIPS cross-binutils.", + ); +const execute = promisify(execFile); +const run = createVerifierRun(); +const workspace = await mkdtemp(join(tmpdir(), "rea-mips-shared-proof-")); +const runtime = join(workspace, "runtime"); +await mkdir(runtime); +const env = { + PATH: process.env.PATH ?? "/usr/bin:/bin", + TMPDIR: runtime, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "ghidra", + REA_PROCESS_RUN_ID: run.run_id, + GHIDRA_INSTALL_DIR: process.env.GHIDRA_INSTALL_DIR, + ...(process.env.JAVA_HOME === undefined + ? {} + : { JAVA_HOME: process.env.JAVA_HOME }), + ...(process.env.GHIDRA_HEADLESS_MAXMEM === undefined + ? {} + : { GHIDRA_HEADLESS_MAXMEM: process.env.GHIDRA_HEADLESS_MAXMEM }), +}; +const reports = []; +let failure; +try { + for (const byteOrder of ["big", "little"]) { + for (const variant of [ + "declared", + "recordless", + "symbol-relative", + "section-relocations", + ]) { + const fixture = await buildMips16SharedFixture( + workspace, + byteOrder, + variant, + ); + const resolved = await parseBinaryTarget(fixture.path); + if (!resolved.ok) throw resolved.error; + assert.equal(resolved.value.sha256, fixture.sha256); + assert.equal(resolved.value.architecture, "mips"); + if (variant === "section-relocations") { + assert.equal(resolved.value.mips.sharedLoad.status, "unavailable"); + assert.match( + resolved.value.mips.sharedLoad.reason, + /section relocation declarations/u, + ); + } else { + assert.equal(resolved.value.mips.sharedLoad.status, "complete"); + assert.deepEqual( + resolved.value.mips.sharedLoad.relocationTypes, + [0, 3], + ); + assert.equal( + resolved.value.mips.sharedLoad.symbolRelativeCount, + variant === "symbol-relative" ? 1 : 0, + ); + } + if (variant === "recordless") + assert.equal(resolved.value.mips.abiFlags, null); + else { + assert.equal(resolved.value.mips.abiFlags.fpAbi, 5); + assert.equal(resolved.value.mips.abiFlags.ases, 0x400); + } + const transport = new StdioClientTransport({ + command: process.execPath, + args: ["scripts/rea.mjs", "mcp"], + cwd: process.cwd(), + env, + stderr: "pipe", + }); + const client = new Client({ + name: "rea-mips16-shared-conformance", + version: "1", + }); + let stderr = ""; + transport.stderr?.on("data", (chunk) => { + stderr = (stderr + chunk.toString()).slice(-262144); + }); + let calls = 0; + const reply = async (name, args = {}) => { + const response = await client.callTool( + { name, arguments: args }, + { timeout: 180000 }, + ); + await writeFile( + join(workspace, `${byteOrder}-${variant}-${++calls}-${name}.json`), + JSON.stringify(response, null, 2), + ); + return response; + }; + const call = async (name, args) => + requireMcpOperationResult(await reply(name, args), name); + const original = await readFile(fixture.path); + try { + await client.connect(transport); + await client.listTools(); + if ( + variant === "symbol-relative" || + variant === "section-relocations" + ) { + const refusal = requireMcpToolError( + await reply("open_binary", { + path: fixture.path, + provider_id: "ghidra", + }), + ); + assert.equal(refusal.category, "unsupported_provider"); + assert.match( + JSON.stringify(refusal), + variant === "symbol-relative" + ? /Symbol-bearing R_MIPS_REL32/u + : /section relocation declarations/u, + ); + assert.deepEqual(await readFile(fixture.path), original); + await client.ping(); + reports.push({ ...fixture, status: "refused", ping: true }); + continue; + } + await call("open_binary", { + path: fixture.path, + provider_id: "ghidra", + }); + const loadedReply = await reply("inspect_native_load_image"); + const loaded = requireMcpOperationResult( + loadedReply, + "inspect_native_load_image", + ); + const evidence = parseEvidence( + JSON.parse( + loadedReply.content.find((item) => item.type === "text").text, + ), + ); + const language = `MIPS:${byteOrder === "little" ? "LE" : "BE"}:32:default`; + assert.equal(loaded.observations.language_id, language); + assert.equal(loaded.observations.compiler_spec_id, "default"); + assert( + loaded.observations.source_files.some( + (file) => file.original_sha256 === fixture.sha256, + ), + ); + assert( + evidence.limitations.some((item) => + item.includes("External GOT bindings"), + ), + ); + if (variant === "recordless") + assert( + evidence.limitations.some((item) => + item.includes("does not establish the floating-point ABI"), + ), + ); + const base = BigInt(loaded.observations.image_base); + const address = (source, clearTag = false) => + `0x${(base + (clearTag ? BigInt(source) & ~1n : BigInt(source))).toString(16)}`; + const compressed = address(fixture.symbols.rea_mips16_probe, true); + const standard = address(fixture.symbols.rea_mips32_probe); + const procedures = await call("list_procedures"); + assert( + procedures.some( + (item) => + item.address === compressed && item.value === "rea_mips16_probe", + ), + ); + assert( + procedures.some( + (item) => + item.address === standard && item.value === "rea_mips32_probe", + ), + ); + let firstInstruction; + const expected = + byteOrder === "little" + ? ["5a6a", "8eea", "074a", "a0e8"] + : ["6a5a", "ea8e", "4a07", "e8a0"]; + for (let i = 0; i < 4; i += 1) { + const instruction = await call("inspect_native_instruction", { + address: `0x${(BigInt(compressed) + BigInt(2 * i)).toString(16)}`, + }); + assert.equal(instruction.status, "decoded"); + assert.equal(instruction.mode, "default"); + assert.equal(instruction.length, 2); + assert.equal(instruction.bytes, expected[i]); + assert.deepEqual(instruction.decoder_context, [ + { + register: "ISA_MODE", + value: "1", + source: "ghidra-program-context", + }, + ]); + if (i === 0) firstInstruction = instruction; + } + const standardInstruction = await call("inspect_native_instruction", { + address: standard, + }); + assert.equal(standardInstruction.length, 4); + assert.equal( + standardInstruction.bytes, + byteOrder === "little" ? "03008224" : "24820003", + ); + assert.equal( + standardInstruction.decoder_context[0].register, + "ISA_MODE", + ); + assert( + [null, "0"].includes(standardInstruction.decoder_context[0].value), + ); + const pointer = await call("read_bytes", { + address: address(fixture.symbols.rea_mips16_pointer), + length: 4, + }); + const pointerBytes = Buffer.from(pointer.bytes_hex, "hex"); + const pointerValue = + byteOrder === "little" + ? pointerBytes.readUInt32LE(0) + : pointerBytes.readUInt32BE(0); + assert.equal( + BigInt(pointerValue), + base + BigInt(fixture.symbols.rea_mips16_probe), + ); + const dossier = await call("analyze_function", { + procedure: compressed, + }); + assert.match(dossier.pseudocode, /\^\s*0x5a/u); + assert.match(dossier.pseudocode, /\+\s*(?:7|0x7)\b/u); + const standardDossier = await call("analyze_function", { + procedure: standard, + }); + assert.match(standardDossier.pseudocode, /\+\s*(?:3|0x3)\b/u); + const snapshotPath = join( + workspace, + `${byteOrder}-${variant}.snapshot.json`, + ); + await call("close_binary", { snapshot_path: snapshotPath }); + const snapshot = parseAnalysisSnapshot( + JSON.parse(await readFile(snapshotPath, "utf8")), + ); + assert.equal( + snapshot.binding.analysis_profile.parameters.language_selection, + "provider-pinned-v1", + ); + assert.equal( + snapshot.binding.analysis_profile.parameters.language_id, + language, + ); + await call("open_binary", { + path: fixture.path, + provider_id: "ghidra", + snapshot_path: snapshotPath, + }); + assert.deepEqual( + await call("inspect_native_instruction", { address: compressed }), + firstInstruction, + ); + await call("close_binary"); + await client.ping(); + const cli = await execute( + process.execPath, + [ + "scripts/rea.mjs", + "inspect-native-instruction", + fixture.path, + compressed, + "--provider", + "ghidra", + "--format", + "json", + ], + { env, timeout: 180000, maxBuffer: 8 * 1024 * 1024 }, + ); + assert.deepEqual( + parseEvidence(JSON.parse(cli.stdout)).normalized_result, + firstInstruction, + ); + assert.deepEqual(await readFile(fixture.path), original); + reports.push({ + ...fixture, + status: "passed", + language_id: language, + mixed_mode: true, + standard_decoder_context: standardInstruction.decoder_context, + pointer_tag_preserved: true, + snapshot_context_preserved: true, + cli_mcp_parity: true, + source_unchanged: true, + ping: true, + }); + } finally { + await client.close(); + await transport.close(); + await writeFile( + join(workspace, `${byteOrder}-${variant}.stderr`), + stderr, + ); + } + } + } +} catch (cause) { + failure = cause; +} +const completed = await completeVerifierRun(run); +if ( + completed.process_lineage.status !== "verified" || + completed.process_lineage.descendants.length !== 0 +) + throw new Error(`Cleanup unverified; retained workspace: ${workspace}`, { + cause: failure, + }); +if (failure !== undefined) + throw new Error( + `MIPS16 shared verification failed; retained workspace: ${workspace}`, + { cause: failure }, + ); +await rm(workspace, { recursive: true, force: true }); +console.log(JSON.stringify({ ...completed, fixtures: reports }, null, 2)); diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index 97ad4ac9e..12400cb94 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -30,6 +30,7 @@ import { requireMcpOperationResult, requireMcpLifecycleResult, requireMcpEvidenceResult, + sortVerifierNames, requireWorkflowEvidenceProvider, } from "./lib/mcp-verifier-results.mjs"; import { @@ -283,8 +284,10 @@ try { const availableTools = initialStatus.tool_availability; if (!Array.isArray(availableTools)) throw new Error("The real session omitted tool availability"); - const expectedNames = availableTools.map(({ name }) => name).sort(); - const actualNames = listed.tools.map(({ name }) => name).sort(); + const expectedNames = sortVerifierNames( + availableTools.map(({ name }) => name), + ); + const actualNames = sortVerifierNames(listed.tools.map(({ name }) => name)); if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { const expected = new Set(expectedNames); const actual = new Set(actualNames); diff --git a/scripts/verify-wasm-artifact.mjs b/scripts/verify-wasm-artifact.mjs index 82ddd0fb0..b45c8ba7d 100644 --- a/scripts/verify-wasm-artifact.mjs +++ b/scripts/verify-wasm-artifact.mjs @@ -247,6 +247,7 @@ try { '"test\nline"', '"fake"\nSection Details:\n Type start=0x0000000a end=0x0000000b (size=0x00000001) count: 0\n"end"', '"unicode-é\\\t"', + '"visible"', ], ); if (debugNames) { diff --git a/scripts/verify-web-network-captures.mjs b/scripts/verify-web-network-captures.mjs index 90c945c63..af58f78ed 100644 --- a/scripts/verify-web-network-captures.mjs +++ b/scripts/verify-web-network-captures.mjs @@ -913,6 +913,30 @@ async function inspect(mode, input, errorCategory) { } assert.notEqual(result.isError, true, mcpTextValue(result)); assertSensitiveLimitations(value, input.sensitive_values ?? []); + if (input.format === "har") { + const surface = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: value.evidence_id }, + view: { kind: "api-surface" }, + }, + }); + if ((input.sensitive_values ?? []).length > 0) { + // Derived view text cannot be checked against undisclosed declarations. + assert.equal(surface.isError, true); + const refused = JSON.parse(mcpTextValue(surface)); + for (const literal of input.sensitive_values) + assert.ok(!JSON.stringify(refused).includes(literal)); + } else { + assert.notEqual(surface.isError, true, mcpTextValue(surface)); + const projected = JSON.parse(mcpTextValue(surface)); + assert.equal(projected.normalized_result.kind, "api-surface"); + assert.equal( + projected.normalized_result.records.examined, + value.normalized_result.records.length, + ); + } + } return value.normalized_result; } const args = [ diff --git a/scripts/verify/android/failure-recovery.mjs b/scripts/verify/android/failure-recovery.mjs index adcf41d74..8e1816aed 100644 --- a/scripts/verify/android/failure-recovery.mjs +++ b/scripts/verify/android/failure-recovery.mjs @@ -1,4 +1,8 @@ -import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpToolError, + sortVerifierNames, +} from "../../lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { access, @@ -273,22 +277,23 @@ const verifyAndroidReadiness = async ({ try { await client.connect(transport); const catalog = await client.listTools(); - const catalogNames = catalog.tools.map(({ name }) => name).sort(); - const status = await client.callTool({ + const catalogNames = sortVerifierNames( + catalog.tools.map(({ name }) => name), + ); + const statusReply = await client.callTool({ name: "binary_session", arguments: {}, }); - assert.notEqual(status.isError, true, JSON.stringify(status)); - const availability = status.structuredContent?.result?.tool_availability; - assert.ok(Array.isArray(availability)); + const status = requireMcpLifecycleResult(statusReply, "binary_session"); + const availability = status.tool_availability; assert.deepEqual( - availability.map(({ name }) => name).sort(), - TOOL_CONTRACTS.map(({ name }) => name).sort(), + sortVerifierNames(availability.map(({ name }) => name)), + sortVerifierNames(TOOL_CONTRACTS.map(({ name }) => name)), "Readiness must retain one availability record for the complete canonical catalog", ); assert.deepEqual( catalogNames, - TOOL_CONTRACTS.map(({ name }) => name).sort(), + sortVerifierNames(TOOL_CONTRACTS.map(({ name }) => name)), "Readiness must not truncate the SDK tool catalog", ); const byName = new Map(availability.map((item) => [item.name, item])); diff --git a/scripts/verify/apple/legacy-simulator.mjs b/scripts/verify/apple/legacy-simulator.mjs index 6ccc10ae2..1d3110cf5 100644 --- a/scripts/verify/apple/legacy-simulator.mjs +++ b/scripts/verify/apple/legacy-simulator.mjs @@ -1,7 +1,6 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { createHash } from "node:crypto"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { promisify } from "node:util"; diff --git a/scripts/verify/architecture-guards.mjs b/scripts/verify/architecture-guards.mjs index ea92bf9d2..967e210de 100644 --- a/scripts/verify/architecture-guards.mjs +++ b/scripts/verify/architecture-guards.mjs @@ -1,4 +1,3 @@ -import assert from "node:assert/strict"; import { readFile, readdir } from "node:fs/promises"; import { join, relative, sep } from "node:path"; import { fileURLToPath } from "node:url"; diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 71b40bdac..1f84681c9 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -100,7 +100,9 @@ const verifyRegions = (evidence) => { ); assert.equal(gzip.reported_size, oracle.length); assert.equal(gzip.size_basis, "provider_reported_validation_unknown"); - return result.regions.map(({ provider_id, ...region }) => region); + return result.regions.map( + ({ provider_id: _provider_id, ...region }) => region, + ); }; const verifyFiles = async (evidence) => { const result = firmwareResultSchemas.extract_firmware.parse( diff --git a/src/adb/AdbDeviceOutput.test.ts b/src/adb/AdbDeviceOutput.test.ts index b85061cae..fe11c2a26 100644 --- a/src/adb/AdbDeviceOutput.test.ts +++ b/src/adb/AdbDeviceOutput.test.ts @@ -239,4 +239,35 @@ describe("parsePmPathOutput", () => { role: "unknown", }); }); + + it("preserves whitespace and the exact POSIX device path", () => { + const devicePath = "/data/app/My App/résumé:final\\draft*.apk"; + const { paths, rejectedPaths, unparsedLines } = parsePmPathOutput( + `package:${devicePath}\r\n`, + ); + expect(paths).toEqual([ + { + device_path: devicePath, + file_name: "résumé:final\\draft*.apk", + role: "unknown", + }, + ]); + expect(rejectedPaths).toEqual([]); + expect(unparsedLines).toEqual([]); + }); + + it("rejects an empty final component without normalizing the source", () => { + const devicePath = "/data/app/pkg/base.apk/"; + const parsed = parsePmPathOutput(`package:${devicePath}\n`); + expect(parsed.paths).toEqual([]); + expect(parsed.rejectedPaths).toEqual([ + { device_path: devicePath, reason: "empty_file_name" }, + ]); + }); + + it("retains malformed lines including surrounding whitespace", () => { + expect( + parsePmPathOutput(" package:/data/app/pkg/base.apk \n").unparsedLines, + ).toEqual([" package:/data/app/pkg/base.apk "]); + }); }); diff --git a/src/adb/AdbDeviceOutput.ts b/src/adb/AdbDeviceOutput.ts index 44bbb53ba..3380de608 100644 --- a/src/adb/AdbDeviceOutput.ts +++ b/src/adb/AdbDeviceOutput.ts @@ -6,6 +6,11 @@ * their evidence basis. */ +import { + adbHostFileName, + type AdbHostFileNameReason, +} from "./AdbHostFileName.js"; + /** One device exactly as `adb devices -l` reports it. */ export interface AdbDeviceEntry { readonly serial: string; @@ -205,11 +210,6 @@ export interface AdbPackagePathEntry { readonly role: "base" | "split" | "unknown"; } -const fileNameOf = (devicePath: string): string => { - const parts = devicePath.split(/\//u); - return parts[parts.length - 1] ?? devicePath; -}; - /** * Parse `pm path PACKAGE` lines. The base/split role is derived from the file * name (`base.apk`, `split_*.apk`), which is the only signal the output @@ -219,20 +219,35 @@ export const parsePmPathOutput = ( stdout: string, ): { paths: readonly AdbPackagePathEntry[]; + rejectedPaths: readonly { + device_path: string; + reason: AdbHostFileNameReason; + }[]; unparsedLines: readonly string[]; } => { const paths: AdbPackagePathEntry[] = []; + const rejectedPaths: { + device_path: string; + reason: AdbHostFileNameReason; + }[] = []; const unparsedLines: string[] = []; for (const rawLine of stdout.split(/\r?\n/u)) { - const line = rawLine.trim(); - if (line === "") continue; - const match = /^package:(\S+)$/u.exec(line); + if (rawLine === "") continue; + const match = /^package:(.*)$/u.exec(rawLine); if (match === null) { - unparsedLines.push(line); + unparsedLines.push(rawLine); continue; } const devicePath = match[1]!; - const fileName = fileNameOf(devicePath); + const validated = adbHostFileName(devicePath); + if (!validated.ok) { + rejectedPaths.push({ + device_path: devicePath, + reason: validated.reason, + }); + continue; + } + const fileName = validated.fileName; paths.push({ device_path: devicePath, file_name: fileName, @@ -244,5 +259,5 @@ export const parsePmPathOutput = ( : "unknown", }); } - return { paths, unparsedLines }; + return { paths, rejectedPaths, unparsedLines }; }; diff --git a/src/adb/AdbHostFileName.test.ts b/src/adb/AdbHostFileName.test.ts index 74a87bf4b..973fcafbb 100644 --- a/src/adb/AdbHostFileName.test.ts +++ b/src/adb/AdbHostFileName.test.ts @@ -14,28 +14,50 @@ describe("adbHostFileName", () => { ], ["/data/local/tmp/résumé.xml", "résumé.xml"], ["/data/local/tmp/a-b_c.d", "a-b_c.d"], - ["/data/local/tmp/", "tmp"], - ])("accepts the portable device path %s", (devicePath, fileName) => { + ["/data/local/tmp/a\\b", "a\\b"], + ["/data/local/tmp/report:2024", "report:2024"], + ["/data/local/tmp/a*b.txt", "a*b.txt"], + ["/data/local/tmp/re\u007fport", "re\u007fport"], + ])("accepts the POSIX device path %s", (devicePath, fileName) => { expect(adbHostFileName(devicePath)).toEqual({ ok: true, fileName }); }); it.each([ ["/", "empty_file_name"], + ["data/local/tmp/file", "not_absolute_path"], + ["/data/local/tmp/", "empty_file_name"], ["/data/local/tmp/..", "dot_segment"], ["/data/local/tmp/.", "dot_segment"], - ["/data/local/tmp/a\\b", "path_separator"], - ["/data/local/tmp/re\u007fport", "control_character"], - ["/data/local/tmp/ta\tb", "control_character"], + ["/data\0/local/tmp/file", "unsupported_character"], + ["/data/local/tmp/a\0b", "unsupported_character"], + ] as const)("refuses %s as %s", (devicePath, reason) => { + expect(adbHostFileName(devicePath)).toEqual({ ok: false, reason }); + }); + + it.each([ + ["/data/local/tmp/a\\b", "unsupported_character"], + ["/data/local/tmp/ta\tb", "unsupported_character"], ["/data/local/tmp/CON", "reserved_name"], ["/data/local/tmp/nul.txt", "reserved_name"], ["/data/local/tmp/com1", "reserved_name"], ["/data/local/tmp/LPT9", "reserved_name"], - ["/data/local/tmp/report:2024", "alternate_data_stream"], + ["/data/local/tmp/COM¹.txt", "reserved_name"], + ["/data/local/tmp/lpt³", "reserved_name"], + ["/data/local/tmp/report:2024", "unsupported_character"], ["/data/local/tmp/report.", "trailing_dot_or_space"], ["/data/local/tmp/report ", "trailing_dot_or_space"], ["/data/local/tmp/a*b.txt", "unsupported_character"], ["/data/local/tmp/a?b.txt", "unsupported_character"], - ] as const)("refuses %s as %s", (devicePath, reason) => { - expect(adbHostFileName(devicePath)).toEqual({ ok: false, reason }); + ] as const)("rejects %s on Windows as %s", (devicePath, reason) => { + expect(adbHostFileName(devicePath, "win32")).toEqual({ ok: false, reason }); + }); + + it("does not impose non-Windows control restrictions on Windows filenames", () => { + expect(adbHostFileName("/data/local/tmp/name\u007f.txt", "win32").ok).toBe( + true, + ); + expect(adbHostFileName("/data/local/tmp/name\u0085.txt", "win32").ok).toBe( + true, + ); }); }); diff --git a/src/adb/AdbHostFileName.ts b/src/adb/AdbHostFileName.ts index ea4d526f2..4ffd7cec1 100644 --- a/src/adb/AdbHostFileName.ts +++ b/src/adb/AdbHostFileName.ts @@ -1,61 +1,53 @@ -import { posix } from "node:path"; - /** Why a device-reported path cannot be used as a host file name. */ export type AdbHostFileNameReason = | "empty_file_name" + | "not_absolute_path" | "dot_segment" - | "path_separator" - | "control_character" | "unsupported_character" | "reserved_name" - | "trailing_dot_or_space" - | "alternate_data_stream"; + | "trailing_dot_or_space"; -/** A portable host file name, or the exact reason the device name was refused. */ export type AdbHostFileNameResult = | { readonly ok: true; readonly fileName: string } | { readonly ok: false; readonly reason: AdbHostFileNameReason }; -/** - * Win32 resolves these basenames to a device regardless of case and of any - * extension (`NUL`, `nul.txt`), so a portable host file name must avoid them on - * every host. `(?:\.|$)` keeps ordinary names such as `console.txt` admissible. - */ const WINDOWS_RESERVED_BASENAME = - /^(?:con|prn|aux|nul|com[1-9]|lpt[1-9])(?:\.|$)/iu; - -/** Characters that are not portable in a host file name on Windows or POSIX. */ -const UNSUPPORTED_CHARACTER = /[*?"<>|]/u; - -/** Control characters (`Cc`), including C0 controls and DEL. */ -const CONTROL_CHARACTER = /\p{Cc}/u; + /^(?:con|prn|aux|nul|com[1-9¹²³]|lpt[1-9¹²³])(?:\.|$)/iu; /** - * Derive a portable host file name from a device-reported path. - * - * The device path is a POSIX path on the device filesystem, so its last - * `/`-separated segment is taken with POSIX semantics rather than the host - * `basename` (which would vary between POSIX and Windows hosts). A name is - * accepted only when every host can store it and address it back unambiguously: - * no separator, control, or reserved-device characters, no trailing dot or - * space, no `:` alternate-data-stream marker, and no `.`/`..` segment. + * Derive the final component of a device POSIX path without normalizing the + * source. A trailing slash has no filename and must not silently become the + * preceding component as `basename` would make it. */ -export const adbHostFileName = (devicePath: string): AdbHostFileNameResult => { - const fileName = posix.basename(devicePath); - if (fileName === "") return { ok: false, reason: "empty_file_name" }; +export const adbHostFileName = ( + devicePath: string, + platform: NodeJS.Platform = process.platform, +): AdbHostFileNameResult => { + if (devicePath === "" || devicePath.endsWith("/")) + return { ok: false, reason: "empty_file_name" }; + if (devicePath.includes("\0")) + return { ok: false, reason: "unsupported_character" }; + if (!devicePath.startsWith("/")) + return { ok: false, reason: "not_absolute_path" }; + const fileName = devicePath.slice(devicePath.lastIndexOf("/") + 1); if (fileName === "." || fileName === "..") return { ok: false, reason: "dot_segment" }; - if (fileName.includes("\\") || fileName.includes("/")) - return { ok: false, reason: "path_separator" }; - if (CONTROL_CHARACTER.test(fileName)) - return { ok: false, reason: "control_character" }; - if (fileName.includes(":")) - return { ok: false, reason: "alternate_data_stream" }; - if (UNSUPPORTED_CHARACTER.test(fileName)) - return { ok: false, reason: "unsupported_character" }; - if (WINDOWS_RESERVED_BASENAME.test(fileName)) - return { ok: false, reason: "reserved_name" }; - if (/[. ]$/u.test(fileName)) - return { ok: false, reason: "trailing_dot_or_space" }; + if (platform === "win32") { + if (/[\\:*?"<>|]/u.test(fileName) || hasWindowsControl(fileName)) + return { ok: false, reason: "unsupported_character" }; + if (WINDOWS_RESERVED_BASENAME.test(fileName)) + return { ok: false, reason: "reserved_name" }; + if (/[. ]$/u.test(fileName)) + return { ok: false, reason: "trailing_dot_or_space" }; + } + return { ok: true, fileName }; }; + +const hasWindowsControl = (fileName: string): boolean => { + for (const character of fileName) { + const code = character.codePointAt(0)!; + if (code > 0 && code < 0x20) return true; + } + return false; +}; diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index 1ac912b27..9ea0f026b 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { chmod, + lstat, mkdir, mkdtemp, readFile, @@ -52,6 +53,28 @@ const refusedBy = async ( return projectAnalysisError(result.error); }; +const namesAliasOnVolume = async ( + parent: string, + firstName: string, + secondName: string, +): Promise => { + const probe = join(parent, "destination-name-probe"); + await mkdir(probe); + try { + await mkdir(join(probe, firstName)); + try { + await mkdir(join(probe, secondName)); + return false; + } catch (cause) { + if (cause instanceof Error && "code" in cause && cause.code === "EEXIST") + return true; + throw cause; + } + } finally { + await rm(probe, { recursive: true, force: true }); + } +}; + it("inspects the adb client without contacting the server", async () => { const { binary } = await writeAdbStub( () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; fi`, @@ -283,6 +306,82 @@ it("refuses to reuse an existing package output directory", async () => { expect(result.error.message).toContain("already exists"); }); +it.each([ + ["an empty filename", "package:/data/app/x/"], + [ + "duplicate local filenames", + "package:/data/app/x/base.apk\npackage:/data/app/y/base.apk", + ], +])("refuses %s before creating output or pulling", async (_, pmOutput) => { + const { binary, root } = await writeAdbStub( + (root) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "-s" ] && [ "$4" = "pm" ]; then printf '%b' '${pmOutput.replaceAll("'", "'\\''").replaceAll("\n", "\\n")}\\n'; exit 0; fi +if [ "$1" = "-s" ] && [ "$3" = "pull" ]; then : > '${join(root, "pull-ran")}'; exit 0; fi +exit 1`, + ); + const output = join(root, "pulls"); + const result = await providerOver(binary).execute({ + operation: "pull_adb_package", + input: { + serial: "emulator-5554", + package: "com.example", + output_directory: output, + }, + }); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.capturedOutput?.stdout).toContain( + pmOutput.split("\n")[0], + ); + await expect(readFile(join(root, "pull-ran"))).rejects.toThrow(); + await expect(lstat(output)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it.each([ + ["case-only names", "base.apk", "BASE.APK"], + ["normalization-equivalent names", "caf\u00e9.apk", "cafe\u0301.apk"], +])( + "checks %s against the actual destination filesystem before pulling", + async (_, firstName, secondName) => { + const { binary, root } = await writeAdbStub((root) => + pullStub( + `'package:/data/app/x/${firstName}' 'package:/data/app/y/${secondName}'`, + `printf 'pulled\\n' >> '${join(root, "pull.log")}' + printf 'apk' > "$5"`, + ), + ); + const aliases = await namesAliasOnVolume(root, firstName, secondName); + const result = await providerOver(binary).execute({ + operation: "pull_adb_package", + input: { + serial: "emulator-5554", + package: "com.example", + output_directory: root, + }, + }); + const pullLog = join(root, "pull.log"); + const packageDirectory = join(root, "com.example"); + if (aliases) { + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.message).toContain("unique local destinations"); + await expect(readFile(pullLog)).rejects.toThrow(); + await expect(lstat(packageDirectory)).rejects.toThrow(); + return; + } + expect(result.ok).toBe(true); + if (!result.ok) throw new Error(result.error.message); + expect( + ( + result.value.result as { artifacts: { file_name: string }[] } + ).artifacts.map((artifact) => artifact.file_name), + ).toEqual([firstName, secondName]); + expect((await readFile(pullLog, "utf8")).trim().split("\n")).toHaveLength( + 2, + ); + }, +); + it("reports an uninstalled package from empty pm path output", async () => { const { binary } = await writeAdbStub( () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi @@ -465,14 +564,14 @@ exit 1`, operation: "pull_adb_file", input: { serial: "emulator-5554", - device_path: "/data/local/tmp/notes.txt", + device_path: "/data/local/tmp/My Notes:r*sumé.txt", output_directory: output, }, }); const digest = createHash("sha256").update("file-bytes").digest("hex"); expect(result).toMatchObject({ - device_path: "/data/local/tmp/notes.txt", - local_path: join(output, "notes.txt"), + device_path: "/data/local/tmp/My Notes:r*sumé.txt", + local_path: join(output, "My Notes:r*sumé.txt"), bytes: 10, sha256: digest, }); @@ -480,10 +579,7 @@ exit 1`, it.each([ ["/data/local/tmp/..", "dot_segment"], - ["/data/local/tmp/a\\b", "path_separator"], - ["/data/local/tmp/CON", "reserved_name"], - ["/data/local/tmp/report.", "trailing_dot_or_space"], - ["/data/local/tmp/report:2024", "alternate_data_stream"], + ["/data/local/tmp/", "empty_file_name"], ] as const)( "refuses the unsafe device path %s before pulling", async (devicePath, reason) => { @@ -510,13 +606,19 @@ exit 0`, { path: ["device_path"], reason: "invalid_value", - message: expect.stringContaining(reason), + message: expect.stringContaining( + `${JSON.stringify(devicePath)} does not name`, + ), }, ], }); + expect( + (projected.details as { issues: { message: string }[] }).issues[0]! + .message, + ).toContain(reason); // No host directory and no adb invocation are effects of a refused pull. await expect(readFile(join(root, "pull-ran"))).rejects.toThrow(); - await expect(readFile(output)).rejects.toThrow(); + await expect(lstat(output)).rejects.toMatchObject({ code: "ENOENT" }); }, ); diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index 82b43b08f..eec316978 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -1,7 +1,15 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { constants as fsConstants } from "node:fs"; -import { access, lstat, mkdir, rm, stat, writeFile } from "node:fs/promises"; +import { + access, + lstat, + mkdir, + rmdir, + rm, + stat, + writeFile, +} from "node:fs/promises"; import { join, resolve } from "node:path"; import type { @@ -72,8 +80,6 @@ export const ADB_PROVIDER_IDENTITY = { version: null, } as const; -const VERSION_TIMEOUT_MS = 10_000; -const VERSION_MAX_OUTPUT_BYTES = 64 * 1024; const DEVICES_TIMEOUT_MS = 20_000; const DEVICES_MAX_OUTPUT_BYTES = 1024 * 1024; const SHELL_TIMEOUT_MS = 30_000; @@ -388,6 +394,68 @@ const removeOwnedPackageDirectory = async ( await rm(path, { recursive: true, force: true }); }; +const reservePackageDestinations = async ( + operation: string, + packageDirectory: string, + entries: readonly { readonly file_name: string }[], + capturedOutput: { + readonly stdout: string; + readonly stderr: string; + readonly truncated: boolean; + }, +): Promise => { + const reservations: { + readonly path: string; + readonly identity: PackageDirectoryIdentity; + }[] = []; + for (const entry of entries) { + const path = join(packageDirectory, entry.file_name); + try { + await mkdir(path); + const created = await lstat(path); + if (!created.isDirectory() || created.isSymbolicLink()) + throw new Error(`Could not reserve local filename ${entry.file_name}`); + reservations.push({ + path, + identity: { + dev: created.dev, + ino: created.ino, + mode: created.mode, + }, + }); + } catch (cause: unknown) { + return new AnalysisOutputError( + operation, + `Package Manager paths cannot be written to unique local destinations at ${JSON.stringify(entry.file_name)}; refusing before pulling any files`, + { cause, capturedOutput }, + ); + } + } + for (const reservation of reservations) { + try { + const current = await lstat(reservation.path); + if ( + !current.isDirectory() || + current.isSymbolicLink() || + current.dev !== reservation.identity.dev || + current.ino !== reservation.identity.ino || + current.mode !== reservation.identity.mode + ) + throw new Error( + `The local filename reservation changed before release: ${reservation.path}`, + ); + await rmdir(reservation.path); + } catch (cause: unknown) { + return new AnalysisOutputError( + operation, + `Could not release a local filename reservation before pulling: ${reservation.path}`, + { cause, capturedOutput }, + ); + } + } + return undefined; +}; + /** * ADB device inspection and acquisition over a caller-selected adb binary. * @@ -1156,6 +1224,19 @@ export class AdbProvider { ); if (!paths.ok) return paths; const parsed = parsePmPathOutput(paths.value.stdout); + const capturedOutput = { + stdout: paths.value.stdout, + stderr: paths.value.stderr, + truncated: false, + }; + if (parsed.rejectedPaths.length > 0) + return err( + new AnalysisOutputError( + operation, + `Package Manager reported paths that cannot be represented as host files: ${parsed.rejectedPaths.map((entry) => `${JSON.stringify(entry.device_path)} (${entry.reason})`).join(", ")}`, + { capturedOutput }, + ), + ); if (parsed.paths.length === 0 && parsed.unparsedLines.length === 0) return err( new AnalysisUnsupportedTargetError( @@ -1168,8 +1249,21 @@ export class AdbProvider { return err( new AnalysisProtocolError( `pm path reported an unrecognized line: ${parsed.unparsedLines[0]}`, + { capturedOutput }, ), ); + const destinationNames = new Set(); + for (const entry of parsed.paths) { + if (destinationNames.has(entry.file_name)) + return err( + new AnalysisOutputError( + operation, + `Package Manager reported multiple paths with the same local destination ${JSON.stringify(entry.file_name)}; refusing before pulling any files`, + { capturedOutput }, + ), + ); + destinationNames.add(entry.file_name); + } const directory = await prepareOutputDirectory( operation, input.output_directory, @@ -1228,6 +1322,43 @@ export class AdbProvider { ); return err(analysisErrorWithCleanupFailure(primary, cleanup, operation)); } + const failOwnedPackage = async ( + primary: AnalysisError, + ): Promise> => { + const owner: PendingPackageDirectoryCleanup = { + path: packageDirectory, + identity: packageDirectoryIdentity, + primary, + remove: () => + removeOwnedPackageDirectory( + packageDirectory, + packageDirectoryIdentity, + ), + }; + try { + await owner.remove(); + } catch (cause: unknown) { + this.#pendingPackageDirectoryCleanup.add(owner); + const cleanup = new ProviderCleanupError( + ADB_PROVIDER_IDENTITY.id, + [packageDirectory], + { reason: cause instanceof Error ? cause.message : String(cause) }, + { operation, cause }, + ); + return err( + analysisErrorWithCleanupFailure(primary, cleanup, operation), + ); + } + return err(primary); + }; + const destinationError = await reservePackageDestinations( + operation, + packageDirectory, + parsed.paths, + capturedOutput, + ); + if (destinationError !== undefined) + return failOwnedPackage(destinationError); const artifacts: { device_path: string; file_name: string; @@ -1273,17 +1404,7 @@ export class AdbProvider { }); return err(error); }; - const seenFileNames = new Set(); for (const entry of parsed.paths) { - if (seenFileNames.has(entry.file_name)) { - failures.push({ - device_path: entry.device_path, - stage: "pull", - message: `Duplicate file name ${entry.file_name} in the APK set; refusing to overwrite`, - }); - continue; - } - seenFileNames.add(entry.file_name); const localPath = join(packageDirectory, entry.file_name); const pulled = await runAdb( operation, @@ -1338,33 +1459,7 @@ export class AdbProvider { const primary = new AnalysisProtocolError( `Every APK pull failed for ${packageName} on ${serial}: ${failures[0]!.message}`, ); - const owner: PendingPackageDirectoryCleanup = { - path: packageDirectory, - identity: packageDirectoryIdentity, - primary, - remove: () => - removeOwnedPackageDirectory( - packageDirectory, - packageDirectoryIdentity, - ), - }; - try { - await owner.remove(); - } catch (cause: unknown) { - this.#pendingPackageDirectoryCleanup.add(owner); - const cleanup = new ProviderCleanupError( - ADB_PROVIDER_IDENTITY.id, - [packageDirectory], - { - reason: cause instanceof Error ? cause.message : String(cause), - }, - { operation: operation, cause }, - ); - return err( - analysisErrorWithCleanupFailure(primary, cleanup, operation), - ); - } - return err(primary); + return failOwnedPackage(primary); } return ok( createAnalysisExecution( @@ -1407,7 +1502,7 @@ export class AdbProvider { { path: ["device_path"], reason: "invalid_value", - message: `The device path does not name a portable host file (${derived.reason}); REA never renames a device-reported file.`, + message: `The device path ${JSON.stringify(input.device_path)} does not name a file that can be written on this host (${derived.reason}); REA never renames a device-reported file.`, }, ]), ); diff --git a/src/android/JadxAnalysis.ts b/src/android/JadxAnalysis.ts index 68fa3d039..d8afed20c 100644 --- a/src/android/JadxAnalysis.ts +++ b/src/android/JadxAnalysis.ts @@ -167,16 +167,26 @@ const searchClasses = async ( "JADX class pagination is incomplete", ); } while (classes.length < total); - if (new Set(classes).size !== classes.length || total !== loaded.class_count) + if (total !== loaded.class_count) throw new AnalysisOutputError( request.operation, - "JADX class inventory is duplicated or differs from the loaded class count", + "JADX class inventory count differs from the loaded class count", ); + const occurrences = new Map(); + for (const name of classes) + occurrences.set(name, (occurrences.get(name) ?? 0) + 1); + const matchingNames = classes.filter((name) => + name.includes(request.input.query), + ); + const duplicateNames = [...occurrences] + .filter(([name, count]) => count > 1 && name.includes(request.input.query)) + .map(([name, count]) => ({ name, occurrences: count })); return androidResultSchemas.search_android_classes.parse({ engine, query: request.input.query, total_classes: total, - matches: classes.filter((name) => name.includes(request.input.query)), + matches: matchingNames, + duplicate_names: duplicateNames, coverage: "complete", }); }; diff --git a/src/apktool/ApktoolDecodeOutput.test.ts b/src/apktool/ApktoolDecodeOutput.test.ts index 067f5ea04..fb258b221 100644 --- a/src/apktool/ApktoolDecodeOutput.test.ts +++ b/src/apktool/ApktoolDecodeOutput.test.ts @@ -26,7 +26,6 @@ const REAL_YML = [ " ids:", " - 1", " tag: null", - "version: 2.7.0-dirty", "versionInfo:", " versionCode: '7'", " versionName: 1.2.3", @@ -44,7 +43,11 @@ describe("parseApktoolYml", () => { }); it("accepts quoted version values some builds emit", () => { - expect(parseApktoolYml("versionName: '2.0.0'\nversionCode: 45\n")).toEqual({ + expect( + parseApktoolYml( + '!!brut.androlib.meta.MetaInfo\nversionInfo:\n versionName: "2.0.0"\n versionCode: 45\n', + ), + ).toEqual({ versionName: "2.0.0", versionCode: "45", minSdkVersion: null, @@ -52,6 +55,49 @@ describe("parseApktoolYml", () => { }); }); + it("reads only the producer's nested fields and preserves scalar spelling", () => { + expect( + parseApktoolYml( + [ + "!!brut.androlib.meta.MetaInfo", + "versionName: decoy", + "versionInfo:", + " versionName: 01.2.3", + " versionCode: 045", + " custom:", + " versionName: unrelated", + "sdkInfo:", + " minSdkVersion: 24", + " targetSdkVersion: '35'", + "", + ].join("\n"), + ), + ).toEqual({ + versionName: "01.2.3", + versionCode: "045", + minSdkVersion: "24", + targetSdkVersion: "35", + }); + }); + + it("distinguishes YAML nulls from quoted null strings in flow mappings", () => { + expect( + parseApktoolYml( + [ + "!!brut.androlib.meta.MetaInfo", + 'versionInfo: { versionName: "null", versionCode: 0045, nested: { versionName: decoy } }', + "sdkInfo: { minSdkVersion: null, targetSdkVersion: ~ }", + "", + ].join("\n"), + ), + ).toEqual({ + versionName: "null", + versionCode: "0045", + minSdkVersion: null, + targetSdkVersion: null, + }); + }); + it("reports null facts for unrelated YAML", () => { expect(parseApktoolYml("version: 2.7.0\n")).toEqual({ versionName: null, @@ -66,7 +112,15 @@ describe("parseManifestPackage", () => { it("reads the package attribute from a decoded manifest", () => { const manifest = ''; - expect(parseManifestPackage(manifest)).toBe("com.rea.apktool.probe"); + expect(parseManifestPackage(`${manifest}`)).toBe( + "com.rea.apktool.probe", + ); + }); + + it("reads package with reordered or single-quoted XML attributes", () => { + expect( + parseManifestPackage(""), + ).toBe("org.rea.app"); }); it("reports null when no package attribute exists", () => { @@ -111,4 +165,78 @@ describe("parseDecodedStrings", () => { "a & b <c>", ); }); + + it("keeps empty, self-closing, and reordered string attributes", () => { + expect( + parseDecodedStrings( + '', + ), + ).toEqual({ + entries: [ + { name: "empty", value: "" }, + { name: "paired", value: "" }, + ], + unparsedCount: 0, + }); + }); + + it("ignores comments without mistaking them for nested markup", () => { + expect( + parseDecodedStrings( + 'beforeafter', + ), + ).toEqual({ + entries: [{ name: "real", value: "beforeafter" }], + unparsedCount: 0, + }); + }); + + it("does not include a trailing document comment in the last string", () => { + expect( + parseDecodedStrings( + 'one', + ), + ).toEqual({ + entries: [{ name: "a", value: "one" }], + unparsedCount: 0, + }); + }); + + it("preserves source whitespace and entity spelling", () => { + expect( + parseDecodedStrings( + ' a & b ', + ).entries, + ).toEqual([{ name: "raw", value: " a & b " }]); + }); + + it("keeps source slices aligned across CRLF and CR line endings", () => { + for (const newline of ["\r\n", "\r"]) { + expect( + parseDecodedStrings( + `${newline} x & y${newline}`, + ), + ).toEqual({ + entries: [{ name: "raw", value: "x & y" }], + unparsedCount: 0, + }); + } + }); + + it("counts a string with no name as unsupported instead of emitting an empty name", () => { + expect( + parseDecodedStrings("value"), + ).toEqual({ entries: [], unparsedCount: 1 }); + }); + + it("reports malformed XML and unsupported string markup as partial", () => { + expect( + parseDecodedStrings("no"), + ).toEqual({ entries: [], unparsedCount: 1 }); + expect( + parseDecodedStrings( + 'Hello world', + ), + ).toEqual({ entries: [], unparsedCount: 1 }); + }); }); diff --git a/src/apktool/ApktoolDecodeOutput.ts b/src/apktool/ApktoolDecodeOutput.ts index d00fb7b0e..05bb54acb 100644 --- a/src/apktool/ApktoolDecodeOutput.ts +++ b/src/apktool/ApktoolDecodeOutput.ts @@ -1,11 +1,17 @@ -/** - * Parsers for the decoded workspace `apktool d --no-src` produces. - * - * Apktool's apktool.yml is a small YAML document; the facts REA projects - * are flat enough that line-oriented extraction stays honest without a - * YAML dependency, and unrecognized content is ignored rather than guessed. - */ +import { + DOMParser, + type Document, + type Element, + type Node, +} from "@xmldom/xmldom"; +import { + isMap, + isScalar, + parseDocument, + type Document as YamlDocument, +} from "yaml"; +/** Project only the Apktool metadata fields REA publishes. */ export interface ApktoolMetadata { readonly versionName: string | null; readonly versionCode: string | null; @@ -13,43 +19,48 @@ export interface ApktoolMetadata { readonly targetSdkVersion: string | null; } -/** - * Project version and SDK facts from apktool.yml. Values keep apktool's - * own spelling (version codes can print as `45` or `45.1`). - */ -export const parseApktoolYml = (yml: string): ApktoolMetadata => { - const metadata: { - versionName: string | null; - versionCode: string | null; - minSdkVersion: string | null; - targetSdkVersion: string | null; - } = { - versionName: null, - versionCode: null, - minSdkVersion: null, - targetSdkVersion: null, - }; - for (const line of yml.split(/\r?\n/u)) { - const versionName = /^\s*versionName:\s*(?:'([^']*)'|(.*))$/u.exec(line); - if (versionName !== null && metadata.versionName === null) - metadata.versionName = (versionName[1] ?? versionName[2] ?? "").trim(); - const versionCode = /^\s*versionCode:\s*(?:'([^']*)'|(.*))$/u.exec(line); - if (versionCode !== null && metadata.versionCode === null) - metadata.versionCode = (versionCode[1] ?? versionCode[2] ?? "").trim(); - const minSdk = /minSdkVersion:\s*'?([0-9]+)'?/u.exec(line); - if (minSdk !== null && metadata.minSdkVersion === null) - metadata.minSdkVersion = minSdk[1]!; - const targetSdk = /targetSdkVersion:\s*'?([0-9]+)'?/u.exec(line); - if (targetSdk !== null && metadata.targetSdkVersion === null) - metadata.targetSdkVersion = targetSdk[1]!; - } - return metadata; +const metaInfoTag = { + tag: "tag:yaml.org,2002:brut.androlib.meta.MetaInfo", + collection: "map" as const, + resolve: (map: unknown) => map, }; -/** Read the package attribute from decoded AndroidManifest.xml text. */ +/** + * Read values from Apktool's tagged metadata document. YAML's core schema + * distinguishes null from quoted strings; numeric values use their source + * range so version codes retain the spelling Apktool emitted. + */ +export const parseApktoolYml = (yml: string): ApktoolMetadata => { + const document = parseDocument(yml, { + schema: "core", + customTags: [metaInfoTag], + strict: true, + uniqueKeys: true, + }); + if (document.errors.length > 0) + throw new TypeError( + `apktool.yml is not valid Apktool metadata: ${document.errors[0]?.message ?? "YAML parse error"}`, + ); + if (!isMap(document.contents)) + throw new TypeError("apktool.yml root must be an Apktool metadata mapping"); + return { + versionName: yamlScalar(document, ["versionInfo", "versionName"], yml), + versionCode: yamlScalar(document, ["versionInfo", "versionCode"], yml), + minSdkVersion: yamlScalar(document, ["sdkInfo", "minSdkVersion"], yml), + targetSdkVersion: yamlScalar( + document, + ["sdkInfo", "targetSdkVersion"], + yml, + ), + }; +}; + +/** Read the package attribute from the decoded Android manifest root. */ export const parseManifestPackage = (manifest: string): string | null => { - const match = /]*\spackage="([^"]+)"/u.exec(manifest); - return match === null ? null : match[1]!; + const document = parseXml(manifest, "AndroidManifest.xml"); + const root = document.documentElement; + if (root === null || root.nodeName !== "manifest") return null; + return root.getAttribute("package") || null; }; export interface DecodedStringEntry { @@ -58,10 +69,10 @@ export interface DecodedStringEntry { } /** - * Project `value` entries from one decoded - * strings.xml. Values are preserved exactly as apktool printed them — - * escaped entities stay escaped — and entries containing nested markup - * are counted as unparsed rather than flattened. + * Project direct `` children from decoded resources XML. DOM parsing + * identifies elements and attributes; text values are sliced from the input + * at parser-reported locations so entity spelling and whitespace are kept. + * Markup-bearing string values are omitted and counted as partial evidence. */ export const parseDecodedStrings = ( stringsXml: string, @@ -69,20 +80,107 @@ export const parseDecodedStrings = ( entries: readonly DecodedStringEntry[]; unparsedCount: number; } => { + let document: Document; + try { + document = parseXml(stringsXml, "strings.xml"); + } catch { + return { entries: [], unparsedCount: 1 }; + } + const root = document.documentElement; + if (root === null || root.nodeName !== "resources") + return { entries: [], unparsedCount: 1 }; + const entries: DecodedStringEntry[] = []; let unparsedCount = 0; - const pattern = /]*>([\s\S]*?)<\/string>/gu; - for ( - let match = pattern.exec(stringsXml); - match !== null; - match = pattern.exec(stringsXml) - ) { - const value = match[2]!; - if (value.includes("<")) { + const offsets = sourceLineOffsets(stringsXml); + for (const child of childNodes(root)) { + if (child.nodeType !== 1) continue; + const element = child as Element; + if (element.nodeName !== "string") continue; + const name = element.getAttribute("name"); + if (name === null || name.length === 0) { unparsedCount += 1; continue; } - entries.push({ name: match[1]!, value: value.trim() }); + const content = childNodes(element); + if (content.some((node) => node.nodeType !== 3 && node.nodeType !== 8)) { + unparsedCount += 1; + continue; + } + if (content.every((node) => node.nodeType === 8)) { + entries.push({ name, value: "" }); + continue; + } + const value = content + .filter((node) => node.nodeType === 3) + .map((node) => { + const start = sourceOffset(node, offsets); + if (start === undefined) return null; + // A literal '<' cannot appear in well-formed XML text. Ending here + // also excludes comments after the document element from this value. + const end = stringsXml.indexOf("<", start); + if (end < start) return null; + return stringsXml.slice(start, end); + }); + if (value.some((part) => part === null)) { + unparsedCount += 1; + continue; + } + entries.push({ name, value: value.join("") }); } return { entries, unparsedCount }; }; + +const yamlScalar = ( + document: YamlDocument, + path: readonly string[], + source: string, +): string | null => { + const node = document.getIn(path, true); + if (!isScalar(node) || node.value === null) return null; + if (typeof node.value === "string") return node.value; + const range = node.range; + return range === null || range === undefined + ? String(node.value) + : source.slice(range[0], range[1]); +}; + +const parseXml = (xml: string, label: string) => { + const errors: string[] = []; + const document = new DOMParser({ + normalizeLineEndings: (source) => source, + onError: (_level, message) => errors.push(message), + }).parseFromString(xml, "application/xml"); + if (errors.length > 0 || document.documentElement === null) + throw new TypeError( + `${label} is not well-formed XML${errors[0] === undefined ? "" : `: ${errors[0]}`}`, + ); + return document; +}; + +const childNodes = (node: Node): Node[] => { + const children: Node[] = []; + for (let child = node.firstChild; child !== null; child = child.nextSibling) + children.push(child); + return children; +}; + +const sourceLineOffsets = (source: string): readonly number[] => { + const offsets = [0]; + for (const match of source.matchAll(/\r\n?|\n/gu)) { + const index = match.index; + offsets.push(index + match[0].length); + } + return offsets; +}; + +const sourceOffset = ( + node: Node, + lineOffsets: readonly number[], +): number | undefined => { + const line = node.lineNumber; + const column = node.columnNumber; + if (line === undefined || column === undefined) return undefined; + const lineStart = lineOffsets[line - 1]; + return lineStart === undefined ? undefined : lineStart + column - 1; +}; diff --git a/src/apktool/ApktoolProvider.test.ts b/src/apktool/ApktoolProvider.test.ts index 23e2bd8b4..3cd86b72a 100644 --- a/src/apktool/ApktoolProvider.test.ts +++ b/src/apktool/ApktoolProvider.test.ts @@ -89,10 +89,12 @@ const decodeStub = ( readonly withStrings?: boolean; readonly withGerman?: boolean; readonly failDecode?: boolean; + readonly stringsXml?: string; } = {}, ): ((root: string) => string) => { const withStrings = options.withStrings ?? true; const withGerman = options.withGerman ?? true; + const stringsXml = options.stringsXml ?? REAL_STRINGS; return () => `if [ "$1" = "--version" ]; then ${VERSION_BLOCK}; exit 0; fi if [ "$1" = "d" ]; then ${ @@ -101,7 +103,7 @@ if [ "$1" = "d" ]; then : `printf '%s\\n' '${REAL_YML.replace(/'/gu, "'\\''")}' > "$5/apktool.yml" printf '%s\\n' '${REAL_MANIFEST.replace(/'/gu, "'\\''")}' > "$5/AndroidManifest.xml" mkdir -p "$5/res/values"${withGerman ? ' "$5/res/values-de"' : ""} - ${withStrings ? `printf '%s\\n' '${REAL_STRINGS.replace(/'/gu, "'\\''")}' > "$5/res/values/strings.xml"` : ""} + ${withStrings ? `printf '%s\\n' '${stringsXml.replace(/'/gu, "'\\''")}' > "$5/res/values/strings.xml"` : ""} ${withGerman ? `printf '%s\\n' '${DE_STRINGS.replace(/'/gu, "'\\''")}' > "$5/res/values-de/strings.xml"` : ""} printf 'I: Using Apktool on stub\\n' exit 0` @@ -173,6 +175,22 @@ it("decodes resources with target identity, metadata, and strings", async () => expect(result.coverage).toBe("complete"); }); +it("preserves empty resource strings and reports markup omissions as partial", async () => { + const stringsXml = + ' A & B Hello world'; + const { command, root } = await writeApktoolStub(decodeStub({ stringsXml })); + const apk = await apkFixture(root); + const result = await evidenceResult(providerOver(command), { + operation: "decode_android_resources", + input: { path: apk.path, include_strings: true }, + }); + expect(result.strings).toEqual([ + { name: "empty", value: "" }, + { name: "raw", value: " A & B " }, + ]); + expect(result.coverage).toBe("partial"); +}); + it("projects one locale's strings on request", async () => { const { command, root } = await writeApktoolStub(decodeStub()); const apk = await apkFixture(root); diff --git a/src/apktool/ApktoolProvider.ts b/src/apktool/ApktoolProvider.ts index 8efb17554..eaf21c3e6 100644 --- a/src/apktool/ApktoolProvider.ts +++ b/src/apktool/ApktoolProvider.ts @@ -1,5 +1,5 @@ import { createHash, randomUUID } from "node:crypto"; -import { createWriteStream, constants as fsConstants } from "node:fs"; +import { createWriteStream } from "node:fs"; import { access, chmod, @@ -717,7 +717,7 @@ export class ApktoolProvider { : []), ...(parsedStrings.unparsedCount > 0 ? [ - `${String(parsedStrings.unparsedCount)} string entries with nested markup were not projected`, + `String entries omitted due to malformed XML or unsupported markup: ${String(parsedStrings.unparsedCount)}`, ] : []), ...(parsedStrings.entries.length > MAX_STRINGS diff --git a/src/application/AnalysisContextQueries.ts b/src/application/AnalysisContextQueries.ts index cdb008e1f..379ebc322 100644 --- a/src/application/AnalysisContextQueries.ts +++ b/src/application/AnalysisContextQueries.ts @@ -3,12 +3,11 @@ import type { AnalysisOperation, AnalysisOperationPort, } from "./AnalysisProvider.js"; -import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { analysisBookmarkSchema } from "../domain/hopperValues.js"; import { resolveAnalysisDocument } from "./AnalysisDocument.js"; -import { err, ok, type Result } from "../domain/result.js"; +import { ok, type Result } from "../domain/result.js"; type Facet = | { readonly state: "available"; readonly value: JsonValue } diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index d88da6df7..87a717795 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -15,6 +15,10 @@ import type { ProgressReporter } from "./ProgressReporter.js"; import type { ProviderRejectionCode } from "../contracts/providerSelection.js"; import type { ProcessLineageObservation } from "../process/ProcessOwnership.js"; import type { ProviderOperationHealth } from "../domain/providerOperationHealth.js"; +import { + relationshipCoverageSchema, + type RelationshipCoverage, +} from "../domain/relationshipCoverage.js"; export interface ExecutionOptions { readonly signal?: AbortSignal; @@ -44,6 +48,8 @@ export interface AnalysisExecution { readonly provider: ProviderIdentity; readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations: readonly string[]; + /** Provider-owned completeness assessment for caller/callee observations. */ + readonly relationshipCoverage?: RelationshipCoverage; readonly locations: readonly EvidenceLocation[]; readonly subject: EvidenceSubjectTarget | null; } @@ -62,6 +68,7 @@ export const createAnalysisExecution = ( readonly rawResult?: unknown; readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations?: readonly string[]; + readonly relationshipCoverage?: RelationshipCoverage; readonly locations?: readonly EvidenceLocation[]; readonly subject?: EvidenceSubjectTarget; } = {}, @@ -76,6 +83,13 @@ export const createAnalysisExecution = ( ? {} : { analysisProfile: structuredClone(options.analysisProfile) }), limitations: [...(options.limitations ?? [])], + ...(options.relationshipCoverage === undefined + ? {} + : { + relationshipCoverage: relationshipCoverageSchema.parse( + options.relationshipCoverage, + ), + }), locations: [...(options.locations ?? [])], subject: options.subject ?? null, }); diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index ca8a4ab73..cd424a7e0 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -17,7 +17,7 @@ import { import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { err, ok, type Result } from "../domain/result.js"; -import { readMipsElfAbiFlags } from "./MipsElfAbiFlags.js"; +import { readMipsElfMetadata } from "./MipsElfMetadata.js"; import { resolveAppBundleExecutable, type ResolvedAppBundle, @@ -26,7 +26,10 @@ import { validateDosComLength, type ExecutableFormatHint, } from "../domain/dosCom.js"; -import { admitRawBinary } from "../domain/rawBinary.js"; +import { + admitRawBinary, + rawBinarySelectionError, +} from "../domain/rawBinary.js"; import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js"; import { mzWindowsHeaderOffset } from "../domain/dosMz.js"; import { @@ -93,6 +96,7 @@ export const parseBinaryTarget = async ( new BinaryTargetError( path, "raw-binary format requires an executable target kind", + { constraint: "load_selection_rejected" }, ), ); const admitted = admitRawBinary({ @@ -101,7 +105,11 @@ export const parseBinaryTarget = async ( fileBytes: metadata.size, }); if (!admitted.ok) - return err(new BinaryTargetError(path, admitted.error)); + return err( + new BinaryTargetError(path, admitted.error.reason, { + constraint: admitted.error.constraint, + }), + ); return ok({ path, sourcePath: canonical, @@ -114,12 +122,16 @@ export const parseBinaryTarget = async ( entryPointSource: admitted.value.entryPointSource, }); } - if (architecture !== undefined || entryPoint !== undefined) + const pairing = rawBinarySelectionError({ + ...(formatHint === undefined ? {} : { format: formatHint }), + architecture, + entryPoint, + }); + if (pairing !== undefined) return err( - new BinaryTargetError( - path, - "architecture and entry_point apply only with format=raw-binary", - ), + new BinaryTargetError(path, pairing, { + constraint: "load_selection_rejected", + }), ); if (formatHint === "dos-com") { if (targetKind !== undefined && targetKind !== "executable") @@ -127,10 +139,16 @@ export const parseBinaryTarget = async ( new BinaryTargetError( path, "DOS COM format requires an executable target kind", + { constraint: "load_selection_rejected" }, ), ); const length = validateDosComLength(metadata.size); - if (!length.ok) return err(new BinaryTargetError(path, length.error)); + if (!length.ok) + return err( + new BinaryTargetError(path, length.error, { + constraint: "file_format_rejected", + }), + ); return ok({ path, sourcePath: canonical, @@ -168,7 +186,12 @@ export const parseBinaryTarget = async ( hostArchitecture, signal, ); - if (!detected.ok) return err(new BinaryTargetError(path, detected.error)); + if (!detected.ok) + return err( + new BinaryTargetError(path, detected.error, { + constraint: "file_format_rejected", + }), + ); return ok({ path, sourcePath: process.platform === "win32" ? candidate : canonical, @@ -234,6 +257,7 @@ const detectArtifactFormat = async ( BinaryTarget["format"], | "analysis-database" | "mach-o" + | "mach-o-universal" | "elf" | "pe" | "dos-mz" @@ -247,6 +271,15 @@ const detectArtifactFormat = async ( const magic = Buffer.alloc(8); const observed = await handle.read(magic, 0, magic.length, 0); throwIfTargetResolutionCancelled(signal); + if ( + observed.bytesRead === 8 && + magic.equals(Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])) + ) + throw new BinaryTargetError( + path, + "WebAssembly v1 artifacts are not supported by the native opening route", + { constraint: "wasm_requires_static_inspection" }, + ); if (hasZipSignature(magic.subarray(0, observed.bytesRead))) { // The HarmonyOS App Pack suffix `.app` is a root-input concept; a nested // ZIP member ending in `.app` keeps the generic zip classification. @@ -290,6 +323,7 @@ const isArchiveFormat = ( BinaryTarget["format"], | "analysis-database" | "mach-o" + | "mach-o-universal" | "elf" | "pe" | "dos-mz" @@ -421,18 +455,23 @@ const readExecutableMetadata = async ( parsed.value.mips.elfClass !== 32 ) return parsed; - const abiFlags = await readMipsElfAbiFlags(handle, parsed.value.mips, () => - throwIfTargetResolutionCancelled(signal), + const declarations = await readMipsElfMetadata( + handle, + parsed.value.mips, + () => throwIfTargetResolutionCancelled(signal), ); - if (!abiFlags.ok) return abiFlags; + if (!declarations.ok) return declarations; return ok({ ...parsed.value, mips: { ...parsed.value.mips, - abiFlags: abiFlags.value.abiFlags, - ...(abiFlags.value.limitations.length === 0 + abiFlags: declarations.value.abiFlags, + ...(declarations.value.sharedLoad === undefined ? {} - : { limitations: abiFlags.value.limitations }), + : { sharedLoad: declarations.value.sharedLoad }), + ...(declarations.value.limitations.length === 0 + ? {} + : { limitations: declarations.value.limitations }), }, }); }; diff --git a/src/application/CallPathTracing.ts b/src/application/CallPathTracing.ts index 66d1aa50c..825c69f44 100644 --- a/src/application/CallPathTracing.ts +++ b/src/application/CallPathTracing.ts @@ -10,6 +10,7 @@ import type { JsonValue } from "../domain/jsonValue.js"; import { ok, type Result } from "../domain/result.js"; import type { EnhancedResult } from "./EnhancedToolTypes.js"; import { resolveProcedureAddress } from "./ProcedureAddressResolution.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; type AnalysisCall = ( name: AnalysisOperation, @@ -51,6 +52,7 @@ interface TraceState { }>; readonly residual: Set; readonly limitations: Set; + readonly relationshipCoverage: Map; queueIndex: number; traversalPath: string[]; } @@ -117,6 +119,7 @@ const createTraceState = (input: CallPathTraceInput): TraceState => ({ failures: [], residual: new Set(), limitations: new Set(), + relationshipCoverage: new Map(), queueIndex: 0, traversalPath: input.goal === input.start ? [input.start] : [], }); @@ -147,12 +150,27 @@ const expandAddress = async ( state.residual.add(`Call relationships were unreadable for ${address}.`); return false; } - for (const limitation of result.value.limitations) { + for (const limitation of result.value.limitations) state.limitations.add(limitation); + const coverage = result.value.relationshipCoverage ?? { + status: "unknown", + reason: "Provider did not report relationship coverage.", + }; + state.relationshipCoverage.set(address, coverage); + for (const diagnostic of "diagnostics" in coverage && + Array.isArray(coverage.diagnostics) + ? coverage.diagnostics + : []) + state.limitations.add(diagnostic); + if (coverage.status === "partial") + for (const limitation of coverage.limitations) + state.residual.add( + `Call relationships for ${address} are limited: ${limitation}`, + ); + else if (coverage.status === "unknown") state.residual.add( - `Call relationships for ${address} are limited: ${limitation}`, + `Call relationship coverage is unknown for ${address}: ${coverage.reason}`, ); - } for (const relatedAddress of sortedUniqueAddresses(related.value)) { recordEdge(input, state, { address, @@ -228,6 +246,11 @@ const projectTrace = (input: CallPathTraceInput, state: TraceState) => ({ traversal: { nodes_visited: state.visited.size, }, + relationship_coverage: Object.fromEntries( + [...state.relationshipCoverage.entries()].sort(([left], [right]) => + compareAddress(left, right), + ), + ), truncated: state.residual.size > 0, residual_unknowns: [...state.residual], limitations: [ diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index a267f2b58..1b8ec6411 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it } from "vitest"; -import { buildCapabilityInventory } from "./CapabilityInventory.js"; +import { + buildCapabilityInventory, + compactToolAvailability, +} from "./CapabilityInventory.js"; +import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; +import { + expandToolAvailability, + toolAvailabilityInventory, + type ToolAvailability, +} from "../contracts/toolOutputSchemaPrimitives.js"; it("reports module syntax and native resolution availability separately", () => { const input = { open: false, capabilities: [] }; @@ -55,22 +64,25 @@ const status = ( open: options.open ?? false, ...(options.kind === undefined ? {} : { kind: options.kind }), ...(options.format === undefined ? {} : { format: options.format }), - capabilities: (options.capabilities ?? []).map((capability) => ({ - ...capability, - availability_code: capability.available - ? null - : (capability.availability_code ?? null), - effects: { - mutates_artifact: false, - launches_process: false, - may_show_ui: false, - may_access_network: false, - may_write_filesystem: false, - changes_permissions: false, - requires_root: false, - }, - limitations: [], - })), + capabilities: + options.capabilities === undefined || options.capabilities.length === 0 + ? [] + : [ + { + provider: { id: "fixture", name: "Fixture", version: "1" }, + shared_limitations: [], + groups: options.capabilities.map((capability) => ({ + operations: [capability.operation], + effects: [], + limitations: [], + available: capability.available, + reason: capability.reason, + availability_code: capability.available + ? null + : (capability.availability_code ?? null), + })), + }, + ], }); const entry = ( @@ -86,6 +98,26 @@ const entry = ( }; describe("capability failure reasons", () => { + it("admits physical universal Mach-O targets for native Mach-O operations", () => { + expect( + entry( + "inspect_native_dispatch_metadata", + status({ + open: true, + kind: "executable", + format: "mach-o-universal", + capabilities: [ + { + operation: "inspect_native_dispatch_metadata", + available: true, + reason: null, + }, + ], + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }); + it.each([ { label: "requires a target", @@ -171,6 +203,152 @@ describe("capability failure reasons", () => { }); }); +describe("compact tool availability", () => { + it("round-trips complete facts, omitted modes, and per-group client requirements", () => { + const full = buildCapabilityInventory(status(), enabledPolicy); + const alternateRequirements = { + required: ["roots"], + optional: [], + missing_required: ["roots"], + missing_optional: [], + } satisfies ToolAvailability["client_requirements"]; + const withOverride = full.map((entry, index) => + index === 1 + ? { + ...entry, + available: false as const, + reason: "client_capability_missing" as const, + remediation: "Use an MCP client with roots support.", + client_requirements: alternateRequirements, + } + : entry, + ); + const compact = compactToolAvailability(withOverride); + const expanded = expandToolAvailability(compact).toSorted((left, right) => + left.name.localeCompare(right.name), + ); + + expect(expanded).toEqual( + withOverride.toSorted((left, right) => + left.name.localeCompare(right.name), + ), + ); + expect(compact.client_requirements).toEqual( + withOverride[0]?.client_requirements, + ); + expect( + compact.groups.find((group) => group.client_requirements !== undefined) + ?.client_requirements, + ).toEqual(alternateRequirements); + expect(compact.groups.every((group) => group.names.length > 0)).toBe(true); + const names = compact.groups.flatMap((group) => group.names).sort(); + expect(names).toEqual(TOOL_CONTRACTS.map(({ name }) => name).sort()); + expect(new Set(names).size).toBe(names.length); + expect(expanded.some((entry) => !Object.hasOwn(entry, "modes"))).toBe(true); + }); + + it("preserves an empty inventory with an explicit empty client baseline", () => { + const compact = compactToolAvailability([]); + expect(compact).toEqual({ + client_requirements: { + required: [], + optional: [], + missing_required: [], + missing_optional: [], + }, + groups: [], + }); + expect(expandToolAvailability(compact)).toEqual([]); + }); + + it("rejects malformed compact facts and accepts an equivalent explicit baseline", () => { + const compact = compactToolAvailability( + buildCapabilityInventory(status(), enabledPolicy), + ); + const firstGroup = compact.groups[0]; + expect(firstGroup).toBeDefined(); + if (firstGroup === undefined) throw new Error("Expected tool groups"); + const availableIndex = compact.groups.findIndex((group) => group.available); + if (availableIndex < 0) throw new Error("Expected an available tool group"); + + const malformedInventories = [ + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 + ? { ...group, effects: [...group.effects, "unknown_effect"] } + : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 + ? { + ...group, + annotations: [...group.annotations, "unknown_hint"], + } + : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 ? { ...group, names: [] } : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 ? { ...group, names: [""] } : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === availableIndex + ? { ...group, reason: "provider_missing" } + : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === availableIndex + ? { ...group, remediation: "unexpected remediation" } + : group, + ), + }, + ]; + for (const malformed of malformedInventories) + expect(toolAvailabilityInventory.safeParse(malformed).success).toBe( + false, + ); + + const inheritedGroup = compact.groups.find( + (group) => group.client_requirements === undefined, + ); + expect(inheritedGroup).toBeDefined(); + if (inheritedGroup === undefined) + throw new Error("Expected an inherited client-requirements group"); + const explicitBaseline = { + ...compact, + groups: compact.groups.map((group) => + group === inheritedGroup + ? { ...group, client_requirements: compact.client_requirements } + : group, + ), + }; + const parsedOverride = + toolAvailabilityInventory.safeParse(explicitBaseline); + expect(parsedOverride.success).toBe(true); + if (parsedOverride.success) + expect(expandToolAvailability(parsedOverride.data)).toEqual( + expandToolAvailability(compact), + ); + }); +}); + describe("partial navigation availability", () => { it("allows explicit documents when the provider has no current-document getter", () => { const capabilities = [ @@ -258,6 +436,36 @@ describe("partial navigation availability", () => { remediation: "Navigation requires macOS.", }); }); + + it("lists shared missing operations once in combined remediation", () => { + const availability = entry( + "get_navigation_context", + status({ open: true, kind: "executable" }), + ); + + expect(availability).toMatchObject({ + reason: "provider_missing", + remediation: + "Configure a provider for one of: current_document, current_address, resolve_containing_procedure.", + modes: [ + { + name: "current_selection", + missing_operations: [ + "current_document", + "current_address", + "resolve_containing_procedure", + ], + }, + { + name: "explicit_document", + missing_operations: [ + "current_address", + "resolve_containing_procedure", + ], + }, + ], + }); + }); }); it("reports host denial before target mismatch but keeps composed workflow prerequisites", () => { diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index d11a582b2..1bf51884e 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -5,15 +5,21 @@ import type { } from "./OptionalObservationProviders.js"; import { - providerCapability, + expandProviderCapabilities, + providerCapabilitySet, + toolAvailabilityInventory, + TOOL_ANNOTATION_NAMES, + TOOL_EFFECT_NAMES, type ClientFeatureAvailability, type ProviderCapability, + type ToolAvailabilityInventory, type ToolAvailability, type ToolAvailabilityMode, type ToolUnavailabilityReason, } from "../contracts/toolOutputSchemaPrimitives.js"; import type { ToolKind } from "../contracts/toolContractTypes.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import { isMachOFormat } from "../domain/binaryTargetTypes.js"; import type { ProviderAvailability } from "./AnalysisProvider.js"; import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; import { @@ -26,7 +32,9 @@ const statusSchema = z.object({ open: z.boolean(), kind: z.enum(["executable", "database", "archive", "artifact"]).optional(), format: z.string().optional(), - capabilities: z.array(providerCapability), + capabilities: z + .array(providerCapabilitySet) + .transform(expandProviderCapabilities), }); type ToolAvailabilityReason = "available" | ToolUnavailabilityReason; @@ -214,6 +222,60 @@ export const buildCapabilityInventory = ( }).sort((left, right) => compareUnicodeCodePoints(left.name, right.name)); }; +type ClientRequirements = ToolAvailability["client_requirements"]; +type MutableAvailabilityGroup = Omit< + ToolAvailabilityInventory["groups"][number], + "names" +> & { names: string[] }; + +const EMPTY_CLIENT_REQUIREMENTS: ClientRequirements = { + required: [], + optional: [], + missing_required: [], + missing_optional: [], +}; + +/** Compact repeated metadata while retaining every operation's current availability inline. */ +export const compactToolAvailability = ( + availability: readonly ToolAvailability[], +): ToolAvailabilityInventory => { + const clientRequirements = + availability[0]?.client_requirements ?? EMPTY_CLIENT_REQUIREMENTS; + const groups = new Map(); + + for (const entry of availability) { + const effects = TOOL_EFFECT_NAMES.filter((name) => entry.effects[name]); + const annotations = TOOL_ANNOTATION_NAMES.filter( + (name) => entry.annotations[name], + ); + const groupFacts = { + surface: entry.surface, + effects, + annotations, + available: entry.available, + reason: entry.reason, + remediation: entry.remediation, + ...(entry.default_mode_available === undefined + ? {} + : { default_mode_available: entry.default_mode_available }), + ...(entry.modes === undefined ? {} : { modes: [...entry.modes] }), + ...(JSON.stringify(entry.client_requirements) === + JSON.stringify(clientRequirements) + ? {} + : { client_requirements: entry.client_requirements }), + }; + const key = JSON.stringify(groupFacts); + const existing = groups.get(key); + if (existing) existing.names.push(entry.name); + else groups.set(key, { ...groupFacts, names: [entry.name] }); + } + + return toolAvailabilityInventory.parse({ + client_requirements: clientRequirements, + groups: [...groups.values()], + }); +}; + const availabilityFor = (context: AvailabilityContext): Availability => { // These built-in workflows accept a caller-selected endpoint; they do not load the optional passive provider. if ( @@ -488,7 +550,7 @@ const targetAvailability = ({ if ( targetOpen && MACHO_TARGET_OPERATIONS.has(name) && - (targetKind !== "executable" || targetFormat !== "mach-o") + (targetKind !== "executable" || !isMachOFormat(targetFormat)) ) return { reason: "target_unsupported", @@ -626,14 +688,15 @@ const navigationContextAvailability = ( }; if (availableMode !== undefined) return { ...facts, reason: "available", remediation: null }; + const missingOperations = [ + ...new Set(modes.flatMap((mode) => mode.missing_operations)), + ]; return { ...facts, reason: failure?.reason ?? "provider_missing", remediation: specificFailureRemediation ?? - `Configure a provider for one of: ${modes - .flatMap((mode) => mode.missing_operations) - .join(", ")}.`, + `Configure a provider for one of: ${missingOperations.join(", ")}.`, }; }; diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index bad6ee52f..65b4ba8de 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -5,6 +5,13 @@ import { stringify as stringifyYaml, } from "yaml"; import { isDeepStrictEqual } from "node:util"; + +import { + parseDshCordisDocument, + serializeDshDocument, + editDshDocument, + dshClientRegistrationEntry, +} from "./DshCordisPatch.js"; import { applyEdits, findNodeAtLocation, @@ -104,11 +111,11 @@ const parseDocument = ( // comments, trailing commas, a BOM, or an empty existing file must be // rejected instead of registering into a document Pi cannot load. if (format === "pi") return objectSchema.parse(JSON.parse(text)); - // Hermes stores MCP servers in `mcp_servers` inside a YAML document. - if (format === "hermes") return objectSchema.parse(parseYaml(text)); // An empty file, which some clients create before any server is added, // holds no settings to preserve. if (isEmptyClientConfigurationText(text)) return {}; + // Hermes stores MCP servers in `mcp_servers` inside a YAML document. + if (format === "hermes") return objectSchema.parse(parseYaml(text)); const errors: ParseError[] = []; // Accept a UTF-8 BOM without shifting diagnostics or editing the original text. const jsonText = text.startsWith("\uFEFF") ? ` ${text.slice(1)}` : text; @@ -131,6 +138,8 @@ export const parseClientConfiguration = ( ): ClientConfigurationDocument => { if (format === undefined || format === "unsupported") throw new TypeError("client does not have a supported MCP config format"); + // DeepSeek Harness stores a Cordis patch list, not a server table. + if (format === "dsh") return parseDshCordisDocument(text); const document = parseDocument(text, format); const serversKey = clientConfigurationServersKey(format); const value = document[serversKey]; @@ -184,6 +193,14 @@ export const withClientServers = ( servers: Record, removeLegacy?: string, ): Record => { + // The Harness patch list upserts or removes by entry id, not by table. + if (parsed.dialect === "dsh") + return Object.hasOwn(servers, PRODUCT_IDENTITY.mcpServerKey) + ? editDshDocument( + parsed, + objectSchema.parse(servers[PRODUCT_IDENTITY.mcpServerKey]), + ) + : editDshDocument(parsed); const [key = "", nested] = parsed.serversPath; if (nested === undefined) return { ...parsed.document, [key]: servers }; const outer = { ...objectSchema.parse(parsed.document[key]) }; @@ -1280,6 +1297,7 @@ export const serializeClientConfiguration = ( return serializeGrokConfiguration(document, originalText); if (format === "hermes") return serializeHermesConfiguration(document, originalText); + if (format === "dsh") return serializeDshDocument(document); // An empty original holds no comments or settings to preserve. Writing a // fresh document also keeps a lone BOM from shifting past the closing brace. if ( @@ -1398,6 +1416,8 @@ export const clientRegistrationEntry = ( args, ...(Object.keys(environment).length === 0 ? {} : { env: environment }), }; + case "dsh": + return dshClientRegistrationEntry(command, environment); case "hermes": // Hermes reads stdio servers from `mcp_servers.` and connects them // unless `enabled` is false, so the entry states it explicitly. diff --git a/src/application/ClientConfigurationFiles.ts b/src/application/ClientConfigurationFiles.ts index 5a665b770..fc3c2d4ea 100644 --- a/src/application/ClientConfigurationFiles.ts +++ b/src/application/ClientConfigurationFiles.ts @@ -10,6 +10,7 @@ import { } from "./ClientConfigurationDocument.js"; import type { SetupClient } from "./SupportedClients.js"; import { clientRegistrationEnvironment } from "./ClientRegistrationEnvironment.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; /** A named source failure that setup can explain without losing the bad path. */ export interface ClientConfigurationFileError { @@ -22,9 +23,15 @@ export interface ClientConfigurationFileError { /** Preserve named read failures and distinguish a missing source from an empty file. */ export const readConfigurationText = async ( path: string, + resources?: ArtifactResourceScope, ): Promise> => { try { - return ok(await readRegularFileText(path)); + return ok( + await readRegularFileText( + path, + resources === undefined ? {} : { resources }, + ), + ); } catch (cause: unknown) { if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return ok(undefined); @@ -43,10 +50,11 @@ export const readConfigurationText = async ( export const readConfigurationFile = async ( path: string, format: SetupClient["format"], + resources?: ArtifactResourceScope, ): Promise< Result > => { - const source = await readConfigurationText(path); + const source = await readConfigurationText(path, resources); if (!source.ok) return source; const content = source.value; if (content === undefined) return ok(undefined); @@ -64,12 +72,17 @@ export const readConfigurationFile = async ( /** Read every applicable user file; malformed lower-priority files also stop setup. */ export const readClientConfigurationFiles = async ( client: SetupClient, + resources?: ArtifactResourceScope, ): Promise< Result > => { const documents: ClientConfigurationDocument[] = []; for (const path of client.configPaths ?? [client.configPath]) { - const document = await readConfigurationFile(path, client.format); + const document = await readConfigurationFile( + path, + client.format, + resources, + ); if (!document.ok) return document; if (document.value !== undefined) { const environment = clientRegistrationEnvironment(document.value); diff --git a/src/application/ClientRegistrationEnvironment.ts b/src/application/ClientRegistrationEnvironment.ts index 6048c2de1..86e95404d 100644 --- a/src/application/ClientRegistrationEnvironment.ts +++ b/src/application/ClientRegistrationEnvironment.ts @@ -9,12 +9,15 @@ import { } from "./ClientConfigurationDocument.js"; import { PRODUCT_IDENTITY } from "../identity.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { dshDocument } from "./DshCordisPatch.js"; /** Read explicit REA overrides without dropping peers when one value is invalid. */ export const clientRegistrationEnvironment = ( parsed: ClientConfigurationDocument | undefined, overrides: Readonly> = {}, ): Result>, ConfigurationError> => { + if (parsed?.dialect === "dsh") + return dshDocument(parsed.document).environment(overrides); const key = parsed?.dialect === "opencode" || parsed?.dialect === "opencode_v2" ? "environment" diff --git a/src/application/ClientRegistrationStatus.ts b/src/application/ClientRegistrationStatus.ts index b40079c45..588b0f9af 100644 --- a/src/application/ClientRegistrationStatus.ts +++ b/src/application/ClientRegistrationStatus.ts @@ -26,6 +26,8 @@ import { } from "./SupportedClients.js"; import type { SetupClient } from "./SupportedClients.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; +import { dshDocument } from "./DshCordisPatch.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; interface ClientRegistrationStatusBase { readonly client: string; @@ -65,6 +67,7 @@ const registrationSchema = z startup_timeout_sec: z.number().positive().optional(), type: z.string().optional(), transport: z.string().optional(), + serverName: z.string().optional(), tools: z.array(z.string()).optional(), disabled: z.boolean().optional(), enabled: z.boolean().optional(), @@ -78,6 +81,7 @@ export const readClientRegistrationStatuses = async ( options: { readonly platform?: NodeJS.Platform; readonly environment?: NodeJS.ProcessEnv; + readonly resources?: ArtifactResourceScope; } = {}, ): Promise => { const statuses: ClientRegistrationStatus[] = []; @@ -110,7 +114,10 @@ export const readClientRegistrationStatuses = async ( continue; } try { - const files = await readClientConfigurationFiles(client); + const files = await readClientConfigurationFiles( + client, + options.resources, + ); if (!files.ok) { statuses.push({ client: client.name, @@ -128,7 +135,12 @@ export const readClientRegistrationStatuses = async ( ); continue; } - const policy = await readClientPolicyBlock(client, parsed); + const policy = await readClientPolicyBlock( + client, + parsed, + undefined, + options.resources, + ); if (!policy.ok) { statuses.push({ client: client.name, @@ -168,6 +180,7 @@ export const readClientRegistrationStatuses = async ( parsed, PRODUCT_IDENTITY.mcpServerKey, ) && + (parsed.dialect !== "dsh" || dshDocument(parsed.document).ready) && policyBlock === undefined ? "aligned" : "stale", @@ -211,6 +224,13 @@ const registrationAligned = ( ) return false; if (client.format === "vscode" && registration.type !== "stdio") return false; + // A Harness registration only connects as the rea stdio server; another + // server name or transport leaves the session without REA tools. + if ( + client.format === "dsh" && + (registration.serverName !== "rea" || registration.transport !== "stdio") + ) + return false; // OMP and Pi infer stdio for a command entry without an explicit type. if ( (client.format === "omp" || client.format === "pi") && @@ -254,6 +274,11 @@ const parseRegistration = ( value: unknown, client: SetupClient, ): z.output => { + if (client.format === "dsh") + return registrationSchema + .pick({ command: true, args: true, serverName: true, transport: true }) + .strip() + .parse(value); if (client.format === "opencode") { const entry = z .object({ diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index 2d5ac1762..c4d130f63 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -15,14 +15,14 @@ import { isManagedToolName, type ManagedToolName, } from "../contracts/managed/managedToolContracts.js"; -import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; +import { + EvidenceFileError, + EvidenceIntegrityError, +} from "../domain/evidenceErrors.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { access } from "node:fs/promises"; -import { - readAnalysisSnapshot, - writeAnalysisSnapshot, -} from "./binary/AnalysisSnapshotFiles.js"; +import { readAnalysisSnapshot } from "./binary/AnalysisSnapshotFiles.js"; import { parseBinaryTarget } from "./BinaryTargetResolver.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { @@ -128,7 +128,7 @@ export const runProviderAnalysis = async ( ] ): Promise => isManagedToolName(tool) - ? runManagedProviderAnalysis(dependencies, path, tool, signal) + ? runManagedProviderAnalysis(dependencies, path, tool, arguments_, signal) : withProcessCancellation(signal, (operationSignal) => runAnalysis(dependencies, path, tool, arguments_, { logger, @@ -143,6 +143,7 @@ export const runManagedProviderExecution = async ( path: string, tool: ManagedToolName, signal?: AbortSignal, + parameters: Readonly> = {}, ): Promise> => withProcessCancellation(signal, async (operationSignal) => { const session = await dependencies.createManagedBinarySession(); @@ -151,7 +152,9 @@ export const runManagedProviderExecution = async ( async () => { const opened = await session.open(path, { signal: operationSignal }); if (!opened.ok) return opened; - return await session.execute(tool, {}, { signal: operationSignal }); + return await session.execute(tool, parameters, { + signal: operationSignal, + }); }, managedAnalysisCleanupFailure, ); @@ -161,6 +164,7 @@ const runManagedProviderAnalysis = async ( dependencies: DirectAnalysisDependencies, path: string, tool: ManagedToolName, + arguments_: Readonly>, signal?: AbortSignal, ): Promise => { const execution = await runManagedProviderExecution( @@ -168,15 +172,19 @@ const runManagedProviderAnalysis = async ( path, tool, signal, + arguments_, ); if (!execution.ok) return cliError(execution.error); const value = execution.value; return createEvidence(value.subject ?? undefined, value.provider, { operation: tool, - parameters: {}, + parameters: arguments_, result: value.result, rawResult: value.rawResult, limitations: value.limitations, + ...(value.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: value.relationshipCoverage }), locations: value.locations, }); }; @@ -331,13 +339,19 @@ const runAnalysis = async ( const snapshot = session.exportAnalysisSnapshot(); if (!snapshot.ok) return cliError(snapshot.error.retainPartialObservation(evidence)); - const written = await writeAnalysisSnapshot( + const written = await session.writeAnalysisSnapshot( snapshot.value, snapshotPath, true, ); - if (!written.ok) - return cliError(written.error.retainPartialObservation(evidence)); + if (!written.ok) { + const recovered = + written.error instanceof EvidenceFileError && + written.error.published !== undefined && + (await session.closeWriteResources()).ok; + if (!recovered) + return cliError(written.error.retainPartialObservation(evidence)); + } } return output; }, @@ -436,6 +450,9 @@ const executeAnalysisTool = async (input: { : { analysisProfile: result.value.analysisProfile }), rawResult: result.value.rawResult, limitations: result.value.limitations, + ...(result.value.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: result.value.relationshipCoverage }), locations: result.value.locations, evidenceLinks: tool === "inspect_artifact" @@ -543,13 +560,8 @@ const analysisProfileForRoute = ( const providerIdentityForRoute = ( route: SessionProviderRoute, operation: string, -): ProviderIdentity => { - const selected = - route.capabilities?.get(operation)?.provider ?? route.identity; - return route.profile?.provider.id === selected.id - ? route.profile.provider - : selected; -}; +): ProviderIdentity => + route.capabilities?.get(operation)?.provider ?? route.identity; const analysisProfileForEvidence = ( session: BinarySession, diff --git a/src/application/Doctor.ts b/src/application/Doctor.ts index 0f4247421..3e8659dca 100644 --- a/src/application/Doctor.ts +++ b/src/application/Doctor.ts @@ -1,5 +1,6 @@ import { constants } from "node:fs"; import { access, readdir, stat } from "node:fs/promises"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js"; import { join } from "node:path"; @@ -100,6 +101,7 @@ export interface DoctorHost { installedSkillIdentity?( clientIds: readonly string[], ): Promise; + close?(): Promise; clientRegistrations?(): Promise; ilspyCmdVersion?(path: string): Promise; } @@ -155,7 +157,41 @@ interface DoctorIdentity { */ export const runDoctor = async ( target?: string, - host: DoctorHost = systemDoctorHost(), + selectedHost?: DoctorHost, + requestedScope?: DoctorScope, +): Promise => + selectedHost === undefined + ? runDoctorOwned(target, systemDoctorHost(), requestedScope) + : inspectDoctor(target, selectedHost, requestedScope); + +/** Inspect with a host whose lifetime belongs to this operation. */ +export const runDoctorOwned = async ( + target: string | undefined, + host: DoctorHost, + requestedScope?: DoctorScope, +): Promise => { + let report: DoctorReport; + try { + report = await inspectDoctor(target, host, requestedScope); + } catch (cause: unknown) { + const cleanup = await closeDoctorHost(host); + if (cleanup !== undefined) + throw new AggregateError( + [cause, new Error(cleanup)], + "Doctor failed and its resource cleanup remains incomplete.", + { cause }, + ); + throw cause; + } + const cleanup = await closeDoctorHost(host); + return cleanup === undefined + ? report + : doctorReportWithCleanup(report, cleanup); +}; + +const inspectDoctor = async ( + target: string | undefined, + host: DoctorHost, requestedScope?: DoctorScope, ): Promise => { const scope = normalizeDoctorScope(requestedScope, target); @@ -197,6 +233,36 @@ export const runDoctor = async ( }; }; +const closeDoctorHost = async ( + host: DoctorHost, +): Promise => { + try { + return await host.close?.(); + } catch (cause: unknown) { + return `Doctor host cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + } +}; + +const doctorReportWithCleanup = ( + report: DoctorReport, + detail: string, +): DoctorReport => { + const check: DoctorCheck = { + name: "doctor-resource-cleanup", + ok: false, + classification: "config_drift", + detail, + remediation: "Resolve the reported cleanup failure, then rerun rea doctor.", + }; + return { + ...report, + healthy: false, + environment_healthy: false, + checks: [...report.checks, check], + scope_checks: [...report.scope_checks, check], + }; +}; + const collectDoctorIdentity = async ( host: DoctorHost, scope: DoctorScopeReport, @@ -307,6 +373,7 @@ export interface SystemDoctorHostOptions { export const systemDoctorHost = ( options: SystemDoctorHostOptions = {}, ): DoctorHost => { + const resources = new ArtifactResourceScope(); const platform = options.platform ?? process.platform; const architecture = options.architecture ?? process.arch; const environment = options.environment ?? process.env; @@ -354,7 +421,7 @@ export const systemDoctorHost = ( name: "target", ok: false, classification: - targetError?.constraint === "directory_requires_file" + targetError?.constraint !== undefined ? "unsupported_target" : "config_drift", detail: path, @@ -446,11 +513,21 @@ export const systemDoctorHost = ( clientIds, environment, platform, + resources, ), + close: async () => { + try { + await resources.close(); + return undefined; + } catch (cause: unknown) { + return `Doctor resource cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + } + }, clientRegistrations: () => readClientRegistrationStatuses(homeDirectory, undefined, { platform, environment, + resources, }), }; }; diff --git a/src/application/DshCordisPatch.ts b/src/application/DshCordisPatch.ts new file mode 100644 index 000000000..81f7428fb --- /dev/null +++ b/src/application/DshCordisPatch.ts @@ -0,0 +1,375 @@ +import { isMap, isSeq, type Node, type YAMLMap } from "yaml"; +import { parseEnvironment } from "../config/environment.js"; +import { ConfigurationError } from "../domain/configurationErrors.js"; +import { err, ok, type Result } from "../domain/result.js"; +import type { ClientConfigurationDocument } from "./ClientConfigurationDocument.js"; +import { CordisExpression, DshCordisYaml } from "./DshCordisYaml.js"; + +export const DSH_ENTRY_ID = "rea-mcp"; +export const DSH_CLIENT_PACKAGE = "@deepseek-ai/dsh-mcp-client"; +const LAUNCHER_KEYS = new Set(["serverName", "transport", "command", "args"]); +const isRecord = (value: unknown): value is Record => + typeof value === "object" && + value !== null && + !Array.isArray(value) && + !(value instanceof CordisExpression); + +interface PatchOperation { + readonly source: unknown; + readonly map: YAMLMap; + readonly ownedRows: readonly number[]; + readonly direct: boolean; +} + +/** One interpretation of identity, ordered patch applicability and source nodes. */ +export class DshCordisDocument { + readonly #yaml: DshCordisYaml; + readonly #operations: readonly PatchOperation[]; + readonly #fields = new Map(); + #entryNode: YAMLMap | undefined; + readonly #insertions: number; + #unknownSelector = false; + readonly config: Record | undefined; + + constructor(text: string) { + this.#yaml = new DshCordisYaml(text); + const root = this.#yaml.document.contents; + if (!isSeq(root)) + throw new ConfigurationError("Cordis patches must be a list"); + const operations: PatchOperation[] = []; + let insertions = 0; + for (const source of root.items) { + const map = this.#map(source); + if (map === undefined) continue; + const inserted = this.#yaml.resolve(this.#yaml.field(map, "insert")); + const targetId = this.#id(map); + const ownedRows = isSeq(inserted) + ? inserted.items.flatMap((entry, index) => + this.#id(entry) === DSH_ENTRY_ID ? [index] : [], + ) + : []; + const direct = !isSeq(inserted) && targetId === DSH_ENTRY_ID; + if (ownedRows.length > 0 && targetId) + throw new ConfigurationError( + "Cannot establish the target group for a Cordis REA insertion from the home patch alone", + ); + operations.push({ source, map, ownedRows, direct }); + if (isSeq(inserted)) { + for (const index of ownedRows) { + const entry = this.#map(inserted.items[index]); + if (entry === undefined) continue; + insertions += 1; + this.#entryNode = entry; + this.#fields.clear(); + this.#unknownSelector = false; + this.#readFields(entry); + } + } else if (direct) { + if (insertions === 0) continue; + // Cordis uses name as a selector, not a rename operation. + const selector = this.#yaml.value(this.#yaml.field(map, "name")); + if ( + selector instanceof CordisExpression || + (selector && this.#field("name") instanceof CordisExpression) + ) { + this.#unknownSelector = true; + continue; + } + if (selector && selector !== this.#field("name")) continue; + this.#readFields(map, new Set(["id", "name"])); + } + } + this.#operations = operations; + this.#insertions = insertions; + const config = this.#field("config"); + // Ownership survives malformed config so uninstall can remove the source row. + this.config = insertions === 0 ? undefined : isRecord(config) ? config : {}; + } + + #map(node: unknown): YAMLMap | undefined { + const map = this.#yaml.resolve(node); + return isMap(map) ? map : undefined; + } + #id(node: unknown): unknown { + const id = this.#yaml.value(this.#yaml.field(this.#map(node), "id")); + if (id instanceof CordisExpression) + throw new ConfigurationError( + "Expression-backed Cordis IDs have unknown ownership; use literal IDs before changing registrations", + ); + return id; + } + #field(key: string): unknown { + return this.#yaml.value(this.#fields.get(key)); + } + #readFields( + map: YAMLMap, + omitted: ReadonlySet = new Set(), + ): void { + for (const pair of map.items) { + const key = this.#yaml.value(pair.key); + if (typeof key === "string" && !omitted.has(key)) + this.#fields.set(key, pair.value); + } + } + get carriesEntry(): boolean { + return this.#operations.some((op) => op.direct || op.ownedRows.length > 0); + } + get ready(): boolean { + const disabled = this.#field("disabled"); + return ( + this.#insertions === 1 && + !this.#unknownSelector && + this.#field("name") === DSH_CLIENT_PACKAGE && + (disabled === undefined || disabled === false) + ); + } + + /** Validate literals without treating expression source as an environment value. */ + environment( + overrides: Readonly> = {}, + ): Result>, ConfigurationError> { + if (this.#unknownSelector) + return err( + new ConfigurationError( + "Expression-backed Cordis name selectors have unknown applicability; use literal selectors before setup", + { + settings: [ + { + setting: "name", + constraint: "Use literal selectors before setup", + }, + ], + }, + ), + ); + const config = this.#field("config"); + if (config instanceof CordisExpression) + return err( + new ConfigurationError( + "Expression-backed Cordis config cannot be safely edited", + { + settings: [ + { + setting: "config", + constraint: + "Use a literal mapping to preserve caller fields during setup", + }, + ], + }, + ), + ); + if (config !== undefined && !isRecord(config)) + return err( + new ConfigurationError("Cordis REA config must be a mapping", { + settings: [ + { + setting: "config", + constraint: "Use a literal mapping before setup", + }, + ], + }), + ); + const raw = this.config?.env; + if (raw !== undefined && !isRecord(raw)) + return err( + new ConfigurationError("Cordis server environment must be a mapping"), + ); + const literals: Record = {}; + for (const [key, value] of Object.entries(raw ?? {})) { + if (value instanceof CordisExpression) continue; + if (typeof value !== "string") + return err( + new ConfigurationError("Invalid Cordis server environment", { + settings: [ + { + setting: `env.${key}`, + constraint: + "Environment values must be strings or Cordis expressions", + }, + ], + }), + ); + literals[key] = value; + } + const environment = { ...literals, ...overrides }; + const validated = parseEnvironment(environment); + return validated.ok ? ok(environment) : validated; + } + + desired( + command: readonly string[], + overrides: Readonly>, + ): Result, ConfigurationError> { + const environment = this.environment(overrides); + if (!environment.ok) return environment; + const env = { + ...(isRecord(this.config?.env) ? this.config.env : {}), + ...overrides, + }; + const desired = { ...this.config, ...dshClientRegistrationEntry(command) }; + delete desired.env; + if (Object.keys(env).length > 0) desired.env = env; + return ok(desired); + } + + /** Plan every replacement from source snapshots before changing an output tree. */ + serialize(desired?: Record): string { + const root = this.#yaml.document.contents; + if (!isSeq(root)) + throw new ConfigurationError("Cordis patches must be a list"); + const replacements = new Map(); + const invalidated = new Set( + this.#operations.flatMap((op) => + op.direct || op.ownedRows.length > 0 + ? this.#yaml.anchors(op.source) + : [], + ), + ); + let carried = false; + for (const op of this.#operations) { + if (op.direct) { + replacements.set(op.source, undefined); + continue; + } + if (op.ownedRows.length === 0) continue; + const copy = this.#yaml.detach(op.map, invalidated); + if (!isMap(copy)) + throw new ConfigurationError("Cordis operation is not a mapping"); + const inserted = this.#yaml.detach( + this.#yaml.resolve(this.#yaml.field(op.map, "insert")), + invalidated, + ); + if (!isSeq(inserted)) + throw new ConfigurationError("Cordis insertion is not a list"); + copy.set("insert", inserted); + const owned = new Set(op.ownedRows); + inserted.items = inserted.items.flatMap((entry, index) => { + if (!owned.has(index)) return [entry]; + if (carried || desired === undefined) return []; + carried = true; + return [this.#entry(desired, invalidated)] as typeof inserted.items; + }); + replacements.set( + op.source, + inserted.items.length === 0 ? undefined : copy, + ); + } + const output = this.#yaml.document.clone(); + const outputRoot = output.contents; + if (!isSeq(outputRoot)) + throw new ConfigurationError("Cordis patches must be a list"); + outputRoot.items = root.items.flatMap((source) => { + const copy = replacements.has(source) + ? replacements.get(source) + : this.#yaml.copy(source, invalidated); + return copy === undefined ? [] : [copy]; + }) as typeof outputRoot.items; + if (!carried && desired !== undefined) + outputRoot.items.push( + output.createNode({ + insert: [ + { id: DSH_ENTRY_ID, name: DSH_CLIENT_PACKAGE, config: desired }, + ], + }) as never, + ); + return output.toString({ lineWidth: 0 }); + } + + #entry( + desired: Record, + invalidated: ReadonlySet, + ): YAMLMap { + const entry = this.#yaml.detach(this.#entryNode, invalidated); + if (!isMap(entry)) + throw new ConfigurationError("Cordis entry is not a mapping"); + this.#yaml.setValue(entry, "id", DSH_ENTRY_ID); + this.#yaml.setValue(entry, "name", DSH_CLIENT_PACKAGE); + entry.delete("disabled"); + for (const [key, value] of this.#fields) { + if (["id", "name", "config", "disabled"].includes(key)) continue; + entry.set(key, this.#yaml.detach(value, invalidated)); + } + const sourceConfig = this.#map(this.#fields.get("config")); + const config = + sourceConfig === undefined + ? this.#yaml.document.createNode({}) + : this.#yaml.detach(sourceConfig, invalidated); + if (!isMap(config)) + throw new ConfigurationError( + "Cordis registration config is not a mapping", + ); + // Caller nodes keep their tags and spelling; only literal launcher values change. + for (const key of LAUNCHER_KEYS) + this.#yaml.setValue(config, key, desired[key]); + const env = this.#map(this.#yaml.field(sourceConfig, "env")); + const desiredEnv = desired.env; + if (isRecord(desiredEnv)) { + const nextEnv = + env === undefined + ? this.#yaml.document.createNode({}) + : this.#yaml.detach(env, invalidated); + if (!isMap(nextEnv)) + throw new ConfigurationError("Cordis environment is not a mapping"); + for (const [key, value] of Object.entries(desiredEnv)) { + this.#yaml.setValue(nextEnv, key, value); + } + config.set("env", nextEnv); + } else config.delete("env"); + entry.set("config", config); + return entry; + } +} + +class DshCordisEdit { + constructor( + readonly source: DshCordisDocument, + readonly desired?: Record, + ) {} +} +export const dshDocument = ( + document: Record, +): DshCordisDocument => { + if (!(document.cordis instanceof DshCordisDocument)) + throw new ConfigurationError("Missing Cordis document owner"); + return document.cordis; +}; +export const parseDshCordisDocument = ( + text: string, +): ClientConfigurationDocument => { + const source = new DshCordisDocument(text); + return { + document: { cordis: source }, + servers: source.config === undefined ? {} : { rea: source.config }, + serversPath: [], + dialect: "dsh", + legacyServers: {}, + }; +}; +export const editDshDocument = ( + parsed: ClientConfigurationDocument, + desired?: Record, +): Record => ({ + cordis: new DshCordisEdit(dshDocument(parsed.document), desired), +}); +export const serializeDshDocument = ( + document: Record, +): string => { + if (!(document.cordis instanceof DshCordisEdit)) + throw new ConfigurationError("Missing Cordis edit plan"); + return document.cordis.source.serialize(document.cordis.desired); +}; +export const dshClientRegistrationEntry = ( + command: readonly string[], + environment: Readonly> = {}, +): Record => { + const [executable = "rea", ...args] = command; + return { + serverName: "rea", + transport: "stdio", + command: executable, + args, + ...(Object.keys(environment).length === 0 + ? {} + : { env: { ...environment } }), + }; +}; diff --git a/src/application/DshCordisYaml.ts b/src/application/DshCordisYaml.ts new file mode 100644 index 000000000..ea84e5e65 --- /dev/null +++ b/src/application/DshCordisYaml.ts @@ -0,0 +1,175 @@ +import { isDeepStrictEqual } from "node:util"; +import { + isAlias, + isMap, + isNode, + isScalar, + isSeq, + parseDocument, + visit, + type Document, + type Node, + type YAMLMap, +} from "yaml"; +import { ConfigurationError } from "../domain/configurationErrors.js"; + +/** Expression source is never a literal value and is never evaluated by REA. */ +export class CordisExpression { + readonly kind = "cordis-expression"; + constructor(readonly source: string) {} +} + +/** Immutable source graph; all alias resolution happens against this document. */ +export class DshCordisYaml { + readonly document: Document; + readonly #values = new Map(); + readonly #visiting = new Set(); + + constructor(text: string) { + this.document = parseDocument(text, { + customTags: [ + { tag: "tag:yaml.org,2002:js", resolve: (source: string) => source }, + ], + logLevel: "silent", + }); + if (this.document.errors.length > 0 || this.document.warnings.length > 0) + throw new ConfigurationError("Invalid or unsupported Cordis YAML", { + cause: this.document.errors[0] ?? this.document.warnings[0], + }); + if (this.document.contents === null) + this.document.contents = this.document.createNode([]); + if (!isSeq(this.document.contents)) + throw new ConfigurationError( + "Cordis patches must be a YAML operation list", + ); + this.value(this.document.contents); + try { + // Retain YAML's native alias-expansion admission before copying edits. + // This value is discarded; source nodes own every Cordis interpretation. + this.document.toJS(); + } catch (cause) { + throw new ConfigurationError("Unsafe Cordis YAML alias expansion", { + cause, + }); + } + } + + resolve(node: unknown): unknown { + if (!isAlias(node)) return node; + const resolved = node.resolve(this.document); + if (resolved === undefined) + throw new ConfigurationError("Cordis YAML contains an unresolved alias"); + return resolved; + } + + value(node: unknown): unknown { + const resolved = this.resolve(node); + if (!isNode(resolved)) return resolved; + if (this.#values.has(resolved)) return this.#values.get(resolved); + if (this.#visiting.has(resolved)) + throw new ConfigurationError( + "Cyclic Cordis YAML cannot be safely edited", + ); + this.#visiting.add(resolved); + let value: unknown; + if (isScalar(resolved)) + value = + resolved.tag === "tag:yaml.org,2002:js" + ? new CordisExpression(String(resolved.value)) + : resolved.value; + else if (isSeq(resolved)) + value = resolved.items.map((item) => this.value(item)); + else if (isMap(resolved)) { + const entries: [string, unknown][] = []; + for (const pair of resolved.items) { + entries.push([this.#key(pair.key), this.value(pair.value)]); + } + value = Object.fromEntries(entries); + } + this.#visiting.delete(resolved); + this.#values.set(resolved, value); + return value; + } + + #key(node: unknown): string { + const key = this.value(node); + if (typeof key === "object" && key !== null) + throw new ConfigurationError( + "Cordis YAML mapping keys must be literal scalars", + ); + // Match YAML's JS object projection while retaining the original key node. + return key === null ? "" : String(key); + } + + field(map: YAMLMap | undefined, key: string): unknown { + return map?.items.findLast((pair) => this.#key(pair.key) === key)?.value; + } + + /** Preserve unchanged nodes and carry their comments when replacing values. */ + setValue(map: YAMLMap, key: string, value: unknown): void { + const original = this.field(map, key); + if (isDeepStrictEqual(this.value(original), value)) return; + const replacement = this.document.createNode(value); + if (isNode(original) && isNode(replacement)) { + if (original.comment !== undefined) + replacement.comment = original.comment; + if (original.commentBefore !== undefined) + replacement.commentBefore = original.commentBefore; + } + map.set(key, replacement); + } + + /** Copy from the source before edits; materialized nodes own no anchors. */ + copy(node: unknown, invalidated?: ReadonlySet): unknown { + if (isAlias(node)) { + const target = this.resolve(node); + return invalidated === undefined || + (isNode(target) && invalidated.has(target)) + ? this.copy(target) + : node.clone(); + } + if (!isNode(node)) return node; + const copied = node.clone(); + if ( + (invalidated === undefined || invalidated.has(node)) && + (isScalar(copied) || isMap(copied) || isSeq(copied)) + ) + delete copied.anchor; + if (isSeq(node) && isSeq(copied)) + copied.items = node.items.map((item) => + this.copy(item, invalidated), + ) as typeof copied.items; + if (isMap(node) && isMap(copied)) + copied.items.forEach((pair, index) => { + pair.key = this.copy(node.items[index]!.key, invalidated); + pair.value = this.copy(node.items[index]!.value, invalidated); + }); + return copied; + } + + /** Replacing an anchor must not change untouched consumers of its old value. */ + detach(node: unknown, invalidated: ReadonlySet): unknown { + const copied = this.copy(node, invalidated); + if (isNode(copied)) + visit(copied, { + Pair: (_key, pair) => { + if (isAlias(pair.key)) pair.key = this.copy(this.resolve(pair.key)); + }, + Value: (_key, value) => { + delete value.anchor; + }, + }); + return copied; + } + + anchors(node: unknown): readonly Node[] { + if (!isNode(node)) return []; + const anchors: Node[] = []; + visit(node, { + Value: (_key, value) => { + if (value.anchor) anchors.push(value); + }, + }); + return anchors; + } +} diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 7e0b1addf..a5d09ecc2 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -33,6 +33,7 @@ import { inspectNativeDispatch } from "./native/NativeDispatchMetadataInspection import { traceNativeValues } from "./native/NativeValueTrace.js"; import { traceNativeUiAction } from "./native/NativeUiActionTrace.js"; import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { invalidEnhancedInput, @@ -343,15 +344,26 @@ export class EnhancedTools { const relation = input.direction === "forward" ? "callees" : "callers"; const tool = input.direction === "forward" ? "procedure_callees" : "procedure_callers"; - const resolved = await resolveProcedureAddress( - this.#call.bind(this), - input.address, + const limitations = new Set(); + const resolved = await this.#observe( + "procedure_address", + { procedure: input.address }, signal, ); if (!resolved.ok) return resolved; - const discovered = new Set([resolved.value]); + for (const limitation of resolved.value.limitations) + limitations.add(limitation); + const startAddress = resolved.value.result; + if (typeof startAddress !== "string" || startAddress.length === 0) + return err( + new AnalysisOutputError( + "procedure_address", + "provider returned no canonical procedure address", + ), + ); + const discovered = new Set([startAddress]); const queue: Array<{ address: string; depth: number }> = [ - { address: resolved.value, depth: 0 }, + { address: startAddress, depth: 0 }, ]; let queueIndex = 0; const graph: Record = {}; @@ -385,13 +397,40 @@ export class EnhancedTools { }); continue; } + for (const limitation of result.value.limitations) + limitations.add(limitation); + const reportedCoverage = result.value.relationshipCoverage; + const coverage = reportedCoverage ?? { + status: "unknown" as const, + reason: "Provider did not report relationship coverage.", + }; + const relationshipLimitations = + reportedCoverage === undefined + ? [ + ...result.value.limitations, + "Provider did not report relationship coverage.", + ] + : [ + ...(reportedCoverage.status === "partial" + ? reportedCoverage.limitations + : reportedCoverage.status === "unknown" + ? [reportedCoverage.reason] + : []), + ...("diagnostics" in reportedCoverage + ? reportedCoverage.diagnostics + : []), + ]; + const nodeLimitations = [...new Set(relationshipLimitations)].sort( + compareUnicodeCodePoints, + ); graph[level].push({ address: current.address, status: "ok", calls: [...related.value], - ...(result.value.limitations.length === 0 + relationship_coverage: coverage, + ...(nodeLimitations.length === 0 ? {} - : { limitations: [...result.value.limitations] }), + : { limitations: nodeLimitations }), }); for (const address of related.value) { if (!discovered.has(address)) { @@ -400,7 +439,10 @@ export class EnhancedTools { } } } - return ok(graph); + return ok({ + graph, + limitations: [...limitations].sort(compareUnicodeCodePoints), + }); } async #analyzeSwiftTypes( diff --git a/src/application/EvidenceBundleCommands.ts b/src/application/EvidenceBundleCommands.ts index 7b91c05c3..a72d17cb5 100644 --- a/src/application/EvidenceBundleCommands.ts +++ b/src/application/EvidenceBundleCommands.ts @@ -1,10 +1,14 @@ -import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; +import { + EvidenceFileError, + EvidenceIntegrityError, +} from "../domain/evidenceErrors.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { EvidenceBundle } from "../domain/evidenceBundle.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; import { err, ok, type Result } from "../domain/result.js"; import { compareBundles } from "../domain/bundleComparison.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { readEvidenceBundle, writeEvidenceBundle, @@ -31,10 +35,33 @@ export const exportEvidenceBundleCommand = async ( ): Promise> => { const loaded = await readEvidenceBundle(sourcePath); if (!loaded.ok) return loaded; - return projectWrite( + const resources = new ArtifactResourceScope(); + const written = await writeEvidenceBundle( loaded.value, - await writeEvidenceBundle(loaded.value, outputPath, overwrite), + outputPath, + overwrite, + resources, ); + if (written.ok) return projectWrite(loaded.value, written); + + let cleanupSucceeded = false; + try { + await resources.close(); + cleanupSucceeded = true; + } catch { + // The original error cause retains the same staged-output owner for diagnostics. + } + if ( + cleanupSucceeded && + written.error instanceof EvidenceFileError && + written.error.published + ) { + return projectWrite(loaded.value, { + ok: true, + value: written.error.published, + }); + } + return written; }; /** Compare two validated canonical Evidence bundles without session state. */ diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.ts index dfc63b884..a9ff27990 100644 --- a/src/application/EvidenceBundleFiles.ts +++ b/src/application/EvidenceBundleFiles.ts @@ -21,6 +21,7 @@ import { } from "../domain/jsonSerialization.js"; import { readJsonFile, writeTextParts } from "./JsonFiles.js"; import { RegularFileCleanupFailure } from "./RegularFileRead.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; type EvidenceReadFailure = | EvidenceFileError @@ -105,6 +106,7 @@ export const writeEvidenceBundle = async ( bundle: EvidenceBundle, path: string, overwrite: boolean, + resources: ArtifactResourceScope, signal?: AbortSignal, ): Promise< Result< @@ -128,6 +130,7 @@ export const writeEvidenceBundle = async ( bufferedJsonParts(canonicalJsonParts(checked)), path, overwrite, + resources, signal === undefined ? undefined : { signal, operation: "export_evidence_bundle" }, diff --git a/src/application/FunctionAnalysisEvidence.ts b/src/application/FunctionAnalysisEvidence.ts index e2ef187d4..00ef766a8 100644 --- a/src/application/FunctionAnalysisEvidence.ts +++ b/src/application/FunctionAnalysisEvidence.ts @@ -46,6 +46,9 @@ export const executeFunctionAnalysisEvidence = async ( rawResult: observation.rawResult, limitations: observation.limitations, locations: observation.locations, + ...(observation.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: observation.relationshipCoverage }), ...(observation.analysisProfile === undefined ? {} : { analysisProfile: observation.analysisProfile }), diff --git a/src/application/GeminiClientSettings.ts b/src/application/GeminiClientSettings.ts index eecb33b53..440ccf9d6 100644 --- a/src/application/GeminiClientSettings.ts +++ b/src/application/GeminiClientSettings.ts @@ -4,6 +4,7 @@ import { stripComments } from "jsonc-parser"; import { z } from "zod"; import { err, ok, type Result } from "../domain/result.js"; import { PRODUCT_IDENTITY } from "../identity.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js"; import { clientConfigurationValuesEqual, @@ -61,6 +62,7 @@ const normalizedPath = async ( const workspaceTrusted = async ( settings: NonNullable, sources: readonly SettingsSource[], + resources?: ArtifactResourceScope, ): Promise> => { let enabled = true; for (const { path, document } of sources) { @@ -70,7 +72,10 @@ const workspaceTrusted = async ( } if (settings.trustOverride !== undefined) return ok(settings.trustOverride); if (!enabled) return ok(true); - const text = await readConfigurationText(settings.trustedFoldersPath); + const text = await readConfigurationText( + settings.trustedFoldersPath, + resources, + ); if (!text.ok) return text; if (text.value === undefined) return ok(false); let rules: z.output; @@ -110,6 +115,7 @@ export const readClientPolicyBlock = async ( client: SetupClient, userDocument?: ClientConfigurationDocument, desiredRegistration?: unknown, + resources?: ArtifactResourceScope, ): Promise> => { if (client.name !== "gemini_cli" || client.geminiSettings === undefined) return ok(undefined); @@ -124,7 +130,7 @@ export const readClientPolicyBlock = async ( const parsed = path === client.configPath && userDocument !== undefined ? ok(userDocument) - : await readConfigurationFile(path, "json"); + : await readConfigurationFile(path, "json", resources); if (!parsed.ok) return parsed; if (parsed.value !== undefined) sources.push({ path, document: parsed.value }); @@ -143,10 +149,10 @@ export const readClientPolicyBlock = async ( ".gemini", "settings.json", ); - const document = await readConfigurationFile(path, "json"); + const document = await readConfigurationFile(path, "json", resources); if (!document.ok) return document; if (document.value !== undefined) { - const trusted = await workspaceTrusted(settings, sources); + const trusted = await workspaceTrusted(settings, sources, resources); if (!trusted.ok) return trusted; if (trusted.value) { const source = { path, document: document.value }; diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index d1da371f5..883d3878c 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -18,7 +18,9 @@ export const workflowAnalysisProfile = ( ): AnalysisProfileCommitment => createAnalysisProfile(REA_WORKFLOW_PROVIDER, { upstream_analysis_profile: upstream, - ...(operation === "binary_overview" || operation === "batch_decompile" + ...(operation === "binary_overview" || + operation === "batch_decompile" || + operation === "get_call_graph" ? { workflow_revision: 2 } : {}), }); diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index 1eb9a7650..76a05f9cb 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -5,12 +5,14 @@ import { open, realpath, rename, - rm, type FileHandle, } from "node:fs/promises"; import { basename, dirname, resolve } from "node:path"; -import { EvidenceFileError } from "../domain/evidenceErrors.js"; +import { + EvidenceFileError, + type PublishedEvidenceFile, +} from "../domain/evidenceErrors.js"; import { AnalysisCancelledError, AnalysisResourceConstraintError, @@ -22,6 +24,10 @@ import { type JsonObjectMemberReader, } from "./JsonInputFile.js"; import { NonRegularFileReadError } from "../filesystem/RegularFile.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; +import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; +import { OwnedStagedOutput } from "../filesystem/OwnedStagedOutput.js"; /** Request control and its owning operation for an interruptible atomic write. */ export interface TextWriteCancellation { @@ -119,28 +125,57 @@ export const writeTextFile = async ( encoded: string, path: string, overwrite: boolean, + resources: ArtifactResourceScope, ): Promise> => - writeTextParts([encoded], path, overwrite); + writeTextParts([encoded], path, overwrite, resources); /** Publish streamed text atomically unless cancellation wins before publication. */ export function writeTextParts( parts: Iterable, path: string, overwrite: boolean, + resources: ArtifactResourceScope, ): Promise>; -/** Pass request control to stop streamed generation before atomic publication. */ export function writeTextParts( parts: Iterable, path: string, overwrite: boolean, + resources: ArtifactResourceScope, cancellation: TextWriteCancellation | undefined, ): Promise; -export async function writeTextParts( +export function writeTextParts( parts: Iterable, path: string, overwrite: boolean, + resources: ArtifactResourceScope, cancellation?: TextWriteCancellation, ): Promise { + return resources + .run(() => + writeTextPartsOwned(parts, path, overwrite, resources, cancellation), + ) + .catch((cause: unknown) => { + const cleanup = + cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined + ? cause.cleanup + : undefined; + return err( + new EvidenceFileError("write", "io", { + cause, + path: resolve(path), + ...(cleanup === undefined ? {} : { cleanup }), + }), + ); + }); +} + +const writeTextPartsOwned = async ( + parts: Iterable, + path: string, + overwrite: boolean, + resources: ArtifactResourceScope, + cancellation?: TextWriteCancellation, +): Promise => { const requestedPath = resolve(path); try { assertWriteActive(cancellation); @@ -165,10 +200,40 @@ export async function writeTextParts( destination, parts, overwrite, + resources, cancellation, ); return ok({ path: requestedPath, bytes }); } catch (cause: unknown) { + if (cause instanceof StagedOutputCleanupFailure) { + const cleanup = { + reason: failureMessage(cause.cleanupCause), + resources: [cause.owner.directory], + }; + if (cause.outcome.ok) + return err( + new EvidenceFileError("write", "io", { + cause, + path: requestedPath, + cleanup, + published: { ...cause.outcome.value, path: requestedPath }, + }), + ); + if (cause.outcome.error instanceof AnalysisCancelledError) + return err( + new AnalysisCancelledError(cause.outcome.error.operation, { + cause, + cleanup, + }), + ); + return err( + new EvidenceFileError("write", missingOrIo(cause.outcome.error), { + cause, + path: requestedPath, + cleanup, + }), + ); + } if ( cancellation?.signal.aborted === true && cause instanceof AnalysisCancelledError && @@ -189,7 +254,7 @@ export async function writeTextParts( }), ); } -} +}; /** A missing path or parent is a selection error, not a permission failure. */ const missingOrIo = (cause: unknown): "missing" | "io" => { @@ -201,33 +266,75 @@ const publishFile = async ( destination: string, parts: Iterable, overwrite: boolean, + resources: ArtifactResourceScope, cancellation?: TextWriteCancellation, ): Promise => { const stagingDirectory = await mkdtemp( resolve(dirname(destination), ".rea-write-"), ); + const owner = await OwnedStagedOutput.create(stagingDirectory); + let outcome: Result; + const staged = resolve(stagingDirectory, "content"); try { - const staged = resolve(stagingDirectory, "content"); const file = await open(staged, "wx", 0o600); - let bytes = 0; + await owner.setFile(new OwnedFileHandle(file)); + let written: Result; try { - bytes = await writeStagedParts(file, parts, cancellation); + const bytes = await writeStagedParts(file, parts, cancellation); assertWriteActive(cancellation); await file.sync(); - } finally { - await file.close(); + written = ok(bytes); + } catch (cause: unknown) { + written = err(cause); } - // Once submitted, the atomic publication can win a later cancellation race. + try { + await owner.closeFile(); + } catch (cause: unknown) { + if (written.ok) written = err(cause); + } + if (!written.ok) throw written.error; + // Once submitted, atomic publication can win a later cancellation race. assertWriteActive(cancellation); - if (overwrite) await rename(staged, destination); + await owner.assertReadyForPublication(staged); + assertWriteActive(cancellation); + if (overwrite) { + await rename(staged, destination); + owner.recordPublishedRename(); + } // A hard link publishes complete bytes atomically and cannot replace an existing name. else await link(staged, destination); - return bytes; - } finally { - await rm(stagingDirectory, { recursive: true, force: true }); + outcome = ok({ path: destination, bytes: written.value }); + } catch (cause: unknown) { + outcome = err(cause); } + + // One cleanup phase owns both the open descriptor (if any) and its staging directory. + const released = await resources.release({ + kind: "staged-output", + resource: owner.directory, + output: owner, + }); + if (released.kind === "failed") + throw new StagedOutputCleanupFailure(outcome, released.cause, owner); + if (!outcome.ok) throw outcome.error; + return outcome.value.bytes; }; +class StagedOutputCleanupFailure extends Error { + constructor( + readonly outcome: Result, + readonly cleanupCause: unknown, + readonly owner: OwnedStagedOutput, + ) { + super("Staged output cleanup failed", { + cause: outcome.ok ? cleanupCause : outcome.error, + }); + } +} + +const failureMessage = (cause: unknown): string => + cause instanceof Error ? cause.message || cause.name : String(cause); + const writeStagedParts = async ( file: FileHandle, parts: Iterable, diff --git a/src/application/MipsElfAbiFlags.ts b/src/application/MipsElfMetadata.ts similarity index 73% rename from src/application/MipsElfAbiFlags.ts rename to src/application/MipsElfMetadata.ts index 6558b7747..8f2d97115 100644 --- a/src/application/MipsElfAbiFlags.ts +++ b/src/application/MipsElfMetadata.ts @@ -4,24 +4,29 @@ import type { MipsElfMetadata, } from "../domain/binaryTargetTypes.js"; import { err, ok, type Result } from "../domain/result.js"; +import { + readMipsElfSharedLoad, + type MipsElfRelocationSection, +} from "./MipsElfSharedLoad.js"; -/** Observed ABI declarations and coverage of their source tables. */ -export interface MipsAbiFlagsInspection { +/** Observed ABI and load declarations, retaining unavailable source coverage. */ +export interface MipsElfInspection { readonly abiFlags: MipsAbiFlags | null; readonly limitations: readonly string[]; + readonly sharedLoad?: MipsElfMetadata["sharedLoad"]; } /** - * Read ELF32 MIPS ABI declarations from the caller's already-open file. + * Read ELF32 MIPS ABI and shared-load declarations from the caller's already-open file. * Inspect both typed tables (not section names), including records beyond the * initial header probe. A missing record is unknown, not a default FP ABI. * The caller supplies its cancellation check so its tagged error is preserved. */ -export const readMipsElfAbiFlags = async ( +export const readMipsElfMetadata = async ( handle: FileHandle, metadata: MipsElfMetadata, checkCancelled: () => void, -): Promise> => { +): Promise> => { checkCancelled(); if (metadata.elfClass !== 32) return err("MIPS ABI record inspection currently supports ELF32 only"); @@ -107,6 +112,9 @@ export const readMipsElfAbiFlags = async ( if (u16(bytes, 44) === 0xffff || u16(bytes, 48) >= 0xff00) return err("extended or reserved MIPS ELF table numbering is unsupported"); let record: Buffer | undefined; + let sharedLoad: MipsElfMetadata["sharedLoad"]; + let programHeaders: Buffer | undefined; + const relocationSections: MipsElfRelocationSection[] = []; const limitations: string[] = []; for (const table of tables) { if (table.count === 0) { @@ -135,12 +143,20 @@ export const readMipsElfAbiFlags = async ( table.label, ); if (!contents.ok) return contents; + if (!table.optional && metadata.type === 3) programHeaders = contents.value; let found = false; for (let i = 0; i < count; i += 1) { checkCancelled(); const position = i * table.entrySize; - if (u32(contents.value, position + table.typeOffset) !== table.recordType) - continue; + const type = u32(contents.value, position + table.typeOffset); + if (metadata.type === 3 && table.optional && (type === 4 || type === 9)) + relocationSections.push({ + type, + offset: u32(contents.value, position + 16), + size: u32(contents.value, position + 20), + entrySize: u32(contents.value, position + 36), + }); + if (type !== table.recordType) continue; if (found) return err(`duplicate MIPS ABI records in ${table.label}`); found = true; const offset = u32(contents.value, position + table.dataOffset); @@ -154,7 +170,37 @@ export const readMipsElfAbiFlags = async ( record = candidate.value; } } - if (record === undefined) return ok({ abiFlags: null, limitations }); + if (metadata.type === 3 && programHeaders !== undefined) { + if (limitations.length > 0) + sharedLoad = { + status: "unavailable", + reason: `MIPS shared-image relocation declaration sources are incomplete. ${limitations.join(" ")}`, + }; + else { + const shared = await readMipsElfSharedLoad( + programHeaders, + little, + fileSize, + read, + checkCancelled, + relocationSections, + ); + sharedLoad = shared.ok + ? shared.value + : { status: "unavailable", reason: shared.error }; + } + } + const shared = + metadata.type === 3 + ? { + sharedLoad: sharedLoad ?? { + status: "unavailable" as const, + reason: "MIPS shared image has no inspected program header table", + }, + } + : {}; + if (record === undefined) + return ok({ abiFlags: null, limitations, ...shared }); // Preserve declarations verbatim, including unknown versions/values. Provider // admission, not parsing, decides which interpretations have been verified. const abiFlags: MipsAbiFlags = { @@ -170,5 +216,5 @@ export const readMipsElfAbiFlags = async ( flags1: u32(record, 16), flags2: u32(record, 20), }; - return ok({ abiFlags, limitations }); + return ok({ abiFlags, limitations, ...shared }); }; diff --git a/src/application/MipsElfSharedLoad.ts b/src/application/MipsElfSharedLoad.ts new file mode 100644 index 000000000..7c4dcf755 --- /dev/null +++ b/src/application/MipsElfSharedLoad.ts @@ -0,0 +1,220 @@ +import type { MipsElfMetadata } from "../domain/binaryTargetTypes.js"; +import { err, ok, type Result } from "../domain/result.js"; + +type ReadRange = ( + offset: number, + length: number, + label: string, +) => Promise>; + +export interface MipsElfRelocationSection { + readonly type: number; + readonly offset: number; + readonly size: number; + readonly entrySize: number; +} +type SharedLoadFacts = Extract< + NonNullable, + { status: "complete" } +>; + +/** Source ELF facts only; the provider owns interpretation and admission. */ +export const readMipsElfSharedLoad = async ( + programHeaders: Buffer, + little: boolean, + fileSize: number, + read: ReadRange, + checkCancelled: () => void, + sections: readonly MipsElfRelocationSection[], +): Promise> => { + const u32 = (bytes: Buffer, offset: number): number => + little ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset); + const loads: { + offset: number; + address: number; + size: number; + memorySize: number; + }[] = []; + let dynamic: { offset: number; address: number; size: number } | undefined; + for (let position = 0; position < programHeaders.length; position += 32) { + checkCancelled(); + const type = u32(programHeaders, position); + const offset = u32(programHeaders, position + 4); + const size = u32(programHeaders, position + 16); + if (type === 1) { + const address = u32(programHeaders, position + 8); + const memorySize = u32(programHeaders, position + 20); + if ( + offset > fileSize - size || + size > memorySize || + address + memorySize > 0x100000000 + ) + return err("invalid or incomplete MIPS PT_LOAD range"); + loads.push({ offset, address, size, memorySize }); + } + if (type === 2) { + if (dynamic !== undefined) return err("duplicate MIPS PT_DYNAMIC tables"); + dynamic = { offset, address: u32(programHeaders, position + 8), size }; + } + } + if (dynamic === undefined) + return err("MIPS shared image has no PT_DYNAMIC table"); + if (dynamic.size === 0 || dynamic.size % 8 !== 0) + return err("invalid MIPS ELF32 dynamic table size"); + if (dynamic.offset > fileSize - dynamic.size) + return err("MIPS PT_DYNAMIC range extends beyond the file"); + const ranges = loads + .filter((load) => load.memorySize > 0) + .sort((left, right) => left.address - right.address); + for (let i = 1; i < ranges.length; i += 1) { + checkCancelled(); + const previous = ranges[i - 1]!; + if (ranges[i]!.address < previous.address + previous.memorySize) + return err( + "overlapping MIPS PT_LOAD memory ranges require separate mapping verification", + ); + } + // One normalized index prevents relocation count * program-header count work. + const mapped = (address: number, length: number, fileBacked: boolean) => { + let low = 0; + let high = ranges.length; + while (low < high) { + const middle = Math.floor((low + high) / 2); + if (ranges[middle]!.address <= address) low = middle + 1; + else high = middle; + } + const range = ranges[low - 1]; + return range !== undefined && + address - range.address <= + (fileBacked ? range.size : range.memorySize) - length + ? range + : undefined; + }; + const dynamicMapping = mapped(dynamic.address, dynamic.size, true); + if ( + dynamicMapping === undefined || + dynamicMapping.offset + dynamic.address - dynamicMapping.address !== + dynamic.offset + ) + return err( + "MIPS PT_DYNAMIC has no unique matching file-backed PT_LOAD range", + ); + + // Bound temporary storage, not admitted artifact size. ELF32 records are + // exact eight-byte structures; every declared record is examined or refused. + const chunkBytes = 64 * 1024; + const relevant = new Set([2, 7, 8, 9, 17, 18, 19, 20, 23]); + const tags = new Map(); + let terminated = false; + for ( + let offset = 0; + offset < dynamic.size && !terminated; + offset += chunkBytes + ) { + const chunk = await read( + dynamic.offset + offset, + Math.min(chunkBytes, dynamic.size - offset), + "dynamic table", + ); + if (!chunk.ok) return chunk; + for (let position = 0; position < chunk.value.length; position += 8) { + checkCancelled(); + const tag = u32(chunk.value, position); + if (tag === 0) { + terminated = true; + break; + } + if (!relevant.has(tag)) continue; + if (tags.has(tag)) return err(`duplicate MIPS dynamic tag ${tag}`); + tags.set(tag, u32(chunk.value, position + 4)); + } + } + if (!terminated) return err("MIPS dynamic table has no DT_NULL terminator"); + if ([2, 7, 8, 9, 23].some((tag) => (tags.get(tag) ?? 0) !== 0)) + return err( + "MIPS shared RELA or PLT relocation declarations require separate verification", + ); + const address = tags.get(17); + const size = tags.get(18); + const stride = tags.get(19); + const checkSections = ( + offset: number, + length: number, + ): string | undefined => { + let observed = 0; + for (const section of sections) { + checkCancelled(); + if (section.size === 0) continue; + if ( + section.type !== 9 || + section.entrySize !== 8 || + section.offset !== offset || + section.size !== length + ) + return "MIPS section relocation declarations outside the dynamic REL table require separate verification"; + observed += 1; + } + return observed > 1 + ? "duplicate MIPS section relocation declarations require separate verification" + : undefined; + }; + const emptyRelocations = (): Result => { + const reason = checkSections(0, 0); + return reason === undefined + ? ok({ + status: "complete", + relocationCount: 0, + relocationTypes: [], + symbolRelativeCount: 0, + }) + : err(reason); + }; + if (address === undefined && size === undefined && stride === undefined) + return emptyRelocations(); + if ( + address === undefined || + size === undefined || + stride !== 8 || + size % 8 !== 0 + ) + return err( + "incomplete or invalid MIPS DT_REL/DT_RELSZ/DT_RELENT declaration", + ); + if (size === 0) return emptyRelocations(); + const mapping = mapped(address, size, true); + if (mapping === undefined) + return err( + "MIPS DT_REL range has no unique complete file-backed PT_LOAD mapping", + ); + const fileOffset = mapping.offset + address - mapping.address; + const sectionReason = checkSections(fileOffset, size); + if (sectionReason !== undefined) return err(sectionReason); + let symbolRelativeCount = 0; + const types = new Set(); + for (let offset = 0; offset < size; offset += chunkBytes) { + const chunk = await read( + fileOffset + offset, + Math.min(chunkBytes, size - offset), + "dynamic REL records", + ); + if (!chunk.ok) return chunk; + for (let position = 0; position < chunk.value.length; position += 8) { + checkCancelled(); + const info = u32(chunk.value, position + 4); + const type = info & 0xff; + const relocationAddress = u32(chunk.value, position); + if (type === 3 && mapped(relocationAddress, 4, false) === undefined) + return err( + "MIPS REL32 destination has no unique complete PT_LOAD memory range", + ); + types.add(type); + if (type === 3 && info >>> 8 !== 0) symbolRelativeCount += 1; + } + } + return ok({ + status: "complete", + relocationCount: size / 8, + relocationTypes: [...types].sort((left, right) => left - right), + symbolRelativeCount, + }); +}; diff --git a/src/application/ReconstructionObligationLedger.closure.test.ts b/src/application/ReconstructionObligationLedger.closure.test.ts index e11bffadd..86dc84474 100644 --- a/src/application/ReconstructionObligationLedger.closure.test.ts +++ b/src/application/ReconstructionObligationLedger.closure.test.ts @@ -9,6 +9,8 @@ import { request, reviewedObligation, } from "./ReconstructionObligationLedger.fixture.js"; +import { deriveReconstructionObligationCandidates } from "./ReconstructionObligationCandidates.js"; +import { evaluateReconstructionObligationLedger } from "./ReconstructionObligationLedgerEvaluation.js"; describe("reconstruction obligation ledger", () => { it("emits the complete deterministic ledger and closure identity", () => { @@ -86,6 +88,34 @@ describe("reconstruction obligation ledger", () => { }); }); +describe("reconstruction obligation ledger Evidence provenance", () => { + it("links observed-case Evidence separately from authority references", () => { + const capture = processEvidence(); + const caseEvidence = proofEvidence("distinct-original-case"); + const bundle = request([capture, caseEvidence]).evidence_bundle; + const generated = deriveReconstructionObligationCandidates(bundle, []); + const candidates = generated.candidates.map((candidate) => ({ + ...candidate, + observed_cases: candidate.observed_cases.map((observedCase) => ({ + ...observedCase, + evidence_id: caseEvidence.evidence_id, + })), + })); + + const result = evaluateReconstructionObligationLedger({ + candidates, + bundle, + manifest: { bindings: [], contradictions: [] }, + generationLimitations: [], + }); + + expect(result.obligations[0]?.authority_references[0]?.evidence_id).toBe( + capture.evidence_id, + ); + expect(result.evidence_links).toContain(caseEvidence.evidence_id); + }); +}); + describe("reconstruction obligation ledger case closure", () => { it("keeps unobserved original cases open", () => { const capture = processEvidence(); @@ -166,45 +196,44 @@ describe("reconstruction obligation ledger case closure", () => { verifier_ids: ["verifier.packaged"], claim_ids: ["claim.process"], }); - const result = build( - request([capture, proof], { - manifest: { - bindings: [ - { - obligation_id: obligation.obligation_id, - owner: { - module_path: "src/processOwner.ts", - symbol: "runProcess", - owner_sha256: "2".repeat(64), - }, - parser_type: null, - original_cases: originalCases(obligation, capture.evidence_id), - fixtures: obligation.required_case_kinds.map((caseKind) => ({ - fixture_id: `packaged.${caseKind}`, - case_kind: caseKind, - authority: "packaged-process", - evidence_ids: [proof.evidence_id], - })), - verifier: { - verifier_id: "verifier.packaged", - claim_id: "claim.process", - command: "npm run verify:package -- process", - authority: "packaged-process", - status: "pass", - result_evidence_id: proof.evidence_id, - enumerated_obligation_ids: [obligation.obligation_id], - nondeterminism: { - mode: "partial-order", - specification: - "Startup precedes teardown; worker completion is unordered.", - }, + const ledgerRequest = request([capture, proof], { + manifest: { + bindings: [ + { + obligation_id: obligation.obligation_id, + owner: { + module_path: "src/processOwner.ts", + symbol: "runProcess", + owner_sha256: "2".repeat(64), + }, + parser_type: null, + original_cases: originalCases(obligation, capture.evidence_id), + fixtures: obligation.required_case_kinds.map((caseKind) => ({ + fixture_id: `packaged.${caseKind}`, + case_kind: caseKind, + authority: "packaged-process", + evidence_ids: [proof.evidence_id], + })), + verifier: { + verifier_id: "verifier.packaged", + claim_id: "claim.process", + command: "npm run verify:package -- process", + authority: "packaged-process", + status: "pass", + result_evidence_id: proof.evidence_id, + enumerated_obligation_ids: [obligation.obligation_id], + nondeterminism: { + mode: "partial-order", + specification: + "Startup precedes teardown; worker completion is unordered.", }, }, - ], - contradictions: [], - }, - }), - ); + }, + ], + contradictions: [], + }, + }); + const result = build(ledgerRequest); expect(result.obligations[0]).toMatchObject({ status: "verified", @@ -212,5 +241,19 @@ describe("reconstruction obligation ledger case closure", () => { }); expect(result.status).toBe("ready"); expect(result.summary.required_open).toBe(0); + + const generated = deriveReconstructionObligationCandidates( + ledgerRequest.evidence_bundle, + ledgerRequest.reviewed_obligations, + ); + const limited = evaluateReconstructionObligationLedger({ + candidates: generated.candidates, + bundle: ledgerRequest.evidence_bundle, + manifest: ledgerRequest.manifest, + generationLimitations: ["A supplementary source was unavailable."], + }); + expect(limited.obligations[0]?.status).toBe("verified"); + expect(limited.status).toBe("unknown"); + expect(limited.coverage.status).toBe("partial"); }); }); diff --git a/src/application/ReconstructionObligationLedgerDiagnostics.ts b/src/application/ReconstructionObligationLedgerDiagnostics.ts index 7d6d84637..5d53d8f4a 100644 --- a/src/application/ReconstructionObligationLedgerDiagnostics.ts +++ b/src/application/ReconstructionObligationLedgerDiagnostics.ts @@ -57,11 +57,6 @@ export const uniqueObligationDiagnostics = ( ), ); -export const hasObligationDiagnostic = ( - diagnostics: readonly Diagnostic[], - code: Diagnostic["code"], -): boolean => diagnostics.some((diagnostic) => diagnostic.code === code); - export const contradictionDiagnostics = ( obligationId: string, contradictions: ReadonlyMap, diff --git a/src/application/ReconstructionObligationLedgerEvaluation.ts b/src/application/ReconstructionObligationLedgerEvaluation.ts index fe7626344..a9366c331 100644 --- a/src/application/ReconstructionObligationLedgerEvaluation.ts +++ b/src/application/ReconstructionObligationLedgerEvaluation.ts @@ -9,6 +9,17 @@ import { type ReconstructionObligationLedger, type ReconstructionObligationManifest, } from "../domain/reconstructionObligationLedgerSchemas.js"; +import { digestCanonicalValue } from "../domain/canonicalDigest.js"; +import { + reconstructionObligationCoverage, + reconstructionObligationDependencyGraph, + reconstructionObligationLedgerClosureDigest, + reconstructionObligationLedgerStatus, + hasReconstructionObligationDiagnostic as hasDiagnostic, + reconstructionObligationOwnershipGraph, + reconstructionObligationReports, + summarizeReconstructionObligations, +} from "../domain/reconstructionObligationLedgerIntegrity.js"; import type { ReconstructionObligationCandidate } from "./ReconstructionObligationCandidates.js"; import { evidenceAuthoritySupportsProof, @@ -20,22 +31,13 @@ import { contradictionDiagnostics, groupManifestBindings, groupManifestContradictions, - hasObligationDiagnostic as hasDiagnostic, uniqueObligationDiagnostics as uniqueDiagnostics, } from "./ReconstructionObligationLedgerDiagnostics.js"; import { originalCaseDiagnostics, originalCasesFor, } from "./ReconstructionObligationOriginalCases.js"; -import { - digestObligationLedgerValue, - obligationLedgerCoverage, - obligationEdgeOrder, - obligationLedgerEvidenceLinks, - obligationLedgerStatus, - obligationReports, - summarizeObligations, -} from "./ReconstructionObligationLedgerProjection.js"; +import { obligationLedgerEvidenceLinks } from "./ReconstructionObligationLedgerProjection.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; type Diagnostic = ReconstructionObligation["diagnostics"][number]; @@ -70,64 +72,41 @@ export const evaluateReconstructionObligationLedger = ({ candidates.map((candidate) => evaluateObligation(candidate, context)), context, ); - const ownershipGraph = obligations - .flatMap((obligation) => - obligation.binding === null - ? [] - : [ - { - obligation_id: obligation.obligation_id, - module_path: obligation.binding.owner.module_path, - symbol: obligation.binding.owner.symbol, - }, - ], - ) - .sort(obligationEdgeOrder); - const dependencyGraph = obligations - .flatMap((obligation) => - obligation.dependency_obligation_ids.map((dependencyId) => ({ - obligation_id: obligation.obligation_id, - depends_on_obligation_id: dependencyId, - })), - ) - .sort(obligationEdgeOrder); + const ownershipGraph = reconstructionObligationOwnershipGraph(obligations); + const dependencyGraph = reconstructionObligationDependencyGraph(obligations); const evidenceLinks = obligationLedgerEvidenceLinks(obligations, manifest); const limitations = ledgerLimitations( generationLimitations, allCandidates.length, orphanManifestIds, ); - const coverage = obligationLedgerCoverage(limitations.length > 0); + const coverage = reconstructionObligationCoverage(limitations); const semantic = { coverage, - summary: summarizeObligations(obligations), - reports: obligationReports(obligations), + summary: summarizeReconstructionObligations(obligations), + reports: reconstructionObligationReports(obligations), ownership_graph: ownershipGraph, dependency_graph: dependencyGraph, obligations, evidence_links: evidenceLinks, limitations, }; - const evaluatedStatus = obligationLedgerStatus(obligations); - const status = - evaluatedStatus === "failed" - ? evaluatedStatus - : limitations.length > 0 - ? "unknown" - : evaluatedStatus; - const closureDigest = digestObligationLedgerValue({ - schema: "ReconstructionObligationClosure", + const status = reconstructionObligationLedgerStatus(obligations, limitations); + const closureDigest = reconstructionObligationLedgerClosureDigest({ status, ...semantic, }); return reconstructionObligationLedgerSchema.parse({ schema: "ReconstructionObligationLedger", - ledger_id: `rol_${digestObligationLedgerValue({ - schema: "ReconstructionObligationLedger", - evidence_ids: bundle.records.map(({ evidence_id: id }) => id), - obligation_ids: obligations.map(({ obligation_id: id }) => id), - closure_digest: closureDigest, - })}`, + ledger_id: `rol_${digestCanonicalValue( + { + schema: "ReconstructionObligationLedger", + evidence_ids: bundle.records.map(({ evidence_id: id }) => id), + obligation_ids: obligations.map(({ obligation_id: id }) => id), + closure_digest: closureDigest, + }, + "Reconstruction obligation ledger", + )}`, closure_digest: closureDigest, status, ...semantic, diff --git a/src/application/ReconstructionObligationLedgerProjection.ts b/src/application/ReconstructionObligationLedgerProjection.ts index f77a6c9d1..4976465bf 100644 --- a/src/application/ReconstructionObligationLedgerProjection.ts +++ b/src/application/ReconstructionObligationLedgerProjection.ts @@ -1,83 +1,9 @@ -import { createHash } from "node:crypto"; - -import canonicalize from "canonicalize"; - -import { canonicalJson } from "../domain/comparisonSemantics.js"; -import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; - import type { ReconstructionObligation, - ReconstructionObligationLedger, ReconstructionObligationManifest, } from "../domain/reconstructionObligationLedgerSchemas.js"; - -type DiagnosticCode = ReconstructionObligation["diagnostics"][number]["code"]; - -export const obligationLedgerCoverage = ( - incomplete: boolean, -): ReconstructionObligationLedger["coverage"] => ({ - status: incomplete ? "partial" : "complete", -}); - -export const summarizeObligations = ( - obligations: readonly ReconstructionObligation[], -): ReconstructionObligationLedger["summary"] => ({ - total: obligations.length, - required: obligations.filter(({ required }) => required).length, - verified: obligations.filter(({ status }) => status === "verified").length, - required_open: obligations.filter( - ({ required, status }) => required && status !== "verified", - ).length, - by_status: countBy( - obligations, - [ - "unowned", - "characterized", - "implemented", - "verified", - "contradicted", - "blocked", - "out-of-scope", - "unknown", - ], - ({ status }) => status, - ), - by_application_layer: countBy( - obligations, - [ - "cli", - "protocol", - "electron", - "persistence", - "process", - "runtime", - "packaging", - "native-abi", - "application", - "other", - ], - ({ application_layer: layer }) => layer, - ), - by_evidence_authority: countBy( - obligations, - ["candidate", "observed", "reviewed", "unknown"], - ({ source_state: state }) => state, - ), -}); - -export const obligationReports = ( - obligations: readonly ReconstructionObligation[], -): ReconstructionObligationLedger["reports"] => ({ - missing_owner_obligation_ids: obligationIdsWith(obligations, "missing-owner"), - missing_verifier_obligation_ids: obligationIdsWith( - obligations, - "missing-verifier", - ), - contradicted_obligation_ids: obligationIdsWith(obligations, "contradiction"), - residual_unknown_ids: [ - ...new Set(obligations.flatMap(({ residual_unknown_ids: ids }) => ids)), - ].sort(), -}); +import { reconstructionObligationEvidenceIds } from "../domain/reconstructionObligationLedgerIntegrity.js"; +import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; export const obligationLedgerEvidenceLinks = ( obligations: readonly ReconstructionObligation[], @@ -85,9 +11,7 @@ export const obligationLedgerEvidenceLinks = ( ): string[] => [ ...new Set([ - ...obligations.flatMap(({ authority_references: references }) => - references.map(({ evidence_id: id }) => id), - ), + ...reconstructionObligationEvidenceIds(obligations), ...manifest.bindings.flatMap((binding) => [ ...binding.original_cases.map(({ evidence_id: id }) => id), ...binding.fixtures.flatMap(({ evidence_ids: ids }) => ids), @@ -97,62 +21,4 @@ export const obligationLedgerEvidenceLinks = ( ]), ...manifest.contradictions.flatMap(({ evidence_ids: ids }) => ids), ]), - ].sort(); - -export const obligationLedgerStatus = ( - obligations: readonly ReconstructionObligation[], -): ReconstructionObligationLedger["status"] => { - if (obligations.length === 0) return "unknown"; - const requiredOpen = obligations.filter( - ({ required, status }) => required && status !== "verified", - ); - if ( - requiredOpen.some( - ({ status, diagnostics }) => - status === "contradicted" || - hasDiagnostic(diagnostics, "verifier-failed"), - ) - ) - return "failed"; - if (requiredOpen.some(({ status }) => status === "unknown")) return "unknown"; - return requiredOpen.length === 0 ? "ready" : "open"; -}; - -export const obligationEdgeOrder = ( - left: { readonly obligation_id: string }, - right: { readonly obligation_id: string }, -): number => - compareUnicodeCodePoints(canonicalJson(left), canonicalJson(right)); - -export const digestObligationLedgerValue = (value: unknown): string => { - const encoded = canonicalize(value); - if (encoded === undefined) - throw new TypeError( - "Reconstruction obligation ledger is not canonical JSON", - ); - return createHash("sha256").update(encoded).digest("hex"); -}; - -const countBy = ( - items: readonly Item[], - values: readonly Value[], - valueFor: (item: Item) => Value, -): { key: Value; count: number }[] => - values.map((value) => ({ - key: value, - count: items.filter((item) => valueFor(item) === value).length, - })); - -const obligationIdsWith = ( - obligations: readonly ReconstructionObligation[], - code: DiagnosticCode, -): string[] => - obligations - .filter(({ diagnostics }) => hasDiagnostic(diagnostics, code)) - .map(({ obligation_id: id }) => id) - .sort(); - -const hasDiagnostic = ( - diagnostics: ReconstructionObligation["diagnostics"], - code: DiagnosticCode, -): boolean => diagnostics.some((diagnostic) => diagnostic.code === code); + ].sort(compareUnicodeCodePoints); diff --git a/src/application/RegularFileRead.ts b/src/application/RegularFileRead.ts index 1c3852326..9183c9f8a 100644 --- a/src/application/RegularFileRead.ts +++ b/src/application/RegularFileRead.ts @@ -16,6 +16,7 @@ import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js" export interface RegularFileReadOptions { readonly signal?: AbortSignal | undefined; readonly symlinks?: "follow" | "reject"; + readonly resources?: ArtifactResourceScope; } export type RegularFileReadOutcome = @@ -68,6 +69,15 @@ export const withRegularFile = async ( path: string, read: (handle: FileHandle, stats: Stats) => Promise, options: RegularFileReadOptions = {}, +): Promise => + options.resources === undefined + ? withRegularFileOwned(path, read, options) + : options.resources.run(() => withRegularFileOwned(path, read, options)); + +const withRegularFileOwned = async ( + path: string, + read: (handle: FileHandle, stats: Stats) => Promise, + options: RegularFileReadOptions, ): Promise => { const { signal } = options; let handle: FileHandle; @@ -78,12 +88,15 @@ export const withRegularFile = async ( }); } catch (cause: unknown) { if (!(cause instanceof RegularFileAdmissionFailure)) throw cause; - throw new RegularFileCleanupFailure( + const failure = new RegularFileCleanupFailure( path, cause.owner, { kind: "failed", cause: cause.cause }, cause.cleanupCause, ); + if (options.resources === undefined) throw failure; + await retryRegularFileCleanup(failure, options.resources); + throw cause.cause; } const owner = new OwnedFileHandle(handle); let outcome: RegularFileReadOutcome; @@ -108,7 +121,16 @@ export const withRegularFile = async ( try { await owner.close(); } catch (cleanupCause: unknown) { - throw new RegularFileCleanupFailure(path, owner, outcome, cleanupCause); + const failure = new RegularFileCleanupFailure( + path, + owner, + outcome, + cleanupCause, + ); + if (options.resources === undefined) throw failure; + const cleanup = await retryRegularFileCleanup(failure, options.resources); + if (cleanup.outcome.kind === "failed") throw cleanup.outcome.cause; + return cleanup.outcome.value; } if (outcome.kind === "failed") throw outcome.cause; return outcome.value; diff --git a/src/application/SetupClientConfiguration.ts b/src/application/SetupClientConfiguration.ts index 3f18d5023..ac493282a 100644 --- a/src/application/SetupClientConfiguration.ts +++ b/src/application/SetupClientConfiguration.ts @@ -12,6 +12,7 @@ import { withClientServers, type ClientConfigurationDocument, } from "./ClientConfigurationDocument.js"; +import { dshDocument, DshCordisDocument } from "./DshCordisPatch.js"; import { constants as fsConstants } from "node:fs"; import { copyFile, mkdir, rm, writeFile } from "node:fs/promises"; import { dirname } from "node:path"; @@ -89,7 +90,10 @@ const configureClientDocument = async ( let parsed: ClientConfigurationDocument; try { parsed = parseClientConfiguration( - original ?? (format === "toml" || format === "grok" ? "" : "{}"), + original ?? + (format === "toml" || format === "grok" || format === "dsh" + ? "" + : "{}"), format, ); const policy = await readClientPolicyBlock( @@ -396,6 +400,7 @@ const registrationCurrent = ( parsed: ClientConfigurationDocument, desired: unknown, ): boolean => + (parsed.dialect !== "dsh" || dshDocument(parsed.document).ready) && clientConfigurationValuesEqual( parsed.servers[PRODUCT_IDENTITY.mcpServerKey], desired, @@ -410,6 +415,17 @@ const clientConfigurationDesired = ( parsed?: ClientConfigurationDocument, ) => { const format = parsed?.dialect ?? client.format ?? "json"; + const launcher = + command.length === 0 ? [PRODUCT_IDENTITY.cliBinary, "mcp"] : command; + if (format === "dsh") { + const desired = ( + parsed === undefined + ? new DshCordisDocument("") + : dshDocument(parsed.document) + ).desired(launcher, providerEnvironment); + if (!desired.ok) throw desired.error; + return desired.value; + } // Preserve explicitly configured server settings, never the ambient process // environment. Newly discovered provider paths win only for their own keys. const environment = clientRegistrationEnvironment( @@ -419,7 +435,7 @@ const clientConfigurationDesired = ( if (!environment.ok) throw environment.error; const registration = clientRegistrationEntry( format, - command.length === 0 ? [PRODUCT_IDENTITY.cliBinary, "mcp"] : command, + launcher, environment.value, ); return { diff --git a/src/application/SetupHost.ts b/src/application/SetupHost.ts index 9f44b8d63..9e2ae95f1 100644 --- a/src/application/SetupHost.ts +++ b/src/application/SetupHost.ts @@ -122,7 +122,7 @@ export const systemSetupHost = ( nodeVersion: process.versions.node, macosVersion: () => doctorHost.macosVersion(), linuxDistribution: readLinuxDistribution, - close: () => closeSetupResources(resources, macHopperHost), + close: () => closeSetupResources(resources, macHopperHost, doctorHost), initialSetupState: async ( scope?: DoctorScope, ): Promise => { @@ -181,9 +181,17 @@ export const systemSetupHost = ( clientIds, environment, platform, + resources, ), installSkill: (clientIds) => - installCanonicalSkill(homeDirectory, clientIds, environment, platform), + installCanonicalSkill( + homeDirectory, + clientIds, + environment, + platform, + undefined, + resources, + ), doctor: (scope) => runDoctor(undefined, doctorHost, scope), }; }; @@ -226,11 +234,14 @@ const clientConfigurationOperations = (resources: ArtifactResourceScope) => ({ const closeSetupResources = async ( resources: ArtifactResourceScope, macHopperHost: ReturnType | undefined, + doctorHost: DoctorHost, ): Promise => { - const [resourceCleanup, hopperCleanup] = await Promise.allSettled([ - resources.close(), - Promise.resolve().then(() => macHopperHost?.close()), - ]); + const [resourceCleanup, hopperCleanup, doctorCleanup] = + await Promise.allSettled([ + resources.close(), + Promise.resolve().then(() => macHopperHost?.close()), + Promise.resolve().then(() => doctorHost.close?.()), + ]); const failures: string[] = []; if (hopperCleanup.status === "rejected") failures.push( @@ -238,6 +249,12 @@ const closeSetupResources = async ( ); else if (hopperCleanup.value !== undefined) failures.push(hopperCleanup.value); + if (doctorCleanup.status === "rejected") + failures.push( + `Doctor host cleanup failed: ${errorMessage(doctorCleanup.reason)}`, + ); + else if (doctorCleanup.value !== undefined) + failures.push(doctorCleanup.value); if (resourceCleanup.status === "rejected") failures.push(errorMessage(resourceCleanup.reason)); return failures.length === 0 ? undefined : failures.join("; "); diff --git a/src/application/SetupSkill.ts b/src/application/SetupSkill.ts index 6bbcdd0c7..83ba96c5a 100644 --- a/src/application/SetupSkill.ts +++ b/src/application/SetupSkill.ts @@ -4,13 +4,18 @@ import { dirname, join } from "node:path"; import writeFileAtomic from "write-file-atomic"; import { err, ok, type Result } from "../domain/result.js"; import { readBoundedFileBytes } from "../process/BoundedFileBytes.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; +import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; import { PRODUCT_IDENTITY } from "../identity.js"; import { NonRegularFileReadError, RegularFileChangedError, } from "../filesystem/RegularFile.js"; -import { withRegularFile } from "./RegularFileRead.js"; +import { + RegularFileCleanupFailure, + withRegularFile, +} from "./RegularFileRead.js"; import { clientSkillDirectories } from "./SupportedClients.js"; export const MANAGED_SKILL_FILES = [ @@ -27,15 +32,21 @@ export const isManagedSkillManifest = (content: string): boolean => content.startsWith(`---\nname: ${PRODUCT_IDENTITY.skillName}\n`) || content.startsWith(`---\r\nname: ${PRODUCT_IDENTITY.skillName}\r\n`); -export const readSkillFile = (path: string): Promise => - readSkillFileState(path).then(({ content }) => content); +export const readSkillFile = ( + path: string, + resources?: ArtifactResourceScope, +): Promise => + readSkillFileState(path, resources).then(({ content }) => content); interface SkillFileState { readonly content: string; readonly mode: number; } -const readSkillFileState = async (path: string): Promise => +const readSkillFileState = async ( + path: string, + resources?: ArtifactResourceScope, +): Promise => withRegularFile( path, async (handle, stats) => { @@ -49,7 +60,10 @@ const readSkillFileState = async (path: string): Promise => throw new RegularFileChangedError(path); return { content: bytes.toString("utf8"), mode: stats.mode & 0o7777 }; }, - { symlinks: "reject" }, + { + symlinks: "reject", + ...(resources === undefined ? {} : { resources }), + }, ); interface CanonicalSkillFile { @@ -173,6 +187,7 @@ export const existingSkillDestinations = async ( home: string, environment: Readonly = {}, platform: NodeJS.Platform = process.platform, + resources?: ArtifactResourceScope, ): Promise => { const existing: SkillDestination[] = []; for (const destination of skillDestinations( @@ -185,11 +200,16 @@ export const existingSkillDestinations = async ( const root = await lstat(destination.path); if (root.isSymbolicLink() || !root.isDirectory()) continue; const manifest = join(destination.path, "SKILL.md"); - const content = await readSkillFile(manifest); + const content = await readSkillFile(manifest, resources); if (isManagedSkillManifest(content)) existing.push(destination); } catch (cause: unknown) { if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") continue; + if ( + cause instanceof RegularFileCleanupFailure || + cause instanceof ArtifactReaderFailure + ) + throw cause; if (cause instanceof NonRegularFileReadError) continue; throw cause; } @@ -199,9 +219,10 @@ export const existingSkillDestinations = async ( const readOptionalSkillState = async ( path: string, + resources?: ArtifactResourceScope, ): Promise => { try { - return await readSkillFileState(path); + return await readSkillFileState(path, resources); } catch (cause: unknown) { if (cause instanceof Error && "code" in cause && cause.code === "ENOENT") return undefined; @@ -214,6 +235,7 @@ const canonicalSkillFiles = async ( clientIds: readonly string[], environment: Readonly = {}, platform: NodeJS.Platform = process.platform, + resources?: ArtifactResourceScope, ): Promise => Promise.all( skillDestinations(home, clientIds, environment, platform).flatMap( @@ -229,7 +251,7 @@ const canonicalSkillFiles = async ( ), "utf8", ), - original: await readOptionalSkillState(destination), + original: await readOptionalSkillState(destination, resources), }; }), ), @@ -241,12 +263,24 @@ export const canonicalSkillNeedsInstall = async ( clientIds: readonly string[] = [], environment: Readonly = {}, platform: NodeJS.Platform = process.platform, + resources?: ArtifactResourceScope, ): Promise => { try { return ( - await canonicalSkillFiles(home, clientIds, environment, platform) + await canonicalSkillFiles( + home, + clientIds, + environment, + platform, + resources, + ) ).some(({ content, original }) => original?.content !== content); } catch (cause: unknown) { + if ( + cause instanceof RegularFileCleanupFailure || + cause instanceof ArtifactReaderFailure + ) + throw cause; // Unreadable skill state fails open to install so setup can repair it. void cause; return true; @@ -277,7 +311,6 @@ const restoreSkillFiles = async ( return failures; }; -/** Transactionally install or upgrade the canonical REA skill and references. */ /** Transactionally install the canonical skill bundle and optional references. */ export const installCanonicalSkill = async ( home: string, @@ -285,6 +318,7 @@ export const installCanonicalSkill = async ( environment: Readonly = {}, platform: NodeJS.Platform = process.platform, operations: SkillInstallFileOperations = systemSkillInstallFileOperations, + resources?: ArtifactResourceScope, ): Promise => { let attempted: CanonicalSkillFile[] = []; let stage: "prepare" | "backup" | "write" | "readback" = "prepare"; @@ -295,6 +329,7 @@ export const installCanonicalSkill = async ( clientIds, environment, platform, + resources, ); const changed = canonical.filter( ({ content, original }) => original?.content !== content, @@ -323,7 +358,7 @@ export const installCanonicalSkill = async ( stage = "readback"; for (const { destination, content } of changed) { failingPath = destination; - if ((await readSkillFile(destination)) !== content) + if ((await readSkillFile(destination, resources)) !== content) throw new Error(`skill readback mismatch: ${destination}`); } return ok("installed"); @@ -351,6 +386,7 @@ export const readInstalledSkillIdentity = async ( clientIds: readonly string[], environment: Readonly = {}, platform: NodeJS.Platform = process.platform, + resources?: ArtifactResourceScope, ): Promise => { const destinations = skillDestinations( home, @@ -361,22 +397,38 @@ export const readInstalledSkillIdentity = async ( let content: string | undefined; for (const destination of destinations) { try { - content = await readSkillFile(join(destination.path, "SKILL.md")); + content = await readSkillFile( + join(destination.path, "SKILL.md"), + resources, + ); break; - } catch { + } catch (cause: unknown) { + if ( + cause instanceof RegularFileCleanupFailure || + cause instanceof ArtifactReaderFailure + ) + throw cause; // Another selected copy may provide metadata; canonical validation below // still marks missing or unreadable copies as stale. } } if (content === undefined) return undefined; const toolCount = /^\s{2}tool_count:\s*(\d+)\s*$/mu.exec(content)?.[1]; - return { - canonical: !(await canonicalSkillNeedsInstall( + let canonical: boolean; + try { + canonical = !(await canonicalSkillNeedsInstall( home, clientIds, environment, platform, - )), + resources, + )); + } catch (cause: unknown) { + if (!(cause instanceof ArtifactReaderFailure)) throw cause; + canonical = false; + } + return { + canonical, version: /^\s{2}version:\s*"([^"]+)"\s*$/mu.exec(content)?.[1] ?? null, toolCount: toolCount === undefined ? null : Number.parseInt(toolCount, 10), }; diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index c4b0bc6e9..b44e5017a 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -38,6 +38,7 @@ export interface SetupClient { | "omp" | "pi" | "hermes" + | "dsh" | "unsupported"; } @@ -129,6 +130,12 @@ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => { const grokDirectory = ({ home, env }: ClientPathContext): string => env.GROK_HOME || join(home, ".grok"); +const dshHomeDirectory = ({ home, env }: ClientPathContext): string => + env.DSH_HOME ?? join(home, ".dsh"); + +const dshPatchDirectory = (context: ClientPathContext): string => + join(dshHomeDirectory(context), "cordis.patch.yml"); + const hermesDirectory = ({ home, platform, env }: ClientPathContext): string => resolveHermesClientHome(home, platform, env); @@ -492,6 +499,17 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ markerPath: hermesDirectory, format: "hermes", }, + { + name: "dsh", + displayName: "DeepSeek Harness", + // The home Cordis patch applies to every profile; the skill directory is + // shared agent storage Harness discovers. + skillPath: ({ home, env }: ClientPathContext) => + join(env.DSH_AGENTS_HOME ?? join(home, ".agents"), "skills"), + configPath: dshPatchDirectory, + markerPath: dshHomeDirectory, + format: "dsh", + }, { name: "grok_bot", displayName: "Grok Bot", diff --git a/src/application/Uninstall.ts b/src/application/Uninstall.ts index d363f950d..d474419a8 100644 --- a/src/application/Uninstall.ts +++ b/src/application/Uninstall.ts @@ -22,8 +22,10 @@ import { skillDestinations, } from "./SetupSkill.js"; import { isOwnedClientRegistrationCommand } from "./ClientRegistrationIdentity.js"; +import { dshDocument } from "./DshCordisPatch.js"; import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js"; import { readRegularFileText } from "./RegularFileRead.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { manualRegistrationRemediation, supportedClients, @@ -50,9 +52,11 @@ export interface UninstallFileSystem { remove(path: string, recursive?: boolean): Promise; } -const systemFileSystem: UninstallFileSystem = { - readText: (path) => readRegularFileText(path), - readRegularText: readSkillFile, +const systemFileSystem = ( + resources: ArtifactResourceScope, +): UninstallFileSystem => ({ + readText: (path) => readRegularFileText(path, { resources }), + readRegularText: (path) => readSkillFile(path, resources), copy: (source, destination) => copyFile(source, destination, fsConstants.COPYFILE_EXCL), writeText: (path, contents) => @@ -60,7 +64,7 @@ const systemFileSystem: UninstallFileSystem = { stat: (path) => lstat(path), realpath: (path) => realpath(path), remove: (path, recursive = true) => rm(path, { recursive }), -}; +}); /** One explicitly classified uninstall action. */ export interface UninstallItem { @@ -83,6 +87,7 @@ export interface UninstallHost { removeClient(client: SetupClient): Promise; removeSkill(): Promise; purgeData(): Promise; + close?(): Promise; } /** @@ -95,35 +100,66 @@ export const runUninstall = async ( purgeData: boolean, host: UninstallHost = systemUninstallHost(), ): Promise => { - const clients = await host.clients(); - const blocked: UninstallItem[] = []; - for (const client of clients) { - const blocking = await host.inspectClient(client); - if (blocking !== undefined) blocked.push(blocking); + let result: UninstallResult | undefined; + let primaryFailure: unknown; + let failed = false; + try { + const clients = await host.clients(); + const blocked: UninstallItem[] = []; + for (const client of clients) { + const blocking = await host.inspectClient(client); + if (blocking !== undefined) blocked.push(blocking); + } + const items: UninstallItem[] = + blocked.length > 0 + ? [ + ...blocked, + item( + "uninstall", + "skipped", + "No REA registration, skill, or data was removed. Repair each failed configuration, then rerun uninstall.", + ), + ] + : await removeAll(clients, purgeData, host); + items.push({ + name: "analysis_engine", + status: "retained", + detail: "Hopper is not owned by REA uninstall.", + }); + const reported = collapseUninstallItems(items); + result = { + status: reported.some(({ status }) => status === "failed") + ? "failed" + : "complete", + items: reported, + }; + } catch (cause: unknown) { + failed = true; + primaryFailure = cause; } - const items: UninstallItem[] = - blocked.length > 0 - ? [ - ...blocked, - item( - "uninstall", - "skipped", - "No REA registration, skill, or data was removed. Repair each failed configuration, then rerun uninstall.", - ), - ] - : await removeAll(clients, purgeData, host); - items.push({ - name: "analysis_engine", - status: "retained", - detail: "Hopper is not owned by REA uninstall.", - }); - const reported = collapseUninstallItems(items); - return { - status: reported.some(({ status }) => status === "failed") - ? "failed" - : "complete", - items: reported, - }; + let cleanupFailure: string | undefined; + try { + cleanupFailure = await host.close?.(); + } catch (cause: unknown) { + cleanupFailure = `Uninstall resource cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + } + if (failed) { + if (cleanupFailure !== undefined) + throw new AggregateError( + [primaryFailure, new Error(cleanupFailure)], + "Uninstall failed and its resource cleanup remains incomplete.", + { cause: primaryFailure }, + ); + throw primaryFailure; + } + if (result === undefined) + throw new Error("Uninstall result was not produced."); + if (cleanupFailure === undefined) return result; + const items = collapseUninstallItems([ + ...result.items, + item("uninstall_cleanup", "failed", cleanupFailure), + ]); + return { status: "failed", items }; }; /** @@ -157,12 +193,14 @@ const removeAll = async ( /** Create uninstall effects contained to detected client configs and REA-owned paths. */ export const systemUninstallHost = ( selectedHome: string | undefined = undefined, - fileSystem: UninstallFileSystem = systemFileSystem, + selectedFileSystem: UninstallFileSystem | undefined = undefined, environment: Readonly = process.env, platform: NodeJS.Platform = process.platform, ): UninstallHost => { const home = selectedHome ?? homeDirectoryFromEnvironment(environment, platform); + const resources = new ArtifactResourceScope(); + const fileSystem = selectedFileSystem ?? systemFileSystem(resources); return { clients: () => Promise.resolve( @@ -185,6 +223,14 @@ export const systemUninstallHost = ( removeSkill: () => removeManagedSkills(home, fileSystem, environment, platform), purgeData: () => purgeManagedData(join(home, ".rea"), fileSystem), + close: async () => { + try { + await resources.close(); + return undefined; + } catch (cause: unknown) { + return `Uninstall resource cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + } + }, }; }; @@ -260,10 +306,15 @@ const removeClient = async ( // Beside a native OpenCode V2 table, OpenCode also loads a V1 `mcp.rea`. const legacyPath = legacyClientServerPath(parsed, name); const hasNative = Object.hasOwn(parsed.servers, name); - if (!hasNative && legacyPath === undefined) + // Harness ownership is the stable entry id. A stale or malformed launcher + // still belongs to that id, and so does a direct patch that projects no row. + const carriesDshEntry = + parsed.dialect === "dsh" && dshDocument(parsed.document).carriesEntry; + if (!hasNative && legacyPath === undefined && !carriesDshEntry) return item(client.name, "skipped", "REA registration is absent."); const removeNative = - hasNative && isOwnedRegistration(parsed.servers[name], client); + carriesDshEntry || + (hasNative && isOwnedRegistration(parsed.servers[name], client)); const removeLegacy = legacyPath !== undefined && isOwnedRegistration(parsed.legacyServers[name], client); @@ -311,6 +362,8 @@ const removeClient = async ( ); if ( (removeNative && Object.hasOwn(readback.servers, name)) || + (readback.dialect === "dsh" && + dshDocument(readback.document).carriesEntry) || (removeLegacy && Object.hasOwn(readback.legacyServers, name)) ) throw new Error("registration readback mismatch"); diff --git a/src/application/Update.test.ts b/src/application/Update.test.ts index 26c580ab0..241cb7cd0 100644 --- a/src/application/Update.test.ts +++ b/src/application/Update.test.ts @@ -18,6 +18,7 @@ import { type NpmInstallationHost, } from "./UpdateRuntime.js"; import type { IntegrationMaintenance } from "./UpdateMaintenance.js"; +import { isUpdateFailure } from "./UpdateFailure.js"; class FakeUpdateHost implements UpdateHost { release: Result = ok("3.3.0"); @@ -37,6 +38,7 @@ class FakeUpdateHost implements UpdateHost { verifiedInstallation: UpdateInstallation | undefined; maintenanceInspected = false; output: UpdateOutput | undefined; + cleanupFailure: string | undefined; installation = (): Promise => { this.installationInspected = true; @@ -63,6 +65,8 @@ class FakeUpdateHost implements UpdateHost { this.maintenanceInspected = true; return Promise.resolve(this.maintenance); }; + close = (): Promise => + Promise.resolve(this.cleanupFailure); } describe("REA update", () => { @@ -79,6 +83,20 @@ describe("REA update", () => { expect(host.maintenanceInspected).toBe(false); }); + it("preserves update status while marking incomplete resource cleanup as failure", async () => { + const host = new FakeUpdateHost(); + host.cleanupFailure = "Regular file remains open: /fixture/skill/SKILL.md"; + + const result = await runUpdate("3.3.0", host); + + expect(result).toMatchObject({ + status: "current", + currentVersion: "3.3.0", + cleanupFailure: host.cleanupFailure, + }); + expect(isUpdateFailure(result)).toBe(true); + }); + it("reports an unknown current installation honestly", async () => { const host = new FakeUpdateHost(); host.installationResult = undefined; diff --git a/src/application/Update.ts b/src/application/Update.ts index 6a92b46a5..b9344bfc1 100644 --- a/src/application/Update.ts +++ b/src/application/Update.ts @@ -49,10 +49,11 @@ export interface UpdateHost { planMaintenance( installation: UpdateInstallation, ): Promise; + close?(): Promise; } /** Verified update outcome, separate from the unapplied integration plan. */ -export type UpdateResult = +export type UpdateResult = ( | { readonly status: "current"; readonly currentVersion: string; @@ -80,7 +81,8 @@ export type UpdateResult = | "verification"; readonly installedVersion?: string; readonly remediation: string; - }; + } +) & { readonly cleanupFailure?: string }; /** Resolve the exact package-manager command for the identified installation and release. */ export const updateInstallCommand = ( @@ -141,6 +143,39 @@ export const runUpdate = async ( currentVersion: string, host: UpdateHost, output: UpdateOutput = "human", +): Promise => { + let result: UpdateResult | undefined; + let primaryFailure: unknown; + let failed = false; + try { + result = await runUpdateWorkflow(currentVersion, host, output); + } catch (cause: unknown) { + failed = true; + primaryFailure = cause; + } + let cleanupFailure: string | undefined; + try { + cleanupFailure = await host.close?.(); + } catch (cause: unknown) { + cleanupFailure = `Update resource cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + } + if (failed) { + if (cleanupFailure !== undefined) + throw new AggregateError( + [primaryFailure, new Error(cleanupFailure)], + "Update failed and its resource cleanup remains incomplete.", + { cause: primaryFailure }, + ); + throw primaryFailure; + } + if (result === undefined) throw new Error("Update result was not produced."); + return cleanupFailure === undefined ? result : { ...result, cleanupFailure }; +}; + +const runUpdateWorkflow = async ( + currentVersion: string, + host: UpdateHost, + output: UpdateOutput, ): Promise => { const installation = await host.installation(); const release = await host.latestVersion(installation); diff --git a/src/application/UpdateFailure.ts b/src/application/UpdateFailure.ts index d7791eaab..85503a3c3 100644 --- a/src/application/UpdateFailure.ts +++ b/src/application/UpdateFailure.ts @@ -2,6 +2,7 @@ import type { UpdateResult } from "./Update.js"; /** Whether the update command left the requested release unapplied. */ export const isUpdateFailure = (result: UpdateResult): boolean => { + if (result.cleanupFailure !== undefined) return true; const status = result.status; switch (status) { case "current": diff --git a/src/application/UpdateMaintenance.ts b/src/application/UpdateMaintenance.ts index 2a5a286d5..06ac1b512 100644 --- a/src/application/UpdateMaintenance.ts +++ b/src/application/UpdateMaintenance.ts @@ -14,6 +14,7 @@ import { existingSkillDestinations } from "./SetupSkill.js"; import type { SetupAction } from "./SetupTypes.js"; import type { Result } from "../domain/result.js"; import { readRegularFileText } from "./RegularFileRead.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; /** Existing REA integrations selected for maintenance, without discovering new targets. */ export interface MaintenanceScope { @@ -58,9 +59,11 @@ export const existingMaintenanceScope = async ( home: string, entryPoint: string, environment: Readonly, + resources?: ArtifactResourceScope, ): Promise => { const registrations = await readClientRegistrationStatuses(home, entryPoint, { environment, + ...(resources === undefined ? {} : { resources }), }); const supported = supportedClients(home, process.platform, environment); const clients: string[] = []; @@ -73,7 +76,10 @@ export const existingMaintenanceScope = async ( const client = supported.find(({ name }) => name === registration.client); if (client === undefined) continue; const parsed = parseClientConfiguration( - await readRegularFileText(client.configPath), + await readRegularFileText( + client.configPath, + resources === undefined ? {} : { resources }, + ), client.format, ); const enabled = z @@ -91,7 +97,12 @@ export const existingMaintenanceScope = async ( ) clients.push(client.name); } - const skillDestinations = await existingSkillDestinations(home, environment); + const skillDestinations = await existingSkillDestinations( + home, + environment, + process.platform, + resources, + ); return { clients, skill: skillDestinations.length > 0, @@ -105,9 +116,15 @@ export const planIntegrationMaintenance = async ( entryPoint: string, environment: Readonly, execute: (command: readonly string[]) => Promise>, + resources?: ArtifactResourceScope, ): Promise => { try { - const scope = await existingMaintenanceScope(home, entryPoint, environment); + const scope = await existingMaintenanceScope( + home, + entryPoint, + environment, + resources, + ); const skillDestinations = scope.skillDestinations ?? []; if (scope.clients.length === 0 && !scope.skill) return { status: "current", plannedActions: [] }; diff --git a/src/application/UpdateRuntime.ts b/src/application/UpdateRuntime.ts index e2e919404..b4986cbcf 100644 --- a/src/application/UpdateRuntime.ts +++ b/src/application/UpdateRuntime.ts @@ -6,6 +6,7 @@ import spawn from "cross-spawn"; import { lt, valid } from "semver"; import { PRODUCT_IDENTITY } from "../identity.js"; +import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { err, ok, type Result } from "../domain/result.js"; import { safeParseJson } from "../domain/safeJson.js"; import { @@ -292,74 +293,86 @@ export const systemUpdateHost = ( packageRoot = fileURLToPath(new URL("../..", import.meta.url)), home: string | undefined = undefined, environment: Readonly = process.env, -): UpdateHost => ({ - installation: async () => - (await detectBunInstallation( - packageRoot, - bunGlobalDirectory(environment, process.platform), - { canonicalPath: realpath, isFile }, - )) ?? - detectNpmInstallation(packageRoot, { - canonicalPath: realpath, - globalRoot: () => - requireCommandOutput(["npm", "root", "--global"], environment), - globalPrefix: () => - requireCommandOutput(["npm", "prefix", "--global"], environment), - }), - latestVersion: async (installation) => { - const context = packageManagerContext(installation, environment); - if (installation?.manager === "bun") { - const version = await runUpdateCommand( - ["bun", "--version"], +): UpdateHost => { + const resources = new ArtifactResourceScope(); + return { + installation: async () => + (await detectBunInstallation( + packageRoot, + bunGlobalDirectory(environment, process.platform), + { canonicalPath: realpath, isFile }, + )) ?? + detectNpmInstallation(packageRoot, { + canonicalPath: realpath, + globalRoot: () => + requireCommandOutput(["npm", "root", "--global"], environment), + globalPrefix: () => + requireCommandOutput(["npm", "prefix", "--global"], environment), + }), + latestVersion: async (installation) => { + const context = packageManagerContext(installation, environment); + if (installation?.manager === "bun") { + const version = await runUpdateCommand( + ["bun", "--version"], + context.environment, + "structured", + context.workingDirectory, + ); + if (!version.ok) + return err({ kind: "unavailable", detail: version.error }); + const bunVersion = valid(version.value); + // bun info was introduced in 1.2.17; older releases reserve the command. + if (bunVersion === null || lt(bunVersion, "1.2.17")) + return err({ + kind: "unsupported-package-manager", + detail: `REA updates for Bun global installations require Bun 1.2.17 or later; bun --version returned ${JSON.stringify(version.value)}. Upgrade Bun manually, then rerun rea update.`, + }); + } + return npmReleaseLookup( + await runUpdateCommand( + installation?.manager === "bun" + ? BUN_LATEST_VERSION_COMMAND + : npmLatestVersionCommand(installation), + context.environment, + "structured", + context.workingDirectory, + ), + ); + }, + installVersion: async (installation, version, output) => { + const context = packageManagerContext(installation, environment); + const result = await runUpdateCommand( + updateInstallCommand(installation, version), context.environment, - "structured", + output, context.workingDirectory, ); - if (!version.ok) - return err({ kind: "unavailable", detail: version.error }); - const bunVersion = valid(version.value); - // bun info was introduced in 1.2.17; older releases reserve the command. - if (bunVersion === null || lt(bunVersion, "1.2.17")) - return err({ - kind: "unsupported-package-manager", - detail: `REA updates for Bun global installations require Bun 1.2.17 or later; bun --version returned ${JSON.stringify(version.value)}. Upgrade Bun manually, then rerun rea update.`, - }); - } - return npmReleaseLookup( - await runUpdateCommand( - installation?.manager === "bun" - ? BUN_LATEST_VERSION_COMMAND - : npmLatestVersionCommand(installation), - context.environment, - "structured", - context.workingDirectory, + return result.ok ? ok(undefined) : result; + }, + installedVersion: (installation) => + runUpdateCommand( + [ + process.execPath, + join(installation.packageRoot, "scripts", "rea.mjs"), + "--version", + ], + environment, ), - ); - }, - installVersion: async (installation, version, output) => { - const context = packageManagerContext(installation, environment); - const result = await runUpdateCommand( - updateInstallCommand(installation, version), - context.environment, - output, - context.workingDirectory, - ); - return result.ok ? ok(undefined) : result; - }, - installedVersion: (installation) => - runUpdateCommand( - [ - process.execPath, + planMaintenance: (installation) => + planIntegrationMaintenance( + home ?? homeDirectoryFromEnvironment(environment, process.platform), join(installation.packageRoot, "scripts", "rea.mjs"), - "--version", - ], - environment, - ), - planMaintenance: (installation) => - planIntegrationMaintenance( - home ?? homeDirectoryFromEnvironment(environment, process.platform), - join(installation.packageRoot, "scripts", "rea.mjs"), - environment, - (command) => runUpdateCommand(command, environment), - ), -}); + environment, + (command) => runUpdateCommand(command, environment), + resources, + ), + close: async () => { + try { + await resources.close(); + return undefined; + } catch (cause: unknown) { + return `Update resource cleanup failed: ${cause instanceof Error ? cause.message : String(cause)}`; + } + }, + }; +}; diff --git a/src/application/analysisView/AnalysisViewService.test.ts b/src/application/analysisView/AnalysisViewService.test.ts index aa7477662..10258d3f9 100644 --- a/src/application/analysisView/AnalysisViewService.test.ts +++ b/src/application/analysisView/AnalysisViewService.test.ts @@ -1,3 +1,8 @@ +import { + analysisViewCaptureFixture, + analysisViewCaptureEvidence, + analysisViewObjectDenseCapture, +} from "../../../tests/fixtures/analysisView.js"; import { describe, expect, it } from "vitest"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js"; @@ -17,6 +22,9 @@ import { import type { Evidence } from "../../domain/evidence.js"; import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js"; import { err, ok } from "../../domain/result.js"; +import { EvidenceLedger } from "../investigation/EvidenceLedger.js"; +import { WEB_NETWORK_CAPTURE_LIMITS } from "../../domain/webNetworkCapture.js"; +import { inspectAnalysisViewInputSchema } from "../../domain/analysisView/analysisView.js"; it.each(["/fixtures/unknown.exe", ""])( "preserves recorded empty native identity fields: path=%j", @@ -346,3 +354,142 @@ describe("summarizeRetainedAnalysis", () => { ).toEqual(err(conflict)); }); }); + +it("seals inferred and literal HAR pages, including a null artifact digest", () => { + const capture = analysisViewCaptureFixture(); + capture.artifact.sha256 = null; + const parent = analysisViewCaptureEvidence(capture); + // Any templated group in the full grouping makes every page inferred. + for (const [offset, confidence] of [ + [0, "inferred"], + [11, "inferred"], + ] as const) { + const result = inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view: { kind: "api-surface", offset, limit: 1 }, + }); + if (!result.ok) throw result.error; + expect(result.value.confidence).toBe(confidence); + expect(result.value.subject).toBeNull(); + expect(parseEvidence(result.value)).toEqual(result.value); + } + expect( + inspectAnalysisView({ + source: { kind: "inline", evidence: analysisViewLayoutEvidence() }, + view: { kind: "api-surface" }, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisUnsupportedTargetError" }, + }); +}); + +it("rejects a null projected digest under a present parent subject", () => { + const capture = analysisViewCaptureFixture(); + capture.artifact.sha256 = null; + const parent = createEvidence( + { path: capture.artifact.path, sha256: "a".repeat(64), format: "file" }, + { id: "fixture", name: "fixture", version: "1" }, + { + operation: "inspect_web_network_capture", + parameters: { sensitive_value_count: 0 }, + result: capture, + confidence: "observed", + }, + ); + expect( + inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view: { kind: "api-surface" }, + }), + ).toMatchObject({ ok: false, error: { _tag: "EvidenceIntegrityError" } }); +}); + +it("refuses an object-dense borrowed capture before making a full schema clone", () => { + const capture = analysisViewObjectDenseCapture(); + expect(Buffer.byteLength(JSON.stringify(capture))).toBeLessThan( + WEB_NETWORK_CAPTURE_LIMITS.inputBytes, + ); + const parent = analysisViewCaptureEvidence(capture); + const ledger = new EvidenceLedger(); + const recorded = ledger.record(parent); + if (!recorded.ok) throw recorded.error; + const borrowed = ledger.forAnalysis(parent.evidence_id)!; + expect(Object.isFrozen(borrowed.normalized_result)).toBe(true); + const result = inspectAnalysisView( + { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "api-surface", limit: 1 }, + }, + (id) => ledger.forAnalysis(id), + ); + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "AnalysisResourceConstraintError", + resource: "memory", + }, + }); + expect(ledger.forAnalysis(parent.evidence_id)).toBe(borrowed); +}); + +it("borrows inline Evidence until admission without invoking nested accessors", () => { + let accessed = false; + const evidence = { + get normalized_result() { + accessed = true; + throw new Error("Evidence must be admitted before parsing"); + }, + }; + const input = { + source: { kind: "inline", evidence }, + view: { kind: "api-surface" }, + }; + const parsed = inspectAnalysisViewInputSchema.parse(input); + if (parsed.source.kind !== "inline") throw new Error("Expected inline input"); + expect(parsed.source.evidence).toBe(evidence); + expect(inspectAnalysisView(input)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(accessed).toBe(false); +}); + +it("keeps inline-envelope admission live alongside capture validation and grouping", () => { + // Each clone alone fits; both graphs coexist while the view is projected. + const capture = analysisViewObjectDenseCapture(40_000); + const parent = analysisViewCaptureEvidence(capture); + const retained = inspectAnalysisView( + { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "api-surface" }, + }, + () => parent, + ); + if (!retained.ok) throw retained.error; + expect( + inspectAnalysisView({ + source: { kind: "inline", evidence: parent }, + view: { kind: "api-surface" }, + }), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); +}); + +it("admits raw_result and parameters before authenticating a portable capture", () => { + const dense = analysisViewObjectDenseCapture(); + const parent = analysisViewCaptureEvidence(); + for (const field of ["raw_result", "parameters"] as const) { + const result = inspectAnalysisView({ + source: { kind: "inline", evidence: { ...parent, [field]: dense } }, + view: { kind: "api-surface" }, + }); + // Refuse expansion before hashing can report the deliberately stale ID. + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError", resource: "memory" }, + }); + } +}); diff --git a/src/application/analysisView/AnalysisViewService.ts b/src/application/analysisView/AnalysisViewService.ts index 380feb59e..bf7e52ff6 100644 --- a/src/application/analysisView/AnalysisViewService.ts +++ b/src/application/analysisView/AnalysisViewService.ts @@ -22,6 +22,7 @@ import { } from "../../domain/analysisView/analysisView.js"; import { jsonValueSchema } from "../../domain/jsonValue.js"; import { err, ok, type Result } from "../../domain/result.js"; +import { WebNetworkApiSurfaceAccumulator } from "../../domain/analysisView/WebNetworkApiSurfaceAccumulator.js"; /** Authenticate a selected-view request and project already completed Evidence. */ export const inspectAnalysisView = ( @@ -41,11 +42,25 @@ export const inspectAnalysisView = ( return inspectAnalysisViewValidated(parsed.data, lookup); }; -/** Project a view from input already parsed by a trusted adapter. */ +/** Authenticate borrowed inline Evidence after adapter envelope validation. */ export const inspectAnalysisViewValidated = ( input: InspectAnalysisViewInput, lookup?: EvidenceLookup, ): Result => { + const inline = input.source.kind === "inline" ? input.source.evidence : null; + const inlineCapture = + typeof inline === "object" && + inline !== null && + Object.getOwnPropertyDescriptor(inline, "operation")?.value === + "inspect_web_network_capture"; + const accumulator = + input.view.kind === "api-surface" || inlineCapture + ? new WebNetworkApiSurfaceAccumulator() + : undefined; + if (input.source.kind === "inline" && accumulator !== undefined) { + const admitted = accumulator.admitInlineEvidence(input.source.evidence); + if (!admitted.ok) return admitted; + } const resolved = resolveAnalysisViewSource(input.source, lookup); if (!resolved.ok) return resolved; const parent = resolved.value; @@ -55,6 +70,7 @@ export const inspectAnalysisViewValidated = ( operation: parent.operation, normalizedResult: parent.normalized_result, limitations: parent.limitations, + parameters: parent.parameters, artifact: parent.subject === null ? null @@ -64,6 +80,7 @@ export const inspectAnalysisViewValidated = ( }, }, input.view, + accumulator, ); if (!projected.ok) return projected; if ( @@ -176,7 +193,11 @@ const createAnalysisViewEvidence = ( }, result: jsonValueSchema.parse(result), rawResult: null, - confidence: "derived", + confidence: + // Paging follows the full grouping, so any templated group makes every page inferred. + result.kind === "api-surface" && result.templated_groups > 0 + ? "inferred" + : "derived", authority: parent.authority, limitations: result.limitations, locations: [ diff --git a/src/application/binary/AnalysisProviderRegistry.ts b/src/application/binary/AnalysisProviderRegistry.ts index 43e6bbb9c..5b7e78813 100644 --- a/src/application/binary/AnalysisProviderRegistry.ts +++ b/src/application/binary/AnalysisProviderRegistry.ts @@ -50,6 +50,8 @@ export interface AnalysisProviderBinding { readonly identity: ProviderIdentity & { readonly version: string }; readonly selectionSource: "request" | "environment" | "auto-single-candidate"; readonly profile: AnalysisProfileCommitment; + /** The provider's capabilities, attributed to its resolved identity. */ + readonly capabilities: readonly CapabilityDescriptor[]; } /** Complete result used for both a bound and intentionally unbound target. */ @@ -361,18 +363,25 @@ const selectedResult = ( }, statuses: readonly AnalysisProviderCandidateStatus[], ): AnalysisProviderSelection => { + const identity = evaluated.profile.provider; + const capabilities = evaluated.status.capabilities.map((capability) => ({ + ...capability, + provider: structuredClone(identity), + })); const selectedStatus = { ...evaluated.status, - provider: structuredClone(evaluated.profile.provider), + provider: structuredClone(identity), selected: true, + capabilities, } satisfies AnalysisProviderCandidateStatus; return { requestedProviderId, binding: { provider: evaluated.candidate, - identity: structuredClone(evaluated.profile.provider), + identity: structuredClone(identity), selectionSource: source, profile: structuredClone(evaluated.profile), + capabilities, }, candidates: replaceCandidate(statuses, selectedStatus), }; diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 15f4e403c..48c2a54bd 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -14,6 +14,7 @@ import { createAnalysisSnapshotWorkflowEntry, parseAnalysisSnapshot, serializeAnalysisSnapshot, + snapshotEvidenceForQuery, snapshotBinding, snapshotTarget, } from "../../domain/analysisSnapshot.js"; @@ -30,6 +31,7 @@ import { SNAPSHOT_CACHE_ENTRY_CEILING, isSnapshotCacheable, } from "./AnalysisSnapshotCache.js"; +import { enhancedOutputSchemas } from "../../contracts/toolOutputSchemaGroups.js"; it("exports independent nested query, workflow and Evidence payloads", () => { const cache = new AnalysisSnapshotCache(); @@ -441,7 +443,7 @@ describe("analysis snapshot cache partitioning", () => { ANALYSIS_SNAPSHOT_PROFILE, createEvidenceBundle([evidence]), ); - if (!exported.ok) throw new Error("snapshot export failed"); + if (!exported.ok) throw exported.error; const tampered = structuredClone(exported.value); const tamperedEntry = tampered.entries[0]; if (tamperedEntry === undefined) @@ -499,3 +501,163 @@ describe("analysis snapshot cache partitioning", () => { ).toBeUndefined(); }); }); + +describe("relationship coverage snapshot replay", () => { + it("authenticates relationship coverage and replays legacy reads as unknown", () => { + const cache = new AnalysisSnapshotCache(); + const operation = "procedure_callees" as const; + const parameters = { procedure: "0x1000", document: "fixture" }; + const coverage = { + status: "complete" as const, + diagnostics: ["The provider retained an unresolved call target."], + }; + const execution = createAnalysisExecution( + ["0x2000"], + ANALYSIS_SNAPSHOT_PROVIDER, + { + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + rawResult: ["0x2000"], + relationshipCoverage: coverage, + subject: { + path: ANALYSIS_SNAPSHOT_TARGET.path, + sha256: ANALYSIS_SNAPSHOT_TARGET.sha256, + format: ANALYSIS_SNAPSHOT_TARGET.format, + ...(ANALYSIS_SNAPSHOT_TARGET.architecture === undefined + ? {} + : { architecture: ANALYSIS_SNAPSHOT_TARGET.architecture }), + }, + }, + ); + cache.record({ + target: ANALYSIS_SNAPSHOT_TARGET, + profile: ANALYSIS_SNAPSHOT_PROFILE, + operation, + parameters, + execution, + }); + const evidence = createEvidence( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROVIDER, + { + operation, + parameters, + result: ["0x2000"], + rawResult: ["0x2000"], + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + relationshipCoverage: coverage, + }, + ); + const exported = cache.export( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + createEvidenceBundle([evidence]), + ); + if (!exported.ok) throw exported.error; + expect( + cache.lookup( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + operation, + parameters, + )?.relationshipCoverage, + ).toEqual(coverage); + + const tampered = structuredClone(exported.value); + const tamperedEntry = tampered.entries[0]; + if (tamperedEntry === undefined) throw new Error("snapshot entry missing"); + tamperedEntry.execution.relationship_coverage = { + status: "complete", + diagnostics: ["forged coverage"], + }; + expect( + new AnalysisSnapshotCache().import(tampered, undefined, () => ok(0)).ok, + ).toBe(false); + + const legacy = structuredClone(exported.value); + const legacyEntry = legacy.entries[0]; + if (legacyEntry === undefined) throw new Error("snapshot entry missing"); + Reflect.deleteProperty(legacyEntry.execution, "relationship_coverage"); + const legacyEvidence = createEvidence( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROVIDER, + { + operation, + parameters, + result: ["0x2000"], + rawResult: ["0x2000"], + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + }, + ); + Reflect.deleteProperty(legacyEvidence, "relationship_coverage"); + legacy.evidence_bundle = createEvidenceBundle([legacyEvidence]); + parseAnalysisSnapshot(legacy); + const replayCache = new AnalysisSnapshotCache(); + expect(replayCache.import(legacy, undefined, () => ok(0)).ok).toBe(true); + expect( + replayCache.lookup( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + operation, + parameters, + )?.relationshipCoverage, + ).toEqual({ + status: "unknown", + reason: "Snapshot entry does not report relationship coverage.", + }); + }); +}); + +it("does not replay pre-envelope call graphs under the revised workflow profile", () => { + const cache = new AnalysisSnapshotCache(); + const target = ANALYSIS_SNAPSHOT_TARGET; + const profile = ANALYSIS_SNAPSHOT_PROFILE; + const operation = "get_call_graph" as const; + const parameters = { address: "0x1000", direction: "forward" }; + const oldProfile = workflowAnalysisProfile(profile); + const revisedProfile = workflowAnalysisProfile(profile, operation); + const oldResult = { "0": [] }; + cache.recordWorkflow({ + target, + profile, + operation, + parameters, + execution: { + ...createAnalysisExecution(oldResult, REA_WORKFLOW_PROVIDER, { + subject: target, + rawResult: oldResult, + }), + analysisProfile: oldProfile, + }, + }); + const evidence = createEvidence(target, REA_WORKFLOW_PROVIDER, { + operation, + confidence: "derived", + parameters, + result: oldResult, + rawResult: oldResult, + analysisProfile: oldProfile, + }); + const exported = cache.export( + target, + profile, + createEvidenceBundle([evidence]), + ); + if (!exported.ok) throw exported.error; + const snapshot = parseAnalysisSnapshot(exported.value); + + expect( + snapshotEvidenceForQuery(snapshot, { + target, + bindingProfile: profile, + operation, + parameters, + provider: REA_WORKFLOW_PROVIDER, + evidenceProfile: revisedProfile, + }), + ).toBeUndefined(); + expect( + enhancedOutputSchemas.get_call_graph?.safeParse({ + normalized_result: oldResult, + }).success, + ).toBe(false); +}); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index cf5083e1a..6440ccf6a 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -34,6 +34,11 @@ import type { import { OFFICIAL_TOOL_CONTRACTS } from "../../contracts/officialToolContracts.js"; import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +const RELATIONSHIP_OPERATIONS: ReadonlySet = new Set([ + "procedure_callers", + "procedure_callees", +]); + const STATEFUL_OPERATIONS: ReadonlySet = new Set([ "health", "current_address", @@ -241,6 +246,7 @@ export class AnalysisSnapshotCache { readonly provider: AnalysisExecution["provider"]; readonly analysisProfile: AnalysisProfileCommitment; readonly limitations: readonly string[]; + readonly relationshipCoverage?: AnalysisExecution["relationshipCoverage"]; readonly locations: AnalysisExecution["locations"]; readonly subject: AnalysisExecution["subject"]; }; @@ -302,6 +308,19 @@ export class AnalysisSnapshotCache { ...cached.execution.limitations, "Loaded from a local REA analysis snapshot; this call did not re-run the provider.", ], + ...(cached.execution.relationship_coverage === null && + RELATIONSHIP_OPERATIONS.has(operation) + ? { + relationshipCoverage: { + status: "unknown", + reason: "Snapshot entry does not report relationship coverage.", + }, + } + : cached.execution.relationship_coverage === null + ? {} + : { + relationshipCoverage: cached.execution.relationship_coverage, + }), locations: cached.execution.locations, subject, }); diff --git a/src/application/binary/AnalysisSnapshotFiles.ts b/src/application/binary/AnalysisSnapshotFiles.ts index 064aae705..174b58bfc 100644 --- a/src/application/binary/AnalysisSnapshotFiles.ts +++ b/src/application/binary/AnalysisSnapshotFiles.ts @@ -16,6 +16,7 @@ import type { AnalysisResourceConstraintError } from "../../domain/analysisError import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js"; import { readJsonObjectMembersFile, writeTextFile } from "../JsonFiles.js"; import type { JsonObjectMemberReader } from "../JsonInputFile.js"; +import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError; @@ -118,6 +119,7 @@ export const writeAnalysisSnapshot = async ( snapshot: AnalysisSnapshot, path: string, overwrite: boolean, + resources: ArtifactResourceScope, ): Promise< Result<{ readonly path: string; readonly bytes: number }, SnapshotFailure> > => { @@ -129,7 +131,7 @@ export const writeAnalysisSnapshot = async ( new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }), ); } - return writeTextFile(encoded, path, overwrite); + return writeTextFile(encoded, path, overwrite, resources); }; const snapshotValidationMessage = (cause: unknown): string => diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index e604256d9..da8d8e679 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -24,7 +24,6 @@ import type { BinarySessionPort, SavedAnalysisSnapshot, } from "./BinarySessionPort.js"; -import { writeAnalysisSnapshot } from "./AnalysisSnapshotFiles.js"; import { SessionProviderRouter, type SessionProviderRoute, @@ -143,14 +142,7 @@ export class BinarySession } } } - const profile = this.#active?.profile; - return structuredClone( - profile !== null && - profile !== undefined && - profile.provider.id === selected.id - ? profile.provider - : selected, - ); + return structuredClone(selected); } /** External live state cannot be replayed by a direct or composed CLI workflow. */ @@ -376,7 +368,7 @@ export class BinarySession await this.#drainCalls(admittedThrough); const snapshot = this.exportAnalysisSnapshot(); if (!snapshot.ok) return snapshot; - const written = await writeAnalysisSnapshot( + const written = await this.writeAnalysisSnapshot( snapshot.value, path, overwrite, @@ -421,7 +413,16 @@ export class BinarySession ); this.clearSessionRecords(); this.#clearRuntimeAvailability(); - return closed; + const fileResources = await this.closeWriteResources(); + if (!closed.ok && !fileResources.ok) + return err( + analysisErrorWithCleanupFailure( + closed.error, + fileResources.error, + "close_binary", + ), + ); + return closed.ok ? fileResources : closed; } async #retireClient( @@ -594,9 +595,22 @@ export class BinarySession name, active.target, ); - this.#observeRuntimeAvailability(name, profiled); - if ( + const execution = profiled.ok && + (name === "procedure_callers" || name === "procedure_callees") && + profiled.value.relationshipCoverage === undefined + ? ok({ + ...profiled.value, + relationshipCoverage: { + status: "unknown" as const, + reason: + "Provider did not report relationship coverage for this observation.", + }, + }) + : profiled; + this.#observeRuntimeAvailability(name, execution); + if ( + execution.ok && cacheable && profile !== undefined && name !== "decode_interface_builder" && @@ -605,16 +619,21 @@ export class BinarySession name !== "trace_dylib_resolution" ) { const evidence = createEvidence( - profiled.value.subject ?? active.target, - profiled.value.provider, + execution.value.subject ?? active.target, + execution.value.provider, { operation: name, parameters: arguments_, - result: profiled.value.result, + result: execution.value.result, analysisProfile: profile, - rawResult: profiled.value.rawResult, - limitations: profiled.value.limitations, - locations: profiled.value.locations, + rawResult: execution.value.rawResult, + limitations: execution.value.limitations, + ...(execution.value.relationshipCoverage === undefined + ? {} + : { + relationshipCoverage: execution.value.relationshipCoverage, + }), + locations: execution.value.locations, }, ); const recorded = this.recordEvidence(evidence); @@ -624,12 +643,12 @@ export class BinarySession profile, operation: name, parameters: arguments_, - execution: profiled.value, + execution: execution.value, }); } else if (profiled.ok && capability.effects.mutatesArtifact) { this.invalidateSnapshot(); } - return profiled; + return execution; } #observeRuntimeAvailability( diff --git a/src/application/binary/BinarySessionRecords.ts b/src/application/binary/BinarySessionRecords.ts index 1f94dcdfd..182d5892f 100644 --- a/src/application/binary/BinarySessionRecords.ts +++ b/src/application/binary/BinarySessionRecords.ts @@ -6,6 +6,7 @@ import type { EvidenceBundle } from "../../domain/evidenceBundle.js"; import type { JsonValue } from "../../domain/jsonValue.js"; import { evidenceBundleForTarget } from "../../domain/evidenceBundle.js"; import { + EvidenceFileError, EvidenceIntegrityError, AnalysisSnapshotMismatchError, } from "../../domain/evidenceErrors.js"; @@ -24,6 +25,9 @@ import type { } from "../AnalysisProvider.js"; import { AnalysisSnapshotCache } from "./AnalysisSnapshotCache.js"; import { InvestigationRecords } from "../investigation/InvestigationRecords.js"; +import { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js"; +import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js"; +import { writeAnalysisSnapshot as writeSnapshotFile } from "./AnalysisSnapshotFiles.js"; const SNAPSHOT_MUTATION_RECOVERY = "Export session observations through export_evidence_bundle if needed. Close the active target without saving a snapshot, then reopen it before importing or saving a snapshot."; @@ -45,6 +49,7 @@ export interface WorkflowSnapshotRecordInput { /** Own binary snapshots and coordinate the composed investigation records. */ export abstract class BinarySessionRecords { readonly #records: InvestigationRecords; + readonly #writeResources = new ArtifactResourceScope(); readonly #snapshot = new AnalysisSnapshotCache(); #snapshotInvalidated = false; readonly #snapshotListeners = new Set<() => void | Promise>(); @@ -89,6 +94,39 @@ export abstract class BinarySessionRecords { return this.#records.evidenceBundleForSerialization(); } + /** Persist one provider-neutral snapshot under this session's resource owner. */ + writeAnalysisSnapshot( + snapshot: AnalysisSnapshot, + path: string, + overwrite: boolean, + ): Promise< + Result<{ readonly path: string; readonly bytes: number }, AnalysisError> + > { + return writeSnapshotFile(snapshot, path, overwrite, this.#writeResources); + } + + /** Retry and release staged snapshot files at the owning session shutdown. */ + async closeWriteResources(): Promise> { + try { + await this.#writeResources.retryCleanup(); + return ok(null); + } catch (cause: unknown) { + const observation = ArtifactReaderFailure.cleanupObservation( + cause, + "analysis snapshot staging", + ); + return err( + new EvidenceFileError("write", "io", { + cause, + cleanup: { + reason: observation.reason, + resources: observation.resources, + }, + }), + ); + } + } + importEvidenceBundle( bundle: unknown, ): Result { diff --git a/src/application/binary/BinarySessionStatus.test.ts b/src/application/binary/BinarySessionStatus.test.ts new file mode 100644 index 000000000..a72b7ed80 --- /dev/null +++ b/src/application/binary/BinarySessionStatus.test.ts @@ -0,0 +1,391 @@ +import { describe, expect, it } from "vitest"; + +import type { + CapabilityDescriptor, + ProviderIdentity, +} from "../AnalysisProvider.js"; +import { + expandProviderCapabilities, + type ProviderCapabilitySet, +} from "../../contracts/toolOutputSchemaPrimitives.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +import type { AnalysisProviderCandidateStatus } from "./AnalysisProviderRegistry.js"; +import type { + SessionProviderRoute, + SessionProviderRouter, +} from "./SessionProviderRouter.js"; +import { binarySessionStatus } from "./BinarySessionStatus.js"; + +type ProjectedStatus = { + capabilities: ProviderCapabilitySet[]; + analysis_provider_candidates: { + provider: ProviderIdentity; + availability: unknown; + target_support: unknown; + selected: boolean; + capabilities: ProviderCapabilitySet[]; + }[]; +}; + +const providerA: ProviderIdentity = { + id: "provider-a", + name: "Provider A", + version: "1.0", +}; +const providerB: ProviderIdentity = { + id: "provider-b", + name: "Provider B", + version: "2.0", +}; + +const MUTATING_EFFECTS = { + mutates_artifact: true, + launches_process: false, + may_show_ui: false, + may_access_network: false, + may_write_filesystem: false, + changes_permissions: false, + requires_root: false, +}; + +type DescriptorOptions = { + readonly launchesProcess?: boolean; +} & ( + | { + readonly available?: true; + readonly reason?: never; + readonly availabilityCode?: never; + } + | { + readonly available: false; + readonly reason?: string; + readonly availabilityCode?: "architecture_unsupported"; + } +); + +const descriptor = ( + provider: ProviderIdentity, + operation: CapabilityDescriptor["operation"], + limitations: readonly string[], + options: DescriptorOptions = {}, +): CapabilityDescriptor => { + const availability = + options.available === false + ? { + available: false as const, + reason: options.reason ?? "Unavailable", + ...(options.availabilityCode === undefined + ? {} + : { availabilityCode: options.availabilityCode }), + } + : { available: true as const, reason: null }; + return { + provider, + operation, + ...availability, + effects: { + mutatesArtifact: true, + launchesProcess: options.launchesProcess ?? false, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: false, + changesPermissions: false, + requiresRoot: false, + }, + limitations, + }; +}; + +const candidate = ( + provider: ProviderIdentity, + capabilities: readonly CapabilityDescriptor[], +): AnalysisProviderCandidateStatus => ({ + provider, + availability: { + status: "available", + code: null, + reason: null, + diagnostics: {}, + }, + targetSupport: { + status: "unknown", + code: null, + reason: "No target is open.", + diagnostics: {}, + }, + selected: true, + capabilities, +}); + +const project = ( + routeCapabilities: readonly CapabilityDescriptor[], + candidates: readonly AnalysisProviderCandidateStatus[], + runtimeUnavailability: ReadonlyMap< + string, + { readonly reason: string } + > = new Map(), +): ProjectedStatus => { + const route = { + capabilities: new Map( + routeCapabilities.map((capability) => [capability.operation, capability]), + ), + binding: null, + } as unknown as SessionProviderRoute; + const router = { + candidateStatuses: () => candidates, + } as unknown as SessionProviderRouter; + return binarySessionStatus({ + target: undefined, + route, + router, + runtimeUnavailability, + runId: undefined, + runtimeLineageSnapshots: [], + requestActivitySnapshots: [], + }) as unknown as ProjectedStatus; +}; + +const expandCanonicalCapabilities = (sets: readonly ProviderCapabilitySet[]) => + [...expandProviderCapabilities(sets)].sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ); + +const selectedCapabilities = [ + descriptor(providerA, "xrefs", ["Shared limitation"]), + descriptor(providerA, "procedure_info", ["Shared limitation"]), + descriptor(providerA, "read_bytes", ["Shared limitation", "beta detail"]), +]; +const otherCapabilities = [ + descriptor(providerB, "read_bytes", ["Different limitation"], { + launchesProcess: true, + available: false, + reason: "Architecture unsupported", + availabilityCode: "architecture_unsupported", + }), +]; + +const expectedRouteCapabilities: ProviderCapabilitySet[] = [ + { + provider: providerA, + shared_limitations: ["Shared limitation"], + groups: [ + { + operations: ["procedure_info"], + effects: ["mutates_artifact"], + limitations: [], + available: false, + reason: "Runtime operation failure", + availability_code: null, + }, + { + operations: ["read_bytes"], + effects: ["mutates_artifact"], + limitations: ["beta detail"], + available: true, + reason: null, + availability_code: null, + }, + { + operations: ["xrefs"], + effects: ["mutates_artifact"], + limitations: [], + available: true, + reason: null, + availability_code: null, + }, + ], + }, +]; +const expectedExpandedRouteCapabilities = [ + { + provider: providerA, + operation: "procedure_info", + available: false, + reason: "Runtime operation failure", + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, + { + provider: providerA, + operation: "read_bytes", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation", "beta detail"], + }, + { + provider: providerA, + operation: "xrefs", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, +]; +const expectedCandidateCapabilities: ProviderCapabilitySet[] = [ + { + provider: providerA, + shared_limitations: ["Shared limitation"], + groups: [ + { + operations: ["procedure_info", "xrefs"], + effects: ["mutates_artifact"], + limitations: [], + available: true, + reason: null, + availability_code: null, + }, + { + operations: ["read_bytes"], + effects: ["mutates_artifact"], + limitations: ["beta detail"], + available: true, + reason: null, + availability_code: null, + }, + ], + }, + { + provider: providerB, + shared_limitations: ["Different limitation"], + groups: [ + { + operations: ["read_bytes"], + effects: ["mutates_artifact", "launches_process"], + limitations: [], + available: false, + reason: "Architecture unsupported", + availability_code: "architecture_unsupported", + }, + ], + }, +]; +const expectedCandidateStatusFacts = [ + expect.objectContaining({ + provider: providerA, + target_support: { + status: "unknown", + code: null, + reason: "No target is open.", + diagnostics: {}, + }, + capabilities: [expectedCandidateCapabilities[0]], + }), + expect.objectContaining({ + provider: providerB, + capabilities: [expectedCandidateCapabilities[1]], + }), +]; +const expectedExpandedCandidateCapabilities = [ + [ + { + provider: providerA, + operation: "procedure_info", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, + { + provider: providerA, + operation: "read_bytes", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation", "beta detail"], + }, + { + provider: providerA, + operation: "xrefs", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, + ], + [ + { + provider: providerB, + operation: "read_bytes", + available: false, + reason: "Architecture unsupported", + availability_code: "architecture_unsupported", + effects: { ...MUTATING_EFFECTS, launches_process: true }, + limitations: ["Different limitation"], + }, + ], +]; + +describe("binary session provider capability projection", () => { + it("preserves provider scope and route overlays while factoring exact prefixes", () => { + const selectedBefore = structuredClone(selectedCapabilities); + const otherBefore = structuredClone(otherCapabilities); + const status = project( + selectedCapabilities, + [ + candidate(providerA, selectedCapabilities), + candidate(providerB, otherCapabilities), + ], + new Map([["procedure_info", { reason: "Runtime operation failure" }]]), + ); + expect(selectedCapabilities).toEqual(selectedBefore); + expect(otherCapabilities).toEqual(otherBefore); + expect(status.capabilities).toEqual(expectedRouteCapabilities); + expect(expandCanonicalCapabilities(status.capabilities)).toEqual( + expectedExpandedRouteCapabilities, + ); + expect(status.analysis_provider_candidates).toEqual( + expectedCandidateStatusFacts, + ); + const selectedCandidate = status.analysis_provider_candidates[0]; + const otherCandidate = status.analysis_provider_candidates[1]; + expect(selectedCandidate).toBeDefined(); + expect(otherCandidate).toBeDefined(); + expect( + expandCanonicalCapabilities(selectedCandidate?.capabilities ?? []), + ).toEqual(expectedExpandedCandidateCapabilities[0]); + expect( + expandCanonicalCapabilities(otherCandidate?.capabilities ?? []), + ).toEqual(expectedExpandedCandidateCapabilities[1]); + }); + + it("keeps empty route and candidate capability collections empty", () => { + const status = project([], [candidate(providerA, [])]); + expect(status.capabilities).toEqual([]); + expect(status.analysis_provider_candidates[0]?.capabilities).toEqual([]); + }); + + it("preserves repeated limitations in their original order", () => { + const capabilities = [ + descriptor(providerA, "procedure_info", [ + "Repeated", + "Repeated", + "left detail", + ]), + descriptor(providerA, "xrefs", ["Repeated", "Repeated", "right detail"]), + ]; + const status = project(capabilities, []); + + expect(status.capabilities[0]).toMatchObject({ + shared_limitations: ["Repeated", "Repeated"], + groups: [ + { operations: ["procedure_info"], limitations: ["left detail"] }, + { operations: ["xrefs"], limitations: ["right detail"] }, + ], + }); + expect( + expandCanonicalCapabilities(status.capabilities).map( + ({ limitations }) => limitations, + ), + ).toEqual([ + ["Repeated", "Repeated", "left detail"], + ["Repeated", "Repeated", "right detail"], + ]); + }); +}); diff --git a/src/application/binary/BinarySessionStatus.ts b/src/application/binary/BinarySessionStatus.ts index 8fbe4aa2b..4794bed49 100644 --- a/src/application/binary/BinarySessionStatus.ts +++ b/src/application/binary/BinarySessionStatus.ts @@ -4,9 +4,14 @@ import type { JsonValue } from "../../domain/jsonValue.js"; import type { ProviderOperationHealth } from "../../domain/providerOperationHealth.js"; import type { CapabilityDescriptor, + ProviderIdentity, ProviderRequestActivitySnapshot, ProviderRuntimeLineageSnapshot, } from "../AnalysisProvider.js"; +import { + PROVIDER_EFFECT_NAMES, + type ProviderCapabilitySet, +} from "../../contracts/toolOutputSchemaPrimitives.js"; import type { SessionProviderRoute, SessionProviderRouter, @@ -38,7 +43,7 @@ export const binarySessionStatus = ({ requestActivitySnapshots, providerOperationHealth, }: BinarySessionStatusInput): JsonValue => { - const capabilities = [...route.capabilities.values()] + const routeCapabilities = [...route.capabilities.values()] .filter(({ operation }) => operation !== "inventory_artifact") .sort((left, right) => compareUnicodeCodePoints(left.operation, right.operation), @@ -52,8 +57,8 @@ export const binarySessionStatus = ({ available: false as const, reason: unavailable.reason, }; - }) - .map(capabilityStatus); + }); + const capabilities = providerCapabilitySets(routeCapabilities); const analysisProviderBinding = route.binding === null ? null @@ -69,13 +74,14 @@ export const binarySessionStatus = ({ availability: structuredClone(candidate.availability), target_support: structuredClone(candidate.targetSupport), selected: candidate.selected, - capabilities: candidate.capabilities - .slice() - .filter(({ operation }) => operation !== "inventory_artifact") - .sort((left, right) => - compareUnicodeCodePoints(left.operation, right.operation), - ) - .map(capabilityStatus), + capabilities: providerCapabilitySets( + candidate.capabilities + .slice() + .filter(({ operation }) => operation !== "inventory_artifact") + .sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ), + ), })); const common = { capabilities, @@ -187,17 +193,109 @@ const processLineageStatus = ( })), }; -const capabilityStatus = ( - descriptor: Omit & { - readonly available: boolean; - readonly reason: string | null; - }, -) => ({ - operation: descriptor.operation, - available: descriptor.available, - reason: descriptor.reason, - availability_code: descriptor.availabilityCode ?? null, - effects: { +type CapabilityEffectName = (typeof PROVIDER_EFFECT_NAMES)[number]; +type CapabilityAvailabilityCode = Exclude< + CapabilityDescriptor["availabilityCode"], + undefined +>; +type CapabilityProjection = Omit & + ( + | { readonly available: true; readonly reason: null } + | { readonly available: false; readonly reason: string } + ); +type ProviderCapabilityGroup = ProviderCapabilitySet["groups"][number]; + +/** Group inline capability facts without merging distinct providers or states. */ +const providerCapabilitySets = ( + descriptors: readonly CapabilityProjection[], +): ProviderCapabilitySet[] => { + const providers = new Map< + string, + { + readonly identity: ProviderIdentity; + readonly descriptors: CapabilityProjection[]; + } + >(); + for (const descriptor of descriptors) { + const key = JSON.stringify([ + descriptor.provider.id, + descriptor.provider.name, + descriptor.provider.version, + ]); + const provider = providers.get(key); + if (provider === undefined) + providers.set(key, { + identity: descriptor.provider, + descriptors: [descriptor], + }); + else provider.descriptors.push(descriptor); + } + return [...providers.values()].map(({ identity, descriptors: owned }) => { + const sharedLimitations = commonLimitationsPrefix(owned); + const groups = new Map(); + for (const descriptor of owned) { + const effects = enabledCapabilityEffects(descriptor); + const limitations = descriptor.limitations.slice( + sharedLimitations.length, + ); + const availabilityCode: CapabilityAvailabilityCode | null = + descriptor.availabilityCode ?? null; + const status = descriptor.available + ? { + available: true as const, + reason: null, + availability_code: null, + } + : { + available: false as const, + reason: descriptor.reason, + availability_code: availabilityCode, + }; + const key = JSON.stringify([ + effects, + limitations, + status.available, + status.reason, + status.availability_code, + ]); + const group = groups.get(key); + if (group === undefined) + groups.set(key, { + operations: [descriptor.operation], + effects, + limitations, + ...status, + }); + else group.operations.push(descriptor.operation); + } + return { + provider: providerSummary(identity), + shared_limitations: sharedLimitations, + groups: [...groups.values()], + }; + }); +}; + +const commonLimitationsPrefix = ( + descriptors: readonly CapabilityProjection[], +): string[] => { + const first = descriptors[0]?.limitations; + if (first === undefined) return []; + let length = first.length; + for (const descriptor of descriptors.slice(1)) { + length = Math.min(length, descriptor.limitations.length); + let index = 0; + while (index < length && first[index] === descriptor.limitations[index]) + index++; + length = index; + } + return first.slice(0, length); +}; + +const enabledCapabilityEffects = ( + descriptor: CapabilityProjection, +): CapabilityEffectName[] => { + const effectValues = { mutates_artifact: descriptor.effects.mutatesArtifact, launches_process: descriptor.effects.launchesProcess, may_show_ui: descriptor.effects.mayShowUi, @@ -205,9 +303,9 @@ const capabilityStatus = ( may_write_filesystem: descriptor.effects.mayWriteFilesystem, changes_permissions: descriptor.effects.changesPermissions, requires_root: descriptor.effects.requiresRoot, - }, - limitations: [...descriptor.limitations], -}); + } satisfies Record; + return PROVIDER_EFFECT_NAMES.filter((name) => effectValues[name]); +}; const providerOperationHealthStatus = ( health: ProviderOperationHealth | null | undefined, diff --git a/src/application/binary/SessionProviderRouter.ts b/src/application/binary/SessionProviderRouter.ts index dac5c183e..cf5653537 100644 --- a/src/application/binary/SessionProviderRouter.ts +++ b/src/application/binary/SessionProviderRouter.ts @@ -125,7 +125,11 @@ const selectableRoute = ( ? undefined : new CompositeProvider(providers); const identity = provider?.identity() ?? emptyProviderIdentity(); - const capabilities = capabilityMap(provider?.capabilities() ?? []); + // The binding attributes deep-provider capabilities to the resolved version. + const capabilities = capabilityMap([ + ...auxiliaryProviders.flatMap((auxiliary) => auxiliary.capabilities()), + ...(binding?.capabilities ?? []), + ]); return { identity, capabilities, diff --git a/src/application/javascript/ElectronBoundaryAnalysis.ts b/src/application/javascript/ElectronBoundaryAnalysis.ts index a718f6ed0..bd090f709 100644 --- a/src/application/javascript/ElectronBoundaryAnalysis.ts +++ b/src/application/javascript/ElectronBoundaryAnalysis.ts @@ -220,5 +220,9 @@ const resolvesToFile = (input: ResolveToFileInput): boolean => { ); }; -const ipcRecordKey = (path: string, finding: ElectronIpcFinding): string => - `${path}\0${String(finding.location.start.line)}:${String(finding.location.start.column)}\0${finding.side}\0${finding.operation}`; +const ipcRecordKey = (path: string, finding: ElectronIpcFinding): string => { + const identity = `${path}\0${String(finding.location.start.line)}:${String(finding.location.start.column)}\0${finding.side}\0${finding.operation}\0${finding.channel ?? finding.channel_expression}\0${JSON.stringify(finding.handler_location)}`; + return finding.invocation_location === undefined + ? identity + : `${identity}\0${String(finding.invocation_location.start.line)}:${String(finding.invocation_location.start.column)}`; +}; diff --git a/src/application/javascript/ElectronBoundaryGraph.ts b/src/application/javascript/ElectronBoundaryGraph.ts index 740c7f001..d77e496fb 100644 --- a/src/application/javascript/ElectronBoundaryGraph.ts +++ b/src/application/javascript/ElectronBoundaryGraph.ts @@ -2,6 +2,7 @@ import type { JavaScriptArtifactGraphContext } from "./JavaScriptArtifactGraphCo import { addElectronIpcBoundaries } from "./ElectronBoundaryGraphIpc.js"; import { addElectronNativeBoundaries } from "./ElectronBoundaryGraphNative.js"; import { addElectronWindowBoundaries } from "./ElectronBoundaryGraphWindows.js"; +import { addElectronCallPaths } from "./ElectronBoundaryGraphCallPaths.js"; /** Add every static Electron process and security-boundary fact to the JAG. */ export const addElectronBoundaries = ( @@ -10,4 +11,5 @@ export const addElectronBoundaries = ( addElectronWindowBoundaries(context); addElectronIpcBoundaries(context); addElectronNativeBoundaries(context); + addElectronCallPaths(context); }; diff --git a/src/application/javascript/ElectronBoundaryGraphCallPaths.ts b/src/application/javascript/ElectronBoundaryGraphCallPaths.ts new file mode 100644 index 000000000..4447d59db --- /dev/null +++ b/src/application/javascript/ElectronBoundaryGraphCallPaths.ts @@ -0,0 +1,193 @@ +import type { ApplicationNode } from "../../domain/javascript/javascriptApplicationGraphSchemas.js"; +import type { ElectronCallPath } from "../../domain/javascript/electronStaticAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { sha256Text } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js"; +import { + collectElectronIpcRecords, + unambiguousElectronIpcPairings, + type ElectronIpcRecord, +} from "./ElectronBoundaryAnalysis.js"; +import { + addElectronInferenceEdge, + electronFindingSourceNode, + electronRangeKey, +} from "./ElectronBoundaryGraphContext.js"; +import { + artifactLocalIdentity, + javascriptAnalysisCoverage, + type JavaScriptArtifactGraphContext, + type JavaScriptArtifactGraphCoverage, +} from "./JavaScriptArtifactGraphContext.js"; +import { staticInferenceEvidence } from "./JavaScriptArtifactGraphEvidence.js"; + +interface ProjectedPath { + readonly file: JavaScriptArtifactFile; + readonly path: ElectronCallPath; + readonly coverage: JavaScriptArtifactGraphCoverage; + readonly nodes: readonly ApplicationNode[]; +} + +/** Retain context-specific wrapper paths, then join only unique IPC records. */ +export const addElectronCallPaths = ( + context: JavaScriptArtifactGraphContext, +): void => { + const projected: ProjectedPath[] = []; + for (const { file, javascript } of context.analysis.files) { + if (javascript === null) continue; + const coverage = javascriptAnalysisCoverage(javascript); + for (const path of javascript.electron.call_paths ?? []) { + const pathId = sha256Text(JSON.stringify(path)); + const nodes = path.steps.map((step, index) => + context.accumulator.addNode({ + kind: "javascript-expression", + identity: artifactLocalIdentity( + file.sha256, + "electron-call-path-step", + `${pathId}:${String(index)}`, + ), + observations: [ + { + label: step.label, + properties: { + path_id: pathId, + entry: path.entry, + entry_kind: path.entry_kind, + channel: path.channel, + terminal: path.terminal, + step_kind: step.kind, + step_index: index, + module_key: step.module_key, + module_runtime: step.module_runtime, + conditions: path.conditions, + resolution: "candidate", + }, + evidence: staticInferenceEvidence({ + sha256: file.sha256, + path: file.path, + range: step.location, + operation: "substitute-electron-wrapper", + coverage, + limitations: [ + "Lexical wrapper substitution is a static candidate; guards, runtime overrides and execution remain unverified.", + ], + }), + }, + ], + }), + ); + const first = nodes[0], + entry = path.steps[0]; + const owner = entry + ? electronFindingSourceNode( + context, + file, + entry.module_key, + entry.module_runtime ?? undefined, + ) + : undefined; + if (first && entry && owner) + addElectronInferenceEdge(context, { + source: owner, + target: first, + file, + range: entry.location, + coverage, + relation: "contains", + operation: "locate-electron-wrapper-path", + properties: { + path_id: pathId, + entry: path.entry, + resolution: "candidate", + }, + }); + for (let i = 1; i < nodes.length; i += 1) { + const source = nodes[i - 1], + target = nodes[i], + previous = path.steps[i - 1], + step = path.steps[i]; + if (!source || !target || !previous || !step) continue; + addElectronInferenceEdge(context, { + source, + target, + file, + range: step.location, + coverage, + relation: ["exposure", "factory", "ipc"].includes(previous.kind) + ? "maps_to" + : "calls", + operation: "trace-electron-wrapper-path", + properties: { + path_id: pathId, + entry: path.entry, + channel: path.channel, + resolution: "candidate", + conditions: path.conditions, + }, + }); + } + projected.push({ file, path, coverage, nodes }); + } + } + for (const { transmission, handler } of unambiguousElectronIpcPairings( + collectElectronIpcRecords(context.analysis), + )) { + const sources = projected.filter( + (p) => p.path.entry_kind === "bridge-member" && matches(p, transmission), + ); + const targets = projected.filter( + (p) => p.path.entry_kind === "ipc-handler" && matches(p, handler), + ); + for (const from of sources) + for (const to of targets) { + const source = from.nodes.at(-1), + target = to.nodes[0]; + if (!source || !target) continue; + addElectronInferenceEdge(context, { + source, + target, + file: from.file, + range: transmission.finding.location, + coverage: from.coverage, + relation: + transmission.finding.mode === "invoke" ? "invokes" : "sends", + operation: "join-electron-wrapper-paths", + properties: { + channel: transmission.finding.channel, + pairing_basis: "unique-exact-literal-channel", + resolution: "candidate", + handler_path: to.file.path, + conditions: to.path.conditions, + }, + limitations: [ + "The unique static channel match does not establish runtime registration, sender acceptance, or dispatch.", + ], + }); + } + } +}; + +const matches = ( + projected: ProjectedPath, + record: ElectronIpcRecord, +): boolean => { + const { path } = projected; + const { finding } = record; + const invocation = finding.invocation_location; + const sourceStepKind = + finding.mode === "send" || finding.mode === "invoke" ? "terminal" : "ipc"; + return ( + projected.file.path === record.file.path && + path.channel === finding.channel && + path.steps.some( + (step) => + step.kind === sourceStepKind && + electronRangeKey(step.location) === electronRangeKey(finding.location), + ) && + (invocation === undefined || + path.steps.some( + (step) => + (step.kind === "call" || step.kind === "exposure") && + electronRangeKey(step.location) === electronRangeKey(invocation), + )) + ); +}; diff --git a/src/application/javascript/ElectronBoundaryGraphContext.ts b/src/application/javascript/ElectronBoundaryGraphContext.ts index cc96f9279..a02b5f016 100644 --- a/src/application/javascript/ElectronBoundaryGraphContext.ts +++ b/src/application/javascript/ElectronBoundaryGraphContext.ts @@ -27,6 +27,7 @@ export interface ElectronGraphEdgeInput { | "exposes" | "sends" | "invokes" + | "calls" | "handles"; readonly operation: string; readonly properties: Readonly>; diff --git a/src/application/javascript/ElectronBoundaryGraphIpc.ts b/src/application/javascript/ElectronBoundaryGraphIpc.ts index d38ba48ae..24dbffba4 100644 --- a/src/application/javascript/ElectronBoundaryGraphIpc.ts +++ b/src/application/javascript/ElectronBoundaryGraphIpc.ts @@ -7,7 +7,10 @@ import { type JavaScriptArtifactGraphContext, type JavaScriptArtifactGraphCoverage, } from "./JavaScriptArtifactGraphContext.js"; -import { astObservationEvidence } from "./JavaScriptArtifactGraphEvidence.js"; +import { + astObservationEvidence, + staticInferenceEvidence, +} from "./JavaScriptArtifactGraphEvidence.js"; import { collectElectronIpcRecords, unambiguousElectronIpcPairings, @@ -120,11 +123,18 @@ const addChannelNode = ( : `literal:${finding.channel}`; return context.accumulator.addNode({ kind: "ipc-channel", - identity: electronObservationIdentity( - context, - "electron-ipc-channel", - identityKey, - ), + identity: + finding.resolution_basis === "wrapper-substitution" + ? artifactLocalIdentity( + file.sha256, + "electron-inferred-ipc-channel", + identityKey, + ) + : electronObservationIdentity( + context, + "electron-ipc-channel", + identityKey, + ), observations: [ { label: finding.channel ?? finding.channel_expression, @@ -135,19 +145,28 @@ const addChannelNode = ( channel: finding.channel, channel_expression: finding.channel_expression, resolution: finding.channel === null ? "dynamic" : "literal", + ...(finding.invocation_location === undefined + ? {} + : { invocation_location: finding.invocation_location }), }, - evidence: astObservationEvidence({ + evidence: (finding.resolution_basis === "wrapper-substitution" + ? staticInferenceEvidence + : astObservationEvidence)({ sha256: file.sha256, path: file.path, range: finding.location, operation: "observe-ipc-channel-expression", coverage, limitations: - finding.channel === null + finding.resolution_basis === "wrapper-substitution" ? [ - "The channel expression is dynamic; its runtime value and pairings remain unknown.", + "The channel is inferred through lexical wrapper substitution; runtime registration and dispatch remain unverified.", ] - : [], + : finding.channel === null + ? [ + "The channel expression is dynamic; its runtime value and pairings remain unknown.", + ] + : [], }), }, ], @@ -176,8 +195,13 @@ const addHandlerNode = (input: IpcHandlerInput): ApplicationNode => { channel_expression: finding.channel_expression, handler_kind: finding.handler_kind, handler_location: finding.handler_location, + ...(finding.invocation_location === undefined + ? {} + : { invocation_location: finding.invocation_location }), }, - evidence: astObservationEvidence({ + evidence: (finding.resolution_basis === "wrapper-substitution" + ? staticInferenceEvidence + : astObservationEvidence)({ sha256: file.sha256, path: file.path, range: handlerRange, @@ -209,6 +233,9 @@ const addHandlerNode = (input: IpcHandlerInput): ApplicationNode => { channel: finding.channel, channel_expression: finding.channel_expression, resolution: finding.channel === null ? "dynamic" : "literal", + ...(finding.invocation_location === undefined + ? {} + : { invocation_location: finding.invocation_location }), }, }); return handler; diff --git a/src/application/javascript/ElectronBoundaryGraphWindows.ts b/src/application/javascript/ElectronBoundaryGraphWindows.ts index f8d192db6..63f62bd12 100644 --- a/src/application/javascript/ElectronBoundaryGraphWindows.ts +++ b/src/application/javascript/ElectronBoundaryGraphWindows.ts @@ -13,7 +13,10 @@ import { type JavaScriptArtifactGraphContext, type JavaScriptArtifactGraphCoverage, } from "./JavaScriptArtifactGraphContext.js"; -import { astObservationEvidence } from "./JavaScriptArtifactGraphEvidence.js"; +import { + astObservationEvidence, + staticInferenceEvidence, +} from "./JavaScriptArtifactGraphEvidence.js"; import { addElectronAstEdge, addElectronInferenceEdge, @@ -190,8 +193,11 @@ const addContextBridge = ( api_status: value.api_status, members: value.members, unknown_members: value.unknown_members, + resolution_basis: value.resolution_basis ?? "direct-syntax", }, - evidence: astObservationEvidence({ + evidence: (value.resolution_basis === "wrapper-substitution" + ? staticInferenceEvidence + : astObservationEvidence)({ sha256: file.sha256, path: file.path, range: value.location, diff --git a/src/application/javascript/JavaScriptApplicationEvidence.ts b/src/application/javascript/JavaScriptApplicationEvidence.ts index aded8f2dd..b812f237e 100644 --- a/src/application/javascript/JavaScriptApplicationEvidence.ts +++ b/src/application/javascript/JavaScriptApplicationEvidence.ts @@ -91,6 +91,12 @@ const parameters = ( ): EvidenceObservation["parameters"] => ({ format: input.format, integrity_policy: input.integrity_policy, + ...(input.max_semantic_nodes === undefined + ? {} + : { max_semantic_nodes: input.max_semantic_nodes }), + ...(input.max_semantic_nodes_per_file === undefined + ? {} + : { max_semantic_nodes_per_file: input.max_semantic_nodes_per_file }), ...(input.source_paths === undefined ? {} : { source_paths: input.source_paths }), diff --git a/src/application/javascript/JavaScriptApplicationService.ts b/src/application/javascript/JavaScriptApplicationService.ts index c7da5df88..ea56ce591 100644 --- a/src/application/javascript/JavaScriptApplicationService.ts +++ b/src/application/javascript/JavaScriptApplicationService.ts @@ -66,6 +66,12 @@ export const analyzeJavaScriptApplicationValidated = async ( input_path: input.input_path, format: input.format, integrity_policy: input.integrity_policy, + ...(input.max_semantic_nodes === undefined + ? {} + : { max_semantic_nodes: input.max_semantic_nodes }), + ...(input.max_semantic_nodes_per_file === undefined + ? {} + : { max_semantic_nodes_per_file: input.max_semantic_nodes_per_file }), ...(input.source_paths === undefined ? {} : { source_paths: input.source_paths }), diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 81714ada0..58bbce3f9 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -146,22 +146,17 @@ export const analyzeAndAdoptJavaScriptArtifactFiles = async ( for (const file of fileSet.files) { try { await setImmediate(); - if (signal?.aborted === true) - throw new AnalysisCancelledError("analyze_javascript_application"); - if ( - !file.text.included && - file.text.reason === "outside-analysis-scope" - ) { - state.files.push({ file, javascript: null, semantic: null }); - if (file.kind === "javascript") - projection.recordUnavailableFile( - file, - "Source was not analyzed because it is outside the caller-selected source_paths.", - null, - "incomplete-module", + if (classifyOutsideAnalysisScope(file, state, projection)) { + if (signal?.aborted === true) { + interruption = new AnalysisCancelledError( + "analyze_javascript_application", ); + break; + } continue; } + if (signal?.aborted === true) + throw new AnalysisCancelledError("analyze_javascript_application"); await beforeFile(file, selectedFileIndex++, selectedFileCount); if (file.kind !== "javascript" || !file.text.included) { const steps = analyzeArtifactFileSteps(file, { @@ -287,20 +282,7 @@ export const analyzeAndAdoptJavaScriptArtifactFiles = async ( }; for (const file of fileSet.files) { if (analyzedPaths.has(file.path)) continue; - if ( - !file.text.included && - file.text.reason === "outside-analysis-scope" - ) { - state.files.push({ file, javascript: null, semantic: null }); - if (file.kind === "javascript") - projection.recordUnavailableFile( - file, - "Source was not analyzed because it is outside the caller-selected source_paths.", - null, - "incomplete-module", - ); - continue; - } + if (classifyOutsideAnalysisScope(file, state, projection)) continue; state.files.push({ file, javascript: null, @@ -370,10 +352,7 @@ function* analyzeArtifactFileSteps< context: ArtifactAnalysisContext, ): Generator { const { state } = context; - if (!file.text.included && file.text.reason === "outside-analysis-scope") { - state.files.push({ file, javascript: null, semantic: null }); - return; - } + if (classifyOutsideAnalysisScope(file, state)) return; addStructuredObservations(file, state); if (file.kind === "html" && file.text.included) state.htmlScripts.push(...parseHtmlScripts(file.path, file.text.value)); @@ -421,6 +400,26 @@ function* analyzeArtifactFileSteps< if (analysis.parse_status === "failed") state.parseFailures += 1; } +const classifyOutsideAnalysisScope = < + SemanticIr extends JavaScriptModuleSemanticIr, +>( + file: JavaScriptArtifactFile, + state: MutableArtifactAnalysis, + projection?: JavaScriptSemanticGraphProjection, +): boolean => { + if (file.text.included || file.text.reason !== "outside-analysis-scope") + return false; + state.files.push({ file, javascript: null, semantic: null }); + if (file.kind === "javascript") + projection?.recordUnavailableFile( + file, + "Source was not analyzed because it is outside the caller-selected source_paths.", + null, + "incomplete-module", + ); + return true; +}; + const addSourceMap = ( file: JavaScriptArtifactFile, context: ArtifactAnalysisContext, @@ -540,6 +539,7 @@ const collectSourceMapOriginals = ( maps: readonly Readonly>[], ): JavaScriptSourceMapObservation => { const sources: JavaScriptSourceMapOriginal[] = []; + let declarationIndex = 0; for (const map of maps) { const names = map.sources; if (!Array.isArray(names)) @@ -557,7 +557,7 @@ const collectSourceMapOriginals = ( ); const root = typeof map.sourceRoot === "string" ? map.sourceRoot : null; for (const [index, raw] of names.entries()) { - if (typeof raw !== "string") + if (raw !== null && typeof raw !== "string") return invalidSourceMap( file, "Source map contains a non-string source name.", @@ -565,7 +565,12 @@ const collectSourceMapOriginals = ( const rawContent = Array.isArray(contents) ? contents[index] : undefined; const content = typeof rawContent === "string" ? rawContent : null; sources.push({ - reference: resolveJavaScriptSourceMapReference(raw, root, file.path), + source_index: index, + declaration_index: declarationIndex++, + reference: + typeof raw === "string" + ? resolveJavaScriptSourceMapReference(raw, root, file.path) + : null, content, content_sha256: content === null ? null : sha256(content), }); diff --git a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts index 630a330c5..ec49d0b9d 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysisTypes.ts @@ -47,7 +47,11 @@ export interface JavaScriptHtmlScriptObservation { /** One original source declared by a local source map. */ export interface JavaScriptSourceMapOriginal { - readonly reference: JavaScriptSourceMapReference; + /** Index within the leaf map's sources and sourcesContent arrays. */ + readonly source_index: number; + /** Index across the flattened map's leaf declarations. */ + readonly declaration_index: number; + readonly reference: JavaScriptSourceMapReference | null; readonly content: string | null; readonly content_sha256: string | null; } diff --git a/src/application/javascript/JavaScriptArtifactGraphDocuments.ts b/src/application/javascript/JavaScriptArtifactGraphDocuments.ts index bdea67b95..e0798bb7b 100644 --- a/src/application/javascript/JavaScriptArtifactGraphDocuments.ts +++ b/src/application/javascript/JavaScriptArtifactGraphDocuments.ts @@ -1,6 +1,7 @@ import { addArtifactContainsEdge, addStaticInferenceEdge, + artifactLocalIdentity, createElectronRoleNode, linkElectronRoleToAsset, type JavaScriptArtifactGraphContext, @@ -118,22 +119,34 @@ export const addJavaScriptSourceMapOriginals = ( if (mapFile === undefined || mapNode === undefined) continue; const coverage = completeApplicationCoverage(); for (const original of sourceMap.sources) { + const reference = original.reference; const node = context.accumulator.addNode({ - kind: "source-module", - identity: { - strategy: "source-map-original", - stability: "source-map-exact", - source_map_sha256: mapFile.sha256, - original_source: original.reference.source_name, - source_root: original.reference.source_root, - source_sha256: original.content_sha256, - }, + kind: reference === null ? "unknown" : "source-module", + identity: + reference === null + ? artifactLocalIdentity( + mapFile.sha256, + "source-map-unnamed-declaration", + String(original.declaration_index), + ) + : { + strategy: "source-map-original", + stability: "source-map-exact", + source_map_sha256: mapFile.sha256, + original_source: reference.source_name, + source_root: reference.source_root, + source_sha256: original.content_sha256, + }, observations: [ { - label: original.reference.source_name || null, - source_map_reference: original.reference, + label: reference?.source_name || null, + source_map_reference: reference, properties: { + source_index: original.source_index, + declaration_index: original.declaration_index, + source_name_available: reference !== null, content_available: original.content !== null, + content_sha256: original.content_sha256, }, evidence: artifactObservationEvidence({ sha256: mapFile.sha256, @@ -151,8 +164,10 @@ export const addJavaScriptSourceMapOriginals = ( target_node_id: node.node_id, relation: "contains", properties: { - original_source: original.reference.source_name, - source_root: original.reference.source_root, + source_index: original.source_index, + declaration_index: original.declaration_index, + original_source: reference?.source_name ?? null, + source_root: reference?.source_root ?? null, }, evidence: artifactObservationEvidence({ sha256: mapFile.sha256, diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts index 8af7aa19d..2a05af6eb 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.test.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.test.ts @@ -67,6 +67,147 @@ describe("artifact-local graph identity", () => { }); }); +describe("loader-owned path lookup", () => { + it("keeps relative ESM imports exact", () => { + const files = fileMap([ + file("app/entry.mjs", "root"), + file("app/feature.js", "root"), + file("app/directory/index.js", "root"), + file("app/feature.js.js", "root"), + ]); + for (const declaredPath of ["./feature", "./directory"]) { + expect( + resolve({ + declaredPath, + sourcePath: "app/entry.mjs", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + } + expect( + resolve({ + declaredPath: "./feature.js", + sourcePath: "app/entry.mjs", + context: "module-specifier", + moduleKind: "import", + files, + }), + ).toMatchObject({ resolved_path: "app/feature.js" }); + }); + + it("does not use .cjs as an extensionless CommonJS probe", () => { + const files = fileMap([ + file("app/entry.cjs", "root"), + file("app/feature.cjs", "root"), + ]); + expect( + resolve({ + declaredPath: "./feature", + sourcePath: "app/entry.cjs", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + expect( + resolve({ + declaredPath: "./feature.cjs", + sourcePath: "app/entry.cjs", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ resolved_path: "app/feature.cjs" }); + }); + + it("treats an empty package main as absent and resolves the root index", () => { + const files = fileMap([ + file("app/src/entry.cjs", "root"), + file("app/node_modules/fixture/package.json", "root", '{"main":""}'), + file("app/node_modules/fixture/index.js", "root"), + ]); + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/src/entry.cjs", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "app/node_modules/fixture/index.js", + }); + }); + + it.each(["import", "require"] as const)( + "limits %s package-main probes to Node's file extensions", + (moduleKind) => { + const files = fileMap([ + file("app/src/entry.js", "root"), + file( + "app/node_modules/fixture/package.json", + "root", + '{"main":"main"}', + ), + file("app/node_modules/fixture/main.cjs", "root"), + file("app/node_modules/fixture/index.mjs", "root"), + ]); + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/src/entry.js", + context: "module-specifier", + moduleKind, + files, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + }, + ); + + it.each(["import", "require"] as const)( + "does not recursively resolve a nested package from a %s package main", + (moduleKind) => { + const files = fileMap([ + file("app/src/entry.js", "root"), + file("app/node_modules/fixture/package.json", "root", '{"main":"sub"}'), + file( + "app/node_modules/fixture/sub/package.json", + "root", + '{"main":"other"}', + ), + file("app/node_modules/fixture/sub/other.js", "root"), + ]); + expect( + resolve({ + declaredPath: "fixture", + sourcePath: "app/src/entry.js", + context: "module-specifier", + moduleKind, + files, + }), + ).toMatchObject({ resolution_status: "not-found", resolved_path: null }); + }, + ); + + it("still rejects non-string package main metadata", () => { + expect( + resolve({ + declaredPath: "./package", + sourcePath: "app/entry.js", + context: "filesystem-expression", + files: fileMap([ + file("app/entry.js", "root"), + file("app/package/package.json", "root", '{"main":false}'), + file("app/package/index.js", "root"), + ]), + }), + ).toMatchObject({ resolution_status: "unavailable", resolved_path: null }); + }); +}); + describe("logical path identity", () => { it("resolves an NFD relative import to the NFC inventory path and rejects a root escape", () => { const nfc = "caf\u00e9".normalize("NFC"); @@ -640,7 +781,6 @@ describe("package subpath directory ownership", () => { it.each([ ["malformed", "{", "not valid package JSON"], ["unavailable", null, "text is unavailable"], - ["cyclic", '{"main":"."}', "cycle"], ])("preserves %s nested directory failures", (_name, text, reason) => { const files = fileMap([ file("app/consumer.js", "root"), @@ -663,10 +803,34 @@ describe("package subpath directory ownership", () => { limitations: [expect.stringContaining(reason)], }); }); + + it("does not recurse through a cyclic nested main during CommonJS lookup", () => { + const files = fileMap([ + file("app/consumer.js", "root"), + file("app/node_modules/fixture/package.json", "root", "{}"), + file( + "app/node_modules/fixture/parser/package.json", + "root", + '{"main":"."}', + ), + ]); + expect( + resolve({ + declaredPath: "fixture/parser", + sourcePath: "app/consumer.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "not-found", + resolved_path: null, + }); + }); }); describe("package subpath legacy lookup and metadata failures", () => { - it("resolves legacy subpath files and indexes without an exports field", () => { + it("resolves CommonJS legacy subpath files and indexes without exports", () => { const files = fileMap([ file("app/src/consumer.js", "root"), file( @@ -674,7 +838,8 @@ describe("package subpath legacy lookup and metadata failures", () => { "root", '{"main":"./index.cjs"}', ), - file("app/node_modules/fixture/direct.cjs", "root"), + file("app/node_modules/fixture/direct.js", "root"), + file("app/node_modules/fixture/explicit.cjs", "root"), file("app/node_modules/fixture/folder/index.js", "root"), ]); @@ -688,7 +853,19 @@ describe("package subpath legacy lookup and metadata failures", () => { }), ).toMatchObject({ resolution_status: "resolved", - resolved_path: "app/node_modules/fixture/direct.cjs", + resolved_path: "app/node_modules/fixture/direct.js", + }); + expect( + resolve({ + declaredPath: "fixture/explicit.cjs", + sourcePath: "app/src/consumer.js", + context: "module-specifier", + moduleKind: "require", + files, + }), + ).toMatchObject({ + resolution_status: "resolved", + resolved_path: "app/node_modules/fixture/explicit.cjs", }); expect( resolve({ @@ -1348,18 +1525,17 @@ describe("directory package entrypoint precedence", () => { }); }); - it("retains cycle diagnostics when directory packages refer to each other", () => { + it("retains cycle diagnostics for generic package-entrypoint inference", () => { const files = fileMap([ - file("app/consumer.js", "root"), + file("app/package.json", "root"), file("app/a/package.json", "root", '{"main":"../b"}'), file("app/b/package.json", "root", '{"main":"../a"}'), ]); expect( resolve({ declaredPath: "./a", - sourcePath: "app/consumer.js", - context: "module-specifier", - moduleKind: "require", + sourcePath: "app/package.json", + context: "package-entrypoint", files, }), ).toMatchObject({ diff --git a/src/application/javascript/JavaScriptArtifactPathResolution.ts b/src/application/javascript/JavaScriptArtifactPathResolution.ts index 035014370..481a40453 100644 --- a/src/application/javascript/JavaScriptArtifactPathResolution.ts +++ b/src/application/javascript/JavaScriptArtifactPathResolution.ts @@ -145,16 +145,21 @@ const resolvePath = (input: ResolveArtifactPathInput): CandidateResolution => { `The exact HTML resource ${confined} was not found among the selected application files.`, ]) ); - return resolveCandidate( - input, - confined, - new Set(), + const isModuleUrlPath = input.context === "module-specifier" && - input.moduleKind !== undefined && - (input.declaredPath.startsWith(".") || input.declaredPath.startsWith("/")) + (input.declaredPath.startsWith(".") || + input.declaredPath.startsWith("/") || + fileUrlPath(input.declaredPath) !== undefined); + const emittedSourceCandidates = + isModuleUrlPath && input.moduleKind !== undefined ? typescriptEmittedSourceCandidates(input.sourcePath, confined) - : [], - ); + : []; + if (isModuleUrlPath && input.moduleKind === "import") + return ( + resolveFileCandidates(input, [confined, ...emittedSourceCandidates]) ?? + notFoundCandidate() + ); + return resolveCandidate(input, confined, new Set(), emittedSourceCandidates); }; const rejectDeclaration = (declared: string): CandidateResolution | null => { @@ -275,7 +280,10 @@ const bareModuleCandidate = ( if (request.dependencyExportsEligible && metadata.kind !== "absent") return resolvePackageDependency(input, candidate, request, metadata); if (request.packageSubpath === ".") { - const direct = resolveFileCandidates(input, fileCandidates(candidate)); + const direct = resolveFileCandidates( + input, + fileCandidates(candidate, input.moduleKind), + ); if (direct !== null) return direct; } if (hasContainerDirectory(input.files, candidate, source?.container_sha256)) @@ -417,8 +425,10 @@ const resolvePackageDependency = ( if (rawExports === undefined) { if (subpath === ".") return ( - resolveFileCandidates(input, fileCandidates(directory)) ?? - resolveDirectory(input, directory, metadata, new Set()) + resolveFileCandidates( + input, + fileCandidates(directory, input.moduleKind), + ) ?? resolveDirectory(input, directory, metadata, new Set()) ); // Resolve legacy ESM subpaths as URLs: suffixes identify the loaded module // but do not belong to its filesystem path. Keep the declared specifier on @@ -753,7 +763,7 @@ const resolveCandidate = ( emittedSourceCandidates: readonly string[] = [], ): CandidateResolution => resolveFileCandidates(input, [ - ...fileCandidates(candidate), + ...fileCandidates(candidate, input.moduleKind), ...emittedSourceCandidates, ]) ?? resolveDirectory( @@ -771,8 +781,10 @@ const resolveDirectory = ( ): CandidateResolution => { if (metadata.kind === "absent") return ( - resolveFileCandidates(input, indexCandidates(directory)) ?? - notFoundCandidate() + resolveFileCandidates( + input, + indexCandidates(directory, input.moduleKind), + ) ?? notFoundCandidate() ); if (metadata.kind === "unavailable") return unresolvedCandidate("unavailable", [metadata.limitation]); @@ -780,15 +792,20 @@ const resolveDirectory = ( return unresolvedCandidate("unavailable", [ `Directory package entrypoint cycle includes ${metadata.path}.`, ]); - const preferred = [ - metadata.main, - ...(input.moduleKind === undefined ? [metadata.module] : []), - ]; - const entry = preferred.find((value) => value !== undefined); + const entry = + metadata.main === "" + ? undefined + : metadata.main !== undefined + ? metadata.main + : input.moduleKind === undefined + ? metadata.module + : undefined; if (entry === undefined) return ( - resolveFileCandidates(input, indexCandidates(directory)) ?? - notFoundCandidate() + resolveFileCandidates( + input, + indexCandidates(directory, input.moduleKind), + ) ?? notFoundCandidate() ); if (typeof entry !== "string" || entry.length === 0) return unresolvedCandidate("unavailable", [ @@ -809,10 +826,12 @@ const resolveDirectory = ( if (typeof confined !== "string") return confined; const selected = resolveFileCandidates(entryInput, [ - ...fileCandidates(confined), - ...indexCandidates(confined), - ]) ?? resolveFileCandidates(input, indexCandidates(directory)); + ...fileCandidates(confined, entryInput.moduleKind), + ...indexCandidates(confined, entryInput.moduleKind), + ]) ?? + resolveFileCandidates(input, indexCandidates(directory, input.moduleKind)); if (selected !== null) return selected; + if (input.moduleKind !== undefined) return notFoundCandidate(); return resolveCandidate( entryInput, confined, @@ -837,13 +856,25 @@ const resolveFileCandidates = ( return null; }; -const fileCandidates = (candidate: string): readonly string[] => [ +const NODE_MODULE_EXTENSIONS = [".js", ".json", ".node"] as const; + +const fileCandidates = ( + candidate: string, + moduleKind?: ResolveArtifactPathInput["moduleKind"], +): readonly string[] => [ candidate, - ...EXTENSIONS.map((extension) => `${candidate}${extension}`), + ...(moduleKind === undefined ? EXTENSIONS : NODE_MODULE_EXTENSIONS).map( + (extension) => `${candidate}${extension}`, + ), ]; -const indexCandidates = (candidate: string): readonly string[] => - EXTENSIONS.map((extension) => posix.join(candidate, `index${extension}`)); +const indexCandidates = ( + candidate: string, + moduleKind?: ResolveArtifactPathInput["moduleKind"], +): readonly string[] => + (moduleKind === undefined ? EXTENSIONS : NODE_MODULE_EXTENSIONS).map( + (extension) => posix.join(candidate, `index${extension}`), + ); type PackageTargetBlockCause = "null" | "empty-array"; diff --git a/src/application/javascript/JavaScriptArtifactReconstruction.ts b/src/application/javascript/JavaScriptArtifactReconstruction.ts index 3b6ebaa13..bd95415c6 100644 --- a/src/application/javascript/JavaScriptArtifactReconstruction.ts +++ b/src/application/javascript/JavaScriptArtifactReconstruction.ts @@ -139,7 +139,14 @@ export const reconstructJavaScriptArtifact = async ( "parse_javascript_sources", `Parsing and projecting ${String(files.files.length - (files.source_selection?.unselected_files ?? 0))} selected application source files`, ); - const semanticProjection = createJavaScriptSemanticGraphProjection(); + const semanticProjection = createJavaScriptSemanticGraphProjection({ + ...(input.max_semantic_nodes === undefined + ? {} + : { totalNodes: input.max_semantic_nodes }), + ...(input.max_semantic_nodes_per_file === undefined + ? {} + : { fileNodes: input.max_semantic_nodes_per_file }), + }); const worker = createJavaScriptSourceAnalysis( { heapMb: diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts index 590f4950b..4816be96c 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.test.ts @@ -6,6 +6,8 @@ import { expect, it } from "vitest"; import { SEMANTIC_GRAPH_FILE_NODE_CEILING, SEMANTIC_GRAPH_NODE_CEILING, + createJavaScriptSemanticGraphProjection, + projectJavaScriptSemanticFileSteps, } from "./JavaScriptSemanticGraphBuilder.js"; import { addSemanticGraphNode, @@ -905,3 +907,48 @@ it.each(["{}", "unknownRoot", "{ a: null }"])( expect(trace.nodes.filter(({ kind }) => kind === "literal")).toEqual([]); }, ); + +it("projects and adopts more than the default per-file capacity when explicitly offered", () => { + const text = "external;\n".repeat(20_001); + const file: JavaScriptArtifactFile = { + path: "large.js", + container_sha256: SHA256, + sha256: SHA256, + bytes: Buffer.byteLength(text), + inventory_artifact_id: `art_${SHA256}`, + kind: "javascript", + unpacked: false, + text: { included: true, value: text }, + }; + const steps = projectJavaScriptSemanticFileSteps( + file, + analyzeJavaScriptSemantics(text), + 20_002, + ); + let result = steps.next(); + while (!result.done) result = steps.next(); + expect(result.value.nodes).toHaveLength(20_002); + expect(result.value.truncated).toBe(false); + const projection = createJavaScriptSemanticGraphProjection({ + totalNodes: 20_010, + fileNodes: 20_002, + }); + projection.adoptFileProjection(result.value); + expect(projection.remainingFileNodeBudget()).toBe(8); + const graph = projection.finish(SHA256, { + graph_id: `jag_${"b".repeat(64)}`, + nodes: [], + }); + expect(graph.coverage).toMatchObject({ + truncated: false, + omitted_nodes: 0, + omitted_relations: 0, + }); + const bounded = createJavaScriptSemanticGraphProjection({ + totalNodes: 20_001, + fileNodes: 20_002, + }); + expect(() => bounded.adoptFileProjection(result.value)).toThrow( + "exceed the offered tree capacity", + ); +}); diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 9bbfe8e15..6a7d0920a 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -41,7 +41,11 @@ import { type ApplicationCoverage, } from "../../domain/javascript/javascriptApplicationEvidenceSchemas.js"; import type { JsonValue } from "../../domain/jsonValue.js"; -import { javaScriptAnalysisCoverageLimits } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; +import { + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS, + JAVASCRIPT_SEMANTIC_NODE_MAXIMUM, + javaScriptAnalysisCoverageLimits, +} from "../../domain/javascript/javascriptAnalysisResourceControls.js"; import { projectSemanticFunctionFingerprints } from "./JavaScriptSemanticGraphFingerprintProjection.js"; import { projectSemanticClosureCaptures, @@ -65,6 +69,8 @@ import { type SemanticGraphProjectionState as BuilderState, } from "./JavaScriptSemanticGraphConstruction.js"; import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; +import type { ElectronCallPath } from "../../domain/javascript/electronStaticAnalysisTypes.js"; +import { projectElectronSemanticCallPaths } from "./JavaScriptSemanticGraphElectronProjection.js"; interface BuilderInput { readonly rootArtifactSha256: string; @@ -105,13 +111,13 @@ interface FileContext extends SemanticFlowProjectionContext { * (`truncated`, `omitted_nodes`, `omitted_relations`, `limits`) rather than * silently dropping data. * - * Sizing: a retained semantic node costs roughly 3 KB once serialized, and V8 - * additionally caps a single string at 512 MB. The result is serialized after the - * object graph is built and validated, so both phases must fit together. A single - * minified vendor bundle can account for most of a tree's nodes, so a whole-tree - * ceiling without a per-file ceiling does not bound the work. Both are applied. + * Defaults bound work even when ample heap is available. Callers may raise the + * capacities explicitly; isolated worker heap, bounded record transport and + * parent result admission independently constrain actual retained memory. The + * result uses incremental serialization rather than a single JSON string. */ -export const SEMANTIC_GRAPH_NODE_CEILING = 100_000; +export const SEMANTIC_GRAPH_NODE_CEILING = + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.semanticNodes; /** * Per-file share of the semantic node budget. @@ -119,7 +125,8 @@ export const SEMANTIC_GRAPH_NODE_CEILING = 100_000; * Bound each source file independently so one large bundled library cannot * consume the whole tree's budget and starve the application's own modules. */ -export const SEMANTIC_GRAPH_FILE_NODE_CEILING = 20_000; +export const SEMANTIC_GRAPH_FILE_NODE_CEILING = + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS.semanticNodesPerFile; export const buildJavaScriptSemanticGraph = ({ rootArtifactSha256, @@ -129,7 +136,11 @@ export const buildJavaScriptSemanticGraph = ({ const projection = createJavaScriptSemanticGraphProjection(); for (const analyzed of analysis.files) { if (analyzed.semantic !== null) - projection.projectFile(analyzed.file, analyzed.semantic.ir); + projection.projectFile( + analyzed.file, + analyzed.semantic.ir, + analyzed.javascript?.electron.call_paths, + ); else if ( !analyzed.file.text.included && analyzed.file.text.reason === "outside-analysis-scope" @@ -162,11 +173,13 @@ export interface JavaScriptSemanticGraphProjection { readonly projectFile: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, + paths?: readonly ElectronCallPath[], ) => void; /** Project one file in steps between its projection passes. */ readonly projectFileSteps: ( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, + paths?: readonly ElectronCallPath[], ) => Generator; readonly finish: ( rootArtifactSha256: string, @@ -179,234 +192,241 @@ export interface JavaScriptSemanticGraphProjection { } /** Accumulate semantic nodes while each source IR is still file-local. */ -export const createJavaScriptSemanticGraphProjection = - (): JavaScriptSemanticGraphProjection => { - const state = emptyState(); - const fingerprints: JavaScriptSemanticFingerprint[] = []; - let truncatedFiles = 0; - let unavailableFiles = 0; - let resourceTruncatedFiles = 0; - const executionLimits = new Map< - string, - ApplicationCoverage["limits"][number] - >(); - const projectSourceSteps = function* ( - file: JavaScriptArtifactFile, - ir: JavaScriptSemanticIr, - ): Generator { - if (state.nodes.size >= SEMANTIC_GRAPH_NODE_CEILING) { - truncatedFiles += 1; - return; - } - const remainingTreeBudget = - SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size; - state.fileNodeBudget = Math.min( - SEMANTIC_GRAPH_FILE_NODE_CEILING, - remainingTreeBudget, - ); - state.fileNodesDropped = false; - fingerprints.push(...(yield* projectFileSteps(file, ir, state))); - // Nodes were dropped only if the budget actually blocked creation. A file - // that exactly fills its share ends with a zero budget without dropping - // anything, so the remaining budget alone cannot decide truncation. - if (state.fileNodesDropped) truncatedFiles += 1; - state.fileNodeBudget = null; - }; - const finishWith = ( - factory: (input: unknown) => Value, - rootArtifactSha256: string, - applicationGraph: BuilderInput["applicationGraph"], - ): Value => { - bindSemanticGraphApplicationNodes(state, applicationGraph); - if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); - const relations = [...state.relations.values()]; - const unknowns = [...state.unknowns.values()]; - type Family = (typeof JAVASCRIPT_SEMANTIC_RELATION_FAMILIES)[number]; - const retainedByFamily = new Map(); - const unknownIdsByFamily = new Map(); - for (const relation of relations) { - const family = JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation]; - retainedByFamily.set(family, (retainedByFamily.get(family) ?? 0) + 1); - } - for (const unknown of unknowns) { - const identifiers = unknownIdsByFamily.get(unknown.family); - if (identifiers === undefined) - unknownIdsByFamily.set(unknown.family, [unknown.unknown_id]); - else identifiers.push(unknown.unknown_id); - } - const graph = factory({ - schema: "JavaScriptSemanticRelationGraph", - root_artifact_sha256: rootArtifactSha256, - application_graph_id: applicationGraph.graph_id, - root_node_ids: [...state.roots], - evidence_contexts: state.evidenceContexts.contexts, - nodes: [...state.nodes.values()], - relations, - // The input is transferred to the factory; `fingerprints` is cleared below. - fingerprints: [...fingerprints], - unknowns, - coverage: { - status: - truncatedFiles > 0 || unavailableFiles > 0 ? "partial" : "unknown", - truncated: truncatedFiles > 0 || resourceTruncatedFiles > 0, - omitted_nodes: truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, +export const createJavaScriptSemanticGraphProjection = ( + limits: { readonly totalNodes?: number; readonly fileNodes?: number } = {}, +): JavaScriptSemanticGraphProjection => { + const nodeCeiling = limits.totalNodes ?? SEMANTIC_GRAPH_NODE_CEILING; + const fileNodeCeiling = limits.fileNodes ?? SEMANTIC_GRAPH_FILE_NODE_CEILING; + for (const value of [nodeCeiling, fileNodeCeiling]) + if ( + !Number.isSafeInteger(value) || + value < 1 || + value > JAVASCRIPT_SEMANTIC_NODE_MAXIMUM + ) + throw new RangeError("Semantic graph requires valid node capacities"); + const state = emptyState(); + const fingerprints: JavaScriptSemanticFingerprint[] = []; + let truncatedFiles = 0; + let unavailableFiles = 0; + let resourceTruncatedFiles = 0; + const executionLimits = new Map< + string, + ApplicationCoverage["limits"][number] + >(); + const projectSourceSteps = function* ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + paths: readonly ElectronCallPath[] = [], + ): Generator { + if (state.nodes.size >= nodeCeiling) { + truncatedFiles += 1; + return; + } + const remainingTreeBudget = nodeCeiling - state.nodes.size; + state.fileNodeBudget = Math.min(fileNodeCeiling, remainingTreeBudget); + state.fileNodesDropped = false; + fingerprints.push(...(yield* projectFileSteps(file, ir, state, paths))); + // Nodes were dropped only if the budget actually blocked creation. A file + // that exactly fills its share ends with a zero budget without dropping + // anything, so the remaining budget alone cannot decide truncation. + if (state.fileNodesDropped) truncatedFiles += 1; + state.fileNodeBudget = null; + }; + const finishWith = ( + factory: (input: unknown) => Value, + rootArtifactSha256: string, + applicationGraph: BuilderInput["applicationGraph"], + ): Value => { + bindSemanticGraphApplicationNodes(state, applicationGraph); + if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); + const relations = [...state.relations.values()]; + const unknowns = [...state.unknowns.values()]; + type Family = (typeof JAVASCRIPT_SEMANTIC_RELATION_FAMILIES)[number]; + const retainedByFamily = new Map(); + const unknownIdsByFamily = new Map(); + for (const relation of relations) { + const family = JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation]; + retainedByFamily.set(family, (retainedByFamily.get(family) ?? 0) + 1); + } + for (const unknown of unknowns) { + const identifiers = unknownIdsByFamily.get(unknown.family); + if (identifiers === undefined) + unknownIdsByFamily.set(unknown.family, [unknown.unknown_id]); + else identifiers.push(unknown.unknown_id); + } + const graph = factory({ + schema: "JavaScriptSemanticRelationGraph", + root_artifact_sha256: rootArtifactSha256, + application_graph_id: applicationGraph.graph_id, + root_node_ids: [...state.roots], + evidence_contexts: state.evidenceContexts.contexts, + nodes: [...state.nodes.values()], + relations, + // The input is transferred to the factory; `fingerprints` is cleared below. + fingerprints: [...fingerprints], + unknowns, + coverage: { + status: + truncatedFiles > 0 || unavailableFiles > 0 ? "partial" : "unknown", + truncated: truncatedFiles > 0 || resourceTruncatedFiles > 0, + omitted_nodes: truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, + omitted_relations: + truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, + limits: [ + ...executionLimits.values(), + ...(truncatedFiles > 0 + ? [ + { + name: "semantic_graph_node_ceiling", + value: nodeCeiling, + unit: "items" as const, + }, + { + name: "semantic_graph_file_node_ceiling", + value: fileNodeCeiling, + unit: "items" as const, + }, + ] + : []), + ], + families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ + family, + status: semanticFamilyStatus(family), + retained_relations: retainedByFamily.get(family) ?? 0, omitted_relations: truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, - limits: [ - ...executionLimits.values(), - ...(truncatedFiles > 0 - ? [ - { - name: "semantic_graph_node_ceiling", - value: SEMANTIC_GRAPH_NODE_CEILING, - unit: "items" as const, - }, - ] - : []), - ], - families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ - family, - status: semanticFamilyStatus(family), - retained_relations: retainedByFamily.get(family) ?? 0, - omitted_relations: - truncatedFiles > 0 || unavailableFiles > 0 ? null : 0, - unknown_ids: unknownIdsByFamily.get(family) ?? [], - })), - }, - limitations: [ - "The semantic graph contains static syntax observations and conservative relationship candidates; it does not claim runtime execution.", - "Local data flow does not claim control-flow-sensitive reaching definitions or arbitrary dynamic property resolution.", - "Promise ownership covers explicit unshadowed Promise construction, static factories, aggregation, chaining, and await syntax only.", - "Function fingerprints are static candidates; equal digests can remain ambiguous and do not prove behavioral equivalence.", - "Event extraction covers EventEmitter-style literal registrations, removals, and dispatch candidates; dynamic names remain unknown.", - "Timer extraction covers global or node:timers scheduling and exact local-handle cancellation.", - "Child-process extraction covers asynchronous node:child_process creation, literal argv/env/stdio options, exit/error listeners, and kill signals.", - "Configuration extraction covers process.env, process.argv, node:fs reads, and direct logical defaults.", - "Request extraction covers fetch, WebSocket, node:http/node:https construction, direct option fields, and exact local response consumers.", - "Boundary extraction covers unshadowed JSON/global coercions plus parse and validation method candidates.", - "Resource extraction covers built-in filesystem/network acquisition and exact local close/destroy/end handles.", - ], - }); - state.nodes.clear(); - state.relations.clear(); - state.unknowns.clear(); - state.roots.clear(); - state.evidenceContexts.clear(); - fingerprints.length = 0; - return graph; - }; - return { - remainingFileNodeBudget: () => - Math.min( - SEMANTIC_GRAPH_FILE_NODE_CEILING, - Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size), - ), - adoptFileProjection: (projection) => { - const newNodes = projection.nodes.filter( - ({ node_id }) => !state.nodes.has(node_id), - ); - if ( - newNodes.length > - Math.min( - SEMANTIC_GRAPH_FILE_NODE_CEILING, - SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size, - ) - ) - throw new RangeError( - "Transferred semantic nodes exceed the offered tree capacity", - ); - for (const context of projection.evidenceContexts) - state.evidenceContexts.adopt(context); - for (const node of newNodes) state.nodes.set(node.node_id, node); - for (const relation of projection.relations) - state.relations.set(relation.relation_id, relation); - for (const unknown of projection.unknowns) - state.unknowns.set(unknown.unknown_id, unknown); - for (const root of projection.roots) state.roots.add(root); - fingerprints.push(...projection.fingerprints); - if (projection.truncated) truncatedFiles += 1; + unknown_ids: unknownIdsByFamily.get(family) ?? [], + })), }, - recordUnavailableFile: (file, reason, limits, unknownReason) => { - const retainedLimits = javaScriptAnalysisCoverageLimits(limits); - for (const limit of retainedLimits) { - executionLimits.set( - JSON.stringify([limit.name, limit.value, limit.unit]), - limit, - ); - } - state.fileNodeBudget = Math.min( - SEMANTIC_GRAPH_FILE_NODE_CEILING, - Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size), - ); - const module = retainNode(state, file, { - kind: "module", - roleKey: "module", - location: null, - label: file.path, - functionNodeId: null, - }); - state.fileNodeBudget = null; - if (module !== null) state.roots.add(module.node_id); - const evidence = { - ...observedSemanticEvidence(file, null), - authority: "unknown" as const, - state: "unknown" as const, - confidence: "unknown" as const, - coverage: partialApplicationCoverage(retainedLimits, null), - limitations: [reason], - }; - for (const family of JAVASCRIPT_SEMANTIC_RELATION_FAMILIES) { - const unknown = createJavaScriptSemanticGraphUnknown( - { - node_id: module?.node_id ?? null, - family, - relation_kinds: JAVASCRIPT_SEMANTIC_RELATIONS.filter( - (relation) => - JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation] === family, - ), - reason: unknownReason, - detail: reason, - candidate_node_ids: [], - evidence, - }, - state.evidenceContexts, - ); - state.unknowns.set(unknown.unknown_id, unknown); - } - unavailableFiles += 1; - if (unknownReason === "resource-limit" && retainedLimits.length > 0) - resourceTruncatedFiles += 1; - }, - projectFile: (file, ir) => { - const steps = projectSourceSteps(file, ir); - while (steps.next().done !== true); - }, - projectFileSteps: projectSourceSteps, - finish: (rootArtifactSha256, applicationGraph) => - finishWith( - createJavaScriptSemanticGraph, - rootArtifactSha256, - applicationGraph, - ), - finishImmutableSteps: (rootArtifactSha256, applicationGraph) => - finishWith( - sealTransferredJavaScriptSemanticGraphSteps, - rootArtifactSha256, - applicationGraph, - ), - }; + limitations: [ + "The semantic graph contains static syntax observations and conservative relationship candidates; it does not claim runtime execution.", + "Local data flow does not claim control-flow-sensitive reaching definitions or arbitrary dynamic property resolution.", + "Promise ownership covers explicit unshadowed Promise construction, static factories, aggregation, chaining, and await syntax only.", + "Function fingerprints are static candidates; equal digests can remain ambiguous and do not prove behavioral equivalence.", + "Event extraction covers EventEmitter-style literal registrations, removals, and dispatch candidates; dynamic names remain unknown.", + "Timer extraction covers global or node:timers scheduling and exact local-handle cancellation.", + "Child-process extraction covers asynchronous node:child_process creation, literal argv/env/stdio options, exit/error listeners, and kill signals.", + "Configuration extraction covers process.env, process.argv, node:fs reads, and direct logical defaults.", + "Request extraction covers fetch, WebSocket, node:http/node:https construction, direct option fields, and exact local response consumers.", + "Boundary extraction covers unshadowed JSON/global coercions plus parse and validation method candidates.", + "Resource extraction covers built-in filesystem/network acquisition and exact local close/destroy/end handles.", + ], + }); + state.nodes.clear(); + state.relations.clear(); + state.unknowns.clear(); + state.roots.clear(); + state.evidenceContexts.clear(); + fingerprints.length = 0; + return graph; }; + return { + remainingFileNodeBudget: () => + Math.min(fileNodeCeiling, Math.max(0, nodeCeiling - state.nodes.size)), + adoptFileProjection: (projection) => { + const newNodes = projection.nodes.filter( + ({ node_id }) => !state.nodes.has(node_id), + ); + if ( + newNodes.length > + Math.min(fileNodeCeiling, nodeCeiling - state.nodes.size) + ) + throw new RangeError( + "Transferred semantic nodes exceed the offered tree capacity", + ); + for (const context of projection.evidenceContexts) + state.evidenceContexts.adopt(context); + for (const node of newNodes) state.nodes.set(node.node_id, node); + for (const relation of projection.relations) + state.relations.set(relation.relation_id, relation); + for (const unknown of projection.unknowns) + state.unknowns.set(unknown.unknown_id, unknown); + for (const root of projection.roots) state.roots.add(root); + fingerprints.push(...projection.fingerprints); + if (projection.truncated) truncatedFiles += 1; + }, + recordUnavailableFile: (file, reason, limits, unknownReason) => { + const retainedLimits = javaScriptAnalysisCoverageLimits(limits); + for (const limit of retainedLimits) { + executionLimits.set( + JSON.stringify([limit.name, limit.value, limit.unit]), + limit, + ); + } + state.fileNodeBudget = Math.min( + fileNodeCeiling, + Math.max(0, nodeCeiling - state.nodes.size), + ); + const module = retainNode(state, file, { + kind: "module", + roleKey: "module", + location: null, + label: file.path, + functionNodeId: null, + }); + state.fileNodeBudget = null; + if (module !== null) state.roots.add(module.node_id); + const evidence = { + ...observedSemanticEvidence(file, null), + authority: "unknown" as const, + state: "unknown" as const, + confidence: "unknown" as const, + coverage: partialApplicationCoverage(retainedLimits, null), + limitations: [reason], + }; + for (const family of JAVASCRIPT_SEMANTIC_RELATION_FAMILIES) { + const unknown = createJavaScriptSemanticGraphUnknown( + { + node_id: module?.node_id ?? null, + family, + relation_kinds: JAVASCRIPT_SEMANTIC_RELATIONS.filter( + (relation) => + JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation] === family, + ), + reason: unknownReason, + detail: reason, + candidate_node_ids: [], + evidence, + }, + state.evidenceContexts, + ); + state.unknowns.set(unknown.unknown_id, unknown); + } + unavailableFiles += 1; + if (unknownReason === "resource-limit" && retainedLimits.length > 0) + resourceTruncatedFiles += 1; + }, + projectFile: (file, ir, paths) => { + const steps = projectSourceSteps(file, ir, paths); + while (steps.next().done !== true); + }, + projectFileSteps: projectSourceSteps, + finish: (rootArtifactSha256, applicationGraph) => + finishWith( + createJavaScriptSemanticGraph, + rootArtifactSha256, + applicationGraph, + ), + finishImmutableSteps: (rootArtifactSha256, applicationGraph) => + finishWith( + sealTransferredJavaScriptSemanticGraphSteps, + rootArtifactSha256, + applicationGraph, + ), + }; +}; /** Project one source with the parent's remaining node capacity, retaining no full IR. */ export function* projectJavaScriptSemanticFileSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, nodeBudget: number, + paths: readonly ElectronCallPath[] = [], ): Generator { if ( !Number.isSafeInteger(nodeBudget) || nodeBudget < 0 || - nodeBudget > SEMANTIC_GRAPH_FILE_NODE_CEILING + nodeBudget > JAVASCRIPT_SEMANTIC_NODE_MAXIMUM ) throw new RangeError( "File semantic projection requires a valid offered node capacity", @@ -414,7 +434,7 @@ export function* projectJavaScriptSemanticFileSteps( const state = emptyState(); state.fileNodeBudget = nodeBudget; state.fileNodesDropped = false; - const fingerprints = yield* projectFileSteps(file, ir, state); + const fingerprints = yield* projectFileSteps(file, ir, state, paths); return { roots: [...state.roots], evidenceContexts: state.evidenceContexts.contexts, @@ -430,6 +450,7 @@ function* projectFileSteps( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, state: BuilderState, + paths: readonly ElectronCallPath[] = [], ): Generator { yield; const moduleNode = retainNode(state, file, { @@ -441,6 +462,7 @@ function* projectFileSteps( }); if (moduleNode === null) return []; state.roots.add(moduleNode.node_id); + yield* projectElectronSemanticCallPaths(file, paths, state); // Node creation keeps source order, which the per-file node budget relies // on; yields between nodes do not change it. const callableNodes = new Map(); diff --git a/src/application/javascript/JavaScriptSemanticGraphElectronProjection.test.ts b/src/application/javascript/JavaScriptSemanticGraphElectronProjection.test.ts new file mode 100644 index 000000000..07a8656da --- /dev/null +++ b/src/application/javascript/JavaScriptSemanticGraphElectronProjection.test.ts @@ -0,0 +1,72 @@ +import { expect, it } from "vitest"; +import { analyzeJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; +import { analyzeJavaScriptSemantics } from "../../domain/javascript/javascriptSemanticAnalysis.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { projectJavaScriptSemanticFileSteps } from "./JavaScriptSemanticGraphBuilder.js"; + +const source = `${Array.from({ length: 40 }, (_, i) => `function library${String(i)}(){return ${String(i)}}`).join("\n")} + const e=require("electron"); + function register(channel,callback){e.ipcMain.addListener(channel,event=>allowed(event)&&callback(event))} + register("test:copy",event=>event.sender.copy());`; +const file: JavaScriptArtifactFile = { + path: "main.js", + sha256: "a".repeat(64), + container_sha256: "b".repeat(64), + bytes: source.length, + inventory_artifact_id: `art_${"a".repeat(64)}`, + kind: "javascript", + unpacked: false, + text: { included: true, value: source }, +}; +const facts = analyzeJavaScriptStaticSource(source); +const ir = analyzeJavaScriptSemantics(source); +const project = (budget: number) => { + const steps = projectJavaScriptSemanticFileSteps( + file, + ir, + budget, + facts.electron.call_paths, + ); + let result = steps.next(); + while (!result.done) result = steps.next(); + return result.value; +}; + +it("retains Electron calls before library definitions consume the node budget", () => { + const result = project(20); + expect(result.nodes).toHaveLength(20); + expect(result.truncated).toBe(true); + const terminal = result.nodes.find( + (node) => node.label === "ipc-event.sender.copy", + ); + expect(terminal).toBeDefined(); + expect( + result.relations.some( + (r) => + r.target_node_id === terminal?.node_id && r.resolution === "candidate", + ), + ).toBe(true); + expect( + result.nodes.some( + (node) => + node.properties.entry === "test:copy" && + Array.isArray(node.properties.conditions) && + node.properties.conditions.length === 1, + ), + ).toBe(true); +}); + +it("reports truncation and keeps referential integrity even within a boundary path", () => { + const result = project(3); + expect(result.nodes).toHaveLength(3); + expect(result.truncated).toBe(true); + expect( + result.nodes.some((node) => node.label === "ipc-event.sender.copy"), + ).toBe(false); + const retained = new Set(result.nodes.map((node) => node.node_id)); + expect( + result.relations.every( + (r) => retained.has(r.source_node_id) && retained.has(r.target_node_id), + ), + ).toBe(true); +}); diff --git a/src/application/javascript/JavaScriptSemanticGraphElectronProjection.ts b/src/application/javascript/JavaScriptSemanticGraphElectronProjection.ts new file mode 100644 index 000000000..53b5bdce6 --- /dev/null +++ b/src/application/javascript/JavaScriptSemanticGraphElectronProjection.ts @@ -0,0 +1,73 @@ +import type { ElectronCallPath } from "../../domain/javascript/electronStaticAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js"; +import { sha256Text } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js"; +import { + addSemanticGraphRelation, + retainSemanticGraphNode, + type SemanticGraphProjectionState, +} from "./JavaScriptSemanticGraphConstruction.js"; +import { inferredSemanticEvidenceAt } from "./JavaScriptSemanticGraphEvidence.js"; + +/** Give discovered process-boundary paths budget before bulk library definitions. */ +export function* projectElectronSemanticCallPaths( + file: JavaScriptArtifactFile, + paths: readonly ElectronCallPath[], + state: SemanticGraphProjectionState, +): Generator { + for (const path of paths) { + yield; + const pathId = sha256Text(JSON.stringify(path)); + let previous: ReturnType = null; + let previousKind: ElectronCallPath["steps"][number]["kind"] | undefined; + for (const [index, step] of path.steps.entries()) { + const node = retainSemanticGraphNode(state, file, { + kind: + step.kind === "function" + ? "function" + : step.kind === "exposure" + ? "boundary" + : "call-site", + roleKey: `electron-path:${pathId}:${String(index)}`, + location: step.location, + label: step.label, + functionNodeId: null, + properties: { + path_id: pathId, + entry: path.entry, + channel: path.channel, + terminal: path.terminal, + step_kind: step.kind, + resolution: "candidate", + conditions: path.conditions.map((c) => ({ + expression: c.expression, + location: { + start: { ...c.location.start }, + end: { ...c.location.end }, + }, + })), + }, + }); + if (index === 0 && node !== null) state.roots.add(node.node_id); + addSemanticGraphRelation(state, { + source: previous, + target: node, + relation: + previousKind === "ipc" + ? "registers-listener" + : previousKind === "exposure" || previousKind === "factory" + ? "aliases" + : "calls", + resolution: "candidate", + evidence: inferredSemanticEvidenceAt(file, step.location), + properties: { + path_id: pathId, + entry: path.entry, + channel: path.channel, + basis: "lexical-wrapper-substitution", + }, + }); + previous = node; + previousKind = step.kind; + } + } +} diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index c51bd7f13..c95cec6d5 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -21,7 +21,10 @@ import { artifactExtractionExecutionSchema, type ArtifactAnalysisOperation, } from "../contracts/artifactToolContracts.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { AnalysisCapabilityUnavailableError, AnalysisInputError, @@ -294,7 +297,7 @@ class ArtifactClient implements AnalysisClient { parameters: Readonly>, options?: ExecutionOptions, ) { - if (this.target.kind !== "executable" || this.target.format !== "mach-o") + if (this.target.kind !== "executable" || !isMachOFormat(this.target.format)) throw this.unsupportedTarget( "trace_dylib_resolution", "trace_dylib_resolution requires an active Mach-O or .app bundle target", diff --git a/src/artifacts/ArtifactResourceScope.ts b/src/artifacts/ArtifactResourceScope.ts index 1f06ed254..59fdcba36 100644 --- a/src/artifacts/ArtifactResourceScope.ts +++ b/src/artifacts/ArtifactResourceScope.ts @@ -1,6 +1,7 @@ import type { JavaScriptSourceAnalysisPort } from "../domain/javascript/javascriptSourceAnalysis.js"; import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js"; import type { OwnedDirectoryHandle } from "../filesystem/OwnedDirectoryHandle.js"; +import type { OwnedStagedOutput } from "../filesystem/OwnedStagedOutput.js"; import type { ArtifactReader } from "./ArtifactReader.js"; import { ArtifactReaderFailure } from "./ArtifactReader.js"; @@ -32,6 +33,11 @@ export type ArtifactResourceOwner = readonly kind: "output-tree"; readonly resource: string; readonly tree: SafeOutputTree; + } + | { + readonly kind: "staged-output"; + readonly resource: string; + readonly output: OwnedStagedOutput; }; export type ArtifactCleanupAttempt = @@ -60,7 +66,7 @@ export class ArtifactResourceScope { "Artifact resource scope is closed", ); if (this.#pending.size > 0) { - await this.#retryCleanup(); + await this.retryCleanup(); continue; } const running = Promise.resolve().then(operation); @@ -86,7 +92,7 @@ export class ArtifactResourceScope { } /** Retry retained owners without closing resource admission. */ - #retryCleanup(): Promise { + retryCleanup(): Promise { if (this.#draining !== undefined) return this.#draining; this.#draining = this.#drainAndRetry().finally(() => { this.#draining = undefined; @@ -97,7 +103,7 @@ export class ArtifactResourceScope { /** Stop admission, wait for active operations, and retry every retained owner. */ close(): Promise { this.#closed = true; - return this.#retryCleanup(); + return this.retryCleanup(); } async #drainAndRetry(): Promise { @@ -152,5 +158,8 @@ const cleanup = async (owner: ArtifactResourceOwner): Promise => { return; case "output-tree": await owner.tree.rollback(); + return; + case "staged-output": + await owner.output.close(); } }; diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index f9e179981..9420bef26 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -8,6 +8,7 @@ import { type NativeCommandRunner, } from "../native/CommandRunner.js"; import { + lipoSubtypeBits, parseLipoArchitectures, type LipoArchitecture, } from "../native/parsers/lipo.js"; @@ -451,14 +452,27 @@ const lipoMatchesSlice = ( (reported.size === null || reported.size === observed.slice_size) && (reported.cpu_type_code === null || reported.cpu_type_code === observedCpuType) && - (reported.cpu_subtype_code === null || - reported.cpu_subtype_code === observedCpuSubtype) && + lipoSubtypeMatches(reported, observedCpuSubtype) && (reported.alignment === null || observed.fat_alignment_exponent === null || reported.alignment === 2 ** observed.fat_alignment_exponent) ); }; +/** + * Compare the complete subtype when lipo printed its capability byte, and + * otherwise only the subtype bits its record names. + */ +const lipoSubtypeMatches = ( + reported: LipoArchitecture, + observedCpuSubtype: number, +): boolean => { + if (reported.cpu_subtype_code !== null) + return reported.cpu_subtype_code === observedCpuSubtype; + const bits = lipoSubtypeBits(reported.cpu_subtype); + return bits === null || bits === (observedCpuSubtype & 0x00ffffff); +}; + /** * Newer lipo names an arm64e ABI variant `arm64e.` (for example * `arm64e.v1` for pointer-auth ABI version 1). The FAT table names that slice diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index bd3fd93e5..5ce2ee417 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -21,6 +21,11 @@ import { type StableRegularFileDescriptor, } from "../filesystem/RegularFile.js"; +const ZIP_UTF8_DECODER = new TextDecoder("utf-8", { + fatal: true, + ignoreBOM: true, +}); + class NodeFileReader extends Reader { #handle: FileHandle | undefined; #owner: OwnedFileHandle | undefined; @@ -148,7 +153,9 @@ export class ZipArtifactReader implements ArtifactReader { async *entries(signal?: AbortSignal): AsyncIterable { let index = 0; - for await (const entry of this.#reader.getEntriesGenerator()) { + for await (const entry of this.#reader.getEntriesGenerator({ + decodeText: decodeZipEntryText, + })) { abortIfNeeded(signal); // ZIP member identity is its directory occurrence, not its filename. // Logical path uniqueness belongs to the artifact inventory boundary. @@ -207,7 +214,7 @@ export class ZipArtifactReader implements ArtifactReader { const projectZipEntry = (entry: Entry, index: number): ArtifactEntry => { const symlink = isSymlink(entry); return { - path: entry.filename, + path: zipEntryPath(entry), kind: entry.directory ? "directory" : symlink ? "symlink" : "file", declaredSize: entry.directory ? null : entry.uncompressedSize, compressedSize: entry.directory ? null : entry.compressedSize, @@ -221,6 +228,34 @@ const projectZipEntry = (entry: Entry, index: number): ArtifactEntry => { }; }; +const decodeZipEntryText = ( + bytes: Uint8Array, + encoding: string, + type: "filename" | "comment", +): string | undefined => { + if (type !== "filename" || encoding !== "utf-8") return undefined; + return decodeStrictZipUtf8(bytes, "ZIP filename is not valid UTF-8"); +}; + +const zipEntryPath = (entry: Entry): string => { + const field = entry.extraFieldUnicodePath; + if (field?.valid !== true) return entry.filename; + return decodeStrictZipUtf8( + field.data.subarray(5), + "ZIP Unicode Path extra field is not valid UTF-8", + ); +}; + +const decodeStrictZipUtf8 = (bytes: Uint8Array, message: string): string => { + try { + // Keep the adapter's path semantics aligned with zip.js's default decoder: + // a leading U+FEFF is part of a member name, not a transport signature. + return ZIP_UTF8_DECODER.decode(bytes); + } catch (cause: unknown) { + throw new ArtifactReaderFailure("format", message, { cause }); + } +}; + const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { const output = new PassThrough(); const controller = new AbortController(); diff --git a/src/artifacts/apple/MachoLoadCommandReader.ts b/src/artifacts/apple/MachoLoadCommandReader.ts index 8a0112fab..14b1aa070 100644 --- a/src/artifacts/apple/MachoLoadCommandReader.ts +++ b/src/artifacts/apple/MachoLoadCommandReader.ts @@ -234,7 +234,7 @@ const readFatSlices = async ( ): Promise => { const slices: MachoSlice[] = []; for (const declaration of declarations) { - const { index, offset, size: length, cpuType, cpuSubtype } = declaration; + const { offset, size: length, cpuType, cpuSubtype } = declaration; const rangeIssue = validateFatSlice(declaration, size, tableEnd); if (rangeIssue !== null) throw new MachoFormatIssue( diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 7dd23bcac..6306a08bf 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -31,7 +31,10 @@ import type { ArtifactResourceOwner, ArtifactResourceScope, } from "../ArtifactResourceScope.js"; -import type { BinaryTarget } from "../../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../../domain/binaryTargetTypes.js"; import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js"; import { scanArtifactInventoryInScope } from "../inventory/ArtifactInventory.js"; import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrityPolicy.js"; @@ -418,7 +421,7 @@ const requireExtractableFormat = async ( if ( !directory && format !== "asar" && - format !== "mach-o" && + !isMachOFormat(format) && !isZipFormat(format) ) throw new AnalysisUnsupportedTargetError( diff --git a/src/browser/CdpCaptureEventBodyShapes.ts b/src/browser/CdpCaptureEventBodyShapes.ts index 53882a819..f3b22bfe7 100644 --- a/src/browser/CdpCaptureEventBodyShapes.ts +++ b/src/browser/CdpCaptureEventBodyShapes.ts @@ -15,19 +15,19 @@ export const ingestResponseBodyShape = ( ): void => { const result = recordValue(value); const body = cdpStringValue(result?.body); - if (body === undefined) { + const base64Encoded = result?.base64Encoded; + if (body === undefined || typeof base64Encoded !== "boolean") { invalidResponseBodyShape(state, requestId); return; } - const decoded = - result?.base64Encoded === true - ? decodeBase64(body)?.toString("utf8") - : body; + const decoded = base64Encoded ? decodeBase64(body)?.toString("utf8") : body; if (decoded === undefined) { invalidResponseBodyShape(state, requestId); return; } - updateResponseBodyShape(state, requestId, inferJsonShape(decoded)); + const shape = inferJsonShape(decoded); + if (shape === null) state.completeness.unavailable("json_body_shapes"); + updateResponseBodyShape(state, requestId, shape); }; export const requestBodyShape = ( @@ -40,6 +40,7 @@ export const requestBodyShape = ( if (!isJsonContentType(recordValue(request.headers)) || body === undefined) return { status: "unavailable", request: null, response: null }; const shape = inferJsonShape(body); + if (shape === null) state.completeness.unavailable("json_body_shapes"); return { status: shape === null ? "unavailable" : "included", request: shape, diff --git a/src/browser/CdpCaptureEvents.redirectBodyShape.test.ts b/src/browser/CdpCaptureEvents.redirectBodyShape.test.ts index fb2da70a3..6faa98368 100644 --- a/src/browser/CdpCaptureEvents.redirectBodyShape.test.ts +++ b/src/browser/CdpCaptureEvents.redirectBodyShape.test.ts @@ -18,6 +18,7 @@ it("preserves an already captured response shape without marking it unavailable" method: "Network.requestWillBeSent", params: { requestId: "request-1", + initiator: { type: "other" }, request: { url: `${origin}/prior`, method: "POST", @@ -43,6 +44,7 @@ it("preserves an already captured response shape without marking it unavailable" base64Encoded: false, }); const capturedShape = events.network.get("request-1")?.body_shapes.response; + expect(capturedShape).toBeDefined(); expect(capturedShape).not.toBeNull(); events.ingest({ @@ -50,6 +52,7 @@ it("preserves an already captured response shape without marking it unavailable" params: { requestId: "request-1", request: { url: `${origin}/continuation`, method: "GET" }, + initiator: { type: "other" }, redirectResponse: null, timestamp: 2, }, diff --git a/src/browser/WebSourceMapFetcher.ts b/src/browser/WebSourceMapFetcher.ts index 3032f20ed..0b17df0cb 100644 --- a/src/browser/WebSourceMapFetcher.ts +++ b/src/browser/WebSourceMapFetcher.ts @@ -567,17 +567,21 @@ const normalizeSourceMap = ( const sourceJsonBytes = new Map(); const nameJsonBytes = new Map(); const originalSources: ParsedSourceMapItem["original_sources"] = []; + let declarationIndex = 0; for (const { leaf, resolvedSources } of decoded) { for (const [index, rawSource] of leaf.map.sources.entries()) { if ((index & 255) === 0) check(); const resolved = - resolvedSources[index] ?? rawSource ?? "[unknown-source]"; - const sanitized = cachedSanitized(sourcesByRaw, resolved); + rawSource === null ? null : (resolvedSources[index] ?? null); + const sanitized = + resolved === null ? null : cachedSanitized(sourcesByRaw, resolved); const content = leaf.map.sourcesContent?.[index] ?? null; consumeExpandedBytes( context.budget, - cachedJsonBytes(sourceJsonBytes, sanitized) + - sourceEntryOverheadBytes() + + (sanitized === null + ? 4 + : cachedJsonBytes(sourceJsonBytes, sanitized)) + + sourceEntryOverheadBytes(index, declarationIndex) + (originalSources.length === 0 ? 0 : 1) + (content === null ? 0 @@ -586,6 +590,8 @@ const normalizeSourceMap = ( 4), ); originalSources.push({ + source_index: index, + declaration_index: declarationIndex++, source: sanitized, artifact: content === null @@ -595,16 +601,21 @@ const normalizeSourceMap = ( } } const mappings: ParsedSourceMapItem["mappings"] = []; - for (const { leaf, rows, resolvedSources } of decoded) + declarationIndex = 0; + for (const { leaf, rows, resolvedSources } of decoded) { + const declarationBase = declarationIndex; + declarationIndex += leaf.map.sources.length; for (const [line, row] of rows.entries()) { if ((line & 255) === 0) check(); for (const segment of row) { if (!isBeforeSourceMapLeafStop(leaf, line, segment[0])) continue; if (segment.length === 1) continue; const rawSource = leaf.map.sources[segment[1]]; + if (rawSource === undefined) continue; const resolved = - resolvedSources[segment[1]] ?? rawSource ?? "[unknown-source]"; - const source = cachedSanitized(sourcesByRaw, resolved); + rawSource === null ? null : (resolvedSources[segment[1]] ?? null); + const source = + resolved === null ? null : cachedSanitized(sourcesByRaw, resolved); const name = segment.length === 5 ? (leaf.map.names[segment[4]] ?? null) : null; const generatedLine = leaf.offset.line + line + 1; @@ -612,9 +623,11 @@ const normalizeSourceMap = ( segment[0] + (line === 0 ? leaf.offset.column : 0); consumeExpandedBytes( context.budget, - cachedJsonBytes(sourceJsonBytes, source) + + (source === null ? 4 : cachedJsonBytes(sourceJsonBytes, source)) + (name === null ? 4 : cachedJsonBytes(nameJsonBytes, name)) + mappingRowBytes( + segment[1], + declarationBase + segment[1], generatedLine, generatedColumn, segment[2] + 1, @@ -623,6 +636,8 @@ const normalizeSourceMap = ( (mappings.length === 0 ? 0 : 1), ); mappings.push({ + source_index: segment[1], + declaration_index: declarationBase + segment[1], generated_line: generatedLine, generated_column: generatedColumn, source, @@ -632,6 +647,7 @@ const normalizeSourceMap = ( }); } } + } const modules = originalModuleEdges(originalSources, check, (edge) => consumeExpandedBytes( context.budget, @@ -705,15 +721,22 @@ const originalModuleEdges = ( for (const source of sources) { check(); if (source.artifact === null) continue; + const sourceName = source.source; + if (sourceName === null) { + incomplete.push( + `unnamed source at declaration ${String(source.declaration_index)}`, + ); + continue; + } const parsed = parseJavaScriptSource(source.artifact.text); check(); if (parsed === null) { - incomplete.push(source.source); + incomplete.push(sourceName); continue; } // A recovered program still yields the imports it did parse, but nodes // after an unrecoverable point are missing, so the edges are a subset. - if (parsed.errors.length > 0) incomplete.push(source.source); + if (parsed.errors.length > 0) incomplete.push(sourceName); let unboundRequires: ReadonlySet | undefined; traverseJavaScriptAst(parsed, { enter: (node) => { @@ -742,14 +765,14 @@ const originalModuleEdges = ( ) return; } - const key = `${source.source}\0${kind}\0${specifier}`; + const key = `${sourceName}\0${kind}\0${specifier}`; if (seen.has(key)) return; seen.add(key); const edge = { - from_source: source.source, + from_source: sourceName, kind, specifier, - resolved_source: resolveOriginalSource(specifier, source.source), + resolved_source: resolveOriginalSource(specifier, sourceName), }; beforeEdge(edge); edges.push(edge); @@ -840,7 +863,8 @@ const preflightResolvedSourceBytes = ( } for (const source of map.sources) { check(); - const sourceKey = source ?? ""; + if (source === null) continue; + const sourceKey = source; let resolved = sourcesForRoot.get(sourceKey); if (resolved === undefined) resolved = resolveSourceMapSource(source, map.sourceRoot, mapUrl); @@ -948,8 +972,13 @@ const sourceMapFailureReservation = ( jsonBytes(status) - jsonBytes("fetch_failed"); -const sourceEntryOverheadBytes = (): number => - Buffer.byteLength('{"source":,"artifact":null}'); +const sourceEntryOverheadBytes = ( + sourceIndex: number, + declarationIndex: number, +): number => + Buffer.byteLength( + `{"source_index":${String(sourceIndex)},"declaration_index":${String(declarationIndex)},"source":,"artifact":null}`, + ); const cachedJsonBytes = (cache: Map, value: string): number => { const cached = cache.get(value); @@ -968,13 +997,15 @@ const cachedSanitized = (cache: Map, value: string): string => { }; const mappingRowBytes = ( + sourceIndex: number, + declarationIndex: number, generatedLine: number, generatedColumn: number, originalLine: number, originalColumn: number, ): number => Buffer.byteLength( - `{"generated_line":${String(generatedLine)},"generated_column":${String(generatedColumn)},"source":,"original_line":${String(originalLine)},"original_column":${String(originalColumn)},"name":}`, + `{"source_index":${String(sourceIndex)},"declaration_index":${String(declarationIndex)},"generated_line":${String(generatedLine)},"generated_column":${String(generatedColumn)},"source":,"original_line":${String(originalLine)},"original_column":${String(originalColumn)},"name":}`, ); const resolveOriginalSource = ( diff --git a/src/browser/history/CaptureFailures.ts b/src/browser/history/CaptureFailures.ts index 78a5f0a43..228541545 100644 --- a/src/browser/history/CaptureFailures.ts +++ b/src/browser/history/CaptureFailures.ts @@ -5,7 +5,6 @@ import { AnalysisAccessDeniedError, AnalysisArtifactChangedError, AnalysisCancelledError, - AnalysisCapabilityUnavailableError, AnalysisInputError, AnalysisOutputError, AnalysisResourceConstraintError, diff --git a/src/browser/history/CaptureRedaction.test.ts b/src/browser/history/CaptureRedaction.test.ts index d6ce3bb1c..26a5fe183 100644 --- a/src/browser/history/CaptureRedaction.test.ts +++ b/src/browser/history/CaptureRedaction.test.ts @@ -42,6 +42,52 @@ it("preserves the same URL-looking text in an ordinary extension", () => { expect(redactor.redactions).toEqual([]); }); +it.each([ + "https:user:password@example.test/path", + "https:///user:password@example.test/path", + "https://\\user:password@example.test/path", + "https://\t/user:password@example.test/path", + "https://\n/user:password@example.test/path", + "https://\r/user:password@example.test/path", + "ht\ttps://user:password@example.test/path", + "\\\\user:password@example.test/path", + "/\\user:password@example.test/path", + "\\/user:password@example.test/path", + "/\t/user:password@example.test/path", +])("excludes credentials whose raw URL span is ambiguous: %s", (value) => { + const redactor = new CaptureRedaction([]); + expect(redactor.text(value, "/request/url", true)).toBeNull(); + expect(redactor.redactions).toEqual([ + { pointer: "/request/url", reason: "transport-credential" }, + ]); + expect(new CaptureRedaction([]).text(value, "/_extension")).toBe(value); +}); + +it("uses non-special URL authority rules to remove the complete userinfo span", () => { + const value = "custom://user:secret@part\\more:password@host/path"; + const redactor = new CaptureRedaction([]); + expect(redactor.text(value, "/response/redirectURL", true)).toBe( + "custom://host/path", + ); + expect(redactor.redactions).toEqual([ + { pointer: "/response/redirectURL", reason: "transport-credential" }, + ]); +}); + +it("preserves noncanonical URLs and path/query userinfo lookalikes without parsed credentials", () => { + for (const value of [ + "https:example.test/path?token=ordinary", + "https://example.test/a/%2e%2e/user:password@literal?url=user:password@literal", + "https://example.test\\user:password@literal", + "https://invalid host\\user:password@literal", + "//invalid host\\user:password@literal", + ]) { + const redactor = new CaptureRedaction([]); + expect(redactor.text(value, "/request/url", true)).toBe(value); + expect(redactor.redactions).toEqual([]); + } +}); + it.each(["\ud800", "\udfff"])( "does not invent replacement UTF-8 bytes for a non-scalar declaration", (literal) => { diff --git a/src/browser/history/CaptureRedaction.ts b/src/browser/history/CaptureRedaction.ts index 86945aed0..450b1bb05 100644 --- a/src/browser/history/CaptureRedaction.ts +++ b/src/browser/history/CaptureRedaction.ts @@ -1,4 +1,5 @@ import type { WebNetworkCaptureRecord } from "../../domain/webNetworkCapture.js"; +import { parseRecordedTransportUrl } from "../../domain/recordedTransportUrl.js"; type Redaction = WebNetworkCaptureRecord["redactions"][number]; const CREDENTIAL_HEADERS = new Set([ @@ -25,10 +26,10 @@ export class CaptureRedaction { text(value: string, pointer: string, transportUrl = false): string | null { let result = value; if (transportUrl) { - const match = - /^([\x00-\x20]*(?:[a-z][a-z0-9+.-]*:)?\/\/)([^/?#]*@)/i.exec(result); - if (match !== null) { - result = `${match[1]}${result.slice(match[0].length)}`; + const { credentials } = parseRecordedTransportUrl(value); + if (credentials === "ambiguous") return this.credential(pointer); + if (credentials !== null) { + result = `${value.slice(0, credentials.start)}${value.slice(credentials.end)}`; this.redactions.push({ pointer, reason: "transport-credential" }); } } diff --git a/src/browser/history/HistoricalCaptureDecoder.ts b/src/browser/history/HistoricalCaptureDecoder.ts index 20258c177..155ba229b 100644 --- a/src/browser/history/HistoricalCaptureDecoder.ts +++ b/src/browser/history/HistoricalCaptureDecoder.ts @@ -29,7 +29,7 @@ import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; import { WEB_NETWORK_CAPTURE_LIMITS, - webNetworkCaptureSchema, + webNetworkCapturePayloadSchema, type InspectWebNetworkCaptureInput, type WebNetworkCapture, } from "../../domain/webNetworkCapture.js"; @@ -38,14 +38,11 @@ import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; import { redactExplicitFailure } from "../../domain/explicitSensitiveFailure.js"; const OPERATION = "inspect_web_network_capture"; -const decodedSchema = webNetworkCaptureSchema.omit({ - artifact: true, - format: true, - runtime_attribution: true, - limitations: true, -}); const replySchema = z.discriminatedUnion("ok", [ - z.strictObject({ ok: z.literal(true), value: decodedSchema }), + z.strictObject({ + ok: z.literal(true), + value: webNetworkCapturePayloadSchema, + }), z.strictObject({ ok: z.literal(false), reason: z.enum([ diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 408ec68e0..050f5c7dc 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -74,15 +74,4 @@ export const CATALOG_IDENTITY = { get digests() { return (catalogDigests ??= createCatalogDigests()); }, - tools: sortedToolContracts.map(({ name, kind, effects, annotations }) => ({ - name, - surface: kind, - effects: { ...effects }, - annotations: { - read_only: annotations.readOnlyHint ?? false, - destructive: annotations.destructiveHint ?? false, - idempotent: annotations.idempotentHint ?? false, - open_world: annotations.openWorldHint ?? true, - }, - })), } as const; diff --git a/src/cli/electronCommands.ts b/src/cli/electronCommands.ts index b3004720c..3f558930b 100644 --- a/src/cli/electronCommands.ts +++ b/src/cli/electronCommands.ts @@ -200,6 +200,15 @@ const registerJavaScriptApplicationCommand = ( input_path: args.path, format: options.artifactFormat, integrity_policy: options.integrityPolicy, + ...(options.maxSemanticNodes === undefined + ? {} + : { max_semantic_nodes: options.maxSemanticNodes }), + ...(options.maxSemanticNodesPerFile === undefined + ? {} + : { + max_semantic_nodes_per_file: + options.maxSemanticNodesPerFile, + }), ...(options.sourcePath === undefined ? {} : { source_paths: options.sourcePath }), diff --git a/src/cli/managedCommands.ts b/src/cli/managedCommands.ts index fdc32634e..7dc74ae3e 100644 --- a/src/cli/managedCommands.ts +++ b/src/cli/managedCommands.ts @@ -54,9 +54,22 @@ const registerInspectManagedMembers = ( args: z.object({ path: z.string().describe("Managed PE executable or assembly path"), }), - run: ({ args }) => + options: z.object({ + type: z + .array(z.string()) + .optional() + .describe( + "Inspect only members declared by this exact type full name, such as System.Threading.Monitor; a nested type is named by its bare name, which selects every nested type of that name (repeatable)", + ), + }), + run: ({ args, options }) => logCliCommand(logger, "inspect-managed-members", () => - runProviderAnalysis(args.path, "inspect_managed_members", {}, logger), + runProviderAnalysis( + args.path, + "inspect_managed_members", + options.type === undefined ? {} : { type_names: options.type }, + logger, + ), ), }); }; diff --git a/src/cli/setupCommands.ts b/src/cli/setupCommands.ts index d21be1c08..15caac321 100644 --- a/src/cli/setupCommands.ts +++ b/src/cli/setupCommands.ts @@ -120,11 +120,12 @@ const registerDoctorCommand = ( }), run: ({ options }) => logCliCommand(logger, "doctor", async () => { - const [{ runDoctor }, { createSystemDoctorHost }] = await Promise.all([ - import("../application/Doctor.js"), - import("../doctorRuntime.js"), - ]); - return runDoctor( + const [{ runDoctorOwned }, { createSystemDoctorHost }] = + await Promise.all([ + import("../application/Doctor.js"), + import("../doctorRuntime.js"), + ]); + return runDoctorOwned( options.target, createSystemDoctorHost(environment), doctorScope(options), diff --git a/src/cliObservationOptions.ts b/src/cliObservationOptions.ts index 52fa94eb6..e7db14e0e 100644 --- a/src/cliObservationOptions.ts +++ b/src/cliObservationOptions.ts @@ -61,6 +61,10 @@ export const electronPageInspectionOptions = z.object({ }); export const javascriptApplicationOptions = z.object({ + maxSemanticNodes: + javaScriptAnalysisResourceControlsSchema.shape.max_semantic_nodes, + maxSemanticNodesPerFile: + javaScriptAnalysisResourceControlsSchema.shape.max_semantic_nodes_per_file, sourcePath: javaScriptSourcePathsSchema .optional() .describe( diff --git a/src/composition/javascriptAnalysisWorkerProcess.ts b/src/composition/javascriptAnalysisWorkerProcess.ts index 3b94f4677..5c66f4afa 100644 --- a/src/composition/javascriptAnalysisWorkerProcess.ts +++ b/src/composition/javascriptAnalysisWorkerProcess.ts @@ -6,7 +6,10 @@ import { } from "../domain/javascript/javascriptSemanticAnalysis.js"; import { failedJavaScriptStaticAnalysis } from "../domain/javascript/javascriptStaticAnalysisHelpers.js"; import { projectJavaScriptModuleSemantics } from "../domain/javascript/javascriptModuleSemanticIr.js"; -import { measureJavaScriptModuleTransfer } from "../javascript/analysis/JavaScriptAnalysisTransfer.js"; +import { + javaScriptTransferByteBudget, + measureJavaScriptModuleTransfer, +} from "../javascript/analysis/JavaScriptAnalysisTransfer.js"; import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; import { projectJavaScriptSemanticFileSteps } from "../application/javascript/JavaScriptSemanticGraphBuilder.js"; import { completeJavaScriptAnalysisSteps } from "../application/javascript/JavaScriptAnalysisControl.js"; @@ -54,7 +57,10 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( ); parsed = null; const module = projectJavaScriptModuleSemantics(ir); - const measured = measureJavaScriptModuleTransfer(module); + const measured = measureJavaScriptModuleTransfer( + module, + javaScriptTransferByteBudget(resultBudgetBytes), + ); if (measured.retentionBytes > resultBudgetBytes) throw new AnalysisResourceConstraintError( "analyze_javascript_application", @@ -67,7 +73,12 @@ const analyze: JavaScriptSourceWorkerAnalyzer = async ( ); await callbacks.phase("projection"); const projection = await completeJavaScriptAnalysisSteps( - projectJavaScriptSemanticFileSteps(file, ir, nodeBudget), + projectJavaScriptSemanticFileSteps( + file, + ir, + nodeBudget, + facts.electron.call_paths, + ), ); return { module, projection }; }; diff --git a/src/config.test.ts b/src/config.test.ts index 68a145e20..eb784802f 100644 --- a/src/config.test.ts +++ b/src/config.test.ts @@ -193,6 +193,7 @@ describe("configuration failure diagnostics", () => { if (result.ok) throw new Error("Expected invalid startup deadline"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "configuration_invalid", + category: "invalid_input", details: { settings: [ { @@ -223,6 +224,7 @@ describe("configuration failure diagnostics", () => { if (result.ok) throw new Error("Expected invalid startup deadline"); expect(projectAnalysisError(result.error)).toMatchObject({ code: "configuration_invalid", + category: "invalid_input", details: { settings: [ { @@ -248,6 +250,7 @@ describe("configuration failure diagnostics", () => { const constraint = `${setting} must be absolute`; expect(projectAnalysisError(result.error)).toMatchObject({ code: "configuration_invalid", + category: "invalid_input", message: `REA configuration is invalid: ${constraint}.`, remediation: { action: expect.stringContaining(`Correct ${setting} `) }, details: { settings: [{ setting, constraint }] }, diff --git a/src/contracts/analysisDetail.test.ts b/src/contracts/analysisDetail.test.ts new file mode 100644 index 000000000..1668ae6f4 --- /dev/null +++ b/src/contracts/analysisDetail.test.ts @@ -0,0 +1,61 @@ +import { expect, it } from "vitest"; +import { z } from "zod"; + +import { + binaryLayoutSummaryViewResultSchema, + javascriptApplicationSummaryViewResultSchema, + projectAnalysisView, +} from "../domain/analysisView/analysisView.js"; +import { + analysisViewJavaScriptEvidence, + analysisViewLayoutEvidence, +} from "../../tests/fixtures/analysisView.js"; +import { toolContract } from "./toolContracts.js"; + +const projectSummary = (parent: { + readonly evidence_id: string; + readonly operation: string; + readonly normalized_result: unknown; + readonly limitations: readonly string[]; +}) => + projectAnalysisView( + { + evidenceId: parent.evidence_id, + operation: parent.operation, + normalizedResult: parent.normalized_result, + limitations: parent.limitations, + }, + { kind: "summary" }, + ); + +it("advertises only that operation's summary from analysis detail outputs", () => { + for (const name of [ + "inspect_binary_layout", + "analyze_javascript_application", + ] as const) { + const schema = JSON.stringify( + z.toJSONSchema(toolContract(name).outputSchema), + ); + expect(schema).not.toContain("api-surface"); + expect(schema).not.toContain("inspect_web_network_capture"); + } + expect( + JSON.stringify( + z.toJSONSchema(toolContract("inspect_analysis_view").outputSchema), + ), + ).toContain("api-surface"); + + const layout = projectSummary(analysisViewLayoutEvidence()); + const javascript = projectSummary(analysisViewJavaScriptEvidence()); + if (!layout.ok || !javascript.ok) throw new Error("Expected summaries"); + expect( + binaryLayoutSummaryViewResultSchema.safeParse(layout.value).success, + ).toBe(true); + expect( + javascriptApplicationSummaryViewResultSchema.safeParse(javascript.value) + .success, + ).toBe(true); + expect( + binaryLayoutSummaryViewResultSchema.safeParse(javascript.value).success, + ).toBe(false); +}); diff --git a/src/contracts/analysisDetail.ts b/src/contracts/analysisDetail.ts index dde2f148e..b6e000164 100644 --- a/src/contracts/analysisDetail.ts +++ b/src/contracts/analysisDetail.ts @@ -1,6 +1,5 @@ import { z } from "zod"; -import { analysisViewResultSchema } from "../domain/analysisView/analysisView.js"; import { evidenceResultOf } from "./toolOutputSchemaPrimitives.js"; /** Caller-selected delivery of completed analysis Evidence that supports retained views. */ @@ -11,7 +10,15 @@ export const analysisDetailSchema = z "complete returns the complete analysis Evidence inline. summary retains that complete Evidence in this session and returns its inspect_analysis_view summary; inspect pages, items or facets through normalized_result.parent_evidence_id.", ); -/** Complete analysis Evidence, or the summary view of its retained record. */ -export const analysisDetailOutputSchemaOf = ( - analysis: Schema, -) => evidenceResultOf(z.union([analysis, analysisViewResultSchema])); +/** + * Complete analysis Evidence, or that operation's summary view of its retained + * record. Callers pass the summary variant the handler actually returns so the + * advertised output does not accept other view kinds. + */ +export const analysisDetailOutputSchemaOf = < + Analysis extends z.ZodType, + Summary extends z.ZodType, +>( + analysis: Analysis, + summary: Summary, +) => evidenceResultOf(z.union([analysis, summary])); diff --git a/src/contracts/analysisViewApiSurfaceExample.ts b/src/contracts/analysisViewApiSurfaceExample.ts new file mode 100644 index 000000000..389ac7268 --- /dev/null +++ b/src/contracts/analysisViewApiSurfaceExample.ts @@ -0,0 +1,73 @@ +/** Generated from the two-request apiSurfaceHar fixture through the HAR decoder and view service. */ +export const API_SURFACE_EXAMPLE = { + input: { + source: { + kind: "retained-evidence", + evidence_id: + "ev_9e218c18bc6159ec9c8492a9130c286496fb1b6d9f03ec2f4ea3321ce2b15097", + }, + view: { + kind: "api-surface", + offset: 0, + limit: 50, + }, + }, + output: { + parent_evidence_id: + "ev_9e218c18bc6159ec9c8492a9130c286496fb1b6d9f03ec2f4ea3321ce2b15097", + parent_operation: "inspect_web_network_capture", + parent_digest: + "9e218c18bc6159ec9c8492a9130c286496fb1b6d9f03ec2f4ea3321ce2b15097", + view_digest: + "47c1859b7a86dd10770061767300bdb6ec177a9254e8dc7863d5ea8603343da6", + artifact: { + path: "/fixtures/api-surface.har", + sha256: + "bf58e2c06e55bfe7e0628157c4ad027ce0aeb81a10e8edef8a7f95b8e5f14225", + }, + coverage: { + status: "page", + examined: 1, + total: 1, + next_offset: null, + exhausted: true, + }, + limitations: [ + "Grouping does not establish handler identity, URL-space schemas or causality.", + "HAR text is producer-decoded Unicode. Original body bytes remain unknown unless content explicitly uses valid base64; declared sizes are preserved independently of retained bytes. mitmproxy 12.2.3 SaveHar may report null optional postData.text; that null is retained and omitted only from the upstream validation copy.", + ], + unknowns: [], + records: { + examined: 2, + total: 2, + excluded: 0, + }, + templated_groups: 1, + groups: [ + { + method: "GET", + origin: "https://api.example.test", + path_template: "/users/{numeric}", + templated_segments: [ + { + segment_index: 1, + rule: "numeric", + basis: "inferred", + observed_values: ["8812", "9031"], + }, + ], + record_ordinals: [0, 1], + statuses: [200, 404], + content_types: ["application/json", "text/plain"], + flags: ["mixed-statuses", "mixed-content-types"], + }, + ], + excluded_records: [], + kind: "api-surface", + view: { + kind: "api-surface", + offset: 0, + limit: 50, + }, + }, +} as const; diff --git a/src/contracts/analysisViewToolContracts.ts b/src/contracts/analysisViewToolContracts.ts index 0edf10f60..9346ef256 100644 --- a/src/contracts/analysisViewToolContracts.ts +++ b/src/contracts/analysisViewToolContracts.ts @@ -1,3 +1,4 @@ +import { API_SURFACE_EXAMPLE } from "./analysisViewApiSurfaceExample.js"; import { analysisViewResultSchema, inspectAnalysisViewInputSchema, @@ -15,10 +16,14 @@ export const ANALYSIS_VIEW_TOOL_CONTRACTS = [ ...toolContractMetadata("inspect_analysis_view"), kind: "application", description: - "Inspect selected views of completed binary-layout, JavaScript application or native analyze_function Evidence without repeating analysis. Use an exact same-session retained evidence_id or portable inline Evidence. Select a summary, layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Native views select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.", + "Inspect selected views of completed binary-layout, JavaScript application or native analyze_function Evidence without repeating analysis. The api-surface view groups retained HAR requests by method, origin and format-rule path template; templated segments are inferences. It reads no headers or bodies and reports no query values. Use an exact same-session retained evidence_id or portable inline Evidence. Select a summary, layout mitigations/linkage facet, one section/symbol/module, or a stable page. Module pages include assets, bundled and source modules; use node_id for ambiguous or unavailable paths. Native views select procedure, pseudocode, assembly, basic blocks, comments, callers/callees, references, unresolved calls, referenced strings/names, native API and high-pcode. Use nonnegative offset and a positive caller-selected limit (defaults 0 and 64); pseudocode uses UTF-16 code units, other native pages use rows. Returns artifact identity, parent Evidence ID, a distinct view digest, coverage, limitations and unknowns. Actual serialized size determines MCP transport admission; choose a smaller page or export the complete retained Evidence if needed.", inputSchema: inspectAnalysisViewInputSchema, outputSchema: evidenceResultOf(analysisViewResultSchema), examples: [ + { + title: "Group requests from retained HAR Evidence", + input: API_SURFACE_EXAMPLE.input, + }, { title: "Summarize retained JavaScript application Evidence", input: { diff --git a/src/contracts/android/androidToolContracts.ts b/src/contracts/android/androidToolContracts.ts index e20b7261b..df2cccf59 100644 --- a/src/contracts/android/androidToolContracts.ts +++ b/src/contracts/android/androidToolContracts.ts @@ -26,7 +26,7 @@ export const ANDROID_TOOL_CONTRACTS = [ name: "search_android_classes", ...toolContractMetadata("search_android_classes"), description: - "Search complete decompiler class names in an explicit local APK by case-sensitive substring. Empty query inventories all classes. Retains artifact and engine identity; no APK execution.", + "Search complete decompiler class-name rows in an explicit local APK by case-sensitive substring. Empty query inventories all rows. Repeated names remain repeated and appear in duplicate_names; the provider accepts only class names for lookup, so these rows do not identify which same-name class node a later lookup selects. Retains artifact and engine identity; no APK execution.", kind: "android-provider", inputSchema: androidInputSchemas.search_android_classes, outputSchema: evidenceResultOf(androidResultSchemas.search_android_classes), diff --git a/src/contracts/javascript/electronToolContracts.ts b/src/contracts/javascript/electronToolContracts.ts index 1c6c26d7c..f7a9ed732 100644 --- a/src/contracts/javascript/electronToolContracts.ts +++ b/src/contracts/javascript/electronToolContracts.ts @@ -11,6 +11,7 @@ import { electronActiveObservationInputSchema, electronActiveObservationResultSchema, } from "../../domain/javascript/electronActiveObservation.js"; +import { javascriptApplicationSummaryViewResultSchema } from "../../domain/analysisView/analysisView.js"; import { analysisDetailOutputSchemaOf, analysisDetailSchema, @@ -34,6 +35,7 @@ export const analyzeJavaScriptApplicationRequestSchema = }); const applicationOutputSchema = analysisDetailOutputSchemaOf( javascriptApplicationAnalysisResultSchema, + javascriptApplicationSummaryViewResultSchema, ); const reconciliationOutputSchema = evidenceResultOf( javascriptRuntimeReconciliationResultSchema, diff --git a/src/contracts/managed/managedToolContracts.ts b/src/contracts/managed/managedToolContracts.ts index 2ff8d36bb..ddcc7a6e3 100644 --- a/src/contracts/managed/managedToolContracts.ts +++ b/src/contracts/managed/managedToolContracts.ts @@ -20,6 +20,28 @@ export const managedTargetInputSchema = z.strictObject({ .optional(), }); +/** Member inspection may select declaring types before members are decoded. */ +export const managedMembersInputSchema = managedTargetInputSchema.extend({ + type_names: z + .array( + z + .string() + .min(1) + .describe( + "Exact declaring-type full_name: namespace-qualified for top-level types (System.Threading.Monitor); nested types report their bare name, so it selects every nested type of that name", + ), + ) + .min(1) + .refine((names) => new Set(names).size === names.length, { + message: "type_names must not repeat a name", + }) + .meta({ uniqueItems: true }) + .describe( + "Inspect only the fields, methods, call edges, and field accesses declared by these exact type full names; omit to inspect every type. Unmatched names are reported in selection.unmatched_type_names.", + ) + .optional(), +}); + const outputSchema = requireOutputSchema( managedOutputSchemas, "inspect_managed_artifact", @@ -54,15 +76,22 @@ export const MANAGED_TOOL_CONTRACTS = [ name: "inspect_managed_members", ...toolContractMetadata("inspect_managed_members"), description: - "Inspect PE/CLI metadata members, signatures, raw CIL hashes, decoded-instruction-tuple hashes, separately reported exception regions, call edges, and field-access anchors without loading or executing target code. Metadata tokens are reported as build-local coordinates bound to the artifact SHA-256 and MVID; the tuple hash does not resolve them or fully commit control flow.", + "Inspect PE/CLI metadata members, signatures, raw CIL hashes, decoded-instruction-tuple hashes, separately reported exception regions, call edges, and field-access anchors without loading or executing target code. Select type_names to decode only the members of those exact declaring types, which keeps large assemblies such as a base class library within the representation budget. Metadata tokens are reported as build-local coordinates bound to the artifact SHA-256 and MVID; the tuple hash does not resolve them or fully commit control flow.", kind: "managed-provider", - inputSchema: managedTargetInputSchema, + inputSchema: managedMembersInputSchema, outputSchema: memberOutputSchema, examples: [ { title: "Example inspect managed members request", input: {}, }, + { + title: "Example inspect selected managed types request", + input: { + path: "/tmp/mscorlib.dll", + type_names: ["System.Threading.Monitor"], + }, + }, ], }, { diff --git a/src/contracts/managed/managedWorkflowExamples.ts b/src/contracts/managed/managedWorkflowExamples.ts index 2a0176dfe..85297d671 100644 --- a/src/contracts/managed/managedWorkflowExamples.ts +++ b/src/contracts/managed/managedWorkflowExamples.ts @@ -53,6 +53,7 @@ const emptyManagedMembers = ( member_refs: [], call_edges: [], field_accesses: [], + selection: null, coverage: { state: "complete", issues: [] }, limitations: [], }); @@ -224,6 +225,7 @@ const boundaryEvidence = () => { const nativeExportEvidence = () => { const result: InspectMacho = { format: "mach-o", + selected_architecture: { name: "x86_64", container_file_offset: null }, endian: "little", word_size: 64, file_type: "dylib", @@ -246,7 +248,9 @@ const nativeExportEvidence = () => { ]), segments: coverage([]), provenance: [], - limitations: [], + limitations: [ + "Selected Mach-O slice container offset is unknown in this synthetic example.", + ], }; return createEvidence( { diff --git a/src/contracts/native/binaryDiagnosticsToolContracts.ts b/src/contracts/native/binaryDiagnosticsToolContracts.ts index 80dea5e62..64a80e42d 100644 --- a/src/contracts/native/binaryDiagnosticsToolContracts.ts +++ b/src/contracts/native/binaryDiagnosticsToolContracts.ts @@ -6,6 +6,7 @@ import { inspectBinaryLayoutInputSchema, binaryLayoutSchema, } from "../../domain/native/binaryLayout.js"; +import { binaryLayoutSummaryViewResultSchema } from "../../domain/analysisView/analysisView.js"; import { analysisDetailOutputSchemaOf, analysisDetailSchema, @@ -46,7 +47,10 @@ export const BINARY_DIAGNOSTICS_TOOL_CONTRACTS = [ description: "Inspect an explicit local binary without launching it as a host process. Returns file-backed section/segment ranges, linked addresses, original symbol/REL/RELA table identities, complete encoded RELR tables and upstream-derived offsets with unknown per-word locations/addends, raw name bytes, dependency/interpreter names and static mitigation inferences inline with artifact SHA-256 Evidence. Initial profile: ELF64 x86-64 little-endian EXEC/DYN/REL on Linux x64 via caller-supplied Python with unchanged pwntools 4.15.0, pyelftools 0.33 and Unicorn 2.1.2. Runtime addresses/library paths, overall relocation inventory completeness and complete derived GOT/PLT coverage remain unknown. Zero executable entry values mean absence; relocatable entries are not applicable. Upstream can emulate PLT instructions with Unicorn for derived static maps. Uses bounded complete output and owned temporary files/processes; malformed, unsupported or oversized objects return no partial success. Select detail summary to retain the complete layout Evidence in this session and receive only its summary view, then inspect sections, symbols, mitigations or linkage with inspect_analysis_view.", inputSchema: inspectBinaryLayoutRequestSchema, - outputSchema: analysisDetailOutputSchemaOf(binaryLayoutSchema), + outputSchema: analysisDetailOutputSchemaOf( + binaryLayoutSchema, + binaryLayoutSummaryViewResultSchema, + ), examples: [ { title: "Inspect ELF linked and file layout", diff --git a/src/contracts/reconstructionReadinessExample.ts b/src/contracts/reconstructionReadinessExample.ts index 0b81cf60b..124ce2a05 100644 --- a/src/contracts/reconstructionReadinessExample.ts +++ b/src/contracts/reconstructionReadinessExample.ts @@ -4,6 +4,8 @@ import { READINESS_REQUIRED_CHECKS, READINESS_STAGE_IDS, } from "../domain/reconstructionReadinessJourneyFindings.js"; +import { reconstructionObligationLedgerClosureDigest } from "../domain/reconstructionObligationLedgerIntegrity.js"; +import type { ReconstructionObligationLedger } from "../domain/reconstructionObligationLedgerSchemas.js"; import type { ReconstructionReadinessInput } from "../domain/reconstructionReadinessSchemas.js"; const HASH = "a".repeat(64); @@ -69,6 +71,129 @@ const EVIDENCE_STATES = [ "reviewed", "unknown", ] as const; +const READINESS_OBLIGATION_LEDGER = (() => { + const ledger: Omit = { + schema: "ReconstructionObligationLedger", + ledger_id: `rol_${HASH}`, + status: "ready", + coverage: { + status: "complete", + }, + summary: { + total: 1, + required: 1, + verified: 1, + required_open: 0, + by_status: STATUSES.map((key) => ({ + key, + count: key === "verified" ? 1 : 0, + })), + by_application_layer: LAYERS.map((key) => ({ + key, + count: key === "process" ? 1 : 0, + })), + by_evidence_authority: EVIDENCE_STATES.map((key) => ({ + key, + count: key === "observed" ? 1 : 0, + })), + }, + reports: { + missing_owner_obligation_ids: [], + missing_verifier_obligation_ids: [], + contradicted_obligation_ids: [], + residual_unknown_ids: [], + }, + ownership_graph: [ + { + obligation_id: "obl.readiness", + module_path: "src/readiness.ts", + symbol: "runReadiness", + }, + ], + dependency_graph: [], + obligations: [ + { + obligation_id: "obl.readiness", + obligation_version: 1, + title: "Preserve the packaged readiness fixture lifecycle", + origin: "reviewed", + application_layer: "process", + family: "packaged-process-lifecycle", + target: { + artifact_sha256: HASH, + application_node_id: null, + semantic_node_id: null, + location: "/fixture", + }, + authority_references: [ + { + evidence_id: EVIDENCE_ID, + authority: "controlled-replay", + state: "observed", + location: "/fixture", + }, + ], + source_state: "observed", + observed_cases: CASES.map((kind) => ({ + kind, + evidence_id: EVIDENCE_ID, + location: `/original/${kind}`, + })), + required: true, + required_case_kinds: [...CASES], + required_original_authority: "process", + required_fixture_authority: "packaged-process", + required_verifier_authority: "packaged-process", + requires_parser_type: false, + dependency_obligation_ids: [], + residual_unknown_ids: [], + unavailable_authority: [], + required_next_evidence: [], + binding: { + obligation_id: "obl.readiness", + owner: { + module_path: "src/readiness.ts", + symbol: "runReadiness", + owner_sha256: HASH, + }, + parser_type: null, + original_cases: CASES.map((kind) => ({ + kind, + evidence_id: EVIDENCE_ID, + location: `/original/${kind}`, + })), + fixtures: CASES.map((kind) => ({ + fixture_id: `fixture.${kind}`, + case_kind: kind, + authority: "packaged-process", + evidence_ids: [EVIDENCE_ID], + })), + verifier: { + verifier_id: "verifier.readiness", + claim_id: "claim.readiness", + command: "npm run verify:readiness", + authority: "packaged-process", + status: "pass", + result_evidence_id: EVIDENCE_ID, + enumerated_obligation_ids: ["obl.readiness"], + nondeterminism: { + mode: "partial-order", + specification: "Startup precedes teardown.", + }, + }, + }, + status: "verified", + diagnostics: [], + }, + ], + evidence_links: [EVIDENCE_ID], + limitations: [], + }; + return { + ...ledger, + closure_digest: reconstructionObligationLedgerClosureDigest(ledger), + }; +})(); /** Complete synthetic input shared by docs, tests, CLI, MCP, and verifier. */ export const RECONSTRUCTION_READINESS_EXAMPLE: ReconstructionReadinessInput = { @@ -217,124 +342,7 @@ export const RECONSTRUCTION_READINESS_EXAMPLE: ReconstructionReadinessInput = { evidence_ids: [EVIDENCE_ID, CONTRADICTING_SOURCE.evidence_id], }, ], - obligation_ledger: { - schema: "ReconstructionObligationLedger", - ledger_id: `rol_${HASH}`, - closure_digest: HASH, - status: "ready", - coverage: { - status: "complete", - }, - summary: { - total: 1, - required: 1, - verified: 1, - required_open: 0, - by_status: STATUSES.map((key) => ({ - key, - count: key === "verified" ? 1 : 0, - })), - by_application_layer: LAYERS.map((key) => ({ - key, - count: key === "process" ? 1 : 0, - })), - by_evidence_authority: EVIDENCE_STATES.map((key) => ({ - key, - count: key === "observed" ? 1 : 0, - })), - }, - reports: { - missing_owner_obligation_ids: [], - missing_verifier_obligation_ids: [], - contradicted_obligation_ids: [], - residual_unknown_ids: [], - }, - ownership_graph: [ - { - obligation_id: "obl.readiness", - module_path: "src/readiness.ts", - symbol: "runReadiness", - }, - ], - dependency_graph: [], - obligations: [ - { - obligation_id: "obl.readiness", - obligation_version: 1, - title: "Preserve the packaged readiness fixture lifecycle", - origin: "reviewed", - application_layer: "process", - family: "packaged-process-lifecycle", - target: { - artifact_sha256: HASH, - application_node_id: null, - semantic_node_id: null, - location: "/fixture", - }, - authority_references: [ - { - evidence_id: EVIDENCE_ID, - authority: "controlled-replay", - state: "observed", - location: "/fixture", - }, - ], - source_state: "observed", - observed_cases: CASES.map((kind) => ({ - kind, - evidence_id: EVIDENCE_ID, - location: `/original/${kind}`, - })), - required: true, - required_case_kinds: [...CASES], - required_original_authority: "process", - required_fixture_authority: "packaged-process", - required_verifier_authority: "packaged-process", - requires_parser_type: false, - dependency_obligation_ids: [], - residual_unknown_ids: [], - unavailable_authority: [], - required_next_evidence: [], - binding: { - obligation_id: "obl.readiness", - owner: { - module_path: "src/readiness.ts", - symbol: "runReadiness", - owner_sha256: HASH, - }, - parser_type: null, - original_cases: CASES.map((kind) => ({ - kind, - evidence_id: EVIDENCE_ID, - location: `/original/${kind}`, - })), - fixtures: CASES.map((kind) => ({ - fixture_id: `fixture.${kind}`, - case_kind: kind, - authority: "packaged-process", - evidence_ids: [EVIDENCE_ID], - })), - verifier: { - verifier_id: "verifier.readiness", - claim_id: "claim.readiness", - command: "npm run verify:readiness", - authority: "packaged-process", - status: "pass", - result_evidence_id: EVIDENCE_ID, - enumerated_obligation_ids: ["obl.readiness"], - nondeterminism: { - mode: "partial-order", - specification: "Startup precedes teardown.", - }, - }, - }, - status: "verified", - diagnostics: [], - }, - ], - evidence_links: [EVIDENCE_ID], - limitations: [], - }, + obligation_ledger: READINESS_OBLIGATION_LEDGER, operation_outcomes: [ { sequence: 0, @@ -404,7 +412,7 @@ export const RECONSTRUCTION_READINESS_EXAMPLE: ReconstructionReadinessInput = { }, { sequence: 1, - ledger_digest: HASH, + ledger_digest: READINESS_OBLIGATION_LEDGER.closure_digest, status: "ready", required_open: 0, newly_verified_obligation_ids: ["obl.readiness"], diff --git a/src/contracts/toolInputReferences.test.ts b/src/contracts/toolInputReferences.test.ts index c661fe040..beb63b7e7 100644 --- a/src/contracts/toolInputReferences.test.ts +++ b/src/contracts/toolInputReferences.test.ts @@ -6,6 +6,34 @@ import { z } from "zod"; import { presentInputJsonSchema } from "./inputSchemaPresentation.js"; import { toolContract } from "./toolContracts.js"; import { toolInputSchemaWithMetadata } from "./toolSchemaMetadata.js"; +import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "./javascript/javascriptRuntimeReconciliationExample.js"; + +it.each(["draft-2020-12", "draft-07"] as const)( + "preserves the complete inline Evidence contract before deferred authentication in %s", + (target) => { + const schema = toolInputSchemaWithMetadata( + toolContract("inspect_analysis_view"), + )["~standard"].jsonSchema.input({ target }); + const ajv = + target === "draft-2020-12" + ? new Ajv2020({ strict: false, validateFormats: false }) + : new Ajv({ strict: false, validateFormats: false }); + expect(ajv.validateSchema(schema)).toBe(true); + const validate = ajv.compile(schema); + const input = { + source: { + kind: "inline", + evidence: JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE, + }, + view: { kind: "summary" }, + }; + expect(validate(input), JSON.stringify(validate.errors)).toBe(true); + expect( + validate({ ...input, source: { kind: "inline", evidence: {} } }), + ).toBe(false); + expect(validate({ ...input, source: { kind: "inline" } })).toBe(false); + }, +); describe("advertised input references", () => { it.each(["draft-2020-12", "draft-07"] as const)( diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index c87c0f09c..3366be8ec 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -59,6 +59,7 @@ import { callPathResultSchema } from "../domain/callPathSchemas.js"; import { staticRuntimeCorrelationResultSchema } from "../domain/staticRuntimeCorrelation.js"; import { reconstructionVerificationResultSchema } from "../domain/reconstructionVerificationSchemas.js"; import { analysisErrorProjectionSchema } from "./errorSchemas.js"; +import { relationshipCoverageSchema } from "../domain/relationshipCoverage.js"; import { nativeDispatchMetadataResultSchema } from "../domain/native/objcSwiftMetadata.js"; import { nativeInvestigationTraceSchema } from "../domain/native/nativeInvestigationGraph.js"; import { @@ -240,6 +241,7 @@ const callPathTraceOutput = evidenceResultOf( error: analysisErrorProjectionSchema, }), ), + relationship_coverage: z.record(z.string(), relationshipCoverageSchema), traversal: z.object({ nodes_visited: z.number().int().min(1), }), @@ -302,7 +304,12 @@ export const enhancedOutputSchemas: Readonly> = { failed: z.number().int().min(0), }), ), - get_call_graph: evidenceResultOf(z.record(z.string(), z.array(graphNode))), + get_call_graph: evidenceResultOf( + z.strictObject({ + graph: z.record(z.string(), z.array(graphNode)), + limitations: z.array(z.string()), + }), + ), analyze_swift_types: evidenceResultOf( z.object({ total: z.number().int().min(0), diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index 1f1a8c700..20c5517ee 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -23,6 +23,7 @@ import { PROVIDER_RETRY_ACTIONS, } from "../domain/providerOperationHealth.js"; import { analysisErrorProjectionSchema } from "./errorSchemas.js"; +import { relationshipCoverageSchema } from "../domain/relationshipCoverage.js"; /** Specialize the single normalized result inside a complete Evidence record. */ export const evidenceResultOf = (schema: Schema) => @@ -52,6 +53,7 @@ export const targetFormatSchema = z.enum([ "hopper", "analysis-database", "mach-o", + "mach-o-universal", "elf", "pe", "dos-mz", @@ -107,12 +109,73 @@ export const providerCapability = z.discriminatedUnion("available", [ ]); export type ProviderCapability = z.output; +/** Listed effects are true; all other members of this closed vocabulary are false. */ +export const PROVIDER_EFFECT_NAMES = Object.keys( + providerCapabilityFacts.effects.shape, +) as (keyof z.output)[]; + export const providerIdentity = z.object({ id: z.string(), name: z.string(), version: z.string().nullable(), }); +const providerCapabilityGroupFacts = { + operations: z.array(z.string().min(1)).min(1), + effects: z.array(providerCapabilityFacts.effects.keyof()), + limitations: z.array(z.string()), +}; + +const providerCapabilityGroup = z.discriminatedUnion("available", [ + z.strictObject({ + ...providerCapabilityGroupFacts, + available: z.literal(true), + reason: z.null(), + availability_code: z.null(), + }), + z.strictObject({ + ...providerCapabilityGroupFacts, + available: z.literal(false), + reason: z.string().nullable(), + availability_code: z.enum(PROVIDER_REJECTION_CODES).nullable(), + }), +]); + +/** Shared limitations precede each group's suffix, preserving exact source order. */ +export const providerCapabilitySet = z.strictObject({ + provider: providerIdentity, + shared_limitations: z.array(z.string()), + groups: z.array(providerCapabilityGroup), +}); +export type ProviderCapabilitySet = z.output; + +/** Decode one status scope without losing provider identity or limitation order. */ +export const expandProviderCapabilities = ( + sets: readonly ProviderCapabilitySet[], +): readonly (ProviderCapability & { + readonly provider: z.output; +})[] => + sets.flatMap((set) => + set.groups.flatMap((group) => + group.operations.map((operation) => ({ + ...providerCapability.parse({ + operation, + available: group.available, + reason: group.reason, + availability_code: group.availability_code, + effects: Object.fromEntries( + PROVIDER_EFFECT_NAMES.map((name) => [ + name, + group.effects.includes(name), + ]), + ), + limitations: [...set.shared_limitations, ...group.limitations], + }), + provider: set.provider, + })), + ), + ); + const processCoordinates = { launcher_pid: z.number().int().min(1), process_group_id: z.number().int().min(1), @@ -333,9 +396,77 @@ export type ToolUnavailabilityReason = z.output< typeof toolUnavailabilityReason >; +export const TOOL_EFFECT_NAMES = Object.keys( + toolAvailabilityFacts.effects.shape, +) as (keyof z.output)[]; +export const TOOL_ANNOTATION_NAMES = Object.keys( + toolAvailabilityFacts.annotations.shape, +) as (keyof z.output)[]; + +const toolAvailabilityGroupFacts = { + names: z.array(z.string().min(1)).min(1), + surface: toolAvailabilityFacts.surface, + default_mode_available: toolAvailabilityFacts.default_mode_available, + modes: toolAvailabilityFacts.modes, + client_requirements: toolAvailabilityFacts.client_requirements.optional(), + effects: z.array(toolAvailabilityFacts.effects.keyof()), + annotations: z.array(toolAvailabilityFacts.annotations.keyof()), +}; + +const toolAvailabilityGroup = z.discriminatedUnion("available", [ + z.strictObject({ + ...toolAvailabilityGroupFacts, + available: z.literal(true), + reason: z.literal("available"), + remediation: z.null(), + }), + z.strictObject({ + ...toolAvailabilityGroupFacts, + available: z.literal(false), + reason: toolUnavailabilityReason, + remediation: z.string().nullable(), + }), +]); + +/** Complete inventory with explicit inherited client requirements and enabled flags. */ +export const toolAvailabilityInventory = z.strictObject({ + client_requirements: toolAvailabilityFacts.client_requirements, + groups: z.array(toolAvailabilityGroup), +}); +export type ToolAvailabilityInventory = z.output< + typeof toolAvailabilityInventory +>; + +/** Expand the wire representation once at the consuming boundary. */ +export const expandToolAvailability = ( + inventory: ToolAvailabilityInventory, +): readonly ToolAvailability[] => + inventory.groups.flatMap((group) => + group.names.map((name) => + toolAvailability.parse({ + ...group, + name, + client_requirements: + group.client_requirements ?? inventory.client_requirements, + effects: Object.fromEntries( + TOOL_EFFECT_NAMES.map((effect) => [ + effect, + group.effects.includes(effect), + ]), + ), + annotations: Object.fromEntries( + TOOL_ANNOTATION_NAMES.map((annotation) => [ + annotation, + group.annotations.includes(annotation), + ]), + ), + }), + ), + ); + export const sessionProvider = z .object({ - capabilities: z.array(providerCapability), + capabilities: z.array(providerCapabilitySet), analysis_run: analysisRun, analysis_activity: analysisActivity, provider_operation_health: providerOperationHealth, @@ -356,10 +487,10 @@ export const sessionProvider = z availability: providerAvailability, target_support: providerTargetSupport, selected: z.boolean(), - capabilities: z.array(providerCapability), + capabilities: z.array(providerCapabilitySet), }), ), - tool_availability: z.array(toolAvailability), + tool_availability: toolAvailabilityInventory, client_features: z.object({ elicitation_form: z.boolean(), elicitation_url: z.boolean(), @@ -387,7 +518,18 @@ export const sessionProvider = z negotiated_protocol_version: z.string().nullable(), client: z.object({ name: z.string(), version: z.string() }).nullable(), skill: z.object({ name: z.string(), expected_version: z.string() }), - catalog: z.record(z.string(), jsonValueSchema), + catalog: z.strictObject({ + counts: z.strictObject({ + cli_commands: z.number().int().min(0), + mcp_tools: z.number().int().min(0), + mcp_prompts: z.number().int().min(0), + }), + digests: z.strictObject({ + tools_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + prompts_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + combined_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + }), + }), protocol_features: z.object({ progress: z.boolean(), cancellation: z.boolean(), @@ -505,6 +647,7 @@ export const graphNode = z.discriminatedUnion("status", [ address: z.string(), status: z.literal("ok"), calls: z.array(z.string()), + relationship_coverage: relationshipCoverageSchema, limitations: z.array(z.string()).optional(), }), z.object({ diff --git a/src/domain/analysisErrorPresentation.ts b/src/domain/analysisErrorPresentation.ts index fa159394d..3e6aa049c 100644 --- a/src/domain/analysisErrorPresentation.ts +++ b/src/domain/analysisErrorPresentation.ts @@ -33,6 +33,8 @@ import type { AnalysisErrorProjection } from "./analysisErrorProjection.js"; export const analysisErrorRemediationAction = ( error: AnalysisError, ): string => { + if (error instanceof EvidenceFileError && error.published !== undefined) + return "Use the published file at details.published_path and resolve the reported staging cleanup failure."; if (error instanceof AnalysisSnapshotMismatchError) return "Run analysis without this snapshot, then save a fresh snapshot using the intended artifact, provider, and analysis profile."; if (error instanceof AnalysisUnsupportedTargetError) @@ -74,6 +76,21 @@ export const analysisErrorRemediationAction = ( error.constraint === "directory_requires_file" ) return "For a JavaScript/Electron application directory, call analyze_javascript_application with input_path or run `rea analyze `. For binary analysis, select its executable file."; + if ( + error instanceof BinaryTargetError && + error.constraint === "load_selection_rejected" + ) + return "Correct the reported load argument (--target-format, --architecture, or --entry-point on the CLI; format, architecture, or entry_point through MCP) and retry."; + if ( + error instanceof BinaryTargetError && + error.constraint === "file_format_rejected" + ) + return "Review the reported file-format reason and select a supported, well-formed target."; + if ( + error instanceof BinaryTargetError && + error.constraint === "wasm_requires_static_inspection" + ) + return `For static inspection, call inspect_wasm_artifact with ${JSON.stringify({ path: error.path })} or run \`rea inspect-wasm-artifact --format json\`. This separate lane requires caller-supplied WABT 1.0.42 via absolute REA_WABT_BIN_DIRECTORY; check its availability in binary_session. REA does not execute WASM or install tools, and the header alone does not establish that the complete module is valid.`; if ( error instanceof AnalysisAccessDeniedError || (error instanceof BinaryTargetError && error.systemCode !== undefined) @@ -93,6 +110,17 @@ export const analysisErrorRemediationAction = ( export const analysisErrorCategory = ( error: AnalysisError, ): AnalysisErrorProjection["category"] => { + if ( + error instanceof BinaryTargetError && + error.constraint === "load_selection_rejected" + ) + return "invalid_input"; + if ( + error instanceof BinaryTargetError && + (error.constraint === "wasm_requires_static_inspection" || + error.constraint === "file_format_rejected") + ) + return "unsupported_target"; if ( error instanceof ProviderSelectionError && error.reason === "provider_unavailable" @@ -156,6 +184,7 @@ const artifactErrorCategory = ( const STATIC_ERROR_CATEGORIES: Readonly< Partial> > = { + ConfigurationError: "invalid_input", AnalysisInputError: "invalid_input", AnalysisAccessDeniedError: "unavailable", AnalysisArtifactChangedError: "integrity_mismatch", @@ -210,10 +239,18 @@ export const analysisErrorUserMessage = (error: AnalysisError): string => { ) .join("; ")}.`; if (error instanceof NoBinaryOpenError) return error.message; + if ( + error instanceof BinaryTargetError && + error.constraint === "wasm_requires_static_inspection" + ) + return `${error.message}. Static WebAssembly inspection uses a separate optional WABT lane, not a native analysis engine.`; if (error instanceof BinaryTargetError) return error.constraint === "directory_requires_file" ? "open_binary accepts files and macOS app bundles. This target is a directory; JavaScript/Electron application directories can be analyzed directly with analyze_javascript_application or `rea analyze `." - : `${error.message}. Check that the path exists, is readable, and points to a supported file.`; + : error.constraint === "file_format_rejected" || + error.constraint === "load_selection_rejected" + ? `${error.message}.` + : `${error.message}. Check that the path exists, is readable, and points to a supported file.`; if (error._tag === "ProcessCaptureError") return ( error.userMessage ?? @@ -368,7 +405,10 @@ const artifactMessage = ({ const evidenceFileMessage = ({ operation, reason, + published, }: EvidenceFileError): string => { + if (published !== undefined) + return "The output file was published, but staging cleanup remains incomplete."; if (reason === "missing") return operation === "read" ? "Evidence file does not exist at the selected path. Check the path and try again." diff --git a/src/domain/analysisErrorProjection.test.ts b/src/domain/analysisErrorProjection.test.ts index eb1de8700..12597fc94 100644 --- a/src/domain/analysisErrorProjection.test.ts +++ b/src/domain/analysisErrorProjection.test.ts @@ -25,6 +25,38 @@ import { ProviderSelectionError } from "./providerSelectionError.js"; import { analysisErrorProjectionSchema } from "../contracts/errorSchemas.js"; import { nativeCallPartialObservationSchema } from "./native/nativeCallPartialObservation.js"; +it("projects a published file receipt alongside retained cleanup failure", () => { + const projected = projectAnalysisError( + new EvidenceFileError("write", "io", { + path: "/tmp/evidence.json", + published: { path: "/tmp/evidence.json", bytes: 42 }, + cleanup: { + reason: "staging directory is still owned", + resources: ["/tmp/.rea-write-abc"], + }, + cause: new Error("cleanup incomplete"), + }), + ); + expect(projected).toMatchObject({ + code: "cleanup_incomplete", + message: + "The output file was published, but staging cleanup remains incomplete.", + remediation: { + action: + "Use the published file at details.published_path and resolve the reported staging cleanup failure.", + }, + details: { + operation: "write", + path: "/tmp/evidence.json", + published: true, + published_path: "/tmp/evidence.json", + bytes: 42, + resources: ["/tmp/.rea-write-abc"], + }, + }); + expect(analysisErrorProjectionSchema.safeParse(projected).success).toBe(true); +}); + const nativeCallPartialObservation = nativeCallPartialObservationSchema.parse({ kind: "native-call-observation", target: { diff --git a/src/domain/analysisErrorProjection.ts b/src/domain/analysisErrorProjection.ts index 2ab2a6c5c..6c0699eb8 100644 --- a/src/domain/analysisErrorProjection.ts +++ b/src/domain/analysisErrorProjection.ts @@ -105,9 +105,11 @@ const underlyingErrorCode = ( if (error instanceof ArtifactOperationError) return artifactOperationCode(error); if (error instanceof BinaryTargetError) - return error.systemCode === undefined - ? "target_unavailable" - : "access_denied"; + return error.constraint === "load_selection_rejected" + ? "invalid_request" + : error.systemCode === undefined + ? "target_unavailable" + : "access_denied"; if (error instanceof EvidenceFileError) return evidenceFileCode(error.reason); if (error instanceof UnknownRegistryError) return unknownRegistryCode(error.reason); @@ -309,6 +311,13 @@ const artifactStateErrorDetails = ( operation: error.operation, reason: error.reason, ...(error.path === undefined ? {} : { path: error.path }), + ...(error.published === undefined + ? {} + : { + published: true, + bytes: error.published.bytes, + published_path: error.published.path, + }), }; return undefined; }; diff --git a/src/domain/analysisSnapshot.test.ts b/src/domain/analysisSnapshot.test.ts index e2c1483c9..b79f38704 100644 --- a/src/domain/analysisSnapshot.test.ts +++ b/src/domain/analysisSnapshot.test.ts @@ -40,6 +40,7 @@ const snapshotWithHistoricalEvidence = () => { raw_result: null, provider: binding.provider, limitations: [], + relationship_coverage: null, locations: [], subject: { path: ANALYSIS_SNAPSHOT_TARGET.path, @@ -69,7 +70,7 @@ const snapshotWithHistoricalEvidence = () => { }; }; -describe("analysis snapshot contract", () => { +describe("analysis snapshot target identity", () => { it("partitions raw snapshots and query identities by the canonical entry", () => { const target: BinaryTarget = { path: "/fixture/image.bin", @@ -117,51 +118,64 @@ describe("analysis snapshot contract", () => { "entry_point", ); }); +}); - it("preserves DOS MZ target and subject formats with the x86 family", () => { - const target = { - path: ANALYSIS_SNAPSHOT_TARGET.path, - sha256: ANALYSIS_SNAPSHOT_TARGET.sha256, - kind: "executable" as const, - format: "dos-mz" as const, - architecture: "x86" as const, - availableArchitectures: ["x86" as const], - }; - const evidence = createEvidence(target, ANALYSIS_SNAPSHOT_PROVIDER, { - operation: "analyze_function", - parameters: {}, - result: null, - analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, - }); - const binding = snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE); - const parsed = analysisSnapshotSchema.parse({ - target: snapshotTarget(target), - binding, - entries: [ - { - query_id: `query_${"0".repeat(64)}`, - operation: "analyze_function", - parameters: {}, - execution: { - result: null, - raw_result: null, - provider: binding.provider, - limitations: [], - locations: [], - subject: target, +describe("analysis snapshot contract", () => { + it.each([ + ["dos-mz", "x86"], + ["mach-o-universal", "arm64"], + ] as const)( + "preserves %s target and subject formats", + (format, architecture) => { + const target = { + path: ANALYSIS_SNAPSHOT_TARGET.path, + sha256: ANALYSIS_SNAPSHOT_TARGET.sha256, + kind: "executable" as const, + format, + architecture, + availableArchitectures: [architecture], + }; + const evidence = createEvidence(target, ANALYSIS_SNAPSHOT_PROVIDER, { + operation: "analyze_function", + parameters: {}, + result: null, + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + }); + const binding = snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE); + const parsed = parseAnalysisSnapshot({ + target: snapshotTarget(target), + binding, + entries: [ + { + query_id: analysisQueryId( + snapshotTarget(target), + binding, + "analyze_function", + {}, + ), + operation: "analyze_function", + parameters: {}, + execution: { + result: null, + raw_result: null, + provider: binding.provider, + limitations: [], + locations: [], + subject: target, + }, }, - }, - ], - workflow_entries: [], - evidence_bundle: createEvidenceBundle([evidence]), - }); - expect(parsed.target).toMatchObject({ - format: "dos-mz", - architecture: "x86", - }); - expect(parsed.entries[0]?.execution.subject?.format).toBe("dos-mz"); - expect(parsed.evidence_bundle.records[0]?.subject?.format).toBe("dos-mz"); - }); + ], + workflow_entries: [], + evidence_bundle: createEvidenceBundle([evidence]), + }); + expect(parsed.target).toMatchObject({ + format, + architecture, + }); + expect(parsed.entries[0]?.execution.subject?.format).toBe(format); + expect(parsed.evidence_bundle.records[0]?.subject?.format).toBe(format); + }, + ); it("rejects a snapshot subject path containing NUL", () => { const binding = snapshotBinding(ANALYSIS_SNAPSHOT_PROFILE); @@ -392,7 +406,10 @@ describe("analysis snapshot Evidence binding", () => { workflow_entries: [], evidence_bundle: createEvidenceBundle([evidence]), }; - expect(parseAnalysisSnapshot(snapshot)).toEqual(snapshot); + const parsedLegacySnapshot = parseAnalysisSnapshot(snapshot); + expect( + parsedLegacySnapshot.entries[0]?.execution.relationship_coverage, + ).toBeNull(); expect(() => parseAnalysisSnapshot({ @@ -420,6 +437,13 @@ describe("analysis snapshot Evidence binding", () => { if (subject !== null && subject !== undefined) subject.path = "/different/path"; }, + (altered: typeof snapshot) => { + const entry = altered.entries[0]; + if (entry === undefined) throw new TypeError("missing fixture entry"); + Reflect.set(entry.execution, "relationship_coverage", { + status: "complete", + }); + }, ]; for (const alter of alterations) { const altered = structuredClone(snapshot); diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index e156144e9..57e876653 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -42,11 +42,16 @@ import { prefixedDigestSchema } from "./../domain/digests.js"; import { canonicalJsonDigestSteps } from "./canonicalJsonDigestSteps.js"; import { freezeJsonSnapshot } from "./immutableJson.js"; import { rawBinaryEntryPointSchema } from "./rawBinary.js"; +import { + relationshipCoverageSchema, + type RelationshipCoverage, +} from "./relationshipCoverage.js"; const architectureSchema = z.enum(BINARY_ARCHITECTURES); const formatSchema = z.enum([ "analysis-database", "mach-o", + "mach-o-universal", "elf", "pe", "dos-mz", @@ -77,7 +82,6 @@ const subjectSchema = z.object({ "directory", "file", "unknown", - "mach-o-universal", "javascript-bundle", "entitlements", ]), @@ -125,6 +129,7 @@ const entrySchema = z.object({ raw_result: jsonValueSchema.nullable(), provider: committedProviderSchema, limitations: z.array(z.string()), + relationship_coverage: relationshipCoverageSchema.nullable().default(null), locations: z.array(evidenceLocationSchema), subject: subjectSchema.nullable(), }), @@ -139,6 +144,7 @@ const workflowEntrySchema = z.object({ provider: committedProviderSchema, analysis_profile: analysisProfileSchema, limitations: z.array(z.string()), + relationship_coverage: relationshipCoverageSchema.nullable().default(null), locations: z.array(evidenceLocationSchema), subject: subjectSchema.nullable(), }), @@ -268,7 +274,7 @@ const snapshotContentDigest = ( rawResult: JsonValue | null, execution: Pick< AnalysisSnapshotEntry["execution"], - "limitations" | "locations" | "subject" + "limitations" | "locations" | "subject" | "relationship_coverage" >, ): string => { const metadata = jsonObjectSchema.parse({ @@ -283,6 +289,9 @@ const snapshotContentDigest = ( format: execution.subject.format, architecture: execution.subject.architecture, }, + ...(execution.relationship_coverage === null + ? {} + : { relationship_coverage: execution.relationship_coverage }), }); return contentDigest({ ...metadata, @@ -309,6 +318,7 @@ interface SnapshotExecution { }; readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations: readonly string[]; + readonly relationshipCoverage?: RelationshipCoverage | null | undefined; readonly locations: readonly EvidenceLocation[]; readonly subject: EvidenceSubjectTarget | null; } @@ -460,6 +470,7 @@ export const createAnalysisSnapshotWorkflowEntry = (input: { provider: execution.analysisProfile.provider, analysis_profile: execution.analysisProfile, limitations: [...execution.limitations], + relationship_coverage: execution.relationshipCoverage ?? null, locations: [...execution.locations], subject: execution.subject === null @@ -522,6 +533,7 @@ export const createAnalysisSnapshotEntry = (input: { raw_result: execution.rawResult, provider: binding.provider, limitations: [...execution.limitations], + relationship_coverage: execution.relationshipCoverage ?? null, locations: [...execution.locations], subject: execution.subject === null @@ -838,19 +850,24 @@ const evidenceMatchesWorkflowEntry = ( evidenceContentDigest(evidence), ]) === workflowEntryEvidenceKey(entry); -const evidenceContentMetadata = (evidence: Evidence) => ({ - limitations: evidence.limitations, - locations: evidence.locations, - subject: - evidence.subject === null - ? null - : { - local_path: evidence.subject.local_path, - sha256: evidence.subject.digest.sha256, - format: evidence.subject.format, - architecture: evidence.subject.architecture, - }, -}); +const evidenceContentMetadata = (evidence: Evidence) => { + return { + limitations: evidence.limitations, + ...(evidence.relationship_coverage === null + ? {} + : { relationship_coverage: evidence.relationship_coverage }), + locations: evidence.locations, + subject: + evidence.subject === null + ? null + : { + local_path: evidence.subject.local_path, + sha256: evidence.subject.digest.sha256, + format: evidence.subject.format, + architecture: evidence.subject.architecture, + }, + }; +}; /** Serialize a validated snapshot with byte-stable canonical entry ordering. */ export const serializeAnalysisSnapshot = (snapshot: AnalysisSnapshot): string => diff --git a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts new file mode 100644 index 000000000..410546543 --- /dev/null +++ b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.test.ts @@ -0,0 +1,173 @@ +import { expect, it } from "vitest"; +import { analysisViewCaptureFixture } from "../../../tests/fixtures/analysisView.js"; +import type { WebNetworkCaptureRecord } from "../webNetworkCapture.js"; +import { WebNetworkApiSurfaceAccumulator } from "./WebNetworkApiSurfaceAccumulator.js"; + +const record = ( + ordinal: number, + path: string, + method = "GET", + mimeType = "application/json", +): WebNetworkCaptureRecord => ({ + ...analysisViewCaptureFixture().records[0]!, + ordinal, + limitations: [], + reported: { + request: { method, url: `https://api.test${path}` }, + response: { status: 200, content: { mimeType } }, + }, +}); + +const add = ( + accumulator: WebNetworkApiSurfaceAccumulator, + sample: WebNetworkCaptureRecord, +) => { + const result = accumulator.add(sample); + if (!result.ok) throw result.error; +}; + +it("admits the capture schema clone and grouping against the same budget", () => { + const sample = record(0, `/${"a".repeat(1200)}`); + const isolated = new WebNetworkApiSurfaceAccumulator(64 * 1024); + add(isolated, sample); + const accumulator = new WebNetworkApiSurfaceAccumulator(64 * 1024); + expect( + accumulator.admitCaptureClone({ + headers: Array.from({ length: 40 }, () => ({ name: "x", value: "y" })), + }).ok, + ).toBe(true); + expect(accumulator.add(sample)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + expect(accumulator.page(0, 1).groups).toEqual([]); +}); + +it("charges non-enumerable fields that the capture schema would clone", () => { + const value = Object.defineProperty({}, "records", { + value: Array.from({ length: 100 }, () => ({ reported: {} })), + }); + expect( + new WebNetworkApiSurfaceAccumulator(64 * 1024).admitCaptureClone(value), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); +}); + +it("bounds many short literal groups, including groups with unavailable MIME types", () => { + const accumulator = new WebNetworkApiSurfaceAccumulator(64 * 1024); + let accepted = 0; + for (; accepted < 1000; accepted += 1) { + const before = accumulator.page(0, 1000); + const result = accumulator.add( + record(accepted, `/route${accepted}`, "GET", ""), + ); + if (result.ok) continue; + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + }); + expect(accumulator.page(0, 1000)).toEqual(before); + break; + } + expect(accepted).toBeGreaterThan(0); + expect(accepted).toBeLessThan(1000); + expect(accumulator.page(0, 1000).coverage.total).toBe(accepted); +}); + +it.each([ + { + name: "templated segments", + path: "/1".repeat(350), + method: "GET", + mime: "application/json", + }, + { + name: "literal path", + path: `/${"a".repeat(30_000)}`, + method: "GET", + mime: "application/json", + }, + { + name: "request identity", + path: "/a", + method: "M".repeat(60_000), + mime: "application/json", + }, + { + name: "response text", + path: "/a", + method: "GET", + mime: `text/${"a".repeat(30_000)}`, + }, +])( + "admits $name together with previously retained state before publishing a new group", + ({ path, method, mime }) => { + const maximum = 1024 * 1024; + const isolated = new WebNetworkApiSurfaceAccumulator(maximum); + add(isolated, record(0, path, method, mime)); + + const accumulator = new WebNetworkApiSurfaceAccumulator(maximum); + for (let ordinal = 0; ordinal < 150; ordinal += 1) + add(accumulator, record(ordinal, `/route${ordinal}`)); + const before = accumulator.page(0, 1000); + expect(accumulator.add(record(150, path, method, mime))).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + expect(accumulator.page(0, 1000)).toEqual(before); + }, +); + +it("retains repeated segment, status and MIME values once across many samples", () => { + const accumulator = new WebNetworkApiSurfaceAccumulator(1024 * 1024); + for (let ordinal = 0; ordinal < 1000; ordinal += 1) + add(accumulator, record(ordinal, "/1".repeat(250))); + const page = accumulator.page(0, 1); + expect(page.groups).toHaveLength(1); + expect(page.groups[0]?.record_ordinals).toHaveLength(1000); + expect(page.groups[0]?.statuses).toEqual([200]); + expect(page.groups[0]?.content_types).toEqual(["application/json"]); + expect(page.groups[0]?.templated_segments).toHaveLength(250); + expect( + page.groups[0]?.templated_segments.every( + (segment) => segment.observed_values.join() === "1", + ), + ).toBe(true); +}); + +it("refuses an entire merge before retaining any novel observed values or its ordinal", () => { + const accumulator = new WebNetworkApiSurfaceAccumulator(1024 * 1024); + for (let ordinal = 0; ordinal < 100; ordinal += 1) + add(accumulator, record(ordinal, `/route${ordinal}`)); + add(accumulator, record(100, "/1".repeat(200))); + const before = accumulator.page(0, 1000); + const result = accumulator.add( + record(101, `/${"2".repeat(80)}`.repeat(200), "GET", "text/plain"), + ); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); + expect(accumulator.page(0, 1000)).toEqual(before); +}); + +it("bounds excluded-record storage and distinct limitations as well as endpoint groups", () => { + const accumulator = new WebNetworkApiSurfaceAccumulator(64 * 1024); + let accepted = 0; + for (; accepted < 1000; accepted += 1) { + const sample = record(accepted, "/a"); + sample.reported = null; + sample.limitations = [`producer caveat ${accepted}`]; + const result = accumulator.add(sample); + if (!result.ok) { + expect(result.error).toMatchObject({ + _tag: "AnalysisResourceConstraintError", + }); + break; + } + } + expect(accepted).toBeGreaterThan(0); + expect(accepted).toBeLessThan(1000); + expect(accumulator.page(0, 1).excluded_records).toHaveLength(accepted); +}); diff --git a/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts new file mode 100644 index 000000000..0abc64675 --- /dev/null +++ b/src/domain/analysisView/WebNetworkApiSurfaceAccumulator.ts @@ -0,0 +1,467 @@ +import { z } from "zod"; +import { + AnalysisInputError, + AnalysisResourceConstraintError, +} from "../analysisErrorCore.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { compareUnicodeCodePoints as compare } from "../unicodeCodePointOrder.js"; +import { parseRecordedTransportUrl } from "../recordedTransportUrl.js"; +import { err, ok, type Result } from "../result.js"; +import { MAX_JSON_DEPTH } from "../jsonValue.js"; +import { + WEB_NETWORK_CAPTURE_LIMITS, + type WebNetworkCaptureRecord, +} from "../webNetworkCapture.js"; +import { pageViewCoverage } from "./analysisViewCoverage.js"; +import type { UnsignedAnalysisView } from "./analysisView.js"; + +type Surface = Extract; +type Group = Surface["groups"][number]; +type Rule = Group["templated_segments"][number]["rule"]; +type Exclusion = Surface["excluded_records"][number]; +interface Segment { + readonly segment_index: number; + readonly rule: Rule; + readonly value: string; +} +interface Route { + readonly template: string; + readonly rules: string; + readonly segments: readonly Segment[]; +} +interface GroupState { + readonly method: string; + readonly origin: string; + readonly path_template: string; + readonly segments: readonly { + readonly segment_index: number; + readonly rule: Rule; + readonly values: Set; + }[]; + readonly ordinals: number[]; + readonly statuses: Set; + readonly contentTypes: Set; +} +type Buckets = Map>>>; + +const requestSchema = z.object({ + request: z.object({ url: z.unknown(), method: z.unknown() }), +}); +const responseSchema = z.object({ + response: z.object({ + status: z.unknown().optional(), + content: z.unknown().optional(), + }), +}); +const contentSchema = z.object({ mimeType: z.unknown().optional() }); +const MEDIA_TYPE_ESSENCE = + /^[A-Za-z0-9!#$%&'*+.^_`|~-]+\/[A-Za-z0-9!#$%&'*+.^_`|~-]+$/u; + +const mediaTypeEssence = (reported: string): string | null => { + const separator = reported.indexOf(";"); + let start = 0; + let end = separator < 0 ? reported.length : separator; + // HTTP OWS is SP/HTAB. Scan from both ends so malformed internal runs of + // whitespace cannot cause repeated trailing-regex searches. + while (start < end && (reported[start] === " " || reported[start] === "\t")) + start += 1; + while ( + end > start && + (reported[end - 1] === " " || reported[end - 1] === "\t") + ) + end -= 1; + const essence = reported.slice(start, end); + // Validate ASCII tokens before case folding can turn Unicode into ASCII. + return MEDIA_TYPE_ESSENCE.test(essence) ? essence.toLowerCase() : null; +}; + +// Conservative representation allowances, not an enforced V8 heap limit. +// Fixed charges include the accumulator, bucket/set entries, array growth and +// page/schema projection. Text includes UTF-16 storage and two worst-case JSON +// encodings (transport text and UTF-8 bytes); no serialization is needed to count. +const GROUP_BYTES = 2048; +const SEGMENT_BYTES = 1024; +const ENTRY_BYTES = 128; +const RECORD_SCRATCH_BYTES = 512; +const SEGMENT_SCRATCH_BYTES = 512; +const CAPTURE_CONTAINER_BYTES = 512; +// Covers cloned member slots and per-member validation/entries/map scratch. +// Scalar strings remain shared references, so their text is not copied here. +const CAPTURE_MEMBER_BYTES = 256; +const textBytes = (length: number): number => 32 + length * 14; + +const segmentRule = (value: string): Rule | null => { + if (/^[0-9]+$/u.test(value)) return "numeric"; + if ( + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test( + value, + ) + ) + return "uuid"; + if (/^[0-9a-f]{32,}$/iu.test(value)) return "hex"; + if (/^[0-9]{4}-[0-9]{2}-[0-9]{2}$/u.test(value)) { + const date = new Date(`${value}T00:00:00.000Z`); + if ( + Number.isFinite(date.getTime()) && + date.toISOString().slice(0, 10) === value + ) + return "iso-date"; + } + return null; +}; + +/** Own group identity, deduplication and admission for the entire projection. */ +export class WebNetworkApiSurfaceAccumulator { + readonly #buckets: Buckets = new Map(); + readonly #groups: GroupState[] = []; + readonly #excluded: Exclusion[] = []; + readonly #limitations = new Set(); + #unavailableResponse = false; + #retainedBytes = 0; + #temporaryBytes = 0; + + constructor(readonly maximumBytes = WEB_NETWORK_CAPTURE_LIMITS.outputBytes) {} + + #retain(bytes: number): boolean { + if (this.#retainedBytes + this.#temporaryBytes + bytes > this.maximumBytes) + return false; + this.#retainedBytes += bytes; + return true; + } + + #temporary(bytes: number): boolean { + if (this.#retainedBytes + this.#temporaryBytes + bytes > this.maximumBytes) + return false; + this.#temporaryBytes += bytes; + return true; + } + + #overBudget(): Result { + return err( + new AnalysisResourceConstraintError( + "inspect_analysis_view", + "memory", + "API-surface retained state and temporary work exceed the projection representation budget.", + { output_bytes: this.maximumBytes }, + ), + ); + } + + /** Reserve schema output and validation scratch while the borrowed parent stays live. */ + admitCaptureClone(value: unknown): Result { + return this.#admitCaptureValue(value, new Set(), 0, false); + } + + /** Admit the complete portable envelope before cloning or authenticating it. */ + admitInlineEvidence(value: unknown): Result { + return this.#admitCaptureValue(value, new Set(), 0, true); + } + + #admitCaptureValue( + value: unknown, + ancestors: Set, + depth: number, + inline: boolean, + ): Result { + // Capture records add three levels before reported; inline Evidence adds one. + if ( + depth > MAX_JSON_DEPTH + (inline ? 4 : 3) || + (typeof value === "object" && value !== null && ancestors.has(value)) + ) + return this.#invalidCapture(); + if (!this.#retain(CAPTURE_MEMBER_BYTES)) return this.#overBudget(); + // Evidence authentication encodes strings and object keys for hashing. + // Keep that allowance live alongside the envelope clone and projection. + if ( + inline && + typeof value === "string" && + !this.#retain(textBytes(value.length)) + ) + return this.#overBudget(); + if (typeof value !== "object" || value === null) return ok(undefined); + if (!this.#retain(CAPTURE_CONTAINER_BYTES)) return this.#overBudget(); + ancestors.add(value); + // Walk depth-first without building a second capture graph. Member charges + // include key enumeration; aliases count each time the schema copies them. + if (Array.isArray(value)) { + for (let index = 0; index < value.length; index += 1) { + const descriptor = Object.getOwnPropertyDescriptor(value, index); + if (descriptor === undefined || !Object.hasOwn(descriptor, "value")) + return this.#invalidCapture(); + const admitted = this.#admitCaptureValue( + descriptor.value, + ancestors, + depth + 1, + inline, + ); + if (!admitted.ok) return admitted; + } + } else { + const prototype: unknown = Object.getPrototypeOf(value); + if (prototype !== Object.prototype && prototype !== null) + return this.#invalidCapture(); + // Zod reads named fields even when they are non-enumerable. + for (const key of Object.getOwnPropertyNames(value)) { + if (inline && !this.#retain(textBytes(key.length))) + return this.#overBudget(); + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if (descriptor === undefined || !Object.hasOwn(descriptor, "value")) + return this.#invalidCapture(); + const admitted = this.#admitCaptureValue( + descriptor.value, + ancestors, + depth + 1, + inline, + ); + if (!admitted.ok) return admitted; + } + } + ancestors.delete(value); + return ok(undefined); + } + + #invalidCapture(): Result { + return err( + new AnalysisInputError("inspect_analysis_view", undefined, [ + { + path: ["source"], + reason: "invalid_value", + message: + "Parent capture must contain bounded ordinary JSON data without cycles or accessors.", + }, + ]), + ); + } + + addLimitations(limitations: readonly string[]): Result { + for (const limitation of limitations) { + if (this.#limitations.has(limitation)) continue; + if (!this.#retain(ENTRY_BYTES + textBytes(limitation.length))) + return this.#overBudget(); + this.#limitations.add(limitation); + } + return ok(undefined); + } + + #exclude( + ordinal: number, + reason: Exclusion["reason"], + ): Result { + if (!this.#retain(ENTRY_BYTES + RECORD_SCRATCH_BYTES)) + return this.#overBudget(); + this.#excluded.push({ ordinal, reason }); + return ok(undefined); + } + + /** Scan literal spans; only matched segments need descriptors or template parts. */ + #route(path: string): Result { + const segments: Segment[] = []; + const parts: string[] = []; + const rules: string[] = []; + let templateLength = path.length; + let rulesLength = 0; + let copied = 0; + let index = 0; + let start = 1; + for (;;) { + const slash = path.indexOf("/", start); + const end = slash < 0 ? path.length : slash; + const value = path.slice(start, end); + const rule = segmentRule(value); + if (rule !== null) { + if (!this.#temporary(SEGMENT_SCRATCH_BYTES)) return this.#overBudget(); + const placeholder = `{${rule}}`; + const identity = `${index}:${rule};`; + segments.push({ segment_index: index, rule, value }); + parts.push(path.slice(copied, start), placeholder); + rules.push(identity); + templateLength += placeholder.length - value.length; + rulesLength += identity.length; + copied = end; + } + if (slash < 0) break; + start = slash + 1; + index += 1; + } + if (segments.length === 0) + return ok({ template: path, rules: "", segments }); + if (!this.#temporary(textBytes(templateLength) + textBytes(rulesLength))) + return this.#overBudget(); + parts.push(path.slice(copied)); + return ok({ template: parts.join(""), rules: rules.join(""), segments }); + } + + add(record: WebNetworkCaptureRecord): Result { + const limitations = this.addLimitations(record.limitations); + if (!limitations.ok) return limitations; + const request = requestSchema.safeParse(record.reported); + if (!request.success) + return this.#exclude(record.ordinal, "request-unavailable"); + const { url, method } = request.data.request; + if (url === null) return this.#exclude(record.ordinal, "url-redacted"); + if (typeof url !== "string") + return this.#exclude(record.ordinal, "url-unparseable"); + // Admission precedes URL parsing, text normalization and path materialization. + if (!this.#temporary(RECORD_SCRATCH_BYTES + textBytes(url.length))) + return this.#overBudget(); + try { + const parsed = parseRecordedTransportUrl(url); + if (parsed.origin === null) + return this.#exclude(record.ordinal, "url-unparseable"); + if (typeof method !== "string" || method.length === 0) + return this.#exclude(record.ordinal, "method-unavailable"); + if (parsed.path === null) + return this.#exclude(record.ordinal, "url-unparseable"); + const response = responseSchema.safeParse(record.reported); + const status = response.success + ? response.data.response.status + : undefined; + const content = contentSchema.safeParse( + response.success ? response.data.response.content : undefined, + ); + const mime = content.success ? content.data.mimeType : undefined; + const rawMime = typeof mime === "string" ? mime : ""; + if (!this.#temporary(textBytes(rawMime.length))) + return this.#overBudget(); + const route = this.#route(parsed.path); + if (!route.ok) return route; + return this.#append( + record.ordinal, + method, + parsed.origin, + route.value, + typeof status === "number" && status !== 0 ? status : null, + mediaTypeEssence(rawMime), + ); + } finally { + // Per-record strings and descriptors never become cumulative scratch. + this.#temporaryBytes = 0; + } + } + + #append( + ordinal: number, + method: string, + origin: string, + route: Route, + status: number | null, + mime: string | null, + ): Result { + let group = this.#buckets + .get(method) + ?.get(origin) + ?.get(route.template) + ?.get(route.rules); + let bytes = ENTRY_BYTES; // One retained record ordinal, including page output. + if (group === undefined) { + bytes += + GROUP_BYTES + + textBytes(method.length) + + textBytes(origin.length) + + textBytes(route.template.length) + + textBytes(route.rules.length); + for (const segment of route.segments) + bytes += SEGMENT_BYTES + ENTRY_BYTES + textBytes(segment.value.length); + } else { + for (const [index, segment] of route.segments.entries()) { + if (!group.segments[index]!.values.has(segment.value)) + bytes += ENTRY_BYTES + textBytes(segment.value.length); + } + } + if (status !== null && !group?.statuses.has(status)) bytes += ENTRY_BYTES; + if (mime !== null && !group?.contentTypes.has(mime)) + bytes += ENTRY_BYTES + textBytes(mime.length); + // Reserve the whole mutation while route scratch is still live. No sets, + // map entries, arrays or new observed values are retained before admission. + if (!this.#retain(bytes)) return this.#overBudget(); + if (group === undefined) { + group = { + method, + origin, + path_template: route.template, + segments: route.segments.map((segment) => ({ + segment_index: segment.segment_index, + rule: segment.rule, + values: new Set([segment.value]), + })), + ordinals: [], + statuses: new Set(), + contentTypes: new Set(), + }; + const byOrigin = this.#buckets.get(method) ?? new Map(); + this.#buckets.set(method, byOrigin); + const byTemplate = byOrigin.get(origin) ?? new Map(); + byOrigin.set(origin, byTemplate); + const byRule = byTemplate.get(route.template) ?? new Map(); + byTemplate.set(route.template, byRule); + byRule.set(route.rules, group); + this.#groups.push(group); + } else { + for (const [index, segment] of route.segments.entries()) + group.segments[index]!.values.add(segment.value); + } + group.ordinals.push(ordinal); + if (status === null || mime === null) this.#unavailableResponse = true; + if (status !== null) group.statuses.add(status); + if (mime !== null) group.contentTypes.add(mime); + return ok(undefined); + } + + page( + offset: number, + limit: number, + ): Pick< + Surface, + | "coverage" + | "groups" + | "templated_groups" + | "excluded_records" + | "limitations" + | "unknowns" + > { + this.#groups.sort( + (a, b) => + compare(a.origin, b.origin) || + compare(a.path_template, b.path_template) || + compare(a.method, b.method), + ); + const groups = this.#groups + .slice(offset, offset + limit) + .map((group): Group => ({ + method: group.method, + origin: group.origin, + path_template: group.path_template, + templated_segments: group.segments.map((segment) => ({ + segment_index: segment.segment_index, + rule: segment.rule, + basis: "inferred", + observed_values: [...segment.values].sort(compare), + })), + record_ordinals: [...group.ordinals], + statuses: [...group.statuses].sort((a, b) => a - b), + content_types: [...group.contentTypes].sort(compare), + flags: [ + ...(group.ordinals.length === 1 ? ["single-sample" as const] : []), + ...(group.statuses.size > 1 ? ["mixed-statuses" as const] : []), + ...(group.contentTypes.size > 1 + ? ["mixed-content-types" as const] + : []), + ], + })); + return { + coverage: pageViewCoverage(offset, groups.length, this.#groups.length), + groups, + templated_groups: this.#groups.reduce( + (count, group) => count + (group.segments.length > 0 ? 1 : 0), + 0, + ), + excluded_records: [...this.#excluded], + limitations: [...this.#limitations].sort(compare), + unknowns: this.#unavailableResponse + ? [ + "Some retained records have unavailable status or content type values.", + ] + : [], + }; + } +} diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index d8bcb45a6..fe2d408c1 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -21,7 +21,13 @@ import { import { analysisInputErrorFromIssues } from "../inputIssueProjection.js"; import { projectBinaryLayoutView } from "./binaryLayoutView.js"; import { projectJavaScriptApplicationView } from "./javascriptApplicationView.js"; +import { + projectWebNetworkApiSurfaceView, + apiSurfaceFields, + apiSurfaceViewSchema, +} from "./webNetworkApiSurfaceView.js"; import { projectNativeFunctionView } from "./nativeFunctionView.js"; +import type { WebNetworkApiSurfaceAccumulator } from "./WebNetworkApiSurfaceAccumulator.js"; /** Public name of the selected-view workflow. */ export const INSPECT_ANALYSIS_VIEW_OPERATION = "inspect_analysis_view" as const; @@ -109,9 +115,17 @@ export const analysisViewRequestSchema = z.discriminatedUnion("kind", [ itemViewSchema, pageViewSchema, nativeViewSchema, + apiSurfaceViewSchema, ]); -/** Inline Evidence or an exact same-session retained reference. */ +// The application owns admission and authenticated Evidence parsing. SDK and +// CLI envelope parsing must borrow this unknown value rather than clone it +// before admission. Advertise the owning Evidence contract in both dialects. +const inlineEvidenceInputSchema = z.unknown().nonoptional(); +inlineEvidenceInputSchema._zod.processJSONSchema = + evidenceSchema._zod.processJSONSchema; + +/** Unauthenticated inline input or an exact same-session retained reference. */ export const analysisViewSourceSchema = z.discriminatedUnion("kind", [ z.strictObject({ kind: z.literal("retained-evidence"), @@ -119,7 +133,7 @@ export const analysisViewSourceSchema = z.discriminatedUnion("kind", [ }), z.strictObject({ kind: z.literal("inline"), - evidence: evidenceSchema, + evidence: inlineEvidenceInputSchema, }), ]); @@ -178,6 +192,32 @@ const javascriptSummarySchema = z.strictObject({ limitation_count: z.number().int().nonnegative(), }); +const summaryViewResultFields = { + kind: z.literal("summary"), + view: summaryViewSchema, + parent_evidence_id: prefixedDigestSchema("ev"), + parent_digest: digestSchema, + view_digest: digestSchema, + artifact: artifactSchema, + coverage: coverageSchema, + limitations: z.array(z.string()), + unknowns: z.array(z.string()), +} as const; + +/** Summary `inspect_binary_layout` detail returns; no other view kind. */ +export const binaryLayoutSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultFields, + summary: layoutSummarySchema, + parent_operation: z.literal("inspect_binary_layout"), +}); + +/** Summary `analyze_javascript_application` detail returns; no other view kind. */ +export const javascriptApplicationSummaryViewResultSchema = z.strictObject({ + ...summaryViewResultFields, + summary: javascriptSummarySchema, + parent_operation: z.literal("analyze_javascript_application"), +}); + const viewResultBase = { parent_evidence_id: prefixedDigestSchema("ev"), parent_operation: z.enum([ @@ -195,6 +235,17 @@ const viewResultBase = { /** Projected facts for one selected view of retained analysis Evidence. */ export const analysisViewResultSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + ...viewResultBase, + ...apiSurfaceFields, + kind: z.literal("api-surface"), + parent_operation: z.literal("inspect_web_network_capture"), + view: apiSurfaceViewSchema, + artifact: z.strictObject({ + path: z.string(), + sha256: digestSchema.nullable(), + }), + }), z.strictObject({ kind: z.literal("summary"), view: summaryViewSchema, @@ -242,6 +293,8 @@ export interface AnalysisViewParent { readonly normalizedResult: unknown; readonly limitations: readonly string[]; readonly artifact?: { readonly path: string; readonly sha256: string } | null; + /** Parent Evidence parameters, when the parent operation records them. */ + readonly parameters?: unknown; } /** Construct a typed input failure for one selected-view constraint. */ @@ -252,6 +305,10 @@ export const analysisViewInputError = ( /** View payload before the content digest is attached. */ export type UnsignedAnalysisView = + | Omit< + Extract, + "view_digest" + > | Omit< Extract, "view_digest" @@ -335,6 +392,7 @@ const parentArtifactPath = (parent: AnalysisViewParent): string => { export const projectAnalysisView = ( parent: AnalysisViewParent, view: AnalysisViewRequest, + apiSurfaceAccumulator?: WebNetworkApiSurfaceAccumulator, ): Result => { const digest = parentDigest(parent.evidenceId); if (!digestSchema.safeParse(digest).success) @@ -347,6 +405,17 @@ export const projectAnalysisView = ( }, ]), ); + if ( + view.kind === "api-surface" || + parent.operation === "inspect_web_network_capture" + ) { + const projected = projectWebNetworkApiSurfaceView( + parent, + view, + apiSurfaceAccumulator, + ); + return projected.ok ? ok(sealAnalysisView(projected.value)) : projected; + } if (parent.operation === "inspect_binary_layout") { const layout = binaryLayoutSchema.safeParse(parent.normalizedResult); if (!layout.success) diff --git a/src/domain/analysisView/binaryLayoutView.ts b/src/domain/analysisView/binaryLayoutView.ts index 13a9b766f..485bfbac6 100644 --- a/src/domain/analysisView/binaryLayoutView.ts +++ b/src/domain/analysisView/binaryLayoutView.ts @@ -149,7 +149,7 @@ const layoutSummary = (layout: BinaryLayout) => ({ export const projectBinaryLayoutView = ( parent: AnalysisViewParent, layout: BinaryLayout, - view: AnalysisViewRequest, + view: Exclude, ): Result => { const shared = parentFields(parent, layout); if (view.kind === "native") diff --git a/src/domain/analysisView/javascriptApplicationView.ts b/src/domain/analysisView/javascriptApplicationView.ts index 60bffb701..fdb0666c4 100644 --- a/src/domain/analysisView/javascriptApplicationView.ts +++ b/src/domain/analysisView/javascriptApplicationView.ts @@ -239,7 +239,7 @@ const selectModule = ( export const projectJavaScriptApplicationView = ( parent: AnalysisViewParent, analysis: JavaScriptApplicationAnalysisResult, - view: AnalysisViewRequest, + view: Exclude, ): Result => { const shared = parentFields(parent, analysis); const modules = moduleEntries(analysis.graph.nodes); diff --git a/src/domain/analysisView/webNetworkApiSurfaceView.test.ts b/src/domain/analysisView/webNetworkApiSurfaceView.test.ts new file mode 100644 index 000000000..f13e392ff --- /dev/null +++ b/src/domain/analysisView/webNetworkApiSurfaceView.test.ts @@ -0,0 +1,460 @@ +import { expect, it } from "vitest"; +import { + analysisViewCaptureFixture, + analysisViewCaptureEvidence, +} from "../../../tests/fixtures/analysisView.js"; +import { projectAnalysisView } from "./analysisView.js"; + +const project = ( + capture = analysisViewCaptureFixture(), + offset = 0, + limit = 50, + parameters: unknown = { sensitive_value_count: 0 }, +) => { + const parent = analysisViewCaptureEvidence(capture); + return projectAnalysisView( + { + evidenceId: parent.evidence_id, + operation: parent.operation, + normalizedResult: parent.normalized_result, + limitations: [], + parameters, + }, + { kind: "api-surface", offset, limit }, + ); +}; +it("groups only exact formats, preserves literal routes, excludes URLs and produces deterministic digests", () => { + const result = project(); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(project()).toEqual(result); + const surface = result.value; + expect(surface.groups.map((group) => group.path_template)).toEqual([ + "/api/CreateDocument", + "/api/ListDocuments", + "/api/Report2", + "/api/Report3", + "/api/v1/users", + "/api/v2/users", + "/dates/2023-02-29", + "/dates/{iso-date}", + "/ids/abc123", + "/ids/{hex}", + "/ids/{uuid}", + "/users/{numeric}", + ]); + expect(surface.groups.at(-1)).toMatchObject({ + record_ordinals: [0, 1], + statuses: [200, 404], + content_types: ["application/json", "text/plain"], + flags: ["mixed-statuses", "mixed-content-types"], + templated_segments: [ + { + segment_index: 1, + rule: "numeric", + basis: "inferred", + observed_values: ["8812", "9031"], + }, + ], + }); + expect(surface.excluded_records).toEqual([ + { ordinal: 13, reason: "url-redacted" }, + { ordinal: 14, reason: "url-unparseable" }, + ]); + expect(JSON.stringify(surface)).not.toContain("not-in-view"); + expect(JSON.stringify(surface)).not.toContain("雪"); +}); +it("pages groups independently of record completeness and preserves null digests", () => { + const capture = analysisViewCaptureFixture(); + capture.artifact.sha256 = null; + capture.total_records = 30; + const result = project(capture, 10, 2); + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + artifact: { sha256: null }, + coverage: { examined: 2, total: 12, exhausted: true, next_offset: null }, + records: { examined: 15, total: 30, excluded: 2 }, + }); + expect(result.value.limitations.join(" ")).toContain("ordinal subset"); + expect(project(capture, 50, 2)).toMatchObject({ + ok: true, + value: { + coverage: { status: "empty", exhausted: true, next_offset: null }, + }, + }); +}); +it("rejects native captures and malformed parents with typed errors", () => { + const capture = analysisViewCaptureFixture(); + capture.format = "mitmproxy"; + expect(project(capture)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisUnsupportedTargetError" }, + }); + expect( + projectAnalysisView( + { + evidenceId: `ev_${"a".repeat(64)}`, + operation: "inspect_web_network_capture", + normalizedResult: {}, + limitations: [], + }, + { kind: "api-surface", offset: 0, limit: 1 }, + ), + ).toMatchObject({ ok: false, error: { _tag: "AnalysisInputError" } }); +}); + +it("preserves partial response facts and explains unavailable requests", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 5); + const values = [ + { + request: { + method: "GET", + url: "https://user:password@api.example.test/1/2?private=value", + }, + response: { status: 200, content: null }, + }, + { request: { method: null, url: "https://api.example.test/" } }, + null, + { request: { method: "GET", url: "data:text/plain,hello" } }, + { + request: { method: "GET", url: "https://api.example.test/1/2" }, + response: { status: null, content: { mimeType: "application/json" } }, + }, + ]; + capture.records.forEach((record, index) => { + record.reported = values[index] ?? null; + }); + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups).toMatchObject([ + { + origin: "https://api.example.test", + path_template: "/{numeric}/{numeric}", + statuses: [200], + content_types: ["application/json"], + templated_segments: [ + { segment_index: 0, observed_values: ["1"] }, + { segment_index: 1, observed_values: ["2"] }, + ], + }, + ]); + expect(result.value.excluded_records).toEqual([ + { ordinal: 1, reason: "method-unavailable" }, + { ordinal: 2, reason: "request-unavailable" }, + { ordinal: 3, reason: "url-unparseable" }, + ]); + expect(result.value.unknowns).not.toEqual([]); + expect(JSON.stringify(result.value)).not.toContain("password"); + expect(JSON.stringify(result.value)).not.toContain("private"); +}); + +it("keeps encoded segments literal and distinguishes origin and method", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 4); + const urls = [ + "https://a.test/%31", + "https://a.test/1", + "https://b.test/1", + "https://a.test/1", + ]; + capture.records.forEach((record, index) => { + record.reported = { + request: { method: index === 3 ? "POST" : "GET", url: urls[index]! }, + response: { + status: 200, + content: { mimeType: "application/json; charset=utf-8" }, + }, + }; + }); + const result = project(capture, 0, 2); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.coverage).toMatchObject({ + total: 4, + next_offset: 2, + exhausted: false, + }); + expect(result.value.groups[0]?.path_template).toBe("/%31"); +}); + +it("refuses parents whose sensitive declarations it cannot check", () => { + for (const parameters of [{ sensitive_value_count: 1 }, {}, null]) { + const result = project(analysisViewCaptureFixture(), 0, 50, parameters); + expect(result).toMatchObject({ + ok: false, + error: { _tag: "AnalysisUnsupportedTargetError" }, + }); + if (result.ok) continue; + expect(result.error).toMatchObject({ + reason: "sensitive_values_unretained", + }); + expect(result.error.message).not.toContain("path templates"); + } +}); + +it("rejects cyclic captures and accessor members without invoking them", () => { + const cyclic: { self?: unknown } = {}; + cyclic.self = cyclic; + let accessed = false; + const accessor = { + get records() { + accessed = true; + return []; + }, + }; + const hiddenAccessor = Object.defineProperty({}, "records", { + get() { + accessed = true; + return []; + }, + }); + const inherited = Object.create({ records: [] }) as unknown; + for (const normalizedResult of [cyclic, accessor, hiddenAccessor, inherited]) + expect( + projectAnalysisView( + { + evidenceId: `ev_${"a".repeat(64)}`, + operation: "inspect_web_network_capture", + normalizedResult, + limitations: [], + }, + { kind: "api-surface", offset: 0, limit: 1 }, + ), + ).toMatchObject({ + ok: false, + error: { _tag: "AnalysisInputError" }, + }); + expect(accessed).toBe(false); +}); + +it("treats HAR missing-response sentinels as unknown", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 1); + capture.records[0]!.reported = { + request: { method: "GET", url: "https://api.example.test/a" }, + response: { status: 0, content: { mimeType: " ; charset=utf-8" } }, + }; + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups[0]).toMatchObject({ + statuses: [], + content_types: [], + }); + expect(result.value.unknowns).not.toEqual([]); +}); + +it("refuses a path whose templated representation exceeds projection capacity", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 1); + capture.records[0]!.reported = { + request: { + method: "GET", + url: `https://api.example.test${"/1".repeat(800_000)}`, + }, + }; + expect(project(capture)).toMatchObject({ + ok: false, + error: { _tag: "AnalysisResourceConstraintError" }, + }); +}); + +it("scans highly segmented literal paths and keeps recorded dot segments", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 3); + const urls = [ + `https://api.example.test${"/a".repeat(90_000)}`, + "https://api.example.test/a/%2e%2e/admin", + "https://api.example.test/admin", + ]; + capture.records.forEach((record, index) => { + record.reported = { request: { method: "GET", url: urls[index] ?? null } }; + }); + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups.map((group) => group.path_template)).toEqual( + expect.arrayContaining(["/a/%2e%2e/admin", "/admin"]), + ); + expect(result.value.groups).toHaveLength(3); +}); + +it("excludes slashless URL spellings instead of reading their paths", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 4); + const urls = [ + "https:user:pass@example.test/path", + "https:123/456", + "https:///user:pass@example.test/path", + "https://\\user:pass@example.test/path", + ]; + capture.records.forEach((record, index) => { + record.reported = { request: { method: "GET", url: urls[index] ?? null } }; + }); + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups).toEqual([]); + expect(result.value.excluded_records).toEqual([ + { ordinal: 0, reason: "url-unparseable" }, + { ordinal: 1, reason: "url-unparseable" }, + { ordinal: 2, reason: "url-unparseable" }, + { ordinal: 3, reason: "url-unparseable" }, + ]); + expect(JSON.stringify(result.value)).not.toContain("pass"); +}); + +it("excludes control-only authorities instead of publishing their userinfo", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 3); + const urls = [ + "https://\t/user:pass@example.test/path", + "https://\n/user:pass@example.test/path", + "https://\r/user:pass@example.test/path", + ]; + capture.records.forEach((record, index) => { + record.reported = { request: { method: "GET", url: urls[index] ?? null } }; + }); + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups).toEqual([]); + expect(result.value.excluded_records).toEqual([ + { ordinal: 0, reason: "url-unparseable" }, + { ordinal: 1, reason: "url-unparseable" }, + { ordinal: 2, reason: "url-unparseable" }, + ]); + expect(JSON.stringify(result.value)).not.toContain("pass"); +}); + +it("keeps distinct content types without repeating a retained essence", () => { + const capture = analysisViewCaptureFixture(); + const record = capture.records[0]!; + const types = ["text/plain", "application/json", "text/plain"]; + capture.records = types.map((mimeType, ordinal) => ({ + ...record, + ordinal, + reported: { + request: { method: "GET", url: "https://api.example.test/a" }, + response: { status: 200, content: { mimeType } }, + }, + })); + capture.total_records = capture.records.length; + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups[0]?.content_types).toEqual([ + "application/json", + "text/plain", + ]); +}); + +it("charges repeated observed values once when merging", () => { + const capture = analysisViewCaptureFixture(); + const record = capture.records[0]!; + const path = "/1".repeat(800); + capture.records = Array.from({ length: 1_000 }, (_, ordinal) => ({ + ...record, + ordinal, + reported: { + request: { method: "GET", url: `https://api.example.test${path}` }, + }, + })); + capture.total_records = capture.records.length; + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups).toHaveLength(1); + expect(result.value.groups[0]?.templated_segments).toHaveLength(800); +}); + +it("keeps a recorded literal placeholder apart from a templated segment", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 2); + const urls = [ + "https://api.example.test/users/{numeric}", + "https://api.example.test/users/123", + ]; + capture.records.forEach((record, index) => { + record.reported = { request: { method: "GET", url: urls[index] ?? null } }; + }); + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups).toHaveLength(2); + expect( + result.value.groups.map((group) => group.templated_segments.length).sort(), + ).toEqual([0, 1]); +}); + +it("treats malformed media types as unknown and keeps source limitations", () => { + const capture = analysisViewCaptureFixture(); + const caveat = + "HAR text is producer-decoded Unicode. Original body bytes remain unknown."; + const captureCaveat = "Producer omitted exchanges before capture startup."; + capture.limitations = [captureCaveat, caveat]; + const malformed = [ + "text /plain", + "unknown", + "\u00a0text/plain\u00a0", + "\u000btext/plain\u000b", + "\u000ctext/plain\u000c", + "text/K", + ]; + capture.records = malformed.map((_, ordinal) => ({ + ...capture.records[0]!, + ordinal, + })); + capture.total_records = capture.records.length; + capture.records.forEach((record, index) => { + record.limitations = [caveat]; + record.reported = { + request: { method: "GET", url: "https://api.example.test/a" }, + response: { + status: 200, + content: { mimeType: malformed[index]! }, + }, + }; + }); + const result = project(capture); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups[0]).toMatchObject({ + content_types: [], + flags: [], + }); + expect(result.value.unknowns).not.toEqual([]); + expect(result.value.limitations).toContain(caveat); + expect(result.value.limitations).toContain(captureCaveat); + expect(result.value.limitations.filter((item) => item === caveat)).toEqual([ + caveat, + ]); +}); + +it("compares media types case-insensitively and counts templated groups across pages", () => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 3); + const reported = [ + ["https://api.example.test/a", "Application/JSON"], + ["https://api.example.test/a", " \tapplication/json\t ; charset=utf-8"], + ["https://api.example.test/users/7", "application/json"], + ]; + capture.records.forEach((record, index) => { + const [url, mimeType] = reported[index] ?? []; + record.reported = { + request: { method: "GET", url: url ?? null }, + response: { status: 200, content: { mimeType: mimeType ?? null } }, + }; + }); + const result = project(capture, 0, 1); + if (!result.ok || result.value.kind !== "api-surface") + throw new Error("Expected surface"); + expect(result.value.groups[0]).toMatchObject({ + path_template: "/a", + content_types: ["application/json"], + flags: [], + }); + expect(result.value.templated_groups).toBe(1); +}); diff --git a/src/domain/analysisView/webNetworkApiSurfaceView.ts b/src/domain/analysisView/webNetworkApiSurfaceView.ts new file mode 100644 index 000000000..a01a92d3f --- /dev/null +++ b/src/domain/analysisView/webNetworkApiSurfaceView.ts @@ -0,0 +1,140 @@ +import { z } from "zod"; +import { AnalysisUnsupportedTargetError } from "../analysisErrorCore.js"; +import type { AnalysisError } from "../analysisErrorBase.js"; +import { err, ok, type Result } from "../result.js"; +import { webNetworkCaptureSchema } from "../webNetworkCapture.js"; +import { + analysisViewInputError, + type AnalysisViewParent, + type AnalysisViewRequest, + type UnsignedAnalysisView, +} from "./analysisView.js"; +import { WebNetworkApiSurfaceAccumulator } from "./WebNetworkApiSurfaceAccumulator.js"; + +export const apiSurfaceViewSchema = z.strictObject({ + kind: z.literal("api-surface"), + offset: z.number().int().nonnegative().default(0), + limit: z.number().int().positive().default(50), +}); +const ruleSchema = z.enum(["numeric", "uuid", "hex", "iso-date"]); +const groupSchema = z.strictObject({ + method: z.string(), + origin: z.string(), + path_template: z.string(), + templated_segments: z.array( + z.strictObject({ + segment_index: z.number().int().nonnegative(), + rule: ruleSchema, + basis: z.literal("inferred"), + observed_values: z.array(z.string()), + }), + ), + record_ordinals: z.array(z.number().int().nonnegative()), + statuses: z.array(z.number()), + content_types: z.array(z.string()), + flags: z.array( + z.enum(["single-sample", "mixed-statuses", "mixed-content-types"]), + ), +}); +export const apiSurfaceFields = { + records: z.strictObject({ + examined: z.number().int().nonnegative(), + total: z.number().int().nonnegative(), + excluded: z.number().int().nonnegative(), + }), + templated_groups: z + .number() + .int() + .nonnegative() + .describe( + "Groups across every page that carry at least one templated segment.", + ), + groups: z.array(groupSchema), + excluded_records: z.array( + z.strictObject({ + ordinal: z.number().int().nonnegative(), + reason: z.enum([ + "url-redacted", + "url-unparseable", + "request-unavailable", + "method-unavailable", + ]), + }), + ), +}; +/** Group retained HAR request lines; format matches do not establish handler identity. */ +export const projectWebNetworkApiSurfaceView = ( + parent: AnalysisViewParent, + view: AnalysisViewRequest, + accumulator = new WebNetworkApiSurfaceAccumulator(), +): Result => { + const unsupported = (reason: string) => + err( + new AnalysisUnsupportedTargetError( + "inspect_analysis_view", + parent.evidenceId, + reason, + ), + ); + if ( + parent.operation !== "inspect_web_network_capture" || + view.kind !== "api-surface" + ) + return unsupported( + `View ${view.kind} is unsupported for parent operation ${parent.operation}; api-surface requires inspect_web_network_capture.`, + ); + const admitted = accumulator.admitCaptureClone(parent.normalizedResult); + if (!admitted.ok) return admitted; + const parsed = webNetworkCaptureSchema.safeParse(parent.normalizedResult); + if (!parsed.success) + return err( + analysisViewInputError([ + { + path: ["source"], + reason: "invalid_value", + message: + "Parent Evidence result does not match inspect_web_network_capture.", + }, + ]), + ); + const capture = parsed.data; + if (capture.format !== "har") + return unsupported( + `api-surface does not support capture format ${capture.format}.`, + ); + const declarations = z + .object({ sensitive_value_count: z.number().int().nonnegative() }) + .safeParse(parent.parameters); + if (!declarations.success || declarations.data.sensitive_value_count > 0) + return unsupported("sensitive_values_unretained"); + const limitations = accumulator.addLimitations([ + ...parent.limitations, + ...capture.limitations, + "Grouping does not establish handler identity, URL-space schemas or causality.", + ...(capture.records.length < capture.total_records + ? [ + "Parent contains an ordinal subset; the surface covers only retained records.", + ] + : []), + ]); + if (!limitations.ok) return limitations; + for (const record of capture.records) { + const added = accumulator.add(record); + if (!added.ok) return added; + } + const page = accumulator.page(view.offset, view.limit); + return ok({ + kind: "api-surface", + view, + parent_evidence_id: parent.evidenceId, + parent_operation: "inspect_web_network_capture", + parent_digest: parent.evidenceId.slice(3), + artifact: { path: capture.artifact.path, sha256: capture.artifact.sha256 }, + records: { + examined: capture.records.length, + total: capture.total_records, + excluded: page.excluded_records.length, + }, + ...page, + }); +}; diff --git a/src/domain/android/androidAnalysis.ts b/src/domain/android/androidAnalysis.ts index a692001d9..e27be2b20 100644 --- a/src/domain/android/androidAnalysis.ts +++ b/src/domain/android/androidAnalysis.ts @@ -121,6 +121,12 @@ export const androidResultSchemas = { query: z.string(), total_classes: z.number().int().nonnegative(), matches: z.array(z.string()), + duplicate_names: z.array( + z.strictObject({ + name: z.string(), + occurrences: z.number().int().min(2), + }), + ), coverage: z.literal("complete"), }), inspect_android_class: z.strictObject({ diff --git a/src/domain/apktool/apktoolResourceAnalysis.ts b/src/domain/apktool/apktoolResourceAnalysis.ts index e4a19634b..df217d040 100644 --- a/src/domain/apktool/apktoolResourceAnalysis.ts +++ b/src/domain/apktool/apktoolResourceAnalysis.ts @@ -2,8 +2,6 @@ import { z } from "zod"; import { localPathStringSchema } from "../localPath.js"; import { digestSchema } from "../digests.js"; -/** Upper bound for one decoded resource projection in one response. */ -const MAX_MANIFEST_BYTES = 512 * 1024; const MAX_STRINGS = 2_000; /** diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index cfb80d91b..f25c8b9b1 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -30,8 +30,6 @@ export const interfaceBuilderAnalysisSchema = z.strictObject({ limitations: z.array(z.string()), }); -type InterfaceBuilderAnalysis = z.infer; - export interface InterfaceBuilderDocumentInput { readonly relativePath: string; readonly archiveSha256: string; diff --git a/src/domain/binaryTarget.fixture.ts b/src/domain/binaryTarget.fixture.ts index bbc1dc5dd..3e2c46147 100644 --- a/src/domain/binaryTarget.fixture.ts +++ b/src/domain/binaryTarget.fixture.ts @@ -22,15 +22,32 @@ export const thinMach = (magic: number, cpu: number): Buffer => { return bytes; }; -/** Build a minimal FAT Mach-O table for binary-target parser tests. */ +/** Build a FAT Mach-O with in-bounds thin slices, or an explicitly truncated table. */ export const fat = ( cpus: readonly number[], declared = cpus.length, ): Buffer => { - const bytes = Buffer.alloc(8 + cpus.length * 20); + const tableSize = 8 + cpus.length * 20; + const slices = cpus.map((cpu) => { + const bytes = thinMach(0xfeedfacf, cpu); + bytes.writeUInt32BE(2, 12); + return bytes; + }); + const sliceStart = Math.ceil(tableSize / 16) * 16; + const bytes = Buffer.alloc( + declared === cpus.length ? sliceStart + cpus.length * 32 : tableSize, + ); bytes.writeUInt32BE(0xcafebabe, 0); bytes.writeUInt32BE(declared, 4); - cpus.forEach((cpu, index) => bytes.writeUInt32BE(cpu, 8 + index * 20)); + cpus.forEach((cpu, index) => { + const entry = 8 + index * 20; + const offset = sliceStart + index * 32; + bytes.writeUInt32BE(cpu, entry); + bytes.writeUInt32BE(offset, entry + 8); + bytes.writeUInt32BE(32, entry + 12); + bytes.writeUInt32BE(4, entry + 16); + if (declared === cpus.length) slices[index]?.copy(bytes, offset); + }); return bytes; }; diff --git a/src/domain/binaryTarget.test.ts b/src/domain/binaryTarget.test.ts index 053232ac0..bf66eaa29 100644 --- a/src/domain/binaryTarget.test.ts +++ b/src/domain/binaryTarget.test.ts @@ -50,20 +50,70 @@ describe("executable header parsing", () => { }); }); + it.each([ + ["Linux x86_64", 0x8664 ^ 0x7b79, "x86_64"], + ["Linux arm64", 0xaa64 ^ 0x7b79, "arm64"], + ["Apple x86_64", 0x8664 ^ 0x4644, "x86_64"], + ["Apple arm64", 0xaa64 ^ 0x4644, "arm64"], + ["FreeBSD x86_64", 0x8664 ^ 0xadc4, "x86_64"], + ["OpenBSD x86_64", 0x8664 ^ 0xadc5, "x86_64"], + ["NetBSD x86_64", 0x8664 ^ 0x1993, "x86_64"], + ["SunOS x86_64", 0x8664 ^ 0x1992, "x86_64"], + ] as const)( + "decodes the %s ReadyToRun OS-override machine on a managed image", + (_label, machine, architecture) => { + const result = parseExecutableHeader( + pe(machine, 64, 0x2002, true), + "x64", + ); + expect(result.ok && result.value).toMatchObject({ + format: "pe", + architecture, + managed: true, + }); + }, + ); + + it("rejects an OS-override machine on a native image", () => { + const result = parseExecutableHeader( + pe(0x8664 ^ 0x7b79, 64, 0x2002), + "x64", + ); + expect(result).toEqual({ + ok: false, + error: "unsupported PE architecture", + }); + }); + it("selects the host architecture from a FAT table", () => { const bytes = fat([0x01000007, 0x0100000c]); const arm = parseExecutableHeader(bytes, "arm64"); const intel = parseExecutableHeader(bytes, "x64"); expect(arm.ok && arm.value).toMatchObject({ + format: "mach-o-universal", architecture: "arm64", availableArchitectures: ["x86_64", "arm64"], }); expect(intel.ok && intel.value).toMatchObject({ + format: "mach-o-universal", architecture: "x86_64", availableArchitectures: ["x86_64", "arm64"], }); }); + it("classifies even a one-slice FAT table as a universal container", () => { + const result = parseExecutableHeader(fat([0x0100000c]), "arm64"); + + expect(result).toMatchObject({ + ok: true, + value: { + format: "mach-o-universal", + architecture: "arm64", + availableArchitectures: ["arm64"], + }, + }); + }); + it("rejects FAT files without a host-compatible slice", () => { expect(parseExecutableHeader(fat([0x01000007]), "arm64")).toMatchObject({ ok: false, @@ -138,4 +188,111 @@ describe("executable header completeness", () => { // Only the slice REA would analyze must be present. expect(parseExecutableHeader(bytes, "x64", 4100).ok).toBe(true); }); + + it.each([ + { + label: "empty range", + corruptRange: (bytes: Buffer, wide: boolean, little: boolean) => + setFatSliceRange(bytes, wide, little, "size", 0n), + error: "FAT slice for arm64 is shorter than its Mach-O header", + }, + { + label: "table overlap", + corruptRange: (bytes: Buffer, wide: boolean, little: boolean) => + setFatSliceRange( + bytes, + wide, + little, + "offset", + BigInt(8 + 2 * (wide ? 32 : 20) - 1), + ), + error: "FAT slice for arm64 overlaps the architecture table", + }, + { + label: "undersized 64-bit header", + corruptRange: (bytes: Buffer, wide: boolean, little: boolean) => + setFatSliceRange(bytes, wide, little, "size", 31n), + error: "FAT slice for arm64 is shorter than its Mach-O header", + }, + ])("rejects a selected FAT slice with $label", ({ corruptRange, error }) => { + for (const [wide, little] of [ + [false, false], + [true, false], + [true, true], + ] as const) { + const bytes = wide + ? fat64([0x01000007, 0x0100000c], little) + : fat([0x01000007, 0x0100000c]); + expect(parseExecutableHeader(bytes, "arm64").ok).toBe(true); + corruptRange(bytes, wide, little); + expect(parseExecutableHeader(bytes, "arm64")).toMatchObject({ + ok: false, + error, + }); + } + }); + + it("uses a 28-byte minimum for a selected 32-bit Mach-O header", () => { + const exactHeader = fat([7]); + setFatSliceRange(exactHeader, false, false, "size", 28n, 0); + expect(parseExecutableHeader(exactHeader, "ia32").ok).toBe(true); + + const undersizedHeader = Buffer.from(exactHeader); + setFatSliceRange(undersizedHeader, false, false, "size", 27n, 0); + + expect(parseExecutableHeader(undersizedHeader, "ia32")).toMatchObject({ + ok: false, + error: "FAT slice for x86 is shorter than its Mach-O header", + }); + }); }); + +const setFatSliceRange = ( + bytes: Buffer, + wide: boolean, + littleEndian: boolean, + field: "offset" | "size", + value: bigint, + index = 1, +): void => { + const entry = 8 + index * (wide ? 32 : 20); + const fieldOffset = entry + (field === "offset" ? 8 : wide ? 16 : 12); + if (wide) { + if (littleEndian) bytes.writeBigUInt64LE(value, fieldOffset); + else bytes.writeBigUInt64BE(value, fieldOffset); + } else { + const numeric = Number(value); + if (littleEndian) bytes.writeUInt32LE(numeric, fieldOffset); + else bytes.writeUInt32BE(numeric, fieldOffset); + } +}; + +const fat64 = (cpus: readonly number[], littleEndian: boolean): Buffer => { + const tableEnd = 8 + cpus.length * 32; + const sliceStart = Math.ceil(tableEnd / 16) * 16; + const bytes = Buffer.alloc(sliceStart + cpus.length * 32); + if (littleEndian) { + bytes.writeUInt32LE(0xcafebabf, 0); + bytes.writeUInt32LE(cpus.length, 4); + } else { + bytes.writeUInt32BE(0xcafebabf, 0); + bytes.writeUInt32BE(cpus.length, 4); + } + cpus.forEach((cpu, index) => { + const entry = 8 + index * 32; + const sliceOffset = BigInt(sliceStart + index * 32); + if (littleEndian) { + bytes.writeUInt32LE(cpu, entry); + bytes.writeBigUInt64LE(sliceOffset, entry + 8); + bytes.writeBigUInt64LE(32n, entry + 16); + bytes.writeUInt32LE(4, entry + 24); + } else { + bytes.writeUInt32BE(cpu, entry); + bytes.writeBigUInt64BE(sliceOffset, entry + 8); + bytes.writeBigUInt64BE(32n, entry + 16); + bytes.writeUInt32BE(4, entry + 24); + } + thinMach(0xfeedfacf, cpu).copy(bytes, Number(sliceOffset)); + }); + return bytes; +}; diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 2d0735b46..66edd2590 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -4,6 +4,7 @@ import type { } from "./binaryTargetTypes.js"; import { err, ok, type Result } from "./result.js"; import { mzWindowsHeaderOffset, parseDosMzHeader } from "./dosMz.js"; +import { decodePeMachine } from "./peMachine.js"; /** Format and architecture facts recovered from an executable header. */ export type ExecutableMetadata = @@ -14,7 +15,12 @@ export type ExecutableMetadata = readonly mips: MipsElfMetadata; } | { - readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com"; + readonly format: + | "mach-o" + | "mach-o-universal" + | "elf" + | "dos-mz" + | "dos-com"; readonly architecture: Exclude; readonly availableArchitectures: readonly Exclude< BinaryArchitecture, @@ -104,7 +110,10 @@ const parseFatMachO = ( if (count === 0 || count > 128 || bytes.length < 8 + count * entrySize) return err("truncated or invalid FAT architecture table"); const architectures: Exclude[] = []; - const sliceEnds = new Map(); + const slices = new Map< + Exclude, + { readonly offset: bigint; readonly size: bigint } + >(); for (let index = 0; index < count; index += 1) { const entry = 8 + index * entrySize; const architecture = machArchitecture(read(entry)); @@ -121,7 +130,7 @@ const parseFatMachO = ( ? bytes.readBigUInt64LE(entry + 16) : bytes.readBigUInt64BE(entry + 16) : BigInt(read(entry + 12)); - sliceEnds.set(architecture, offset + size); + slices.set(architecture, { offset, size }); } const preferred = host === "arm64" @@ -133,15 +142,23 @@ const parseFatMachO = ( : host === "arm" ? "arm" : undefined; - if (preferred === undefined || !architectures.includes(preferred)) + const slice = preferred === undefined ? undefined : slices.get(preferred); + if (preferred === undefined || slice === undefined) return err(`FAT binary has no host-compatible ${host} architecture`); - const sliceEnd = sliceEnds.get(preferred) ?? 0n; + const tableEnd = BigInt(8 + count * entrySize); + if (slice.offset < tableEnd) + return err(`FAT slice for ${preferred} overlaps the architecture table`); + const minimumHeaderSize = + preferred === "arm64" || preferred === "x86_64" ? 32n : 28n; + if (slice.size < minimumHeaderSize) + return err(`FAT slice for ${preferred} is shorter than its Mach-O header`); + const sliceEnd = slice.offset + slice.size; if (sliceEnd > BigInt(fileSize)) return err( `truncated FAT slice: the ${preferred} slice ends at byte ${sliceEnd}; the file has ${fileSize}`, ); return ok({ - format: "mach-o", + format: "mach-o-universal", architecture: preferred, availableArchitectures: architectures, }); @@ -216,14 +233,17 @@ export const parsePeRecord = ( record.toString("binary", 0, 4) !== "PE\u0000\u0000" ) return err("invalid or truncated PE header"); - const architecture = peArchitecture(record.readUInt16LE(4)); - if (architecture === undefined) return err("unsupported PE architecture"); + const machine = decodePeMachine(record.readUInt16LE(4)); + if (machine === undefined) return err("unsupported PE architecture"); + const { architecture } = machine; const optionalHeaderSize = record.readUInt16LE(20); if (optionalHeaderSize > 4096 || record.length < 24 + optionalHeaderSize) return err("invalid or truncated PE optional header"); const optionalHeader = record.subarray(24, 24 + optionalHeaderSize); const managed = peManagedStatus(optionalHeader, architecture); if (!managed.ok) return managed; + if (machine.nativeOsOverride && !managed.value) + return err("unsupported PE architecture"); const characteristics = record.readUInt16LE(22); return ok({ format: "pe", @@ -299,21 +319,3 @@ const elfArchitecture = (machine: number): BinaryArchitecture | undefined => { } return undefined; }; - -const peArchitecture = ( - machine: number, -): Exclude | undefined => { - switch (machine) { - case 0x14c: - return "x86"; - case 0x8664: - return "x86_64"; - case 0x1c0: - case 0x1c2: - case 0x1c4: - return "arm"; - case 0xaa64: - return "arm64"; - } - return undefined; -}; diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index 5d7deba68..3bd79144f 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -18,6 +18,15 @@ export interface MipsElfMetadata { readonly abiFlags?: MipsAbiFlags | null; /** Unavailable declaration sources; missing records there remain unknown. */ readonly limitations?: readonly string[]; + /** Complete dynamic REL declarations, or the reason they could not be inspected. */ + readonly sharedLoad?: + | { + readonly status: "complete"; + readonly relocationCount: number; + readonly relocationTypes: readonly number[]; + readonly symbolRelativeCount: number; + } + | { readonly status: "unavailable"; readonly reason: string }; } /** Declared fields of the 24-byte MIPS ABI flags record, not inferred semantics. */ @@ -74,7 +83,12 @@ export type BinaryTarget = readonly managed?: never; }) | (ExecutableTarget & { - readonly format: "mach-o" | "elf" | "dos-mz" | "dos-com"; + readonly format: + | "mach-o" + | "mach-o-universal" + | "elf" + | "dos-mz" + | "dos-com"; readonly executableRole?: never; readonly managed?: never; }) @@ -121,3 +135,9 @@ export type BinaryTarget = readonly kind: "artifact"; readonly format: "plist" | "javascript" | "source-map"; }); + +/** Whether an executable target contains a thin or universal Mach-O image. */ +export const isMachOFormat = ( + format: unknown, +): format is "mach-o" | "mach-o-universal" => + format === "mach-o" || format === "mach-o-universal"; diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index d10995525..e0f343fca 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -1,14 +1,10 @@ import { describe, expect, it } from "vitest"; -import { enhancedInputSchemas } from "../contracts/enhancedInputs.js"; import { buildCallPath } from "./callPath.js"; -import { - callPathInputSchema, - callPathResultSchema, -} from "./callPathSchemas.js"; +import { callPathResultSchema, type CallPathInput } from "./callPathSchemas.js"; import { createEvidence, type Evidence } from "./evidence.js"; -import { functionDossierSchema } from "./hopperValues.js"; -import { jsonValueSchema } from "./jsonValue.js"; +import type { JsonValue } from "./jsonValue.js"; +import type { RelationshipCoverage } from "./relationshipCoverage.js"; const observe = ( address: string, @@ -17,9 +13,10 @@ const observe = ( options: { readonly evidenceLimitations?: readonly string[]; readonly dossierLimitations?: readonly string[]; + readonly relationshipCoverage?: RelationshipCoverage; } = {}, ): Evidence => { - const value = functionDossierSchema.parse({ + const value = { procedure: { address, name: `fn_${address.slice(2)}`, @@ -54,34 +51,36 @@ const observe = ( native_value_flow: null, basic_blocks: [], limitations: [...(options.dossierLimitations ?? [])], - }); + } satisfies JsonValue; return createEvidence( { path: "/tmp/a", sha256: "a".repeat(64), format: "mach-o" }, { id: provider, name: provider, version: "1" }, { operation: "analyze_function", - parameters: enhancedInputSchemas.analyze_function.parse({ - procedure: address, - }), - result: jsonValueSchema.parse(value), + parameters: { procedure: address }, + result: value, confidence: "derived", authority: "shipped-artifact", ...(options.evidenceLimitations === undefined ? {} : { limitations: options.evidenceLimitations }), + ...(options.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: options.relationshipCoverage }), }, ); }; -const run = (functions: Evidence[], overrides: Record = {}) => - buildCallPath( - callPathInputSchema.parse({ - functions, - start: { address: "0x1000" }, - goal: { address: "0x4000" }, - ...overrides, - }), - ); +const run = ( + functions: Evidence[], + overrides: Partial> = {}, +) => + buildCallPath({ + functions, + start: { address: "0x1000" }, + goal: { address: "0x4000" }, + ...overrides, + }); describe("call path reconstruction", () => { it("returns deterministic shortest-first cited paths", () => { @@ -171,6 +170,21 @@ describe("call path reconstruction", () => { }); describe("call path topology and capacity", () => { + it("retains every distinct generic limitation beyond the push argument limit", () => { + const limitations = Array.from( + { length: 150_000 }, + (_, index) => `context-${index.toString().padStart(7, "0")}`, + ); + const result = run([ + observe("0x1000", [], "rea-workflow", { + evidenceLimitations: [...limitations].reverse(), + }), + ]); + expect(result.status).toBe("unknown"); + expect(result.search_scope.exhaustive).toBe(false); + expect(result.limitations).toEqual(limitations); + }, 30_000); + it("retains every shortest path through merging branches and cycles", () => { const result = run([ observe("0x1000", ["0x3000", "0x2000", "0x1000"]), @@ -314,6 +328,93 @@ describe("call path topology and capacity", () => { }); }); +describe("explicit call relationship coverage", () => { + it("accumulates 150,000 partial reasons without argument-stack exhaustion", () => { + const result = run([ + observe("0x1000", [], "rea-workflow", { + relationshipCoverage: { + status: "partial", + limitations: Array(150_000).fill("omitted"), + }, + evidenceLimitations: [], + dossierLimitations: [], + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + limitations: ["omitted"], + }); + }); + + it("retains contextual limits on a found path without reducing complete coverage", () => { + const limitation = "Provider results reflect the current database state."; + const result = run([ + observe("0x1000", ["0x4000"], "rea-workflow", { + relationshipCoverage: { status: "complete" }, + evidenceLimitations: [limitation], + }), + ]); + expect(result).toMatchObject({ + status: "found", + shortest_hops: 1, + search_scope: { exhaustive: true }, + }); + expect(result.limitations).toContain(limitation); + }); + + it("retains a found path while its expanded relationship coverage remains partial", () => { + const limitation = "Other callees were omitted by the provider cap."; + const result = run([ + observe("0x1000", ["0x4000"], "rea-workflow", { + relationshipCoverage: { status: "partial", limitations: [limitation] }, + }), + ]); + expect(result).toMatchObject({ + status: "found", + shortest_hops: 1, + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(limitation); + }); + + it("retains an unknown goal coverage reason without expanding the terminal goal", () => { + const reason = "The goal's outgoing relationship coverage is unknown."; + const result = run([ + observe("0x1000", ["0x4000"], "rea-workflow", { + relationshipCoverage: { status: "complete" }, + }), + observe("0x4000", ["0x5000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "found", + search_scope: { exhaustive: true }, + explored: { nodes: 2 }, + }); + expect(result.limitations).toEqual([reason]); + }); + + it("ignores incomplete relationship coverage outside the reachable search closure", () => { + const reason = "An unrelated function's callees were not observed."; + const result = run([ + observe("0x1000", [], "rea-workflow", { + relationshipCoverage: { status: "complete" }, + }), + observe("0x2000", ["0x5000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "not_found", + search_scope: { exhaustive: true }, + explored: { nodes: 1 }, + }); + expect(result.limitations).toEqual([]); + }); +}); + describe("call path evidence admission", () => { it("normalizes hex addresses and rejects duplicates and mixed providers", () => { expect( diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 1da429625..2186c8010 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -236,12 +236,31 @@ const inspectSearch = ({ const depth = reached.get(node) ?? 0; const snapshot = snapshots.get(node); if (snapshot !== undefined) { - limitations.push(...snapshot.limitations); + const coverage = snapshot.evidence.relationship_coverage; + for (const limitation of snapshot.limitations) + limitations.push(limitation); + if (coverage?.status === "partial") + for (const limitation of coverage.limitations) + limitations.push(limitation); + else if (coverage?.status === "unknown") + limitations.push(coverage.reason); + for (const diagnostic of coverage !== null && "diagnostics" in coverage + ? coverage.diagnostics + : []) + limitations.push(diagnostic); // The requested goal is a terminal for this search. Its outgoing edges // cannot add a path to the goal, so its dossier limits are contextual // but do not make the searched start-to-goal closure incomplete. if (node === goal) continue; - if (snapshot.limitations.length > 0) incompleteExpandedCoverage = true; + // Explicit, Evidence-bound relationship coverage owns completeness; + // generic provider/dossier limitations remain contextual. Legacy Evidence + // without that metadata retains its conservative limitation-based rule. + if ( + coverage === null + ? snapshot.limitations.length > 0 + : coverage.status !== "complete" + ) + incompleteExpandedCoverage = true; } else if (node === goal) { continue; } diff --git a/src/domain/configurationErrors.ts b/src/domain/configurationErrors.ts index 88078d0be..686e5c42f 100644 --- a/src/domain/configurationErrors.ts +++ b/src/domain/configurationErrors.ts @@ -38,7 +38,11 @@ export class NoBinaryOpenError extends AnalysisError { /** Additional constraints identified while admitting a filesystem target. */ export interface BinaryTargetErrorOptions extends AnalysisErrorOptions { - readonly constraint?: "directory_requires_file"; + readonly constraint?: + | "directory_requires_file" + | "wasm_requires_static_inspection" + | "file_format_rejected" + | "load_selection_rejected"; } /** A supplied target path could not be safely opened as a supported app or binary. */ diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index fc2f19921..c5898aecc 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -19,6 +19,39 @@ const TARGET: BinaryTarget = { const PROVIDER = { id: "fixture", name: "Fixture provider", version: "1" }; const PROFILE = createAnalysisProfile(PROVIDER, { loader: "default" }); +it("preserves omitted relationship diagnostics and authenticates reported diagnostics", () => { + const observation = { + operation: "procedure_callees", + parameters: { procedure: "0x1000" }, + result: ["0x2000"], + }; + const historical = createEvidence(TARGET, PROVIDER, { + ...observation, + relationshipCoverage: { status: "complete" }, + }); + expect(historical.relationship_coverage).toEqual({ status: "complete" }); + expect(parseEvidence(JSON.parse(JSON.stringify(historical)))).toEqual( + historical, + ); + const diagnosed = createEvidence(TARGET, PROVIDER, { + ...observation, + relationshipCoverage: { + status: "complete", + diagnostics: ["Only resolved direct call references are observed."], + }, + }); + expect(parseEvidence(JSON.parse(JSON.stringify(diagnosed)))).toEqual( + diagnosed, + ); + expect(diagnosed.evidence_id).not.toBe(historical.evidence_id); + expect(() => + parseEvidence({ + ...historical, + relationship_coverage: diagnosed.relationship_coverage, + }), + ).toThrow("semantic identifier"); +}); + it("preserves identity and bytes when reusing an owned immutable result", () => { const value = { nested: { values: ["observed", 1] } }; const mutable = createEvidence(TARGET, PROVIDER, { diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index dcfee98fd..8e21686e5 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -27,6 +27,10 @@ import { } from "./immutableJson.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; +import { + relationshipCoverageSchema, + type RelationshipCoverage, +} from "./relationshipCoverage.js"; /** Provider identity schema shared by evidence-bearing persistence formats. */ export const providerSchema = z.object({ @@ -118,6 +122,7 @@ const evidenceBaseSchema = z authority: evidenceAuthoritySchema, environment: executionEnvironmentSchema.nullable(), limitations: z.array(z.string()), + relationship_coverage: relationshipCoverageSchema.nullable().default(null), locations: z.array(evidenceLocationSchema), evidence_links: z.array(prefixedDigestSchema("ev")), }) @@ -163,9 +168,11 @@ const authenticatedEvidenceSchema = evidenceSchema.superRefine( ); /** Complete observation whose normalized payload retains its operation-specific type. */ -export type Evidence = z.infer< - typeof evidenceSchema -> & { readonly normalized_result: Result }; +type EvidenceSchemaValue = z.infer; +export type Evidence = + EvidenceSchemaValue & { + readonly normalized_result: Result; + }; const immutableEvidenceSnapshots = new WeakMap(); const immutableResultSchema = z .custom(isImmutableJsonSnapshot) @@ -215,13 +222,14 @@ export interface EvidenceObservation { readonly authority?: EvidenceAuthority; readonly environment?: ExecutionEnvironment | null; readonly limitations?: readonly string[]; + readonly relationshipCoverage?: RelationshipCoverage; /** Reason the subject identity is unavailable, when no target identity is supplied. */ readonly subjectUnavailableReason?: string; readonly locations?: readonly EvidenceLocation[]; readonly evidenceLinks?: readonly string[]; } -type EvidenceWithoutId = Omit; +type EvidenceWithoutId = Omit; const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({ subject: @@ -243,6 +251,9 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({ authority: evidence.authority, environment: evidence.environment, limitations: evidence.limitations, + ...(evidence.relationship_coverage === null + ? {} + : { relationship_coverage: evidence.relationship_coverage }), locations: evidence.locations, evidence_links: evidence.evidence_links, }); @@ -404,6 +415,9 @@ const normalizeEvidenceObservation = ( : []), ...(observation.limitations ?? []), ], + ...(observation.relationshipCoverage === undefined + ? { relationship_coverage: null } + : { relationship_coverage: observation.relationshipCoverage }), locations: [...(observation.locations ?? [])], evidence_links: [...(observation.evidenceLinks ?? [])], } satisfies JsonValue; diff --git a/src/domain/evidenceErrors.ts b/src/domain/evidenceErrors.ts index 37fe1127c..364cbe79d 100644 --- a/src/domain/evidenceErrors.ts +++ b/src/domain/evidenceErrors.ts @@ -3,6 +3,12 @@ import { type AnalysisErrorOptions, } from "./analysisErrorBase.js"; +/** Receipt retained when publication succeeded but staging cleanup did not. */ +export interface PublishedEvidenceFile { + readonly path: string; + readonly bytes: number; +} + /** Evidence identity, schema, or bundle manifests failed integrity checks. */ export class EvidenceIntegrityError extends AnalysisError { readonly _tag = "EvidenceIntegrityError"; @@ -42,13 +48,19 @@ export class EvidenceFileError extends AnalysisError { /** Caller-selected path that failed, resolved against the working directory. */ readonly path: string | undefined; + /** Present only when a write committed before cleanup failed. */ + readonly published: PublishedEvidenceFile | undefined; constructor( readonly operation: "read" | "write", readonly reason: "not-file" | "exists" | "invalid-json" | "missing" | "io", - options?: ErrorOptions & { readonly path?: string }, + options?: AnalysisErrorOptions & { + readonly path?: string; + readonly published?: PublishedEvidenceFile; + }, ) { super(`Evidence bundle ${operation} failed: ${reason}`, options); this.path = options?.path; + this.published = options?.published; } } diff --git a/src/domain/evm/evmInterface.ts b/src/domain/evm/evmInterface.ts index 75ada1f97..6db3f7e16 100644 --- a/src/domain/evm/evmInterface.ts +++ b/src/domain/evm/evmInterface.ts @@ -1,14 +1,15 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; + const byteOffset = z.number().int().nonnegative(); const sha256 = z.string().regex(/^[a-f0-9]{64}$/); /** Select the carrier representation explicitly; no RPC or deployed-contract lookup is implied. */ export const inspectEvmInterfaceInputSchema = z.strictObject({ - path: z - .string() - .min(1) - .describe("Absolute path to caller-selected local bytecode carrier"), + path: localPathStringSchema.describe( + "Absolute path to caller-selected local bytecode carrier", + ), encoding: z .enum(["raw", "hex"]) .describe( diff --git a/src/domain/javascript/electronStaticAnalysisIpc.ts b/src/domain/javascript/electronStaticAnalysisIpc.ts index 0b43f0727..5cebf0cc9 100644 --- a/src/domain/javascript/electronStaticAnalysisIpc.ts +++ b/src/domain/javascript/electronStaticAnalysisIpc.ts @@ -175,7 +175,7 @@ const addValidation = ( }); }; -const ipcDescriptor = ( +export const ipcDescriptor = ( path: readonly string[] | undefined, ): ElectronIpcDescriptor | undefined => { if (path?.length !== 2) return undefined; @@ -201,6 +201,7 @@ const ipcDescriptor = ( operation: "send-to-host", mode: "send", }; + case "addListener": case "on": return { side: "renderer", operation: "on", mode: "listen" }; case "once": @@ -209,6 +210,7 @@ const ipcDescriptor = ( } const main = object === "ipcMain" ? operation : undefined; switch (main) { + case "addListener": case "on": return { side: "main", operation: "on", mode: "listen" }; case "once": diff --git a/src/domain/javascript/electronStaticAnalysisTypes.ts b/src/domain/javascript/electronStaticAnalysisTypes.ts index 6a4bbf0bf..b8d2c03fb 100644 --- a/src/domain/javascript/electronStaticAnalysisTypes.ts +++ b/src/domain/javascript/electronStaticAnalysisTypes.ts @@ -52,6 +52,7 @@ export type ElectronContextBridgeApiKey = /** One contextBridge API declaration without executing its API object. */ export type ElectronContextBridgeFinding = ElectronContextBridgeApiKey & { + readonly resolution_basis?: "wrapper-substitution"; readonly world: "main" | "isolated"; readonly world_id: ElectronStaticValue | null; readonly api_status: "object-literal" | "dynamic" | "missing"; @@ -112,10 +113,14 @@ type ElectronIpcHandler = }; interface ElectronIpcFindingContext { + /** Absent on direct syntax observations; wrappers require inferred evidence. */ + readonly resolution_basis?: "wrapper-substitution"; readonly module_key: string | null; /** Recovered factory runtime; absent in older or unbundled findings. */ readonly module_runtime?: string; readonly location: JavaScriptSourceRange; + /** Outer syntax invocation when a wrapper produced this underlying IPC call. */ + readonly invocation_location?: JavaScriptSourceRange; } /** One statically visible Electron IPC send, invocation, listener, or handler. */ @@ -192,4 +197,34 @@ export interface ElectronStaticFindings { readonly sender_validations: readonly ElectronSenderValidationFinding[]; readonly utility_processes: readonly ElectronUtilityProcessFinding[]; readonly native_addon_bindings: readonly ElectronNativeAddonBindingFinding[]; + /** Source-bound candidate paths through bundled exports and IPC closures. */ + readonly call_paths?: readonly ElectronCallPath[]; +} + +export interface ElectronCallPathStep { + readonly kind: + | "exposure" + | "factory" + | "function" + | "call" + | "ipc" + | "terminal"; + readonly label: string; + readonly location: JavaScriptSourceRange; + readonly module_key: string | null; + readonly module_runtime: string | null; +} + +/** A statically substituted candidate, never a runtime reachability claim. */ +export interface ElectronCallPath { + readonly entry_kind: "bridge-member" | "ipc-handler"; + readonly entry: string; + readonly channel: string | null; + readonly terminal: string; + readonly resolution: "candidate"; + readonly steps: readonly ElectronCallPathStep[]; + readonly conditions: readonly { + readonly expression: string; + readonly location: JavaScriptSourceRange; + }[]; } diff --git a/src/domain/javascript/electronWrappedCallPaths.test.ts b/src/domain/javascript/electronWrappedCallPaths.test.ts new file mode 100644 index 000000000..b39ac7b69 --- /dev/null +++ b/src/domain/javascript/electronWrappedCallPaths.test.ts @@ -0,0 +1,552 @@ +import { describe, expect, it } from "vitest"; +import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; + +const bundle = (entry: string) => `(() => { + const modules = { + 2(module) { module.exports = require("electron"); }, + 5(module,exports,load) { + const electron=load(2); + Object.defineProperty(exports,"__esModule",{value:true}); + exports.sender = function(channel) {return (...args)=>send(channel,...args)}; + function send(channel,...args) {electron.ipcRenderer.send(channel,...args)} + exports.events = {addListener(channel,handler) { + const guarded = (event,...args) => {if (allowed(event)) handler(event,...args)}; + return electron.ipcMain.addListener(channel,guarded); + }}; + }, + 9(module,exports,load) {const electron=load(2), helper=load(5); ${entry}} + }; + function load(id) {const m={exports:{}};modules[id].call(m.exports,m,m.exports,load);return m.exports;} + load(9); +})();`; + +describe("wrapped Electron control-flow alternatives", () => { + it.each(["flag ? copy : cut", "flag ? copy : unknownHandler", "choose()"])( + "does not select one IPC handler from %s", + (handler) => { + const result = analyzeJavaScriptStaticSource(` + const e = require("electron"); + const copy = event => event.sender.copy(); + const cut = event => event.sender.cut(); + function choose() { if (flag) return copy; } + function register(channel, handler) { e.ipcMain.on(channel, handler); } + register("test:copy", ${handler}); + `); + expect(result.electron.ipc).toContainEqual( + expect.objectContaining({ + channel: "test:copy", + side: "main", + handler_kind: "dynamic-expression", + resolution_basis: "wrapper-substitution", + }), + ); + expect(result.electron.call_paths ?? []).toEqual([]); + expect(result.parse_status).toBe("partial"); + }, + ); + it.each(["'ready'", "false", "0", "''", "({})", "(()=>{})"])( + "short-circuits nullish fallback for %s in callbacks and seed calls", + (left) => { + for (const source of [ + `e.ipcMain.on("test:copy", event => { ${left} ?? event.sender.copy(); });`, + `${left} ?? e.ipcMain.on("test:copy", event => event.sender.copy());`, + `e.ipcMain.on("test:copy", event => { const selected = ${left} ?? (()=>event.sender.copy()); selected(); });`, + ]) + expect( + analyzeJavaScriptStaticSource(`const e=require("electron");${source}`) + .electron.call_paths ?? [], + ).toEqual([]); + }, + ); + it.each(["null", "undefined", "void 0"])( + "follows nullish fallback for %s", + (left) => { + const result = analyzeJavaScriptStaticSource(`const e=require("electron"); + ${left} ?? e.ipcMain.on("test:copy", event => { ${left} ?? event.sender.copy(); });`); + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ + terminal: "ipc-event.sender.copy", + conditions: [], + }), + ); + }, + ); + it("retains a nullish guard for unknown fallback rather than a falsy guard", () => { + for (const source of [ + `e.ipcMain.on("test:copy", event => { value ?? event.sender.copy(); });`, + `value ?? e.ipcMain.on("test:copy", event => event.sender.copy());`, + ]) { + const result = analyzeJavaScriptStaticSource( + `const e=require("electron");${source}`, + ); + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ + terminal: "ipc-event.sender.copy", + conditions: [ + expect.objectContaining({ + expression: "(value) === null || (value) === undefined", + }), + ], + }), + ); + } + }); + it.each([ + "if (flag) return copy; else return cut; return dead;", + "if (flag) { return copy; } return cut;", + "if (flag) { if (other) return copy; else return cut; } else return cut; return dead;", + ])("retains every possible factory return: %s", (body) => { + const result = analyzeJavaScriptStaticSource(`const e=require("electron"); + const copy=()=>e.ipcRenderer.send("test:copy"), cut=()=>e.ipcRenderer.send("test:cut"), dead=()=>e.ipcRenderer.send("test:dead"); + function choose(){${body}} + e.contextBridge.exposeInMainWorld("desktop", {copy:choose()});`); + expect( + [ + ...new Set(result.electron.call_paths?.map((path) => path.channel)), + ].sort(), + ).toEqual(["test:copy", "test:cut"]); + const paths = result.electron.call_paths ?? []; + expect( + paths + .find(({ channel }) => channel === "test:copy") + ?.conditions.map(({ expression }) => expression), + ).toContain("flag"); + expect( + paths + .filter(({ channel }) => channel === "test:cut") + .some((path) => + path.conditions.some(({ expression }) => expression === "!(flag)"), + ), + ).toBe(true); + }); +}); + +it("preserves an observed empty contextBridge API object", () => { + const result = analyzeJavaScriptStaticSource(` + const electron = require("electron"); + electron.contextBridge.exposeInMainWorld("desktop", {}); + `); + expect(result.electron.context_bridge_apis).toContainEqual( + expect.objectContaining({ + api_key: "desktop", + api_status: "object-literal", + members: [], + unknown_members: 0, + }), + ); + expect(result.electron.context_bridge_apis?.[0]).not.toHaveProperty( + "resolution_basis", + ); + expect(result.parse_status).toBe("complete"); +}); + +it("connects exposed closures to IPC sends and text clipboard calls", () => { + const result = analyzeJavaScriptStaticSource( + bundle(` + const text = value => navigator.clipboard.writeText(value); + const api={copy:helper.sender("example:copy"),copyText:text}; + electron.contextBridge.exposeInMainWorld("desktop",api); + `), + ); + expect(result.electron.context_bridge_apis).toMatchObject([ + { api_key: "desktop", members: ["copy", "copyText"] }, + ]); + expect(result.electron.ipc).toContainEqual( + expect.objectContaining({ + side: "renderer", + channel: "example:copy", + operation: "send", + }), + ); + expect(result.electron.call_paths).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entry: "desktop.copy", + channel: "example:copy", + terminal: "electron.ipcRenderer.send", + }), + expect.objectContaining({ + entry: "desktop.copyText", + terminal: "navigator.clipboard.writeText", + }), + ]), + ); + const ipcFinding = result.electron.ipc.find( + ({ channel }) => channel === "example:copy", + ); + const ipcPath = result.electron.call_paths?.find( + ({ entry }) => entry === "desktop.copy", + ); + expect(ipcFinding).toBeDefined(); + expect(ipcPath?.steps.at(-1)).toBeDefined(); + expect(ipcFinding?.location).toEqual(ipcPath?.steps.at(-1)?.location); +}); +it("resolves a helper factory listed after its bundle consumer", () => { + const result = analyzeJavaScriptStaticSource(`(() => { + const modules = { + 2(module) { module.exports = require("electron"); }, + 9(module,exports,load) { + const electron=load(2),helper=load(5); + electron.contextBridge.exposeInMainWorld("desktop", {copy:helper.sender("after:copy")}); + }, + 5(module,exports,load) { + const electron=load(2); + exports.sender=channel=>()=>electron.ipcRenderer.send(channel); + } + }; + function load(id) { const m={exports:{}}; modules[id].call(m.exports,m,m.exports,load); return m.exports; } + load(9); + })();`); + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ + entry: "desktop.copy", + channel: "after:copy", + terminal: "electron.ipcRenderer.send", + }), + ); +}); +it("substitutes guarded listener callbacks and preserves guard evidence", () => { + const result = analyzeJavaScriptStaticSource( + bundle( + `helper.events.addListener("example:copy",event=>event.sender.copy());`, + ), + ); + expect(result.electron.ipc).toContainEqual( + expect.objectContaining({ + side: "main", + channel: "example:copy", + operation: "on", + }), + ); + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ + entry_kind: "ipc-handler", + channel: "example:copy", + terminal: "ipc-event.sender.copy", + conditions: [expect.objectContaining({ expression: "allowed(event)" })], + }), + ); + const ipcFinding = result.electron.ipc.find( + ({ channel, operation }) => + channel === "example:copy" && operation === "on", + ); + const ipcStep = result.electron.call_paths + ?.find(({ entry_kind }) => entry_kind === "ipc-handler") + ?.steps.find(({ kind }) => kind === "ipc"); + expect(ipcFinding).toBeDefined(); + expect(ipcStep).toBeDefined(); + expect(ipcFinding?.location).toEqual(ipcStep?.location); +}); +it("retains distinct wrapper invocation ranges for one IPC API source", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + const copy=event=>event.sender.copy(); + function register(channel,handler){e.ipcMain.on(channel,handler);} + register("same:channel",copy); + register("same:channel",copy); + `); + const findings = result.electron.ipc.filter( + ({ resolution_basis, channel }) => + resolution_basis === "wrapper-substitution" && channel === "same:channel", + ); + expect(findings).toHaveLength(2); + expect(findings[0]?.location).toEqual(findings[1]?.location); + expect(findings[0]?.invocation_location).toBeDefined(); + expect(findings[1]?.invocation_location).toBeDefined(); + expect(findings[0]?.invocation_location).not.toEqual( + findings[1]?.invocation_location, + ); +}); +it("keeps a directly observed renderer operation singular when tracing its exposed callback", () => { + const result = analyzeJavaScriptStaticSource(` + const {contextBridge,ipcRenderer}=require("electron"); + contextBridge.exposeInMainWorld("desktop",{ping:()=>ipcRenderer.invoke("ping")}); + `); + expect(result.electron.ipc).toHaveLength(1); + expect(result.electron.ipc[0]).toMatchObject({ + channel: "ping", + operation: "invoke", + }); + expect(result.electron.ipc[0]).not.toHaveProperty("resolution_basis"); + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ entry: "desktop.ping", channel: "ping" }), + ); +}); +it("does not identify overwritten Electron namespaces or shadowed require", () => { + for (const source of [ + `const e=require("electron");e.ipcMain={on(){}};e.ipcMain.on("bad",event=>event.sender.copy());`, + `function require(){return {ipcMain:{on(){}}}};const e=require("electron");e.ipcMain.on("bad",event=>event.sender.copy());`, + ]) + expect( + analyzeJavaScriptStaticSource(source).electron.call_paths ?? [], + ).toEqual([]); +}); +it("preserves guarded alternatives in direct exposed object literals", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + const copy=()=>e.ipcRenderer.send("copy"); + const cut=()=>e.ipcRenderer.send("cut"); + e.contextBridge.exposeInMainWorld("desktop", {action: flag ? copy : cut}); + `); + expect(result.electron.call_paths).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entry: "desktop.action", + channel: "copy", + conditions: [expect.objectContaining({ expression: "flag" })], + }), + expect.objectContaining({ + entry: "desktop.action", + channel: "cut", + conditions: [expect.objectContaining({ expression: "!(flag)" })], + }), + ]), + ); +}); +it("preserves conditions when a wrapper dispatches through conditional external methods", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + const send=flag ? e.ipcRenderer.send : e.ipcRenderer.invoke; + e.contextBridge.exposeInMainWorld("desktop", {action:()=>send("channel")}); + `); + expect(result.electron.call_paths).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + entry: "desktop.action", + channel: "channel", + terminal: "electron.ipcRenderer.send", + conditions: [expect.objectContaining({ expression: "flag" })], + }), + expect.objectContaining({ + entry: "desktop.action", + channel: "channel", + terminal: "electron.ipcRenderer.invoke", + conditions: [expect.objectContaining({ expression: "!(flag)" })], + }), + ]), + ); + expect( + result.electron.ipc.filter( + ({ resolution_basis, channel }) => + resolution_basis === "wrapper-substitution" && channel === "channel", + ), + ).toEqual([]); + expect(result.parse_status).toBe("partial"); +}); +it("keeps conditional IPC channels unknown instead of reporting them unconditionally", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + const channel=flag ? "copy" : "cut"; + e.contextBridge.exposeInMainWorld("desktop", {action:()=>e.ipcRenderer.send(channel)}); + `); + expect(result.electron.call_paths ?? []).toEqual([]); + expect(result.parse_status).toBe("partial"); +}); +it("retains enclosing branch predicates without emitting unconditional wrapper findings", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + function expose(key,api){e.contextBridge.exposeInMainWorld(key,api);} + function register(channel,handler){e.ipcMain.on(channel,handler);} + if(flag){ + expose("conditional",{copy:()=>navigator.clipboard.writeText("copy")}); + register("only:flag",event=>event.sender.copy()); + } + `); + expect(result.electron.context_bridge_apis).not.toContainEqual( + expect.objectContaining({ + api_key: "conditional", + resolution_basis: "wrapper-substitution", + }), + ); + expect(result.electron.ipc).not.toContainEqual( + expect.objectContaining({ + channel: "only:flag", + resolution_basis: "wrapper-substitution", + }), + ); + for (const entry of ["conditional.copy", "only:flag"]) + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ + entry, + conditions: [expect.objectContaining({ expression: "flag" })], + }), + ); + expect(result.parse_status).toBe("partial"); +}); +it("preserves the raw bridge observation for a guarded object value", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + const api=flag ? {copy:()=>e.ipcRenderer.send("copy")} : {}; + e.contextBridge.exposeInMainWorld("desktop", api); + `); + expect(result.electron.context_bridge_apis).toContainEqual( + expect.objectContaining({ + api_key: "desktop", + api_status: "dynamic", + }), + ); + expect(result.parse_status).toBe("partial"); +}); +it("preserves the source expression for a guarded bridge world key", () => { + const result = analyzeJavaScriptStaticSource(` + const e=require("electron"); + const world=flag ? "desktop" : "shell"; + e.contextBridge.exposeInMainWorld(world, {copy:()=>e.ipcRenderer.send("copy")}); + `); + expect(result.electron.context_bridge_apis).toContainEqual( + expect.objectContaining({ api_key: null, api_key_expression: "world" }), + ); + expect(result.parse_status).toBe("partial"); +}); +it.each(["const", "var"])( + "does not use a %s callback initializer before exposure", + (declaration) => { + const result = analyzeJavaScriptStaticSource( + bundle(` + const electron=load(2); + electron.contextBridge.exposeInMainWorld("desktop", {copy: callback}); + ${declaration} callback=()=>electron.ipcRenderer.send("late"); + `), + ); + expect(result.electron.call_paths ?? []).toEqual([]); + expect(result.parse_status).toBe("partial"); + }, +); +it("does not use an original Electron module factory after its slot is replaced", () => { + const result = analyzeJavaScriptStaticSource( + bundle(` + modules[2] = function(module) { + module.exports = {contextBridge:{exposeInMainWorld(){}},ipcRenderer:{send(){}}}; + }; + const electron=load(2); + electron.contextBridge.exposeInMainWorld("desktop", {copy:()=>electron.ipcRenderer.send("fake")}); + `), + ); + expect(result.electron.call_paths ?? []).toEqual([]); +}); +it.each(["({})", "load(2)"])( + "does not emit the impossible logical-RHS terminal after %s", + (left) => { + const result = analyzeJavaScriptStaticSource( + bundle(` + electron.ipcMain.on("guard", event => { ${left} || event.sender.copy(); }); + `), + ); + expect(result.electron.call_paths ?? []).toEqual([]); + }, +); +it("does not follow a statically false callback branch", () => { + const result = analyzeJavaScriptStaticSource( + bundle( + `helper.events.addListener("example:copy",event=>{if(false)event.sender.copy()});`, + ), + ); + expect(result.electron.call_paths ?? []).toEqual([]); +}); +it("retains early rejection guards and excludes calls after definite returns", () => { + const guarded = analyzeJavaScriptStaticSource( + `const e=require("electron");e.ipcMain.on("guard",event=>{if(!allowed(event))return;event.sender.copy()});`, + ); + expect(guarded.electron.call_paths).toContainEqual( + expect.objectContaining({ + conditions: [ + expect.objectContaining({ expression: "!(!allowed(event))" }), + ], + }), + ); + for (const body of [ + "return;event.sender.copy()", + "if(true)return;event.sender.copy()", + "if(false)event.sender.copy()", + ]) + expect( + analyzeJavaScriptStaticSource( + `const e=require("electron");e.ipcMain.on("dead",event=>{${body}});`, + ).electron.call_paths ?? [], + ).toEqual([]); + expect( + analyzeJavaScriptStaticSource( + `const e=require("electron");if(false)e.ipcMain.on("dead",event=>event.sender.copy());`, + ).electron.call_paths ?? [], + ).toEqual([]); +}); +it("rejects dynamic channels, computed overwrites, unknown spreads and reassigned callback parameters", () => { + for (const entry of [ + `helper.events.addListener(runtimeChannel,event=>event.sender.copy());`, + `electron[runtimeKey]={};electron.ipcMain.on("bad",event=>event.sender.copy());`, + `const api={copy:helper.sender("bad"),...unknownApi};electron.contextBridge.exposeInMainWorld("desktop",api);`, + `helper.events.addListener("bad",event=>{event=other;event.sender.copy()});`, + ]) + expect( + analyzeJavaScriptStaticSource(bundle(entry)).electron.call_paths ?? [], + ).toEqual([]); +}); +it("resolves renamed default and namespace import helpers from their syntax", () => { + const result = analyzeJavaScriptStaticSource( + bundle(` + const defaultImport=x=>x&&x.__esModule?x:{default:x}; + function namespaceImport(x){if(x&&x.__esModule)return x;return unknownWrap(x)} + const E=defaultImport(load(2)),H=namespaceImport(load(5)); + E.default.contextBridge.exposeInMainWorld("renamed",{copy:(0,H.sender)("test:renamed")}); + `), + ); + expect(result.electron.call_paths).toContainEqual( + expect.objectContaining({ + entry: "renamed.copy", + channel: "test:renamed", + terminal: "electron.ipcRenderer.send", + }), + ); + expect( + result.electron.call_paths + ?.flatMap((p) => p.steps) + .every((step) => step.label.length > 0), + ).toBe(true); +}); +it("does not trust helper names or follow returns after a definite throw", () => { + for (const helper of [ + `function __toESM() {return {ipcRenderer:{send(){}}}}`, + `function __toESM(value) {throw Error("inert");return value;}`, + `function __toESM(value) {if(true){throw Error("inert")}return value;}`, + ]) { + const result = analyzeJavaScriptStaticSource(` + const electron = require("electron"); + ${helper} + const wrapped = __toESM(electron); + electron.contextBridge.exposeInMainWorld("desktop",{ + copy:()=>wrapped.ipcRenderer.send("test:copy") + }); + `); + expect(result.electron.call_paths ?? []).toEqual([]); + } +}); + +describe("wrapped Electron API identity", () => { + it("does not restore rejected Electron identities through wrapper substitution", () => { + for (const source of [ + `const e=require("electron");e.ipcMain={on(){}};e.ipcMain.on("bad",event=>event.sender.copy());`, + `function require(){return {ipcMain:{on(){}}}};const e=require("electron");e.ipcMain.on("bad",event=>event.sender.copy());`, + `const e=require("electron");const ipc=e.ipcRenderer,alias=ipc;delete alias.send;function send(channel){ipc.send(channel)}send("bad");`, + `require("electron/renderer").ipcRenderer={};const e=require("electron");function send(channel){e.ipcRenderer.send(channel)}send("bad");`, + `const e=require("electron");[e.ipcRenderer]=replacements;function send(channel){e.ipcRenderer.send(channel)}send("bad");`, + `const e=require("electron");for(e.contextBridge of replacements){}function expose(api){e.contextBridge.exposeInMainWorld("api",api)}expose({copy:()=>navigator.clipboard.writeText("bad")});`, + `const e=require("electron");const ipc=e.ipcRenderer;Reflect.deleteProperty(ipc,"send");function send(channel){ipc.send(channel)}send("bad");`, + ]) { + const { electron } = analyzeJavaScriptStaticSource(source); + expect(electron.call_paths ?? []).toEqual([]); + expect(electron.ipc).toEqual([]); + expect(electron.context_bridge_apis).toEqual([]); + } + }); + it("keeps unaffected Electron methods available after a sibling method write", () => { + const { electron } = analyzeJavaScriptStaticSource(` + const e=require("electron"); + delete e.ipcRenderer.send; + function invoke(channel){e.ipcRenderer.invoke(channel)} + invoke("good"); + `); + expect(electron.ipc).toContainEqual( + expect.objectContaining({ channel: "good", operation: "invoke" }), + ); + }); +}); diff --git a/src/domain/javascript/electronWrappedCallPaths.ts b/src/domain/javascript/electronWrappedCallPaths.ts new file mode 100644 index 000000000..b7cc873b4 --- /dev/null +++ b/src/domain/javascript/electronWrappedCallPaths.ts @@ -0,0 +1,548 @@ +import * as t from "@babel/types"; +import { JavaScriptSymbolicBindings } from "./javascriptSymbolicBindings.js"; +import { + symbolicVariants, + symbolicTruth, + symbolicUnknown, + type SymbolicValue, + type SymbolicEnvironment, + type SymbolicFunction, +} from "./javascriptSymbolicValues.js"; +import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; +import type { JavaScriptElectronIdentity } from "./javascriptSemanticAnalysis.js"; +import type { JavaScriptAnalysisAccumulator } from "./javascriptStaticAnalysisState.js"; +import type { + ElectronCallPath, + ElectronCallPathStep, + ElectronIpcDescriptor, + ElectronIpcFinding, +} from "./electronStaticAnalysisTypes.js"; +import { + childNodes, + traverseJavaScriptAst, +} from "./javascriptSemanticTraversal.js"; +import { range, calleeName } from "./javascriptStaticAnalysisHelpers.js"; +import { ipcDescriptor } from "./electronStaticAnalysisIpc.js"; +import { + symbolicDefiniteExit, + symbolicLogicalRight, +} from "./javascriptSymbolicControlFlow.js"; + +const callLabel = (node: t.Node): string => + t.isSequenceExpression(node) + ? callLabel(node.expressions.at(-1) ?? node) + : calleeName(node) || "local call"; + +interface PathContext { + readonly entryNode: t.Node; + readonly entry: string; + readonly entryKind: ElectronCallPath["entry_kind"] | null; + readonly channel: string | null; + readonly steps: readonly ElectronCallPathStep[]; + readonly conditions: ElectronCallPath["conditions"]; + readonly active: ReadonlySet; +} + +/** Substitutes inert wrapper syntax; every emitted path remains a candidate. */ +export const collectElectronWrappedCallPaths = ( + program: t.Program, + state: JavaScriptSemanticAnalysisState, + source: string, + accumulator: JavaScriptAnalysisAccumulator, + identity: JavaScriptElectronIdentity, +): readonly ElectronCallPath[] => { + // Bounds repeated interprocedural expansion independently of bundle bytes. + // Exhaustion is reported as unknown coverage, never as a negative finding. + let remainingVisits = 1_000_000; + const resolver = new JavaScriptSymbolicBindings(state, program, identity); + const paths: ElectronCallPath[] = []; + const seen = new Set(); + const step = ( + node: t.Node, + kind: ElectronCallPathStep["kind"], + label: string, + ): ElectronCallPathStep => { + const module = accumulator.moduleRangeIndex.find(node.start); + return { + kind, + label, + location: range(node), + module_key: module?.key ?? null, + module_runtime: module?.runtime ?? null, + }; + }; + const condition = ( + context: PathContext, + node: t.Node, + positive: boolean | "nullish" | "not-nullish" = true, + ): PathContext => { + const expression = source + .slice(node.start ?? 0, node.end ?? 0) + .slice(0, 512); + return { + ...context, + conditions: [ + ...context.conditions, + { + expression: + positive === "nullish" + ? `(${expression}) === null || (${expression}) === undefined` + : positive === "not-nullish" + ? `(${expression}) !== null && (${expression}) !== undefined` + : positive + ? expression + : `!(${expression})`, + location: range(node), + }, + ], + }; + }; + const metadataContext = ( + context: PathContext, + value: SymbolicValue, + ): PathContext | null => { + if (value.conditionUnresolved) { + accumulator.unknownFindings += 1; + return null; + } + return ( + value.conditions?.reduce( + (current, branch) => condition(current, branch.node, branch.positive), + context, + ) ?? context + ); + }; + const hasConditions = (value: SymbolicValue): boolean => + symbolicVariants(value).some((variant) => variant.conditions?.length); + const emit = ( + context: PathContext, + node: t.Node, + terminal: string, + channel = context.channel, + ): void => { + if (context.entryKind === null) return; + const path: ElectronCallPath = { + entry_kind: context.entryKind, + entry: context.entry, + channel, + terminal, + resolution: "candidate", + steps: [...context.steps, step(node, "terminal", terminal)], + conditions: context.conditions, + }; + const key = JSON.stringify(path); + if (seen.has(key)) return; + seen.add(key); + paths.push(path); + }; + const functionBody = ( + fn: SymbolicFunction, + args: readonly SymbolicValue[], + context: PathContext, + invocationNode: t.Node, + ): void => { + if (context.active.has(fn.node)) return; + if (fn.conditionUnresolved === true) { + accumulator.unknownFindings += 1; + return; + } + const name = + (t.isFunctionExpression(fn.node) || t.isFunctionDeclaration(fn.node)) && + fn.node.id + ? fn.node.id.name + : "closure"; + walk(fn.node.body, resolver.bind(fn, args, invocationNode), { + ...context, + steps: [ + ...context.steps, + ...fn.origins.map((n) => + step( + n, + "factory", + t.isCallExpression(n) ? callLabel(n.callee) : "factory", + ), + ), + step(fn.node, "function", name), + ], + active: new Set([...context.active, fn.node]), + }); + }; + const ipc = ( + node: t.CallExpression | t.OptionalCallExpression, + descriptor: ElectronIpcDescriptor, + args: readonly SymbolicValue[], + context: PathContext, + endpoint: string, + guardedDispatch: boolean, + ): void => { + const first = args[0]; + if (first?.kind !== "literal" || typeof first.value !== "string") { + accumulator.unknownFindings += 1; + return; + } + const channelContext = metadataContext(context, first); + if (!channelContext) return; + const handlerValue = args[1] ?? symbolicUnknown("missing handler"); + const handlerContext = metadataContext(channelContext, handlerValue); + if (!handlerContext) return; + const channel = first.value; + const location = range(node); + const invocationLocation = + JSON.stringify(range(context.entryNode)) === JSON.stringify(location) + ? undefined + : range(context.entryNode); + const findingContext = { + module_key: null, + location, + ...(invocationLocation === undefined + ? {} + : { invocation_location: invocationLocation }), + }; + const handlers = symbolicVariants(handlerValue); + // A channel registration must not acquire one arbitrary handler from a union. + // Keep the unresolved registration so it still competes with other handlers. + const fn = + handlers.length === 1 && handlers[0]?.kind === "function" + ? handlers[0] + : undefined; + const handlerNode = node.arguments[1]; + const finding: ElectronIpcFinding = + descriptor.mode === "send" || descriptor.mode === "invoke" + ? { + ...descriptor, + channel, + channel_expression: null, + handler_kind: null, + handler_location: null, + ...findingContext, + } + : fn + ? { + ...descriptor, + channel, + channel_expression: null, + handler_kind: "inline-function", + handler_location: range(fn.node), + ...findingContext, + } + : handlerNode + ? { + ...descriptor, + channel, + channel_expression: null, + handler_kind: "dynamic-expression", + handler_location: range(handlerNode), + ...findingContext, + } + : { + ...descriptor, + channel, + channel_expression: null, + handler_kind: "missing", + handler_location: null, + ...findingContext, + }; + // A fully observed transmission stays one source fact when its callback is + // traced. Registrations retain invocation identity because handlers compete. + const duplicate = accumulator.ipc.some( + ({ value }) => + value.side === finding.side && + value.operation === finding.operation && + value.channel === channel && + JSON.stringify(value.location) === JSON.stringify(location) && + ((value.resolution_basis === undefined && + (value.mode === "send" || value.mode === "invoke")) || + JSON.stringify(value.invocation_location) === + JSON.stringify(invocationLocation)), + ); + const findingGuarded = + context.conditions.length > 0 || hasConditions(first) || guardedDispatch; + if (!duplicate && !findingGuarded) { + accumulator.ipc.push({ + offset: node.start ?? -1, + value: { ...finding, resolution_basis: "wrapper-substitution" }, + }); + if (descriptor.mode !== "send" && descriptor.mode !== "invoke" && !fn) + accumulator.unknownFindings += 1; + } else if (findingGuarded) accumulator.unknownFindings += 1; + if (descriptor.mode === "send" || descriptor.mode === "invoke") { + emit(channelContext, node, endpoint, channel); + return; + } + if (descriptor.side === "main" && fn) + functionBody( + fn, + [ + { kind: "external", path: ["ipc-event"] }, + symbolicUnknown("IPC payload"), + ], + { + ...handlerContext, + entryKind: "ipc-handler", + entry: channel, + channel, + steps: [...context.steps, step(node, "ipc", endpoint)], + }, + node, + ); + }; + const bridge = ( + node: t.CallExpression | t.OptionalCallExpression, + args: readonly SymbolicValue[], + context: PathContext, + guardedDispatch: boolean, + ): void => { + const key = args[0]; + const object = args[1]; + if (key?.kind !== "literal" || typeof key.value !== "string" || !object) { + accumulator.unknownFindings += 1; + return; + } + const keyContext = metadataContext(context, key); + if (!keyContext || !metadataContext(keyContext, object)) return; + const members = [...resolver.members(object)].sort(); + const location = range(node); + const existing = accumulator.contextBridgeApis.find( + ({ value }) => + JSON.stringify(value.location) === JSON.stringify(location), + ); + const finding = { + world: "main" as const, + world_id: null, + api_key: key.value, + api_key_expression: null, + api_status: "dynamic" as const, + resolution_basis: "wrapper-substitution" as const, + members, + unknown_members: resolver.unknownMembers(object), + module_key: null, + location, + }; + // This report shape has no predicate field, so a guarded world or object + // cannot be recorded as an unconditional exposure fact. + const guardedKey = hasConditions(key); + const guardedObject = hasConditions(object); + const guarded = + context.conditions.length > 0 || + guardedKey || + guardedObject || + guardedDispatch; + if (guarded) accumulator.unknownFindings += 1; + // Existing entries are raw syntax observations and remain valid even when + // symbolic substitution can only describe this call under a predicate. + if (!guarded && existing) + (existing.value.api_status !== "object-literal" || + existing.value.unknown_members > 0) && + accumulator.contextBridgeApis.splice( + accumulator.contextBridgeApis.indexOf(existing), + 1, + { offset: node.start ?? -1, value: finding }, + ); + else if (!guarded) + accumulator.contextBridgeApis.push({ + offset: node.start ?? -1, + value: finding, + }); + for (const name of members) { + const value = resolver.property(object, name); + for (const fn of symbolicVariants(value)) + if (fn.kind === "function") { + const memberContext = metadataContext(keyContext, fn); + if (!memberContext) continue; + functionBody( + fn, + fn.node.params.map(() => symbolicUnknown("public API argument")), + { + ...memberContext, + entryNode: node, + entryKind: "bridge-member", + entry: `${key.value}.${name}`, + channel: null, + steps: [step(node, "exposure", `${key.value}.${name}`)], + active: new Set(), + }, + node, + ); + } + } + }; + const call = ( + node: t.CallExpression | t.OptionalCallExpression, + env: SymbolicEnvironment, + context: PathContext, + ): void => { + const value = resolver.evaluate(node.callee, env); + const args = resolver.arguments(node, env); + for (const callee of symbolicVariants(value)) { + const branchContext = metadataContext(context, callee); + if (!branchContext) continue; + if (callee.kind === "function") { + functionBody( + callee, + args, + { + ...branchContext, + steps: [ + ...branchContext.steps, + step(node, "call", callLabel(node.callee)), + ], + }, + node, + ); + } else if (callee.kind === "external") { + const path = callee.path; + const endpoint = path.join("."); + if (identity.intactExport(path)) { + const descriptor = ipcDescriptor(path.slice(1)); + if (descriptor) + ipc( + node, + descriptor, + args, + branchContext, + endpoint, + hasConditions(callee), + ); + else if ( + path.slice(1).join(".") === "contextBridge.exposeInMainWorld" + ) + bridge(node, args, branchContext, hasConditions(callee)); + else if (path[1] === "clipboard") emit(branchContext, node, endpoint); + } else if ( + (path[0] === "navigator" && path[1] === "clipboard") || + (path[0] === "ipc-event" && path[1] === "sender") + ) + emit(branchContext, node, endpoint); + } + } + }; + function walk( + node: t.Node, + env: SymbolicEnvironment, + context: PathContext, + ): void { + if (--remainingVisits < 0 || t.isFunction(node) || t.isClass(node)) return; + if (t.isBlockStatement(node)) { + let next = context; + for (const statement of node.body) { + walk(statement, env, next); + if (symbolicDefiniteExit(statement)) break; + if (t.isIfStatement(statement)) { + const truth = symbolicTruth(resolver.evaluate(statement.test, env)); + if (truth !== undefined) { + if ( + symbolicDefiniteExit( + truth ? statement.consequent : statement.alternate, + ) + ) + break; + } else if (symbolicDefiniteExit(statement.consequent)) + next = condition(next, statement.test, false); + else if (symbolicDefiniteExit(statement.alternate)) + next = condition(next, statement.test); + } + } + return; + } + if (t.isIfStatement(node) || t.isConditionalExpression(node)) { + const truth = symbolicTruth(resolver.evaluate(node.test, env)); + if (truth !== false) + walk( + node.consequent, + env, + truth === undefined ? condition(context, node.test) : context, + ); + if (truth !== true && node.alternate) + walk( + node.alternate, + env, + truth === undefined ? condition(context, node.test, false) : context, + ); + return; + } + if (t.isLogicalExpression(node)) { + walk(node.left, env, context); + const right = symbolicLogicalRight( + node.operator, + resolver.evaluate(node.left, env), + ); + if (right === false) return; + walk( + node.right, + env, + right === undefined + ? condition( + context, + node.left, + node.operator === "??" ? "nullish" : node.operator === "&&", + ) + : context, + ); + return; + } + if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) + call(node, env, context); + for (const child of childNodes(node)) walk(child, env, context); + } + traverseJavaScriptAst(program, { + enter: (node) => { + if (!t.isCallExpression(node) && !t.isOptionalCallExpression(node)) + return; + if (!t.isStringLiteral(node.arguments[0])) return; + const env = resolver.environmentAt(node); + let context: PathContext = { + entryNode: node, + entry: "", + entryKind: null, + channel: null, + steps: [], + conditions: [], + active: new Set(), + }; + for ( + let child: t.Node = node, parent = state.parentsByNode.get(child); + parent && !t.isFunction(parent); + child = parent, parent = state.parentsByNode.get(child) + ) { + if (t.isIfStatement(parent) || t.isConditionalExpression(parent)) { + if (child === parent.test) continue; + const truth = symbolicTruth(resolver.evaluate(parent.test, env)); + const positive = child === parent.consequent; + if (truth !== undefined && truth !== positive) return; + if (truth === undefined) + context = condition(context, parent.test, positive); + } else if (t.isLogicalExpression(parent) && child === parent.right) { + const right = symbolicLogicalRight( + parent.operator, + resolver.evaluate(parent.left, env), + ); + if (right === false) return; + if (right === undefined) + context = condition( + context, + parent.left, + parent.operator === "??" ? "nullish" : parent.operator === "&&", + ); + } else if (t.isBlockStatement(parent)) { + const index = parent.body.findIndex( + (statement) => statement === child, + ); + if ( + index >= 0 && + parent.body + .slice(0, index) + .some((statement) => symbolicDefiniteExit(statement)) + ) + return; + } + } + call(node, env, context); + }, + }); + if (remainingVisits < 0 || resolver.truncated) { + accumulator.unknownFindings += 1; + accumulator.electronCallPathLimitReached = true; + } + if (resolver.unresolved) accumulator.unknownFindings += 1; + return paths; +}; diff --git a/src/domain/javascript/javascriptAnalysisResourceControls.test.ts b/src/domain/javascript/javascriptAnalysisResourceControls.test.ts new file mode 100644 index 000000000..e5f98bda3 --- /dev/null +++ b/src/domain/javascript/javascriptAnalysisResourceControls.test.ts @@ -0,0 +1,22 @@ +import { expect, it } from "vitest"; +import { javaScriptAnalysisResourceControlsSchema } from "./javascriptAnalysisResourceControls.js"; + +it("keeps omitted operational choices absent and validates explicit semantic budgets", () => { + expect(javaScriptAnalysisResourceControlsSchema.parse({})).toEqual({}); + expect( + javaScriptAnalysisResourceControlsSchema.parse({ + max_semantic_nodes: 750_000, + max_semantic_nodes_per_file: 700_000, + }), + ).toEqual({ + max_semantic_nodes: 750_000, + max_semantic_nodes_per_file: 700_000, + }); + for (const value of [0, -1, 1.5, 1_000_001, Infinity, NaN]) { + for (const key of ["max_semantic_nodes", "max_semantic_nodes_per_file"]) + expect( + javaScriptAnalysisResourceControlsSchema.safeParse({ [key]: value }) + .success, + ).toBe(false); + } +}); diff --git a/src/domain/javascript/javascriptAnalysisResourceControls.ts b/src/domain/javascript/javascriptAnalysisResourceControls.ts index a457dc2e9..c627ad880 100644 --- a/src/domain/javascript/javascriptAnalysisResourceControls.ts +++ b/src/domain/javascript/javascriptAnalysisResourceControls.ts @@ -6,8 +6,13 @@ import type { ApplicationCoverage } from "./javascriptApplicationEvidenceSchemas export const JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS = { heapMb: 1024, timeoutMs: 300_000, + semanticNodes: 100_000, + semanticNodesPerFile: 20_000, } as const; +/** Maximum caller-selected node capacity; heap and transfer admission still apply. */ +export const JAVASCRIPT_SEMANTIC_NODE_MAXIMUM = 1_000_000; + /** Project observed execution limits without inventing unmeasured values. */ export const javaScriptAnalysisCoverageLimits = ( limits: Readonly> | null, @@ -37,6 +42,24 @@ export const javaScriptAnalysisCoverageLimits = ( /** Optional operational choices; omission keeps them out of Evidence parameters. */ export const javaScriptAnalysisResourceControlsSchema = z.strictObject({ + max_semantic_nodes: z + .number() + .int() + .min(1) + .max(JAVASCRIPT_SEMANTIC_NODE_MAXIMUM) + .optional() + .describe( + "Maximum retained semantic nodes across the application; defaults to 100000. Worker heap and parent result admission remain enforced.", + ), + max_semantic_nodes_per_file: z + .number() + .int() + .min(1) + .max(JAVASCRIPT_SEMANTIC_NODE_MAXIMUM) + .optional() + .describe( + "Maximum retained semantic nodes per source file; defaults to 20000, additionally bounded by max_semantic_nodes and memory/transfer limits.", + ), max_heap_mb: z .number() .int() diff --git a/src/domain/javascript/javascriptApplicationGraphSchemas.ts b/src/domain/javascript/javascriptApplicationGraphSchemas.ts index 17e19fb10..90af9ee43 100644 --- a/src/domain/javascript/javascriptApplicationGraphSchemas.ts +++ b/src/domain/javascript/javascriptApplicationGraphSchemas.ts @@ -24,6 +24,7 @@ export const JAVASCRIPT_APPLICATION_NODE_KINDS = [ "javascript-asset", "javascript-chunk", "javascript-module", + "javascript-expression", "source-map", "source-module", "browser-window", diff --git a/src/domain/javascript/javascriptApplicationProjectionResources.ts b/src/domain/javascript/javascriptApplicationProjectionResources.ts index 0e5ba7570..3e5280c9e 100644 --- a/src/domain/javascript/javascriptApplicationProjectionResources.ts +++ b/src/domain/javascript/javascriptApplicationProjectionResources.ts @@ -37,6 +37,14 @@ export const estimateJavaScriptStaticApplicationProjection = ( } for (const binding of facts.electron.native_addon_bindings) records += binding.members.length * 3; + for (const path of facts.electron.call_paths ?? []) { + // One contextual node and edge per step, with repeated guards and identity. + records += path.steps.length * 4 + 8; + repeatedBytes += + Buffer.byteLength(JSON.stringify(path.conditions)) * + path.steps.length * + 2; + } const metadataBytes = 2048 + Buffer.byteLength(sourcePath) * 4; return (checkpointBytes * 4 + repeatedBytes + records * metadataBytes) * 8; }; diff --git a/src/domain/javascript/javascriptBootstrap.ts b/src/domain/javascript/javascriptBootstrap.ts new file mode 100644 index 000000000..cdb33d024 --- /dev/null +++ b/src/domain/javascript/javascriptBootstrap.ts @@ -0,0 +1,150 @@ +import * as t from "@babel/types"; + +import { semanticStaticPropertyKey } from "./javascriptAstValues.js"; +import { moduleFactory } from "./javascriptStaticAnalysisHelpers.js"; +import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { + resolveSemanticBindingState, + semanticStableInitializer, + type JavaScriptSemanticAnalysisState, +} from "./javascriptSemanticState.js"; + +/** A literal module table dispatched by a standalone CommonJS-style runtime. */ +interface JavaScriptBootstrap { + readonly body: t.BlockStatement; + readonly table: t.ObjectExpression; + readonly loaderName: string; + readonly loaderBindingId: string; + readonly tableBindingId: string; +} + +/** Recognize syntax, never invoke a bootstrap or infer it from variable names. */ +export const javascriptBootstrapAt = ( + call: t.CallExpression, + state: JavaScriptSemanticAnalysisState, +): JavaScriptBootstrap | undefined => { + if ( + (!t.isFunctionExpression(call.callee) && + !t.isArrowFunctionExpression(call.callee)) || + !t.isBlockStatement(call.callee.body) || + call.callee.params.length !== 0 || + call.arguments.length !== 0 + ) + return undefined; + const body = call.callee.body; + const tables = new Map(); + for (const statement of body.body) { + if (!t.isVariableDeclaration(statement)) continue; + for (const declaration of statement.declarations) { + if ( + t.isIdentifier(declaration.id) && + t.isObjectExpression(declaration.init) && + declaration.init.properties.length > 0 && + declaration.init.properties.every( + (property) => moduleFactory(property) !== undefined, + ) + ) { + const binding = resolveSemanticBindingState( + state, + declaration.id, + declaration.id.name, + ); + if ( + binding !== undefined && + semanticStableInitializer(binding, state)?.node === declaration.init + ) + tables.set(binding.bindingId, declaration.init); + } + } + } + if (tables.size === 0) return undefined; + for (const loader of body.body) { + if ( + !t.isFunctionDeclaration(loader) || + loader.id === null || + loader.id === undefined || + loader.params.length !== 1 + ) + continue; + const parameter = loader.params[0]; + if (!t.isIdentifier(parameter)) continue; + const loaderName = loader.id.name; + const loaderBindingId = bindingId(state, loader.id); + const parameterBindingId = bindingId(state, parameter); + if (loaderBindingId === undefined || parameterBindingId === undefined) + continue; + let tableBindingId: string | undefined; + traverseJavaScriptAst(loader.body, { + enter: (node, _parent, ancestors) => { + if ( + !t.isCallExpression(node) || + ancestors().some((node) => t.isFunction(node)) + ) + return; + const candidate = dispatchedTableBinding( + node, + loaderBindingId, + parameterBindingId, + state, + ); + if (candidate !== undefined && tables.has(candidate)) + tableBindingId = candidate; + }, + }); + const table = + tableBindingId === undefined ? undefined : tables.get(tableBindingId); + if (table !== undefined && tableBindingId !== undefined) + return { body, table, loaderName, loaderBindingId, tableBindingId }; + } + return undefined; +}; + +const dispatchedTableBinding = ( + call: t.CallExpression, + loaderBindingId: string, + keyBindingId: string, + state: JavaScriptSemanticAnalysisState, +): string | undefined => { + const member = call.callee; + const viaCall = + t.isMemberExpression(member) && + semanticStaticPropertyKey(member.property, member.computed) === "call"; + const dispatch = + viaCall && t.isMemberExpression(member) ? member.object : member; + if ( + !t.isMemberExpression(dispatch) || + !dispatch.computed || + !t.isIdentifier(dispatch.object) || + !t.isIdentifier(dispatch.property) || + bindingId(state, dispatch.property) !== keyBindingId + ) + return undefined; + const args = viaCall ? call.arguments.slice(1) : call.arguments; + const [module, exports, require] = args; + if ( + args.length !== 3 || + !t.isIdentifier(module) || + bindingId(state, module) === undefined || + !t.isMemberExpression(exports) || + !t.isIdentifier(exports.object) || + bindingId(state, exports.object) !== bindingId(state, module) || + semanticStaticPropertyKey(exports.property, exports.computed) !== + "exports" || + !t.isIdentifier(require) || + bindingId(state, require) !== loaderBindingId + ) + return undefined; + if ( + viaCall && + (!t.isNode(call.arguments[0]) || + !t.isNodesEquivalent(call.arguments[0], exports)) + ) + return undefined; + return bindingId(state, dispatch.object); +}; + +const bindingId = ( + state: JavaScriptSemanticAnalysisState, + identifier: t.Identifier, +): string | undefined => + resolveSemanticBindingState(state, identifier, identifier.name)?.bindingId; diff --git a/src/domain/javascript/javascriptFeatureSeed.ts b/src/domain/javascript/javascriptFeatureSeed.ts index 534285068..929733f00 100644 --- a/src/domain/javascript/javascriptFeatureSeed.ts +++ b/src/domain/javascript/javascriptFeatureSeed.ts @@ -82,7 +82,12 @@ const kindMatches = ( ({ properties }) => properties.endpoint_kind === "route", ) ); - if (kind === "api") return node.kind === "context-bridge-api"; + if (kind === "api") + return ( + node.kind === "context-bridge-api" || + (node.kind === "javascript-expression" && + node.observations.some((o) => o.properties.step_kind === "exposure")) + ); if (kind === "channel") return node.kind === "ipc-channel"; if (kind === "module") return ["javascript-asset", "javascript-module", "source-module"].includes( diff --git a/src/domain/javascript/javascriptFeatureTrace.ts b/src/domain/javascript/javascriptFeatureTrace.ts index e535080a9..47333bab6 100644 --- a/src/domain/javascript/javascriptFeatureTrace.ts +++ b/src/domain/javascript/javascriptFeatureTrace.ts @@ -217,7 +217,9 @@ const pathTo = ( const isTerminal = (node: ApplicationNode): boolean => ["endpoint", "storage", "native-addon", "native-export", "unknown"].includes( node.kind, - ); + ) || + (node.kind === "javascript-expression" && + node.observations.some((o) => o.properties.step_kind === "terminal")); const factSummary = ( nodes: readonly ApplicationNode[], diff --git a/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts index ecd3e99a6..ed98d2626 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.reference_lifetimes.test.ts @@ -9,6 +9,85 @@ const resultValue = (body: string) => "result", ).returnSites[0]?.value; +describe("bounded paths through dynamic selectors and class origins", () => { + it("bounds linked-list walks through class instances", () => { + const start = performance.now(); + const analysis = analyzeJavaScriptSemantics(` + class List { tail = null; } + export function walk() { + const list = new List(); + let node = list.tail; + while (node) { use(node.value.parent); node = node.previous; } + return 42; + } + `); + expect(onlyCallable(analysis, "walk").returnSites[0]?.value).toEqual({ + status: "literal", + value: 42, + }); + expect(performance.now() - start).toBeLessThan(2000); + }, 30000); + + it("retains effects through class getter aliases while bounding loop paths", () => { + expect( + resultValue(` + const shared = { value: 1 }; + class List { get tail() { return { value: shared }; } } + let node = new List().tail; + while (node) { use(node.value); node = node.previous; } + return shared.value; + `)?.status, + ).toBe("unknown"); + }); + + it("bounds dynamic-key setter loops that also create literal containers", () => { + const start = performance.now(); + const analysis = analyzeJavaScriptSemantics(` + export function set(root, keys, value) { + let node = root; + for (const [index, key] of keys.entries()) { + if (index === keys.length - 1) { node[key] = value; return root; } + const child = node[key]; + if (isObject(child)) { node = child; continue; } + const fresh = isIndex(keys[index + 1]) ? [] : {}; + node[key] = fresh; + node = fresh; + } + return root; + } + export function unaffected() { return 42; } + `); + expect(onlyCallable(analysis, "unaffected").returnSites[0]?.value).toEqual({ + status: "literal", + value: 42, + }); + expect(onlyCallable(analysis, "set").returnSites[0]?.value.status).toBe( + "unknown", + ); + expect(performance.now() - start).toBeLessThan(2000); + }, 30000); + + it("retains shared-child mutations reached by dynamic-key alias loops", () => { + expect( + resultValue(` + const shared = { value: 1 }; + const source = { child: { next: shared }, keep: 3 }; + let node = source.child; + while (flag) { node = node[key] || {}; node.value = 2; } + return shared.value; + `)?.status, + ).toBe("unknown"); + expect( + resultValue(` + const source = { child: { value: 1 }, keep: 3 }; + let node = source.child; + while (flag) { node = node[key] || {}; node.value = 2; } + return source.keep; + `), + ).toEqual({ status: "literal", value: 3 }); + }); +}); + describe("bounded member path propagation", () => { it("bounds cyclic member paths with no observed container origin", () => { const branches = Array.from( diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 517ed5836..b8e2a24ae 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -121,6 +121,15 @@ export const analyzeParsedJavaScriptReferences = ( return collectSemanticReferences(file.program, state, name); }; +/** Share the lexical index with bounded, demand-driven boundary analyses. */ +export function* indexJavaScriptBindingsSteps( + program: t.Program, +): Generator { + const state = createState(program); + yield* collectDefinitionsSteps(program, state); + return state; +} + /** Recover semantics from an already parsed JavaScript artifact. */ export const analyzeParsedJavaScriptSemantics = ( file: ParsedJavaScriptSource, @@ -369,9 +378,24 @@ export const classifyParsedJavaScriptElectronBindings = ( export function* classifyParsedJavaScriptElectronBindingsSteps( file: ParsedJavaScriptSource, ): Generator> { - const facts = new Map(); const state = createState(file.program); yield* collectDefinitionsSteps(file.program, state); + return (yield* indexJavaScriptElectronIdentitySteps(file.program, state)) + .bindings; +} + +/** Shared lexical identity and visible-write proof for direct and wrapped calls. */ +export interface JavaScriptElectronIdentity { + readonly bindings: ReadonlyMap; + readonly intactExport: (path: readonly (string | null)[]) => boolean; + readonly exportPath: (root: t.Node) => readonly string[] | undefined; +} + +export function* indexJavaScriptElectronIdentitySteps( + program: t.Program, + state: JavaScriptSemanticAnalysisState, +): Generator { + const facts = new Map(); const interop = createSemanticModuleInterop(state); const writes = new SemanticPropertyPathCoverage(); let collectedWrites = false; @@ -402,7 +426,7 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( : undefined; }; yield* collectJavaScriptIdentityWritesSteps( - file.program, + program, (root, kind) => { if (kind === "opaque-call") return interop.opaqueMutationPath(root); const exported = kind === "binding" ? undefined : rootExport(root); @@ -422,7 +446,10 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( }, ); collectedWrites = true; - yield* traverseJavaScriptAstSteps(file.program, { + const intactExport = (path: readonly (string | null)[]): boolean => + ELECTRON_MODULE.test(path[0] ?? "") && + !writes.covers(["electron", ...path.slice(1)]); + yield* traverseJavaScriptAstSteps(program, { enter: (node) => { if ( !t.isCallExpression(node) && @@ -434,12 +461,12 @@ export function* classifyParsedJavaScriptElectronBindingsSteps( const exported = rootExport(root); if ( exported !== undefined && - !writes.covers(["electron", ...exported, ...members]) + intactExport(["electron", ...exported, ...members]) ) facts.set(root.start ?? -1, exported); }, }); - return facts; + return { bindings: facts, intactExport, exportPath: rootExport }; } // Read import and require origins directly; general value evaluation is diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index 35f3a0ca9..e3f3a2942 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -718,6 +718,9 @@ const referencedValues = ( type MutationObservableOrigin = | t.ObjectExpression | t.ArrayExpression + | t.ClassDeclaration + | t.ClassExpression + | t.NewExpression | "ambient" | "array" | "array-prototype" @@ -808,11 +811,6 @@ const observableMutationPaths = ( const observer = bindingNodes.get(binding.bindingId); if (observer === undefined) continue; for (const initializer of binding.initializers) { - const expression = unwrapJavaScriptExpression(initializer.node).node; - // Preserve member paths through class values and local instances so - // getter-result aliases can be resolved at the accessor boundary. - if (t.isClass(expression) || t.isNewExpression(expression)) - (observer.origins ??= new Set()).add("opaque"); link( project(expressionNode(initializer.node), initializer.projection), observer, @@ -837,7 +835,14 @@ const observableMutationPaths = ( const selected = pendingExpressions.pop(); if (selected === undefined) break; const { expression, observer } = selected; - if (t.isObjectExpression(expression) || t.isArrayExpression(expression)) + // Keep class/getter projections distinct so cycle pruning does not + // discard observable escapes through a shared getter result. + if ( + t.isObjectExpression(expression) || + t.isArrayExpression(expression) || + t.isClass(expression) || + t.isNewExpression(expression) + ) (observer.origins ??= new Set()).add(expression); else if (t.isIdentifier(expression)) { const binding = resolveSemanticBindingState( diff --git a/src/domain/javascript/javascriptStaticAnalysis.ts b/src/domain/javascript/javascriptStaticAnalysis.ts index 50af5c765..5586adf84 100644 --- a/src/domain/javascript/javascriptStaticAnalysis.ts +++ b/src/domain/javascript/javascriptStaticAnalysis.ts @@ -1,4 +1,5 @@ import * as t from "@babel/types"; +import { collectElectronWrappedCallPaths } from "./electronWrappedCallPaths.js"; import { ELECTRON_IDENTITY_LIMITATION, ELECTRON_MODULE, @@ -27,6 +28,7 @@ import { } from "./javascriptStaticAnalysisCalls.js"; import { inspectBundlerRegistration, + inspectBundlerBootstrap, inspectEsbuildWrapper, } from "./javascriptStaticAnalysisBundler.js"; import { finalizeLocatedFindings } from "./javascriptStaticAnalysisFindings.js"; @@ -36,8 +38,10 @@ import { type JavaScriptFindingContext, } from "./javascriptStaticAnalysisState.js"; import type { JavaScriptStaticAnalysis } from "./javascriptStaticAnalysisTypes.js"; +import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; import { - classifyParsedJavaScriptElectronBindingsSteps, + indexJavaScriptElectronIdentitySteps, + indexJavaScriptBindingsSteps, classifyParsedJavaScriptRpcClientRootsSteps, classifyParsedJavaScriptOpenReceivers, type JavaScriptOpenReceiverFact, @@ -80,16 +84,30 @@ export function* analyzeParsedJavaScriptStaticSourceSteps( openReceiverFacts?: ReadonlyMap, ): Generator { const accumulator = createJavaScriptAnalysisAccumulator(source.length); - const electronBindings = - yield* classifyParsedJavaScriptElectronBindingsSteps(file); + const bindings = yield* indexJavaScriptBindingsSteps(file.program); + const electronIdentity = yield* indexJavaScriptElectronIdentitySteps( + file.program, + bindings, + ); const clientRoots = yield* classifyParsedJavaScriptRpcClientRootsSteps(file); - yield* traverseStaticSourceSteps(file, { + yield* traverseStaticSourceSteps( + file, + { + source, + accumulator, + ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), + electronBindings: electronIdentity.bindings, + rpcScan: createJavaScriptRpcScan(source, clientRoots), + }, + bindings, + ); + accumulator.electronCallPaths = collectElectronWrappedCallPaths( + file.program, + bindings, source, accumulator, - ...(openReceiverFacts === undefined ? {} : { openReceiverFacts }), - electronBindings, - rpcScan: createJavaScriptRpcScan(source, clientRoots), - }); + electronIdentity, + ); addSourceMapDirectives(source, file.comments ?? [], accumulator); return finalizeStaticAnalysis(source, file, accumulator); } @@ -97,11 +115,12 @@ export function* analyzeParsedJavaScriptStaticSourceSteps( function* traverseStaticSourceSteps( file: ParsedJavaScriptSource, findings: JavaScriptFindingContext, + bindings: JavaScriptSemanticAnalysisState, ): Generator { yield* traverseJavaScriptAstSteps(file, { enter: (node) => { findings.accumulator.visitedNodes += 1; - inspectNode(node, findings); + inspectNode(node, findings, bindings); return undefined; }, }); @@ -114,6 +133,16 @@ const finalizeStaticAnalysis = ( ): JavaScriptStaticAnalysis => { const parserErrors = file.errors.length; const limitations = [ + ...(accumulator.electronCallPaths?.length + ? [ + "Bundled Electron call paths use static argument and closure substitutions. Conditions, ambient helper overrides, runtime reachability and arbitrary side effects remain unproven.", + ] + : []), + ...(accumulator.electronCallPathLimitReached + ? [ + "Electron wrapper expansion reached its resource budget (1000000 visits/substitutions or depth 128); additional paths remain unknown.", + ] + : []), ...(parserErrors === 0 ? [] : [ @@ -162,6 +191,9 @@ const finalizeStaticAnalysis = ( source_map_urls: accumulator.sourceMaps, vendors: detectVendors(source), electron: { + ...(accumulator.electronCallPaths?.length + ? { call_paths: accumulator.electronCallPaths } + : {}), browser_windows: finalizeLocatedFindings( accumulator.browserWindows, accumulator.moduleRangeIndex, @@ -194,10 +226,12 @@ const finalizeStaticAnalysis = ( const inspectNode = ( node: t.Node, findings: JavaScriptFindingContext, + bindings: JavaScriptSemanticAnalysisState, ): void => { const { source, accumulator } = findings; inspectElectronStaticNode(node, findings); if (t.isCallExpression(node)) { + inspectBundlerBootstrap(source, node, accumulator, bindings); inspectBundlerRegistration(source, node, accumulator); inspectEsbuildWrapper(source, node, accumulator); inspectCall(source, node, findings); diff --git a/src/domain/javascript/javascriptStaticAnalysisBundler.ts b/src/domain/javascript/javascriptStaticAnalysisBundler.ts index 4e141b58e..bf357a722 100644 --- a/src/domain/javascript/javascriptStaticAnalysisBundler.ts +++ b/src/domain/javascript/javascriptStaticAnalysisBundler.ts @@ -1,4 +1,9 @@ import * as t from "@babel/types"; +import { javascriptBootstrapAt } from "./javascriptBootstrap.js"; +import { + resolveSemanticBindingState, + type JavaScriptSemanticAnalysisState, +} from "./javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; @@ -77,6 +82,64 @@ export const inspectBundlerRegistration = ( ); }; +/** Recover standalone Webpack tables whose factories are called by a local loader. */ +export const inspectBundlerBootstrap = ( + source: string, + call: t.CallExpression, + accumulator: AnalysisAccumulator, + bindings: JavaScriptSemanticAnalysisState, +): void => { + const bootstrap = javascriptBootstrapAt(call, bindings); + if (bootstrap === undefined) return; + const runtime = `webpack-bootstrap@${String(call.start ?? 0)}`; + const recovered = recoverBundlerModules( + source, + bootstrap.table, + runtime, + accumulator, + ); + const entries: string[] = []; + let unknownEntries = 0; + traverseJavaScriptAst(bootstrap.body, { + enter: (node, _parent, ancestors) => { + if ( + !t.isCallExpression(node) || + ancestors().some((node) => t.isFunction(node)) || + !t.isIdentifier(node.callee) || + resolveSemanticBindingState(bindings, node.callee, node.callee.name) + ?.bindingId !== bootstrap.loaderBindingId + ) + return; + const value = argumentValue(node.arguments[0]); + if (value === undefined) unknownEntries++; + else entries.push(value); + }, + }); + const registration: JavaScriptBundlerRegistration = { + bundler: "webpack", + runtime, + chunk_keys: [`bootstrap@${String(call.start ?? 0)}`], + unknown_chunk_keys: 0, + runtime_require_name: bootstrap.loaderName, + runtime_module_cache_status: "not-observed", + entry_module_keys: uniqueValues(entries).values, + unknown_entry_module_keys: unknownEntries, + async_chunk_keys: recovered.asyncChunkKeys, + unknown_async_chunk_keys: recovered.unknownAsyncChunkKeys, + modules: recovered.modules.sort((left, right) => + compareUnicodeCodePoints(left.module_key, right.module_key), + ), + location: range(call), + }; + accumulator.unknownFindings += + unknownEntries + recovered.unknownAsyncChunkKeys; + addFindingOnce( + accumulator, + `registration\0${registrationKey(registration)}`, + () => accumulator.registrations.push(registration), + ); +}; + /** Inspect esbuild helper wrappers and recover their literal module tables. */ export const inspectEsbuildWrapper = ( source: string, diff --git a/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts b/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts index ddee2ce32..f5f51aa1d 100644 --- a/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts +++ b/src/domain/javascript/javascriptStaticAnalysisModuleIndex.test.ts @@ -3,6 +3,98 @@ import { expect, it } from "vitest"; import { JavaScriptModuleRangeIndex } from "./javascriptStaticAnalysisFindings.js"; import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js"; +it.each(["direct", "call"])( + "recovers standalone module factories dispatched with %s", + (style) => { + const dispatch = + style === "call" + ? "modules[id].call(module.exports, module, module.exports, load)" + : "modules[id](module, module.exports, load)"; + const source = `(() => { + const modules = { + 17(module, exports, req) { exports.read = req(23); }, + 23(module) { module.exports = require("electron"); } + }; + const cache = {}; + function load(id) { + if (cache[id]) return cache[id].exports; + const module = cache[id] = { exports: {} }; + ${dispatch}; + return module.exports; + } + load(17); + })();`; + const analysis = analyzeJavaScriptStaticSource(source); + expect(analysis.bundler_registrations).toMatchObject([ + { + bundler: "webpack", + entry_module_keys: ["17"], + modules: [{ module_key: "17" }, { module_key: "23" }], + }, + ]); + expect(analysis.references).toEqual( + expect.arrayContaining([ + expect.objectContaining({ specifier: "23", module_key: "17" }), + expect.objectContaining({ specifier: "electron", module_key: "23" }), + ]), + ); + }, +); + +it("does not infer a bootstrap from a function table without a matching loader", () => { + const analysis = analyzeJavaScriptStaticSource(`(() => { + const handlers = { 17(module) { module.exports = require("electron"); } }; + function load(id) { return handlers[id](userInput); } + load(17); + })();`); + expect(analysis.bundler_registrations).toEqual([]); +}); + +it("does not confuse a shadowed dispatch table with the outer module table", () => { + const analysis = analyzeJavaScriptStaticSource(`(() => { + const modules = { 17(module) { module.exports = require("electron"); } }; + function load(id) { + const modules = userInput; + const module = { exports: {} }; + modules[id](module, module.exports, load); + return module.exports; + } + load(17); + })();`); + expect(analysis.bundler_registrations).toEqual([]); +}); + +it("retains a stable var module table without treating declaration mutability as a write", () => { + const analysis = analyzeJavaScriptStaticSource(`(() => { + var modules = { 17(module) { module.exports = require("electron"); } }; + function load(id) { + const module = { exports: {} }; + modules[id](module, module.exports, load); + return module.exports; + } + load(17); + })();`); + expect(analysis.bundler_registrations).toMatchObject([ + { entry_module_keys: ["17"], modules: [{ module_key: "17" }] }, + ]); +}); + +it("only attributes bootstrap entries to the recovered loader binding", () => { + const analysis = analyzeJavaScriptStaticSource(`(() => { + const modules = { 17(module) { module.exports = require("electron"); } }; + function load(id) { + const module = { exports: {} }; + modules[id](module, module.exports, load); + return module.exports; + } + { const load = userInput; load(99); } + load(17); + })();`); + expect(analysis.bundler_registrations).toMatchObject([ + { entry_module_keys: ["17"] }, + ]); +}); + it("preserves first insertion precedence and inclusive interval bounds", () => { const nested = new JavaScriptModuleRangeIndex(100); expect(nested.find(0)).toBeUndefined(); diff --git a/src/domain/javascript/javascriptStaticAnalysisState.ts b/src/domain/javascript/javascriptStaticAnalysisState.ts index 44f6f3184..3c93d2fa3 100644 --- a/src/domain/javascript/javascriptStaticAnalysisState.ts +++ b/src/domain/javascript/javascriptStaticAnalysisState.ts @@ -10,6 +10,7 @@ import type { JavaScriptStaticStorage, } from "./javascriptStaticAnalysisTypes.js"; import type { + ElectronCallPath, ElectronBrowserWindowFinding, ElectronContextBridgeFinding, ElectronIpcFinding, @@ -36,6 +37,8 @@ export interface LocatedJavaScriptFinding { /** Mutable collections used during one inert AST traversal. */ export interface JavaScriptAnalysisAccumulator { + electronCallPaths?: readonly ElectronCallPath[]; + electronCallPathLimitReached?: boolean; readonly references: LocatedJavaScriptFinding[]; readonly endpoints: LocatedJavaScriptFinding[]; readonly storage: LocatedJavaScriptFinding[]; diff --git a/src/domain/javascript/javascriptSymbolicBindings.test.ts b/src/domain/javascript/javascriptSymbolicBindings.test.ts new file mode 100644 index 000000000..a4248e30b --- /dev/null +++ b/src/domain/javascript/javascriptSymbolicBindings.test.ts @@ -0,0 +1,440 @@ +import { expect, it } from "vitest"; +import * as t from "@babel/types"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; +import { indexJavaScriptBindingsSteps } from "./javascriptSemanticAnalysis.js"; +import { + completeSemanticSteps, + traverseJavaScriptAst, +} from "./javascriptSemanticTraversal.js"; +import { JavaScriptSymbolicBindings } from "./javascriptSymbolicBindings.js"; +import { + symbolicTruth, + symbolicVariants, + type SymbolicValue, +} from "./javascriptSymbolicValues.js"; + +const resolverFor = (source: string) => { + const parsed = parseJavaScriptSource(source); + if (parsed === null) throw new Error("symbolic source fixture did not parse"); + const state = completeSemanticSteps( + indexJavaScriptBindingsSteps(parsed.program), + ); + return { + parsed, + resolver: new JavaScriptSymbolicBindings(state, parsed.program), + }; +}; + +/** Parse a fixture and collect the first argument of each inspect call. */ +const inspectFixture = ( + source: string, + evaluate: ( + resolver: JavaScriptSymbolicBindings, + node: t.CallExpression, + ) => SymbolicValue = (resolver, node) => resolver.evaluate(node.arguments[0]), +) => { + const { parsed, resolver } = resolverFor(source); + const values: SymbolicValue[] = []; + traverseJavaScriptAst(parsed.program, { + enter: (node) => { + if ( + t.isCallExpression(node) && + t.isIdentifier(node.callee, { name: "inspect" }) + ) { + values.push(evaluate(resolver, node)); + } + }, + }); + return { parsed, resolver, values }; +}; + +const inspectedValues = (source: string): SymbolicValue[] => + inspectFixture(source).values; + +const inspectOne = (source: string): SymbolicValue => { + const values = inspectedValues(source); + expect(values).toHaveLength(1); + const value = values[0]; + if (value === undefined) throw new Error("inspect call result missing"); + return value; +}; + +it("substitutes numeric modules, named exports and captured wrapper arguments", () => { + const { resolver, values } = inspectFixture(`(() => { + const modules = { + 4(module) { module.exports = require("electron"); }, + 8(module, exports, load) { + const electron = load(4); + exports.sender = function(channel) { return (...args) => electron.ipcRenderer.send(channel,...args); }; + }, + 12(module, exports, load) { const helpers = load(8); const api = {copy: helpers.sender("test:copy")}; inspect(api.copy); } + }; + function load(id) { const m = {exports:{}}; modules[id](m,m.exports,load); return m.exports; } + load(12); + })();`); + expect(values).toHaveLength(1); + const value = values[0]; + if (value === undefined) throw new Error("inspect call result missing"); + const fn = symbolicVariants(value).find((v) => v.kind === "function"); + expect(fn?.kind).toBe("function"); + if (fn?.kind !== "function" || !t.isCallExpression(fn.node.body)) + throw new Error("closure missing"); + const env = resolver.bind(fn, []); + expect(resolver.evaluate(fn.node.body.callee, env)).toEqual({ + kind: "external", + path: ["electron", "ipcRenderer", "send"], + }); + expect(resolver.arguments(fn.node.body, env)[0]).toEqual({ + kind: "literal", + value: "test:copy", + }); +}); + +it("checks block lexical initializers against their containing function call", () => { + const { parsed, resolver } = resolverFor(`run(); + function run() { { const value = require("electron"); inspect(value); } }`); + let resolved: SymbolicValue | undefined; + traverseJavaScriptAst(parsed.program, { + enter: (node) => { + if ( + !t.isCallExpression(node) || + !t.isIdentifier(node.callee, { name: "inspect" }) + ) + return; + const fn = parsed.program.body.find( + (statement): statement is t.FunctionDeclaration => + t.isFunctionDeclaration(statement) && statement.id?.name === "run", + ); + if (!fn) throw new Error("run declaration missing"); + const call = parsed.program.body[0]; + if ( + !t.isExpressionStatement(call) || + !t.isCallExpression(call.expression) + ) + throw new Error("run invocation missing"); + resolved = resolver.evaluate( + node.arguments[0], + resolver.bind( + { + kind: "function", + node: fn, + environment: { values: new Map() }, + origins: [], + }, + [], + call.expression, + ), + ); + }, + }); + expect(resolved).toEqual({ kind: "external", path: ["electron"] }); +}); + +it("keeps same-factory reads before initialization unresolved", () => { + const { values, resolver } = inspectFixture( + `(() => { + const modules = { 9(module) { inspect(value); const value = require("electron"); } }; + function load(id) { const m={exports:{}}; modules[id](m); return m.exports; } + load(9); + })();`, + (bindings, node) => + bindings.evaluate(node.arguments[0], bindings.environmentAt(node)), + ); + expect(values).toHaveLength(1); + const resolved = values[0]; + if (resolved === undefined) throw new Error("inspect call result missing"); + expect(resolved).toEqual({ + kind: "unknown", + reason: "initializer not available at this read", + }); + expect(resolver.unresolved).toBe(true); +}); + +it("invalidates a bootstrap module slot mutated through an alias", () => { + const values = inspectedValues(`(() => { + const modules = { + 1(module) { module.exports = require("electron"); }, + 2(module, exports, load) { inspect(load(1)); inspect(load(3)); }, + 3(module) { module.exports = require("electron"); } + }; + const table = modules; + function load(id) { const m = {exports:{}}; modules[id](m,m.exports,load); return m.exports; } + table[1] = function(module) { module.exports = require("other"); }; + delete table[3]; + load(2); + })();`); + expect(values).toHaveLength(2); + expect(values).toEqual([ + expect.objectContaining({ kind: "unknown" }), + expect.objectContaining({ kind: "unknown" }), + ]); +}); + +it.each([ + `const electron=require("electron"); const ipc=electron.ipcMain; ipc.handle=()=>null; inspect(electron.ipcMain.handle);`, + `require("electron").ipcMain.handle=()=>null; inspect(require("electron").ipcMain.handle);`, + `const electron=require("electron"); ({ipcMain:electron.ipcMain}=replacement); inspect(electron.ipcMain);`, + `const electron=require("electron"); for(electron.ipcMain of replacements){} inspect(electron.ipcMain);`, +])("invalidates visibly overwritten Electron members: %s", (source) => { + const values = inspectedValues(source); + expect(values).toHaveLength(1); + expect(values[0]?.kind).toBe("unknown"); +}); + +it("does not substitute a lexical or var initializer before its source write", () => { + for (const source of [ + `inspect(value); const value = require("electron");`, + `function run() { inspect(value); var value = require("electron"); }`, + ]) { + const values = inspectedValues(source); + expect(values).toHaveLength(1); + const value = values[0]; + if (value === undefined) throw new Error("inspect call result missing"); + expect(value).toMatchObject({ kind: "unknown" }); + } +}); + +it("keeps function declarations hoisted and checks closures at invocation", () => { + const inspected = inspectedValues(` + inspect(read); + function read() { return value; } + function localRead() { + const local = require("electron"); + return local; + } + const closure = () => value; + const value = require("electron"); + inspect(closure()); + inspect(localRead()); + `); + expect(inspected[0]).toMatchObject({ kind: "function" }); + expect(inspected[1]).toEqual({ kind: "external", path: ["electron"] }); + expect(inspected[2]).toEqual({ kind: "external", path: ["electron"] }); +}); + +it("keeps a synchronous cross-function callback before initialization unresolved", () => { + const { parsed, resolver } = resolverFor(`function outer() { + const value = require("electron"); + return use(() => value); + } + function use(callback) { return callback(); } + inspect(outer());`); + const outer = parsed.program.body.find( + (statement): statement is t.FunctionDeclaration => + t.isFunctionDeclaration(statement) && statement.id?.name === "outer", + ); + const outerCall = parsed.program.body.find( + (statement): statement is t.ExpressionStatement => + t.isExpressionStatement(statement) && + t.isCallExpression(statement.expression) && + t.isIdentifier(statement.expression.callee, { name: "inspect" }), + ); + if (!outer || !outerCall || !t.isCallExpression(outerCall.expression)) + throw new Error("outer fixture missing"); + const invocation = outer.body.body.find( + (statement): statement is t.ReturnStatement => + t.isReturnStatement(statement), + ); + if (!invocation || !t.isCallExpression(invocation.argument)) + throw new Error("synchronous callback call missing"); + const result = resolver.evaluate( + invocation.argument, + resolver.bind( + { + kind: "function", + node: outer, + environment: { values: new Map() }, + origins: [], + }, + [], + outerCall.expression, + ), + ); + expect(result).toMatchObject({ kind: "unknown" }); +}); + +it("uses completed activation provenance for a returned closure", () => { + const result = inspectOne(`inspect(use(outer())); + function use(callback) { return callback(); } + function outer() { + const value = require("electron"); + return () => value; + }`); + expect(result).toEqual({ kind: "external", path: ["electron"] }); +}); + +it("does not treat statements after a returned closure as initialized", () => { + const result = inspectOne(`inspect(outer()()); + function outer() { + return () => value; + const value = require("electron"); + }`); + expect(result).toMatchObject({ kind: "unknown" }); +}); + +it("does not treat statements after a returned module factory as initialized", () => { + const result = inspectOne(`(() => { + const modules = { + 5(module, exports) { + exports.read = () => electron; + return; + const electron = require("electron"); + }, + 9(module, exports, load) { const helper = load(5); inspect(helper.read()); } + }; + function load(id) { const m={exports:{}}; modules[id](m,m.exports,load); return m.exports; } + load(9); + })();`); + expect(result).toMatchObject({ kind: "unknown" }); +}); + +it("treats known symbolic records and loaders as truthy", () => { + expect(symbolicTruth({ kind: "record", values: new Map() })).toBe(true); + expect(symbolicTruth({ kind: "loader", modules: new Map() })).toBe(true); + expect( + symbolicTruth({ + kind: "union", + values: [ + { kind: "record", values: new Map() }, + { kind: "loader", modules: new Map() }, + ], + }), + ).toBe(true); +}); + +it("retains branch predicates on conditionally selected wrapper functions", () => { + const value = inspectOne(` + const selected = enabled + ? () => navigator.clipboard.writeText("yes") + : () => navigator.clipboard.writeText("no"); + inspect(selected); + `); + const functions = symbolicVariants(value).filter( + (candidate) => candidate.kind === "function", + ); + expect(functions).toHaveLength(2); + expect(functions.map((fn) => fn.conditions?.[0]?.positive)).toEqual([ + true, + false, + ]); +}); + +it("retains fallthrough return guards after a one-sided early return", () => { + const value = inspectOne(` + function choose() { + if (enabled) return () => navigator.clipboard.writeText("early"); + return () => navigator.clipboard.writeText("late"); + } + inspect(choose()); + `); + const functions = symbolicVariants(value).filter( + (candidate) => candidate.kind === "function", + ); + expect(functions).toHaveLength(2); + expect(functions[0]).toMatchObject({ conditions: [{ positive: true }] }); + expect(functions[1]).toMatchObject({ conditions: [{ positive: false }] }); +}); + +it("composes fallthrough guards across successive conditional returns", () => { + const value = inspectOne(` + function choose() { + if (first) return () => navigator.clipboard.writeText("first"); + if (second) return () => navigator.clipboard.writeText("second"); + return () => navigator.clipboard.writeText("last"); + } + inspect(choose()); + `); + const functions = symbolicVariants(value).filter( + (candidate) => candidate.kind === "function", + ); + expect(functions).toHaveLength(3); + expect(functions[0]?.conditions?.map(({ positive }) => positive)).toEqual([ + true, + ]); + expect(functions[1]?.conditions?.map(({ positive }) => positive)).toEqual([ + false, + true, + ]); + expect(functions[2]?.conditions?.map(({ positive }) => positive)).toEqual([ + false, + false, + ]); +}); + +it("marks return guards unresolved when a nested join cannot be represented", () => { + const value = inspectOne(` + function choose() { + if (enabled) { + if (other) return () => navigator.clipboard.writeText("early"); + } + return () => navigator.clipboard.writeText("late"); + } + inspect(choose()); + `); + const functions = symbolicVariants(value).filter( + (candidate) => candidate.kind === "function", + ); + expect(functions).toHaveLength(2); + expect(functions[1]).toMatchObject({ conditionUnresolved: true }); +}); + +it.each([ + [ + "duplicate literal keys use the last value", + `const object = { value: "first", value: "last" }; inspect(object.value);`, + { kind: "literal", value: "last" }, + ], + [ + "a final accessor is unknown", + `const object = { value: "first", get value() { return "accessor"; } }; inspect(object.value);`, + { kind: "unknown", reason: "accessor property" }, + ], + [ + "a later literal overwrites an accessor", + `const object = { get value() { return "accessor"; }, value: "last" }; inspect(object.value);`, + { kind: "literal", value: "last" }, + ], + [ + "a later unknown computed key can overwrite an earlier value", + `const object = { value: "known", [dynamic]: "other" }; inspect(object.value);`, + { kind: "unknown", reason: "dynamic property overwrite" }, + ], + [ + "a later static key overrides an earlier unknown computed key", + `const object = { [dynamic]: "other", value: "last" }; inspect(object.value);`, + { kind: "literal", value: "last" }, + ], + [ + "a known spread can override an earlier static value", + `const replacement = { value: "spread" }; const object = { value: "first", ...replacement }; inspect(object.value);`, + { kind: "literal", value: "spread" }, + ], +] as const)("resolves object properties when %s", (_case, source, expected) => { + expect(inspectOne(source)).toEqual(expected); +}); + +it("propagates branch guards through object properties and function calls", () => { + const inspected = inspectedValues(` + const direct = flag + ? { action: () => navigator.clipboard.writeText("direct-yes") } + : { action: () => navigator.clipboard.writeText("direct-no") }; + function choose() { + return flag + ? () => navigator.clipboard.writeText("call-yes") + : () => navigator.clipboard.writeText("call-no"); + } + const called = { action: choose() }; + inspect(direct.action); + inspect(called.action); + `); + for (const value of inspected) { + const functions = symbolicVariants(value).filter( + (candidate) => candidate.kind === "function", + ); + expect(functions).toHaveLength(2); + expect(functions.map((fn) => fn.conditions?.[0]?.positive)).toEqual([ + true, + false, + ]); + } +}); diff --git a/src/domain/javascript/javascriptSymbolicBindings.ts b/src/domain/javascript/javascriptSymbolicBindings.ts new file mode 100644 index 000000000..b1f37269f --- /dev/null +++ b/src/domain/javascript/javascriptSymbolicBindings.ts @@ -0,0 +1,989 @@ +import * as t from "@babel/types"; +import { + semanticStaticPropertyKey, + unwrapJavaScriptExpression, +} from "./javascriptAstValues.js"; +import { javascriptBootstrapAt } from "./javascriptBootstrap.js"; +import { ELECTRON_MODULE } from "./javascriptElectronMemberWrites.js"; +import { + indexJavaScriptElectronIdentitySteps, + type JavaScriptElectronIdentity, +} from "./javascriptSemanticAnalysis.js"; +import { symbolicLogicalRight } from "./javascriptSymbolicControlFlow.js"; +import { + moduleFactory, + modulePropertyName, +} from "./javascriptStaticAnalysisHelpers.js"; +import { + childNodes, + completeSemanticSteps, + traverseJavaScriptAst, +} from "./javascriptSemanticTraversal.js"; +import { + resolveSemanticBindingState, + semanticResolutionBlocked, + type JavaScriptSemanticAnalysisState, +} from "./javascriptSemanticState.js"; +import { + symbolicCondition, + symbolicUnion, + symbolicUnknown, + symbolicVariants, + symbolicConditionsUnresolved, + symbolicTruth, + type SymbolicCondition, + type SymbolicValue, + type SymbolicEnvironment, + type SymbolicFunction, +} from "./javascriptSymbolicValues.js"; + +interface PropertyWrite { + readonly path: readonly string[]; + readonly value: t.Node; + readonly site: t.Node; +} +interface SymbolicReturnFlow { + readonly values: readonly SymbolicValue[]; + readonly fallsThrough: boolean; +} +const emptyEnvironment: SymbolicEnvironment = { values: new Map() }; + +/** Demand-driven substitutions for local bindings, closures and literal module tables. */ +export class JavaScriptSymbolicBindings { + readonly #writes = new Map(); + readonly #environments = new WeakMap(); + readonly #cache = new WeakMap< + SymbolicEnvironment, + WeakMap + >(); + readonly #active = new Set(); + readonly #activeFunctions = new Set(); + readonly #exportAliases = new Map(); + readonly #moduleBindings = new Set(); + readonly #replacements = new Map(); + readonly #propertyActive = new WeakMap>(); + readonly #literalPropertyIndex = new WeakMap< + t.ObjectExpression, + ReadonlyMap | null + >(); + readonly #mutated = new Set(); + readonly #electronIdentity: JavaScriptElectronIdentity; + readonly #namespaceOwners = new Map(); + readonly #unsafeExports = new Set(); + readonly #aliasesByBinding = new Map(); + readonly #bootstrapTables: Array<{ + readonly tableAliases: ReadonlySet; + readonly modules: Map; + }> = []; + #remainingEvaluations = 1_000_000; + #depth = 0; + truncated = false; + unresolved = false; + + constructor( + readonly state: JavaScriptSemanticAnalysisState, + program: t.Program, + electronIdentity = completeSemanticSteps( + indexJavaScriptElectronIdentitySteps(program, state), + ), + ) { + this.#electronIdentity = electronIdentity; + this.#indexBindingAliases(); + traverseJavaScriptAst(program, { + enter: (node) => { + if (t.isCallExpression(node)) this.#indexBootstrap(node); + }, + }); + traverseJavaScriptAst(program, { + enter: (node) => { + if (t.isAssignmentExpression(node)) + this.#indexWrite(node.left, node.right, node); + if (t.isUpdateExpression(node)) + this.#indexWrite(node.argument, node, node); + if (t.isUnaryExpression(node, { operator: "delete" })) + this.#indexWrite(node.argument, node, node); + if ( + t.isCallExpression(node) && + t.isMemberExpression(node.callee) && + t.isIdentifier(node.callee.object, { name: "Object" }) && + !resolveSemanticBindingState(state, node.callee.object, "Object") && + semanticStaticPropertyKey( + node.callee.property, + node.callee.computed, + ) === "defineProperty" + ) { + const [target, key, descriptor] = node.arguments; + if ( + t.isIdentifier(target) && + t.isStringLiteral(key) && + t.isObjectExpression(descriptor) + ) { + const value = descriptor.properties.find( + (p) => + t.isObjectProperty(p) && + semanticStaticPropertyKey(p.key, p.computed) === "value", + ); + if (t.isObjectProperty(value)) + this.#addWrite(target, [key.value], value.value, node); + } + } + }, + }); + for (const table of this.#bootstrapTables) + this.#invalidateBootstrapTable(table.tableAliases, table.modules); + } + + environmentAt(node: t.Node): SymbolicEnvironment { + let current: t.Node | undefined = node; + while (current) { + const found = this.#environments.get(current); + if (found) return found; + current = this.state.parentsByNode.get(current); + } + return emptyEnvironment; + } + + evaluate( + input: t.Node | null | undefined, + environment?: SymbolicEnvironment, + ): SymbolicValue { + if (!input) return { kind: "literal", value: undefined }; + if (--this.#remainingEvaluations < 0 || this.#depth >= 128) { + this.truncated = true; + return symbolicUnknown("symbolic resource limit"); + } + const node = unwrapJavaScriptExpression(input).node; + const env = environment ?? this.environmentAt(node); + const cache = this.#cache.get(env) ?? new WeakMap(); + this.#cache.set(env, cache); + const cached = cache.get(node); + if (cached) return cached; + if (this.#active.has(node)) return symbolicUnknown("cyclic binding"); + this.#active.add(node); + this.#depth += 1; + try { + const value = this.#evaluate(node, env); + cache.set(node, value); + return value; + } finally { + this.#active.delete(node); + this.#depth -= 1; + } + } + + property(value: SymbolicValue, key: string): SymbolicValue { + let result: SymbolicValue; + if (value.kind === "union") + result = symbolicUnion( + value.values.map((variant) => this.property(variant, key)), + ); + else result = this.#property(value, key); + for (const condition of value.conditions ?? []) + result = symbolicCondition(result, condition); + if (value.conditionUnresolved) + result = symbolicConditionsUnresolved(result); + return result; + } + + #property(value: SymbolicValue, key: string): SymbolicValue { + const active = this.#propertyActive.get(value) ?? new Set(); + if (active.has(key)) return symbolicUnknown("cyclic property"); + active.add(key); + this.#propertyActive.set(value, active); + try { + return this.#propertyValue(value, key); + } finally { + active.delete(key); + } + } + + #propertyValue(value: SymbolicValue, key: string): SymbolicValue { + if (value.kind === "external") { + const path = [...value.path, key]; + if ( + ELECTRON_MODULE.test(path[0] ?? "") && + !this.#electronIdentity.intactExport(path) + ) { + return symbolicUnknown("overwritten Electron API member"); + } + return { kind: "external", path }; + } + if (value.kind === "record") + return value.values.get(key) ?? symbolicUnknown("missing property"); + if (value.kind === "namespace") { + if ( + this.#mutated.has(value.bindingId) || + this.#unsafeExports.has(`${value.bindingId}:${key}`) + ) + return symbolicUnknown("ambiguous export writes"); + const replacement = this.#replacements.get(value.bindingId); + if (replacement) + return this.property( + this.evaluate(replacement, value.environment), + key, + ); + const writes = (this.#writes.get(value.bindingId) ?? []).filter( + (w) => w.path.length === 1 && w.path[0] === key, + ); + const write = writes.at(-1); + return write + ? this.evaluate(write.value, value.environment) + : symbolicUnknown("unresolved module export"); + } + if (value.kind === "object") { + let index = this.#literalPropertyIndex.get(value.node); + if (index === undefined) { + const properties = new Map(); + index = properties; + for (const property of value.node.properties) { + const name = t.isSpreadElement(property) + ? null + : semanticStaticPropertyKey(property.key, property.computed); + if (name === null || t.isSpreadElement(property)) { + index = null; + break; + } + properties.set(name, property); + } + this.#literalPropertyIndex.set(value.node, index); + } + if (index !== null) { + const property = index.get(key); + return property === undefined + ? symbolicUnknown("missing property") + : this.#objectPropertyValue(property, value.environment); + } + for (const p of [...value.node.properties].reverse()) { + if (t.isSpreadElement(p)) { + const spread = this.evaluate(p.argument, value.environment); + const found = this.property(spread, key); + if (found.kind !== "unknown") return found; + if ( + this.unknownMembers(spread) > 0 || + this.members(spread).includes(key) + ) + return found; + } else if (semanticStaticPropertyKey(p.key, p.computed) === key) { + return this.#objectPropertyValue(p, value.environment); + } else if (semanticStaticPropertyKey(p.key, p.computed) === null) + return symbolicUnknown("dynamic property overwrite"); + } + } + if (value.kind === "array" && /^\d+$/u.test(key)) + return value.values[Number(key)] ?? symbolicUnknown("missing argument"); + return symbolicUnknown("unresolved property"); + } + + #objectPropertyValue( + property: t.ObjectMethod | t.ObjectProperty, + environment: SymbolicEnvironment, + ): SymbolicValue { + return t.isObjectMethod(property) && property.kind !== "method" + ? symbolicUnknown("accessor property") + : this.evaluate( + t.isObjectProperty(property) ? property.value : property, + environment, + ); + } + + members( + value: SymbolicValue, + active = new Set(), + ): readonly string[] { + if (active.has(value)) return []; + active.add(value); + if (value.kind === "union") + return [...new Set(value.values.flatMap((v) => this.members(v, active)))]; + if (value.kind === "record") return [...value.values.keys()]; + if (value.kind === "namespace") { + const replacement = this.#replacements.get(value.bindingId); + return replacement + ? this.members(this.evaluate(replacement, value.environment), active) + : [ + ...new Set( + (this.#writes.get(value.bindingId) ?? []).flatMap((w) => + w.path.length === 1 ? w.path : [], + ), + ), + ]; + } + if (value.kind !== "object") return []; + return [ + ...new Set( + value.node.properties.flatMap((p) => + t.isSpreadElement(p) + ? this.members(this.evaluate(p.argument, value.environment), active) + : [semanticStaticPropertyKey(p.key, p.computed)].filter( + (k): k is string => k !== null, + ), + ), + ), + ]; + } + + unknownMembers( + value: SymbolicValue, + active = new Set(), + ): number { + if (active.has(value)) return 1; + active.add(value); + if (value.kind === "union") + return value.values.reduce( + (sum, v) => sum + this.unknownMembers(v, active), + 0, + ); + if (value.kind === "record") return 0; + if (value.kind === "namespace") { + if (this.#mutated.has(value.bindingId)) return 1; + const replacement = this.#replacements.get(value.bindingId); + return replacement + ? this.unknownMembers( + this.evaluate(replacement, value.environment), + active, + ) + : 0; + } + if (value.kind !== "object") return 1; + return value.node.properties.reduce( + (sum, p) => + sum + + (t.isSpreadElement(p) + ? this.unknownMembers( + this.evaluate(p.argument, value.environment), + new Set(active), + ) + : Number(semanticStaticPropertyKey(p.key, p.computed) === null)), + 0, + ); + } + + bind( + fn: SymbolicFunction, + args: readonly SymbolicValue[], + invocationNode?: t.Node, + ): SymbolicEnvironment { + const values = new Map(); + fn.node.params.forEach((param, index) => { + const target = t.isRestElement(param) ? param.argument : param; + if (!t.isIdentifier(target)) return; + const binding = resolveSemanticBindingState( + this.state, + target, + target.name, + ); + if (binding) + values.set( + binding.bindingId, + t.isRestElement(param) + ? { kind: "array", values: args.slice(index) } + : (args[index] ?? symbolicUnknown("runtime parameter")), + ); + }); + const scopeId = + this.state.scopeByNode.get(fn.node)?.scopeId ?? + this.state.scopeByNode.get(fn.node.body)?.scopeId; + return { + values, + parent: fn.environment, + point: invocationNode ?? fn.node, + scopeNode: fn.node, + ...(scopeId === undefined ? {} : { scopeId }), + }; + } + + arguments( + node: t.CallExpression | t.OptionalCallExpression, + env: SymbolicEnvironment, + ): readonly SymbolicValue[] { + return node.arguments.flatMap((a) => { + if (!t.isSpreadElement(a)) return [this.evaluate(a, env)]; + const value = this.evaluate(a.argument, env); + return value.kind === "array" + ? value.values + : [symbolicUnknown("dynamic spread arguments")]; + }); + } + + #evaluate(node: t.Node, env: SymbolicEnvironment): SymbolicValue { + if ( + t.isStringLiteral(node) || + t.isNumericLiteral(node) || + t.isBooleanLiteral(node) + ) + return { kind: "literal", value: node.value }; + if (t.isNullLiteral(node)) return { kind: "literal", value: null }; + if (t.isFunction(node)) + return { kind: "function", node, environment: env, origins: [] }; + if (t.isObjectExpression(node)) + return { kind: "object", node, environment: env }; + if (t.isIdentifier(node)) return this.#identifier(node, env); + if (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) { + const key = semanticStaticPropertyKey(node.property, node.computed); + return key === null + ? symbolicUnknown("dynamic member") + : this.property(this.evaluate(node.object, env), key); + } + if (t.isSequenceExpression(node)) + return this.evaluate(node.expressions.at(-1), env); + if (t.isAssignmentExpression(node)) return this.evaluate(node.right, env); + if (t.isAwaitExpression(node)) return this.evaluate(node.argument, env); + if (t.isUnaryExpression(node)) { + const value = this.evaluate(node.argument, env); + if (node.operator === "void") + return { kind: "literal", value: undefined }; + const truth = symbolicTruth(value); + if (node.operator === "!" && truth !== undefined) + return { kind: "literal", value: !truth }; + } + if (t.isConditionalExpression(node)) { + const truth = symbolicTruth(this.evaluate(node.test, env)); + return truth === undefined + ? symbolicUnion([ + symbolicCondition(this.evaluate(node.consequent, env), { + node: node.test, + positive: true, + }), + symbolicCondition(this.evaluate(node.alternate, env), { + node: node.test, + positive: false, + }), + ]) + : this.evaluate(truth ? node.consequent : node.alternate, env); + } + if (t.isLogicalExpression(node)) { + const left = this.evaluate(node.left, env); + const right = symbolicLogicalRight(node.operator, left); + if (right !== undefined) + return right ? this.evaluate(node.right, env) : left; + const leftCondition: SymbolicCondition = { + node: node.left, + positive: + node.operator === "&&" + ? false + : node.operator === "||" + ? true + : "not-nullish", + }; + const rightCondition: SymbolicCondition = { + node: node.left, + positive: + node.operator === "&&" + ? true + : node.operator === "||" + ? false + : "nullish", + }; + return symbolicUnion([ + symbolicCondition(left, leftCondition), + symbolicCondition(this.evaluate(node.right, env), rightCondition), + ]); + } + if (t.isCallExpression(node) || t.isOptionalCallExpression(node)) + return this.#call(node, env); + return symbolicUnknown(`unsupported ${node.type}`); + } + + #identifier(node: t.Identifier, env: SymbolicEnvironment): SymbolicValue { + const binding = resolveSemanticBindingState(this.state, node, node.name); + if ( + !binding && + node.name === "undefined" && + !semanticResolutionBlocked(this.state, node, node.name) + ) + return { kind: "literal", value: undefined }; + if (!binding) + return !semanticResolutionBlocked(this.state, node, node.name) && + ["require", "navigator", "String", "Boolean"].includes(node.name) + ? { kind: "external", path: [node.name] } + : symbolicUnknown("unbound identifier"); + if (binding.definitions.some((d) => d.kind === "assignment")) + return symbolicUnknown("reassigned binding"); + // Electron member identity is checked at the selected property. A write + // to one method must not erase the module origin or unaffected siblings. + const electronOrigin = this.#electronIdentity.exportPath(node); + if (electronOrigin === undefined && this.#mutated.has(binding.bindingId)) + return symbolicUnknown("dynamic member writes"); + for (let e: SymbolicEnvironment | undefined = env; e; e = e.parent) { + const value = e.values.get(binding.bindingId); + if (value) return value; + } + if (electronOrigin === undefined && this.#writes.has(binding.bindingId)) + return symbolicUnknown("overwritten object members"); + const initializer = + binding.initializers.length === 1 ? binding.initializers[0] : undefined; + if (!initializer) + return symbolicUnknown("missing or ambiguous initializer"); + const point = this.#initializerReadPoint(binding.scopeId, env, node); + if ( + !initializer.entryBody && + !binding.definitions.every(({ kind }) => kind === "function") && + (initializer.node.end == null || + point == null || + initializer.node.end > point) + ) { + this.unresolved = true; + return symbolicUnknown("initializer not available at this read"); + } + if (this.state.conditionalInitializers.has(initializer.node)) + return symbolicUnknown("conditional initializer"); + let value = this.evaluate(initializer.node, env); + for (const key of initializer.projection) { + if (key === null) return symbolicUnknown("dynamic destructuring"); + value = this.property(value, String(key)); + } + return value; + } + + #initializerReadPoint( + bindingScopeId: string, + environment: SymbolicEnvironment, + readNode: t.Node, + ): number | undefined { + let current: SymbolicEnvironment | undefined = environment; + let child: SymbolicEnvironment | undefined; + while (current) { + if (this.#scopeContains(bindingScopeId, current.scopeId)) { + const point = child?.point ?? readNode; + if (current.completedAt !== undefined) + return current.completedAt.end ?? undefined; + if (current.scopeNode && !this.#nodeWithin(point, current.scopeNode)) + return undefined; + return point.start ?? undefined; + } + child = current; + current = current.parent; + } + let outermost = environment; + while (outermost.parent) outermost = outermost.parent; + const point = outermost.point ?? environment.point ?? readNode; + return point.start ?? undefined; + } + + #scopeContains(bindingScopeId: string, environmentScopeId?: string): boolean { + let scopeId: string | null = bindingScopeId; + while (scopeId !== null) { + if (scopeId === environmentScopeId) return true; + scopeId = this.state.scopesById.get(scopeId)?.parentScopeId ?? null; + } + return false; + } + + #nodeWithin(node: t.Node, ancestor?: t.Node): boolean { + if (!ancestor) return false; + let current: t.Node | undefined = node; + while (current) { + if (current === ancestor) return true; + current = this.state.parentsByNode.get(current); + } + return false; + } + + #completedEnvironment( + environment: SymbolicEnvironment, + activation: SymbolicEnvironment, + returnPoint: t.Node, + ): SymbolicEnvironment | undefined { + if (environment === activation) + return { ...environment, completedAt: returnPoint }; + if (environment.parent === undefined) return undefined; + const parent = this.#completedEnvironment( + environment.parent, + activation, + returnPoint, + ); + return parent === undefined ? undefined : { ...environment, parent }; + } + + #completeReturnedFunction( + value: SymbolicFunction, + activation: SymbolicEnvironment, + returnPoint: t.Node, + ): SymbolicFunction { + const environment = this.#completedEnvironment( + value.environment, + activation, + returnPoint, + ); + if (environment === undefined) return value; + const { returnPoint: _returnPoint, ...returned } = value; + return { ...returned, environment }; + } + + #moduleCompletionPoint(factory: t.Function): t.Node { + let earliestReturn: t.ReturnStatement | undefined; + const pending: t.Node[] = [factory.body]; + while (pending.length > 0) { + const node = pending.pop(); + if (!node) continue; + if (node !== factory.body && (t.isFunction(node) || t.isClass(node))) + continue; + if ( + t.isReturnStatement(node) && + (earliestReturn === undefined || + (node.start ?? Infinity) < (earliestReturn.start ?? Infinity)) + ) + earliestReturn = node; + pending.push(...childNodes(node)); + } + return earliestReturn ?? factory.body; + } + + #call( + node: t.CallExpression | t.OptionalCallExpression, + env: SymbolicEnvironment, + ): SymbolicValue { + const callee = this.evaluate(node.callee, env); + const args = this.arguments(node, env); + return symbolicUnion( + symbolicVariants(callee).map((value) => { + let result: SymbolicValue; + if (value.kind === "loader") { + const key = args[0]; + const loaded = + key?.kind === "literal" + ? (value.modules.get(String(key.value)) ?? + symbolicUnknown("missing module")) + : symbolicUnknown("dynamic module"); + if (loaded.kind === "namespace") { + const factory = this.#namespaceOwners.get(loaded.bindingId); + result = + factory === undefined + ? loaded + : { + ...loaded, + environment: { + ...loaded.environment, + completedAt: this.#moduleCompletionPoint(factory), + }, + }; + } else result = loaded; + } else if ( + value.kind === "external" && + value.path.join(".") === "require" + ) { + const spec = args[0]; + result = + spec?.kind === "literal" && typeof spec.value === "string" + ? { kind: "external", path: [spec.value] } + : symbolicUnknown("dynamic require"); + } else if ( + value.kind !== "function" || + this.#activeFunctions.has(value.node) + ) { + result = symbolicUnknown("unresolved or recursive call"); + } else { + this.#activeFunctions.add(value.node); + try { + const bound = this.bind(value, args, node); + const returned = t.isBlockStatement(value.node.body) + ? this.#returns(value.node.body.body, bound) + : this.evaluate(value.node.body, bound); + result = symbolicUnion( + symbolicVariants(returned).map((v) => + v.kind === "function" + ? { + ...this.#completeReturnedFunction( + v, + bound, + v.returnPoint ?? value.node.body, + ), + origins: [...value.origins, node, ...v.origins], + } + : v, + ), + ); + } finally { + this.#activeFunctions.delete(value.node); + } + } + for (const condition of value.conditions ?? []) + result = symbolicCondition(result, condition); + if (value.conditionUnresolved) + result = symbolicConditionsUnresolved(result); + return result; + }), + ); + } + + #returns( + statements: readonly t.Statement[], + env: SymbolicEnvironment, + ): SymbolicValue { + const flow = this.#returnFlow(statements, env); + return symbolicUnion([ + ...flow.values, + ...(flow.fallsThrough + ? [{ kind: "literal" as const, value: undefined }] + : []), + ]); + } + + #returnFlow( + statements: readonly t.Statement[], + env: SymbolicEnvironment, + depth = 0, + ): SymbolicReturnFlow { + const possible: SymbolicValue[] = []; + let unresolvedFallthrough = false; + const continuationConditions: SymbolicCondition[] = []; + const guardCurrentPath = (value: SymbolicValue): SymbolicValue => { + let guarded = value; + for (const condition of continuationConditions) + guarded = symbolicCondition(guarded, condition); + return unresolvedFallthrough + ? symbolicConditionsUnresolved(guarded) + : guarded; + }; + for (const statement of statements) { + if (--this.#remainingEvaluations < 0 || depth >= 128) { + this.truncated = true; + return { + values: [...possible, symbolicUnknown("symbolic resource limit")], + fallsThrough: false, + }; + } + if (t.isThrowStatement(statement)) + return { + values: [...possible, symbolicUnknown("wrapper exits by throwing")], + fallsThrough: false, + }; + if (t.isReturnStatement(statement)) { + const value = this.evaluate(statement.argument, env); + const returned = symbolicUnion( + symbolicVariants(value).map((variant) => + variant.kind === "function" + ? { ...variant, returnPoint: statement } + : variant, + ), + ); + return { + values: [...possible, guardCurrentPath(returned)], + fallsThrough: false, + }; + } + if (t.isBlockStatement(statement)) { + const flow = this.#returnFlow(statement.body, env, depth + 1); + possible.push(...flow.values.map(guardCurrentPath)); + if (!flow.fallsThrough) + return { values: possible, fallsThrough: false }; + } + if (t.isIfStatement(statement)) { + const truth = symbolicTruth(this.evaluate(statement.test, env)); + const branches = + truth === undefined + ? [statement.consequent, statement.alternate] + : [truth ? statement.consequent : statement.alternate]; + const flows = branches.map((branch) => + this.#returnFlow(branch ? [branch] : [], env, depth + 1), + ); + possible.push( + ...flows.flatMap((flow, index) => + flow.values.map((value) => { + const guarded = guardCurrentPath(value); + return truth === undefined + ? symbolicCondition(guarded, { + node: statement.test, + positive: index === 0, + }) + : guarded; + }), + ), + ); + if (flows.every((flow) => !flow.fallsThrough)) + return { values: possible, fallsThrough: false }; + if (truth === undefined) { + const fallthrough = flows.flatMap((flow, index) => + flow.fallsThrough ? [{ flow, index }] : [], + ); + if ( + fallthrough.length === 1 && + fallthrough[0]?.flow.values.length === 0 + ) + continuationConditions.push({ + node: statement.test, + positive: fallthrough[0].index === 0, + }); + else if ( + fallthrough.length > 0 && + (fallthrough.length < flows.length || + fallthrough.some(({ flow }) => flow.values.length > 0)) + ) + unresolvedFallthrough = true; + } + } + } + return { values: possible, fallsThrough: true }; + } + + #indexBootstrap(call: t.CallExpression): void { + const bootstrap = javascriptBootstrapAt(call, this.state); + if (!bootstrap) return; + const modules = new Map(); + const tableAliases = new Set([bootstrap.tableBindingId]); + const pendingAliases = [bootstrap.tableBindingId]; + while (pendingAliases.length > 0) { + const source = pendingAliases.pop()!; + for (const alias of this.#aliasesByBinding.get(source) ?? []) + if (!tableAliases.has(alias)) { + tableAliases.add(alias); + pendingAliases.push(alias); + } + } + for (const property of bootstrap.table.properties) { + const factory = moduleFactory(property); + if (!factory) continue; + const moduleKey = modulePropertyName(property); + const values = new Map(); + const environment: SymbolicEnvironment = { + values, + point: factory.body, + scopeNode: factory, + ...(this.state.scopeByNode.get(factory)?.scopeId === undefined + ? {} + : { scopeId: this.state.scopeByNode.get(factory)!.scopeId }), + }; + const first = factory.params[0]; + const second = factory.params[1]; + const third = factory.params[2]; + const binding = (n: t.Node | undefined) => + t.isIdentifier(n) + ? resolveSemanticBindingState(this.state, n, n.name)?.bindingId + : undefined; + const moduleId = binding(first); + const exportsId = binding(second); + const loaderId = binding(third); + const namespaceId = `${String(factory.start)}:exports`; + this.#namespaceOwners.set(namespaceId, factory); + const namespace: SymbolicValue = { + kind: "namespace", + bindingId: namespaceId, + environment, + }; + if (moduleId) { + values.set(moduleId, { + kind: "record", + values: new Map([["exports", namespace]]), + }); + this.#exportAliases.set(moduleId, namespaceId); + this.#moduleBindings.add(moduleId); + } + if (exportsId) { + values.set(exportsId, namespace); + this.#exportAliases.set(exportsId, namespaceId); + } + if (loaderId) values.set(loaderId, { kind: "loader", modules }); + this.#environments.set(factory, environment); + modules.set(moduleKey, namespace); + } + this.#bootstrapTables.push({ tableAliases, modules }); + } + + #indexBindingAliases(): void { + for (const binding of this.state.bindingsById.values()) { + const initializer = binding.initializers[0]; + if ( + !initializer || + initializer.entryBody !== undefined || + initializer.projection.length !== 0 || + !t.isIdentifier(initializer.node) + ) + continue; + const source = resolveSemanticBindingState( + this.state, + initializer.node, + initializer.node.name, + ); + if (!source) continue; + const aliases = this.#aliasesByBinding.get(source.bindingId) ?? []; + aliases.push(binding.bindingId); + this.#aliasesByBinding.set(source.bindingId, aliases); + } + } + + #invalidateBootstrapTable( + tableAliases: ReadonlySet, + modules: Map, + ): void { + let invalidateAll = false; + const invalidSlots = new Set(); + for (const bindingId of tableAliases) { + if (this.#mutated.has(bindingId)) invalidateAll = true; + for (const definition of this.state.bindingsById.get(bindingId) + ?.definitions ?? []) + if (definition.kind === "assignment") invalidateAll = true; + for (const write of this.#writes.get(bindingId) ?? []) { + if (write.path.length === 0) invalidateAll = true; + else invalidSlots.add(write.path[0]!); + } + } + if (invalidateAll) modules.clear(); + else for (const key of invalidSlots) modules.delete(key); + } + + #indexWrite(target: t.Node, value: t.Node, site: t.Node): void { + const path: string[] = []; + let root = target; + let dynamic = false; + while (t.isMemberExpression(root)) { + const key = semanticStaticPropertyKey(root.property, root.computed); + if (key === null) dynamic = true; + else path.unshift(key); + root = root.object; + } + if (dynamic && t.isIdentifier(root)) { + const binding = resolveSemanticBindingState(this.state, root, root.name); + if (binding) + this.#mutated.add( + this.#exportAliases.get(binding.bindingId) ?? binding.bindingId, + ); + return; + } + if (t.isIdentifier(root)) this.#addWrite(root, path, value, site); + } + + #addWrite( + root: t.Identifier, + path: readonly string[], + value: t.Node, + site: t.Node, + ): void { + const binding = resolveSemanticBindingState(this.state, root, root.name); + if (!binding) return; + const alias = this.#exportAliases.get(binding.bindingId); + if (this.#moduleBindings.has(binding.bindingId) && path[0] !== "exports") + return; + const key = alias ?? binding.bindingId; + const selected = this.#moduleBindings.has(binding.bindingId) + ? path.slice(1) + : path; + if (alias) { + const owner = this.#namespaceOwners.get(alias); + for ( + let parent = this.state.parentsByNode.get(site); + parent && parent !== owner; + parent = this.state.parentsByNode.get(parent) + ) { + if ( + t.isFunction(parent) || + t.isIfStatement(parent) || + t.isConditionalExpression(parent) || + t.isLogicalExpression(parent) || + t.isLoop(parent) || + t.isSwitchCase(parent) || + t.isTryStatement(parent) + ) { + if (selected.length === 0) this.#mutated.add(key); + else this.#unsafeExports.add(`${key}:${selected[0] ?? ""}`); + return; + } + } + } + if (alias && selected.length === 0) { + this.#replacements.set(key, value); + return; + } + const writes = this.#writes.get(key) ?? []; + writes.push({ path: selected, value, site }); + this.#writes.set(key, writes); + } +} diff --git a/src/domain/javascript/javascriptSymbolicControlFlow.ts b/src/domain/javascript/javascriptSymbolicControlFlow.ts new file mode 100644 index 000000000..79bccd0a4 --- /dev/null +++ b/src/domain/javascript/javascriptSymbolicControlFlow.ts @@ -0,0 +1,53 @@ +import * as t from "@babel/types"; +import { + symbolicTruth, + type SymbolicValue, +} from "./javascriptSymbolicValues.js"; + +/** Whether short-circuit evaluation reaches the right operand; unknown stays unknown. */ +export const symbolicLogicalRight = ( + operator: t.LogicalExpression["operator"], + left: SymbolicValue, +): boolean | undefined => { + if (left.kind === "union") { + const choices = left.values.map((value) => + symbolicLogicalRight(operator, value), + ); + return choices.length > 0 && + choices.every((choice) => choice === choices[0]) + ? choices[0] + : undefined; + } + if (operator === "??") { + if (left.kind === "literal") + return left.value === null || left.value === undefined; + return [ + "function", + "object", + "namespace", + "array", + "record", + "loader", + ].includes(left.kind) + ? false + : undefined; + } + const truth = symbolicTruth(left); + return truth === undefined ? undefined : operator === "&&" ? truth : !truth; +}; + +/** Only definite synchronous exits; unsupported control flow stays unknown. */ +export const symbolicDefiniteExit = ( + node: t.Node | null | undefined, + depth = 0, +): boolean => { + if (!node || depth >= 128) return false; + if (t.isReturnStatement(node) || t.isThrowStatement(node)) return true; + if (t.isBlockStatement(node)) + return node.body.some((child) => symbolicDefiniteExit(child, depth + 1)); + return ( + t.isIfStatement(node) && + symbolicDefiniteExit(node.consequent, depth + 1) && + symbolicDefiniteExit(node.alternate, depth + 1) + ); +}; diff --git a/src/domain/javascript/javascriptSymbolicValues.ts b/src/domain/javascript/javascriptSymbolicValues.ts new file mode 100644 index 000000000..f267c2209 --- /dev/null +++ b/src/domain/javascript/javascriptSymbolicValues.ts @@ -0,0 +1,118 @@ +import type * as t from "@babel/types"; + +/** A lexical substitution environment, never a JavaScript execution context. */ +export interface SymbolicEnvironment { + readonly values: ReadonlyMap; + readonly parent?: SymbolicEnvironment; + /** Source point at which the current function is being symbolically executed. */ + readonly point?: t.Node; + /** Lexical function scope for point-sensitive local initializer checks. */ + readonly scopeId?: string; + /** Function activation captured by this environment, when one exists. */ + readonly scopeNode?: t.Function; + /** Completion boundary proven for this captured activation. */ + readonly completedAt?: t.Node; +} + +export interface SymbolicCondition { + readonly node: t.Node; + readonly positive: boolean | "nullish" | "not-nullish"; +} + +export interface SymbolicMetadata { + /** Predicates required for this value to be selected. */ + readonly conditions?: readonly SymbolicCondition[]; + /** A control-flow join prevented a sound predicate from being retained. */ + readonly conditionUnresolved?: true; +} + +export interface SymbolicFunction extends SymbolicMetadata { + readonly kind: "function"; + readonly node: t.Function; + readonly environment: SymbolicEnvironment; + readonly origins: readonly t.Node[]; + /** Return statement completing the activation that produced this value. */ + readonly returnPoint?: t.Node; +} + +export type SymbolicValue = ( + | { readonly kind: "unknown"; readonly reason: string } + | { + readonly kind: "literal"; + readonly value: string | number | boolean | null | undefined; + } + | { readonly kind: "external"; readonly path: readonly string[] } + | { + readonly kind: "object"; + readonly node: t.ObjectExpression; + readonly environment: SymbolicEnvironment; + } + | { + readonly kind: "namespace"; + readonly bindingId: string; + readonly environment: SymbolicEnvironment; + } + | { + readonly kind: "loader"; + readonly modules: ReadonlyMap; + } + | { readonly kind: "array"; readonly values: readonly SymbolicValue[] } + | { + readonly kind: "record"; + readonly values: ReadonlyMap; + } + | { readonly kind: "union"; readonly values: readonly SymbolicValue[] } + | SymbolicFunction +) & + SymbolicMetadata; + +export const symbolicUnknown = (reason: string): SymbolicValue => ({ + kind: "unknown", + reason, +}); +export const symbolicVariants = ( + value: SymbolicValue, +): readonly SymbolicValue[] => + value.kind === "union" ? value.values : [value]; +export const symbolicUnion = ( + values: readonly SymbolicValue[], +): SymbolicValue => { + const unique = [...new Set(values.flatMap(symbolicVariants))]; + return unique.length === 1 ? unique[0]! : { kind: "union", values: unique }; +}; +export const symbolicTruth = (value: SymbolicValue): boolean | undefined => { + if (value.kind === "union") { + const choices = value.values.map(symbolicTruth); + return choices.length > 0 && + choices.every((choice) => choice === choices[0]) + ? choices[0] + : undefined; + } + return value.kind === "literal" + ? Boolean(value.value) + : ["function", "object", "namespace", "array", "record", "loader"].includes( + value.kind, + ) + ? true + : undefined; +}; + +/** Attach selection predicates to values while preserving their representation. */ +export const symbolicCondition = ( + value: SymbolicValue, + condition: SymbolicCondition, +): SymbolicValue => { + if (value.kind === "union") + return symbolicUnion( + value.values.map((variant) => symbolicCondition(variant, condition)), + ); + return { ...value, conditions: [...(value.conditions ?? []), condition] }; +}; + +export const symbolicConditionsUnresolved = ( + value: SymbolicValue, +): SymbolicValue => { + if (value.kind === "union") + return symbolicUnion(value.values.map(symbolicConditionsUnresolved)); + return { ...value, conditionUnresolved: true }; +}; diff --git a/src/domain/managed/managedApplicationGraph.ts b/src/domain/managed/managedApplicationGraph.ts index 3e4ffea37..4ae40acd9 100644 --- a/src/domain/managed/managedApplicationGraph.ts +++ b/src/domain/managed/managedApplicationGraph.ts @@ -335,6 +335,11 @@ const projectionLimitations = ( ...(omissions.partialInput ? ["At least one supplied managed Evidence record has partial coverage."] : []), + ...(omissions.selectedTypes === null + ? [] + : [ + `Managed member Evidence selected only these declaring types: ${omissions.selectedTypes.join(", ")}; type, method, and field nodes of other types are omitted, not absent.`, + ]), ...(omissions.unscopedTokenRelationships ? [ "Managed boundary tokens were not linked to member nodes because matching module MVID scope was not established.", diff --git a/src/domain/managed/managedApplicationGraphCoverage.ts b/src/domain/managed/managedApplicationGraphCoverage.ts index dc15a6462..0e842ba80 100644 --- a/src/domain/managed/managedApplicationGraphCoverage.ts +++ b/src/domain/managed/managedApplicationGraphCoverage.ts @@ -19,8 +19,15 @@ export interface ManagedGraphCoverageSources { export interface ManagedGraphProjectionOmissions { readonly partialInput: boolean; readonly unscopedTokenRelationships: boolean; + /** Declaring types the member Evidence selected; null when it inspected every type. */ + readonly selectedTypes: readonly string[] | null; } +const omitsAny = (omissions: ManagedGraphProjectionOmissions): boolean => + omissions.partialInput || + omissions.unscopedTokenRelationships || + omissions.selectedTypes !== null; + const completeCoverage = (): ApplicationGraphEvidence["coverage"] => ({ status: "complete", truncated: false, @@ -61,13 +68,14 @@ export const assessManagedGraphOmissions = ( sources.boundaries?.coverage.state, sources.artifact?.coverage.state, ].some((state) => state !== undefined && state !== "complete"), + selectedTypes: sources.members?.selection?.type_names ?? null, }); /** Derive graph coverage solely from the coverage of supplied source evidence. */ export const managedGraphEvidenceCoverage = ( omissions: ManagedGraphProjectionOmissions, ): ApplicationGraphEvidence["coverage"] => - omissions.partialInput || omissions.unscopedTokenRelationships + omitsAny(omissions) ? { status: "partial", truncated: false, @@ -80,8 +88,7 @@ export const managedGraphEvidenceCoverage = ( export const managedGraphResultCoverage = ( omissions: ManagedGraphProjectionOmissions, ) => ({ - status: - omissions.partialInput || omissions.unscopedTokenRelationships - ? ("partial" as const) - : ("complete-within-inputs" as const), + status: omitsAny(omissions) + ? ("partial" as const) + : ("complete-within-inputs" as const), }); diff --git a/src/domain/managed/managedArtifact.ts b/src/domain/managed/managedArtifact.ts index 66a3b1761..297b60a4e 100644 --- a/src/domain/managed/managedArtifact.ts +++ b/src/domain/managed/managedArtifact.ts @@ -440,6 +440,14 @@ export const managedMemberInspectionSchema = z.object({ member_refs: z.array(managedMemberReferenceSchema), call_edges: z.array(managedCallEdgeSchema), field_accesses: z.array(managedFieldAccessSchema), + /** Caller-selected declaring types; null when every type was inspected. */ + selection: z + .object({ + type_names: z.array(z.string().min(1)).min(1), + unmatched_type_names: z.array(z.string().min(1)), + }) + .nullable() + .default(null), coverage: z.object({ state: z.enum(["complete", "partial", "unavailable"]), issues: z.array(managedParseIssueSchema), diff --git a/src/domain/managed/managedMemberComparison.ts b/src/domain/managed/managedMemberComparison.ts index 5197c1755..77e382dc7 100644 --- a/src/domain/managed/managedMemberComparison.ts +++ b/src/domain/managed/managedMemberComparison.ts @@ -213,6 +213,11 @@ export interface ManagedMemberComparisonSide { readonly result: ManagedMemberInspection; } +const selectedTypes = ( + result: ManagedMemberInspection, +): ReadonlySet | null => + result.selection === null ? null : new Set(result.selection.type_names); + /** Compare managed members with constrained exact and structural identities. */ export const compareManagedMembers = ( left: ManagedMemberComparisonSide, @@ -224,6 +229,8 @@ export const compareManagedMembers = ( rightEvidenceId: right.evidenceId, leftComplete: left.result.metadata.status === "complete", rightComplete: right.result.metadata.status === "complete", + leftSelection: selectedTypes(left.result), + rightSelection: selectedTypes(right.result), }; const methodItems = buildMethodItems(methodMatches, itemContext); const fieldItems = buildFieldItems(fieldMatches, itemContext); diff --git a/src/domain/managed/managedMemberComparisonCoverage.ts b/src/domain/managed/managedMemberComparisonCoverage.ts index 9d3daafe0..7f28cc88e 100644 --- a/src/domain/managed/managedMemberComparisonCoverage.ts +++ b/src/domain/managed/managedMemberComparisonCoverage.ts @@ -16,7 +16,9 @@ export const buildComparisonCoverage = ({ const leftStatus = sideCoverageStatus(left); const rightStatus = sideCoverageStatus(right); const unknownInput = - left.coverage.state !== "complete" || right.coverage.state !== "complete"; + left.coverage.state !== "complete" || + right.coverage.state !== "complete" || + !sameSelection(left, right); return { status: !unknownInput && leftStatus === "complete" && rightStatus === "complete" @@ -27,6 +29,20 @@ export const buildComparisonCoverage = ({ }; }; +/** Whether both observations inspected the same declaring types. */ +const sameSelection = ( + left: ManagedMemberInspection, + right: ManagedMemberInspection, +): boolean => { + if (left.selection === null || right.selection === null) + return left.selection === right.selection; + const rightNames = new Set(right.selection.type_names); + return ( + rightNames.size === new Set(left.selection.type_names).size && + left.selection.type_names.every((name) => rightNames.has(name)) + ); +}; + const sideCoverageStatus = ( result: ManagedMemberInspection, ): ManagedMemberComparisonResult["coverage"]["left_status"] => @@ -96,6 +112,14 @@ export const comparisonLimitations = ( ]; if (left.coverage.state !== "complete" || right.coverage.state !== "complete") limitations.push("At least one managed member observation is partial."); + if (!sameSelection(left, right)) + limitations.push( + "The observations select different declaring types; a member outside the other side's selection is unknown, not added or removed.", + ); + else if (left.selection !== null) + limitations.push( + `Both observations cover only the selected declaring types (${left.selection.type_names.join(", ")}); member totals and statuses describe those types, not the whole assembly.`, + ); if (left.exported_types === null || right.exported_types === null) limitations.push( "At least one managed member observation predates ExportedType inspection; its exported and forwarded types are unknown.", diff --git a/src/domain/managed/managedMemberComparisonItems.ts b/src/domain/managed/managedMemberComparisonItems.ts index 5cd9aba76..23b4d600b 100644 --- a/src/domain/managed/managedMemberComparisonItems.ts +++ b/src/domain/managed/managedMemberComparisonItems.ts @@ -26,8 +26,21 @@ interface ComparisonItemContext { readonly rightEvidenceId: string; readonly leftComplete: boolean; readonly rightComplete: boolean; + /** Declaring types each side selected; null when it inspected every type. */ + readonly leftSelection: ReadonlySet | null; + readonly rightSelection: ReadonlySet | null; } +/** Whether an inventory observed every member of this member's declaring type. */ +const inventoryCovers = ( + complete: boolean, + selection: ReadonlySet | null, + member: Method | Field, +): boolean => + complete && + (selection === null || + (member.declaring_type !== null && selection.has(member.declaring_type))); + /** Declared type and name of each unpaired member, per side, whose signature was not decoded. */ interface UndecodedOneSidedNames { readonly left: ReadonlySet; @@ -123,7 +136,11 @@ const methodOnlyItem = ( ): MethodItem => { if (member.left !== undefined) { const limitations = absenceLimitations(member.left, { - oppositeComplete: context.rightComplete, + oppositeComplete: inventoryCovers( + context.rightComplete, + context.rightSelection, + member.left, + ), oppositeUndecodedNames: undecoded.right, opposite: "right", inventory: "method", @@ -141,7 +158,11 @@ const methodOnlyItem = ( } const limitations = absenceLimitations(member.right, { - oppositeComplete: context.leftComplete, + oppositeComplete: inventoryCovers( + context.leftComplete, + context.leftSelection, + member.right, + ), oppositeUndecodedNames: undecoded.left, opposite: "left", inventory: "method", @@ -233,7 +254,11 @@ const fieldOnlyItem = ( ): FieldItem => { if (member.left !== undefined) { const limitations = absenceLimitations(member.left, { - oppositeComplete: context.rightComplete, + oppositeComplete: inventoryCovers( + context.rightComplete, + context.rightSelection, + member.left, + ), oppositeUndecodedNames: undecoded.right, opposite: "right", inventory: "field", @@ -250,7 +275,11 @@ const fieldOnlyItem = ( } const limitations = absenceLimitations(member.right, { - oppositeComplete: context.leftComplete, + oppositeComplete: inventoryCovers( + context.leftComplete, + context.leftSelection, + member.right, + ), oppositeUndecodedNames: undecoded.left, opposite: "left", inventory: "field", diff --git a/src/domain/managed/managedMemberComparisonMatch.ts b/src/domain/managed/managedMemberComparisonMatch.ts index 2c9e0cf04..f8e2b1d43 100644 --- a/src/domain/managed/managedMemberComparisonMatch.ts +++ b/src/domain/managed/managedMemberComparisonMatch.ts @@ -250,22 +250,74 @@ const groupBy = ( return grouped; }; +/** Whether a side's inventory observed members of this declaring type. */ +const observedBy = + (side: ManagedMemberComparisonSide) => + ({ declaring_type }: { readonly declaring_type: string | null }): boolean => + side.result.selection === null || + (declaring_type !== null && + side.result.selection.type_names.includes(declaring_type)); + +/** + * Pair only members whose declaring type both sides observed. A member outside + * the other side's selection stays one-sided, so a type selection never yields + * a remap to a member of a type the other side did not inspect. + */ +const matchObserved = ( + left: readonly Item[], + right: readonly Item[], + sides: { + readonly inLeft: (item: Item) => boolean; + readonly inRight: (item: Item) => boolean; + }, + match: ( + left: readonly Keyed[], + right: readonly Keyed[], + ) => ReturnType>, + key: (item: Item) => Keyed, +): ReturnType> => { + const matched = match( + left.filter(sides.inRight).map(key), + right.filter(sides.inLeft).map(key), + ); + return { + ...matched, + leftOnly: [ + ...matched.leftOnly, + ...left.filter((item) => !sides.inRight(item)).map(key), + ], + rightOnly: [ + ...matched.rightOnly, + ...right.filter((item) => !sides.inLeft(item)).map(key), + ], + }; +}; + export const keyMembers = ( left: ManagedMemberComparisonSide, right: ManagedMemberComparisonSide, ): { readonly methodMatches: ReturnType; readonly fieldMatches: ReturnType; -} => ({ - methodMatches: matchMethods( - left.result.methods.map(keyMethod), - right.result.methods.map(keyMethod), - ), - fieldMatches: matchFields( - left.result.fields.map(keyField), - right.result.fields.map(keyField), - ), -}); +} => { + const sides = { inLeft: observedBy(left), inRight: observedBy(right) }; + return { + methodMatches: matchObserved( + left.result.methods, + right.result.methods, + sides, + matchMethods, + keyMethod, + ), + fieldMatches: matchObserved( + left.result.fields, + right.result.fields, + sides, + matchFields, + keyField, + ), + }; +}; export const parseManagedMemberEvidence = ( evidence: unknown, diff --git a/src/domain/native/binaryLayout.ts b/src/domain/native/binaryLayout.ts index e2a8d20cf..62ebb34f2 100644 --- a/src/domain/native/binaryLayout.ts +++ b/src/domain/native/binaryLayout.ts @@ -1,5 +1,6 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; import { CANONICAL_BASE64_PATTERN } from "../stringPatterns.js"; const unsignedHex = z.string().regex(/^0x(?:0|[1-9a-f][0-9a-f]{0,15})$/); @@ -16,10 +17,9 @@ const name = z.strictObject({ /** Explicit local object selection, independent of an active disassembler target. */ export const inspectBinaryLayoutInputSchema = z.strictObject({ - path: z - .string() - .min(1) - .describe("Absolute filesystem path to the selected binary"), + path: localPathStringSchema.describe( + "Absolute filesystem path to the selected binary", + ), }); /** File and linked-image evidence; numeric ELF values never pass through unsafe JSON numbers. */ diff --git a/src/domain/native/nativeInspection.ts b/src/domain/native/nativeInspection.ts index 072e8d272..c0b7a3d46 100644 --- a/src/domain/native/nativeInspection.ts +++ b/src/domain/native/nativeInspection.ts @@ -28,7 +28,12 @@ const architectureSchema = z.object({ name: z.string(), cpu_type: z.string().nullable(), cpu_subtype: z.string().nullable(), - file_offset: z.number().int().min(0).nullable(), + file_offset: z + .number() + .int() + .min(0) + .nullable() + .describe("Offset from the start of the input Mach-O container."), size: z.number().int().min(0).nullable(), alignment: z.number().int().min(0).nullable(), }); @@ -43,7 +48,12 @@ const sectionSchema = z.object({ name: z.string(), address: hexAddress.nullable(), size: z.number().int().min(0).nullable(), - file_offset: z.number().int().min(0).nullable(), + file_offset: z + .number() + .int() + .min(0) + .nullable() + .describe("Offset from the start of the selected Mach-O slice."), alignment: z.number().int().min(0).nullable(), flags: z.array(z.string()), }); @@ -51,7 +61,12 @@ const segmentSchema = z.object({ name: z.string(), vm_address: hexAddress.nullable(), vm_size: z.number().int().min(0).nullable(), - file_offset: z.number().int().min(0).nullable(), + file_offset: z + .number() + .int() + .min(0) + .nullable() + .describe("Offset from the start of the selected Mach-O slice."), file_size: z.number().int().min(0).nullable(), maximum_permissions: permissionSchema, initial_permissions: permissionSchema, @@ -84,12 +99,39 @@ const buildMetadataSchema = z.object({ export const inspectMachoSchema = z.object({ format: z.literal("mach-o"), + /** Exact lipo slice selected for inspection and its offset in the input container. */ + selected_architecture: z.object({ + name: z + .string() + .nullable() + .describe( + "Exact architecture name selected from lipo output; null when selection is unknown.", + ), + container_file_offset: z + .number() + .int() + .min(0) + .nullable() + .describe( + "Offset of the selected slice from the start of the input Mach-O container; null when unknown.", + ), + }), endian: z.enum(["little", "big"]).nullable(), word_size: z.union([z.literal(32), z.literal(64)]).nullable(), file_type: z.string().nullable(), flags: z.array(z.string()), uuid: z.string().nullable(), - entrypoints: coverage(z.object({ file_offset: z.number().int().min(0) })), + entrypoints: coverage( + z.object({ + file_offset: z + .number() + .int() + .min(0) + .describe( + "LC_MAIN entry offset from the start of the selected Mach-O slice.", + ), + }), + ), architectures: coverage(architectureSchema), build_metadata: coverage(buildMetadataSchema), load_commands: coverage(loadCommandSchema), diff --git a/src/domain/native/nativeInstruction.ts b/src/domain/native/nativeInstruction.ts index e4c112cd1..652aa014b 100644 --- a/src/domain/native/nativeInstruction.ts +++ b/src/domain/native/nativeInstruction.ts @@ -53,7 +53,23 @@ export const nativeInstructionSchema = z.strictObject({ ]), procedure: z.string().nullable(), architecture: z.string(), - mode: z.string(), + mode: z + .string() + .describe( + "Provider-reported mode label. Per-address decoder context, when available, is reported separately; a program-wide label does not establish runtime encoding mode.", + ), + decoder_context: z + .array( + z.strictObject({ + register: z.string().min(1), + value: z.string().min(1).nullable(), + source: z.string().min(1), + }), + ) + .optional() + .describe( + "Observed per-address decoder context. Null means no value was available; omission means the provider or older capture does not expose this facet. The separate mode field retains the program language variant.", + ), bytes: z.string().nullable(), length: z.number().int().positive().nullable(), mnemonic: z.string().nullable(), diff --git a/src/domain/native/nativeInvestigationGraph.ts b/src/domain/native/nativeInvestigationGraph.ts index cc5073d79..0dd0dbe8b 100644 --- a/src/domain/native/nativeInvestigationGraph.ts +++ b/src/domain/native/nativeInvestigationGraph.ts @@ -38,10 +38,6 @@ export const nativeInvestigationNodeKindSchema = z.enum([ "unknown", ]); -type NativeInvestigationNodeKind = z.infer< - typeof nativeInvestigationNodeKindSchema ->; - /** One UI, dispatch, or value entity with optional decoded source location. */ export const nativeInvestigationNodeSchema = z.strictObject({ id: z.string().min(1), @@ -51,7 +47,6 @@ export const nativeInvestigationNodeSchema = z.strictObject({ attributes: z.record(z.string(), jsonValueSchema), evidence: z.array(nativeMetadataEvidenceSchema), }); -type NativeInvestigationNode = z.infer; /** Relationship types preserve UI wiring, native dispatch, and value flow. */ export const nativeInvestigationRelationSchema = z.enum([ @@ -72,10 +67,6 @@ export const nativeInvestigationRelationSchema = z.enum([ "controls", ]); -type NativeInvestigationRelation = z.infer< - typeof nativeInvestigationRelationSchema ->; - /** An observed, inferred, or unresolved relationship between entities. */ export const nativeInvestigationEdgeSchema = z.discriminatedUnion( "resolution", diff --git a/src/domain/native/peResources.ts b/src/domain/native/peResources.ts index 80fcc80bb..9ed9c3552 100644 --- a/src/domain/native/peResources.ts +++ b/src/domain/native/peResources.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; + const uint = z.number().int().min(0).max(0xffffffff); const range = z.strictObject({ offset: uint, bytes: uint }); const identity = z.discriminatedUnion("kind", [ @@ -14,7 +16,9 @@ const identity = z.discriminatedUnion("kind", [ /** Complete bounded inspection of a caller-selected PE image, without execution. */ export const inspectPeResourcesInputSchema = z.strictObject({ - path: z.string().min(1).describe("Absolute filesystem path to a PE image"), + path: localPathStringSchema.describe( + "Absolute filesystem path to a PE image", + ), max_file_bytes: z .number() .int() diff --git a/src/domain/native/recordedCrash.ts b/src/domain/native/recordedCrash.ts index adef5a7ea..110bd1d91 100644 --- a/src/domain/native/recordedCrash.ts +++ b/src/domain/native/recordedCrash.ts @@ -1,5 +1,6 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; import { CANONICAL_BASE64_PATTERN } from "../stringPatterns.js"; import { recordedNoteCoverageIssues } from "./recordedCrashNoteCoverage.js"; @@ -50,10 +51,9 @@ const supportedRegisters = new Set(RECORDED_LINUX_AMD64_REGISTERS); /** Select an explicit recording and optionally request debugger-derived mapping context. */ export const inspectRecordedCrashInputSchema = z.strictObject({ - path: z - .string() - .min(1) - .describe("Absolute filesystem path to a supplied Linux ELF core"), + path: localPathStringSchema.describe( + "Absolute filesystem path to a supplied Linux ELF core", + ), include_debugger_context: z .boolean() .default(false) diff --git a/src/domain/peMachine.ts b/src/domain/peMachine.ts new file mode 100644 index 000000000..16a5ace37 --- /dev/null +++ b/src/domain/peMachine.ts @@ -0,0 +1,50 @@ +import type { BinaryArchitecture } from "./binaryTargetTypes.js"; + +export type PeArchitecture = Exclude; + +/** A decoded COFF machine and whether it carried a ReadyToRun OS override. */ +export interface PeMachine { + readonly architecture: PeArchitecture; + readonly nativeOsOverride: boolean; +} + +const architectureForMachine = ( + machine: number, +): PeArchitecture | undefined => { + switch (machine) { + case 0x014c: + return "x86"; + case 0x8664: + return "x86_64"; + case 0x01c0: + case 0x01c2: + case 0x01c4: + return "arm"; + case 0xaa64: + return "arm64"; + } + return undefined; +}; + +/** + * ReadyToRun images compiled for a non-Windows OS store the COFF machine XORed + * with IMAGE_FILE_MACHINE_NATIVE_OS_OVERRIDE (dotnet/runtime + * src/coreclr/inc/pedecoder.h): Apple, FreeBSD, OpenBSD, Linux, NetBSD, SunOS. + */ +const NATIVE_OS_OVERRIDES = [0x4644, 0xadc4, 0xadc5, 0x7b79, 0x1993, 0x1992]; + +/** + * Decode a COFF machine. Only managed images carry an OS override, so callers + * must reject `nativeOsOverride` when the image has no CLI header. + */ +export const decodePeMachine = (machine: number): PeMachine | undefined => { + const direct = architectureForMachine(machine); + if (direct !== undefined) + return { architecture: direct, nativeOsOverride: false }; + for (const override of NATIVE_OS_OVERRIDES) { + const architecture = architectureForMachine(machine ^ override); + if (architecture !== undefined) + return { architecture, nativeOsOverride: true }; + } + return undefined; +}; diff --git a/src/domain/propertyListKeys.test.ts b/src/domain/propertyListKeys.test.ts index 15eb1bd6d..c6c157a01 100644 --- a/src/domain/propertyListKeys.test.ts +++ b/src/domain/propertyListKeys.test.ts @@ -9,6 +9,10 @@ it.each([ "__proto__", "", "__proto__", + "__proto__", + "__proto__", + "__proto__", + "", ])("omits and counts a dictionary keyed %s", (key) => { const parsed = parseXmlPropertyList( plist( @@ -64,3 +68,56 @@ it("omits own __proto__ entries that JSON decoding creates", () => { omittedPrototypeKeys: 2, }); }); + +it("preserves the pinned decoder's first-child key semantics", () => { + const text = plist( + "__prokept__proto__omitted", + ); + expect(parseXmlPropertyList(text)).toEqual({ + value: { __pro: "kept" }, + omittedPrototypeKeys: 1, + }); +}); + +it("preserves OpenStep dictionary and array autodetection", () => { + expect( + parseXmlPropertyList( + " { CFBundleExecutable = App; values = (one, two); __rea_prototype_key__ = kept; }", + ), + ).toEqual({ + value: { + CFBundleExecutable: "App", + values: ["one", "two"], + __rea_prototype_key__: "kept", + }, + omittedPrototypeKeys: 0, + }); + expect(parseXmlPropertyList("(one, two)")).toEqual({ + value: ["one", "two"], + omittedPrototypeKeys: 0, + }); +}); + +it.each(["\n", "\r\n", "\r", "\u0085", "\u2028", "\u2029"])( + "preserves unrelated source values around %j line endings", + (newline) => { + const text = plist( + `${newline}kept keya b…c
d${newline}__proto__omitted`, + ); + expect(parseXmlPropertyList(text)).toEqual({ + value: { "kept\rkey": "a\rb\u0085c\u2028d" }, + omittedPrototypeKeys: 1, + }); + }, +); + +it("locates sparse omitted keys across newline-dense source", () => { + const lines = "\n".repeat(64 * 1024); + const text = plist( + `${lines}__proto__omittednested${lines}__proto__omittedkept`, + ); + expect(parseXmlPropertyList(text)).toEqual({ + value: { nested: { kept: true } }, + omittedPrototypeKeys: 2, + }); +}); diff --git a/src/domain/propertyListKeys.ts b/src/domain/propertyListKeys.ts index 2c537f38d..778af9295 100644 --- a/src/domain/propertyListKeys.ts +++ b/src/domain/propertyListKeys.ts @@ -1,3 +1,4 @@ +import { DOMParser, normalizeLineEndings } from "@xmldom/xmldom"; import { parse } from "plist"; /** The dictionary key REA results cannot carry: validation strips it. */ @@ -46,10 +47,6 @@ export const omitPrototypeKeys = ( value: unknown, ): PropertyListWithoutPrototypeKeys => withoutKey(value, PROTOTYPE_KEY); -// Comments and CDATA text are consumed whole so only element keys match. -const KEY_TOKEN = - /||([^<]*)<\/key>|<\/key>/gu; - const XML_ENTITIES: Readonly> = { amp: "&", apos: "'", @@ -87,18 +84,54 @@ export const decodeXmlText = (text: string): string => export const parseXmlPropertyList = ( text: string, ): PropertyListWithoutPrototypeKeys => { - // The decoded text holds every key the decoder can produce, including - // entity-encoded spellings of the placeholder, so no source key aliases it. - // Decoding comment or CDATA text too can only lengthen the placeholder. - const decoded = decodeXmlText(text); - let placeholder = "__rea_prototype_key__"; - while (decoded.includes(placeholder)) placeholder = `_${placeholder}`; - const substituted = text.replace( - KEY_TOKEN, - (token, plain?: string, cdata?: string) => - (plain === undefined ? cdata : decodeXmlText(plain)) === PROTOTYPE_KEY - ? `${placeholder}` - : token, + // Preserve the pinned decoder's existing OpenStep autodetection, also used + // by app Info.plist resolution. Only XML needs the key-element substitution. + const first = text.trimStart()[0]; + if (first === "{" || first === "(") return omitPrototypeKeys(parse(text)); + const { substituted, placeholder } = substitutePrototypeKeys( + normalizeLineEndings(text), ); return withoutKey(parse(substituted), placeholder); }; + +/** Locate keys with the same XML parser and first-child semantics as plist. */ +const substitutePrototypeKeys = (text: string) => { + // Normalize only the literal line endings the pinned decoder normalizes. + // Preserve character references in the source: serializing a DOM would turn + // e.g. an encoded carriage return into a literal newline on the second parse. + const document = new DOMParser().parseFromString(text, "text/xml"); + const keys = Array.from(document.getElementsByTagName("key")); + const decodedKeys = new Set(keys.map((key) => key.firstChild?.nodeValue)); + let placeholder = "__rea_prototype_key__"; + while (decodedKeys.has(placeholder)) placeholder = `_${placeholder}`; + + const omitted = keys + .map((key) => key.firstChild) + .filter((node) => node?.nodeValue === PROTOTYPE_KEY); + if (omitted.length === 0) return { substituted: text, placeholder }; + + // xmldom uses one-based locators in document order. Scan forward only to + // needed lines instead of retaining an offset for every source newline. + let line = 1; + let lineStart = 0; + const parts: string[] = []; + let cursor = 0; + for (const node of omitted) { + const nodeLine = node?.lineNumber; + const column = node?.columnNumber; + if (nodeLine === undefined || column === undefined) + throw new TypeError("XML plist key has no source location"); + while (line < nodeLine) { + const newline = text.indexOf("\n", lineStart); + if (newline === -1) + throw new TypeError("XML plist key has an invalid source location"); + lineStart = newline + 1; + line += 1; + } + const offset = lineStart + column - 1; + parts.push(text.slice(cursor, offset), ``); + cursor = offset; + } + parts.push(text.slice(cursor)); + return { substituted: parts.join(""), placeholder }; +}; diff --git a/src/domain/rawBinary.test.ts b/src/domain/rawBinary.test.ts index ca4d7f0d6..f22e5fced 100644 --- a/src/domain/rawBinary.test.ts +++ b/src/domain/rawBinary.test.ts @@ -55,15 +55,20 @@ describe("raw-binary admission", () => { }, }); }); - it("rejects an empty file", () => { + it("rejects an empty file as a target refusal while naming the extent", () => { expect(validateRawBinaryLength(0).ok).toBe(false); - expect( - admitRawBinary({ - architecture: "x86", - entryPoint: "0x1000", - fileBytes: 0, - }).ok, - ).toBe(false); + const result = admitRawBinary({ + architecture: "x86", + entryPoint: "0x1000", + fileBytes: 0, + }); + expect(result).toEqual({ + ok: false, + error: { + reason: "raw-binary requires at least one file byte", + constraint: "file_format_rejected", + }, + }); }); it("rejects a 32-bit mapping that exceeds the architecture width", () => { const result = admitRawBinary({ @@ -73,7 +78,8 @@ describe("raw-binary admission", () => { }); expect(result.ok).toBe(false); if (result.ok) throw new Error("expected overflow rejection"); - expect(result.error).toContain("arm address width"); + expect(result.error.reason).toContain("arm address width"); + expect(result.error.constraint).toBe("load_selection_rejected"); }); it("allows a 64-bit mapping that would overflow 32-bit ARM", () => { expect( @@ -92,11 +98,27 @@ describe("raw-binary admission", () => { admitRawBinary({ architecture, entryPoint, fileBytes: 4 }), ).toEqual({ ok: false, - error: `raw-binary ${architecture} entry_point ${entryPoint} must be 4-byte aligned`, + error: { + reason: `raw-binary ${architecture} entry_point ${entryPoint} must be 4-byte aligned`, + constraint: "load_selection_rejected", + }, }); } }, ); + it.each([ + ["malformed entry", { architecture: "arm", entryPoint: "0xgg" }], + ["missing entry", { architecture: "arm", entryPoint: undefined }], + [ + "missing architecture", + { architecture: undefined, entryPoint: "0x8000000" }, + ], + ])("classifies a %s as caller input", (_name, selection) => { + const result = admitRawBinary({ ...selection, fileBytes: 4 }); + expect(result.ok).toBe(false); + if (result.ok) throw new Error("expected caller-input rejection"); + expect(result.error.constraint).toBe("load_selection_rejected"); + }); it.each(["x86", "x86_64"])( "admits byte-aligned %s entries", (architecture) => { @@ -105,7 +127,7 @@ describe("raw-binary admission", () => { entryPoint: "0x1001", fileBytes: 1, }); - if (!admitted.ok) throw new Error(admitted.error); + if (!admitted.ok) throw new Error(admitted.error.reason); expect(admitted.value.entryPoint).toBe("0x1001"); }, ); diff --git a/src/domain/rawBinary.ts b/src/domain/rawBinary.ts index a389bf7c3..d00af8d49 100644 --- a/src/domain/rawBinary.ts +++ b/src/domain/rawBinary.ts @@ -60,6 +60,16 @@ export interface RawBinaryAdmission { readonly entryPointSource: string; } +/** Why a headerless load was refused, with its owning classification. */ +export interface RawBinaryAdmissionRejection { + readonly reason: string; + /** + * `load_selection_rejected` names a caller-input selection problem; + * `file_format_rejected` names a refusal caused by the target file itself. + */ + readonly constraint: "load_selection_rejected" | "file_format_rejected"; +} + const addressWidth = (architecture: BinaryArchitecture): bigint => architecture === "x86" || architecture === "arm" ? 0xffffffffn @@ -70,33 +80,47 @@ export const admitRawBinary = (input: { readonly architecture: unknown; readonly entryPoint: unknown; readonly fileBytes: number; -}): Result => { +}): Result => { const architecture = binaryArchitectureSchema.safeParse(input.architecture); if (!architecture.success) - return err( - "raw-binary format requires architecture (x86, x86_64, arm, or arm64)", - ); + return err({ + reason: + "raw-binary format requires architecture (x86, x86_64, arm, or arm64)", + constraint: "load_selection_rejected", + }); if (typeof input.entryPoint !== "string") - return err( - "raw-binary format requires entry_point as a hexadecimal address of the form 0x…", - ); + return err({ + reason: + "raw-binary format requires entry_point as a hexadecimal address of the form 0x…", + constraint: "load_selection_rejected", + }); const entry = parseRawBinaryEntryPoint(input.entryPoint); - if (!entry.ok) return entry; + if (!entry.ok) + return err({ + reason: entry.error, + constraint: "load_selection_rejected", + }); if ( (architecture.data === "arm" || architecture.data === "arm64") && entry.value.value % 4n !== 0n ) - return err( - `raw-binary ${architecture.data} entry_point ${entry.value.canonical} must be 4-byte aligned`, - ); + return err({ + reason: `raw-binary ${architecture.data} entry_point ${entry.value.canonical} must be 4-byte aligned`, + constraint: "load_selection_rejected", + }); const length = validateRawBinaryLength(input.fileBytes); - if (!length.ok) return length; + if (!length.ok) + return err({ + reason: length.error, + constraint: "file_format_rejected", + }); const last = entry.value.value + BigInt(input.fileBytes) - 1n; const max = addressWidth(architecture.data); if (entry.value.value > max || last > max) - return err( - `raw-binary mapping from ${entry.value.canonical} must fit in the ${architecture.data} address width`, - ); + return err({ + reason: `raw-binary mapping from ${entry.value.canonical} must fit in the ${architecture.data} address width`, + constraint: "load_selection_rejected", + }); return ok({ architecture: architecture.data, entryPoint: entry.value.canonical, diff --git a/src/domain/reconstructionObligationLedgerIntegrity.ts b/src/domain/reconstructionObligationLedgerIntegrity.ts new file mode 100644 index 000000000..c0f9dd099 --- /dev/null +++ b/src/domain/reconstructionObligationLedgerIntegrity.ts @@ -0,0 +1,212 @@ +import { digestCanonicalValue } from "./canonicalDigest.js"; +import { canonicalJson } from "./comparisonSemantics.js"; +import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; + +import type { + ReconstructionObligation, + ReconstructionObligationLedger, +} from "./reconstructionObligationLedgerSchemas.js"; + +/** Derive the ledger summary from the obligation facts. */ +export const summarizeReconstructionObligations = ( + obligations: readonly ReconstructionObligation[], +): ReconstructionObligationLedger["summary"] => ({ + total: obligations.length, + required: obligations.filter(({ required }) => required).length, + verified: obligations.filter(({ status }) => status === "verified").length, + required_open: obligations.filter( + ({ required, status }) => required && status !== "verified", + ).length, + by_status: countBy( + obligations, + [ + "unowned", + "characterized", + "implemented", + "verified", + "contradicted", + "blocked", + "out-of-scope", + "unknown", + ], + ({ status }) => status, + ), + by_application_layer: countBy( + obligations, + [ + "cli", + "protocol", + "electron", + "persistence", + "process", + "runtime", + "packaging", + "native-abi", + "application", + "other", + ], + ({ application_layer }) => application_layer, + ), + by_evidence_authority: countBy( + obligations, + ["candidate", "observed", "reviewed", "unknown"], + ({ source_state }) => source_state, + ), +}); + +/** Derive closure status from required obligations and their diagnostic facts. */ +export const reconstructionObligationLedgerStatus = ( + obligations: readonly ReconstructionObligation[], + limitations: readonly string[], +): ReconstructionObligationLedger["status"] => { + if (obligations.length === 0) return "unknown"; + const requiredOpen = obligations.filter( + ({ required, status }) => required && status !== "verified", + ); + if ( + requiredOpen.some( + ({ status, diagnostics }) => + status === "contradicted" || + hasReconstructionObligationDiagnostic(diagnostics, "verifier-failed"), + ) + ) + return "failed"; + if (limitations.length > 0) return "unknown"; + if (requiredOpen.some(({ status }) => status === "unknown")) return "unknown"; + return requiredOpen.length === 0 ? "ready" : "open"; +}; + +export const reconstructionObligationCoverage = ( + limitations: readonly string[], +): ReconstructionObligationLedger["coverage"] => ({ + status: limitations.length === 0 ? "complete" : "partial", +}); + +export const reconstructionObligationReports = ( + obligations: readonly ReconstructionObligation[], +): ReconstructionObligationLedger["reports"] => ({ + missing_owner_obligation_ids: obligationIdsWith(obligations, "missing-owner"), + missing_verifier_obligation_ids: obligationIdsWith( + obligations, + "missing-verifier", + ), + contradicted_obligation_ids: obligationIdsWith(obligations, "contradiction"), + residual_unknown_ids: [ + ...new Set(obligations.flatMap(({ residual_unknown_ids: ids }) => ids)), + ].sort(compareUnicodeCodePoints), +}); + +export const reconstructionObligationOwnershipGraph = ( + obligations: readonly ReconstructionObligation[], +): ReconstructionObligationLedger["ownership_graph"] => + obligations + .flatMap((obligation) => + obligation.binding === null + ? [] + : [ + { + obligation_id: obligation.obligation_id, + module_path: obligation.binding.owner.module_path, + symbol: obligation.binding.owner.symbol, + }, + ], + ) + .sort(obligationEdgeOrder); + +export const reconstructionObligationDependencyGraph = ( + obligations: readonly ReconstructionObligation[], +): ReconstructionObligationLedger["dependency_graph"] => + obligations + .flatMap((obligation) => + obligation.dependency_obligation_ids.map((dependencyId) => ({ + obligation_id: obligation.obligation_id, + depends_on_obligation_id: dependencyId, + })), + ) + .sort(obligationEdgeOrder); + +/** Evidence attached to facts serialized on an obligation. */ +export const reconstructionObligationEvidenceIds = ( + obligations: readonly ReconstructionObligation[], +): string[] => + [ + ...new Set( + obligations.flatMap((obligation) => [ + ...obligation.authority_references.map(({ evidence_id: id }) => id), + ...obligation.observed_cases.map(({ evidence_id: id }) => id), + ...(obligation.binding === null + ? [] + : [ + ...obligation.binding.original_cases.map( + ({ evidence_id: id }) => id, + ), + ...obligation.binding.fixtures.flatMap( + ({ evidence_ids: ids }) => ids, + ), + ...(obligation.binding.verifier === null + ? [] + : [obligation.binding.verifier.result_evidence_id]), + ]), + ]), + ), + ].sort(compareUnicodeCodePoints); + +const obligationEdgeOrder = ( + left: { readonly obligation_id: string }, + right: { readonly obligation_id: string }, +): number => + compareUnicodeCodePoints(canonicalJson(left), canonicalJson(right)); + +const obligationIdsWith = ( + obligations: readonly ReconstructionObligation[], + code: ReconstructionObligation["diagnostics"][number]["code"], +): string[] => + obligations + .filter(({ diagnostics }) => + hasReconstructionObligationDiagnostic(diagnostics, code), + ) + .map(({ obligation_id: id }) => id) + .sort(compareUnicodeCodePoints); + +export const hasReconstructionObligationDiagnostic = ( + diagnostics: Readonly, + code: ReconstructionObligation["diagnostics"][number]["code"], +): boolean => diagnostics.some((diagnostic) => diagnostic.code === code); + +/** Recompute the closure commitment from the fields carried by a ledger. */ +export const reconstructionObligationLedgerClosureDigest = (ledger: { + readonly status: ReconstructionObligationLedger["status"]; + readonly coverage: ReconstructionObligationLedger["coverage"]; + readonly summary: ReconstructionObligationLedger["summary"]; + readonly reports: ReconstructionObligationLedger["reports"]; + readonly ownership_graph: ReconstructionObligationLedger["ownership_graph"]; + readonly dependency_graph: ReconstructionObligationLedger["dependency_graph"]; + readonly obligations: ReconstructionObligationLedger["obligations"]; + readonly evidence_links: ReconstructionObligationLedger["evidence_links"]; + readonly limitations: ReconstructionObligationLedger["limitations"]; +}): string => + digestCanonicalValue( + { + schema: "ReconstructionObligationClosure", + status: ledger.status, + coverage: ledger.coverage, + summary: ledger.summary, + reports: ledger.reports, + ownership_graph: ledger.ownership_graph, + dependency_graph: ledger.dependency_graph, + obligations: ledger.obligations, + evidence_links: ledger.evidence_links, + limitations: ledger.limitations, + }, + "Reconstruction obligation ledger", + ); + +const countBy = ( + items: readonly Item[], + values: readonly Value[], + valueFor: (item: Item) => Value, +): { key: Value; count: number }[] => + values.map((value) => ({ + key: value, + count: items.filter((item) => valueFor(item) === value).length, + })); diff --git a/src/domain/reconstructionObligationLedgerSchemas.test.ts b/src/domain/reconstructionObligationLedgerSchemas.test.ts index c2aedd024..005758210 100644 --- a/src/domain/reconstructionObligationLedgerSchemas.test.ts +++ b/src/domain/reconstructionObligationLedgerSchemas.test.ts @@ -3,7 +3,12 @@ import { describe, expect, it } from "vitest"; import { reconstructionObligationLedgerSchema, reconstructionObligationManifestSchema, + type ReconstructionObligationLedger, } from "./reconstructionObligationLedgerSchemas.js"; +import { + reconstructionObligationLedgerClosureDigest, + summarizeReconstructionObligations, +} from "./reconstructionObligationLedgerIntegrity.js"; const digest = "a".repeat(64); @@ -32,20 +37,13 @@ describe("reconstruction obligation ledger schemas", () => { reconstructionObligationManifestSchema.parse(manifest).bindings, ).toHaveLength(10_001); - const ledger = { + const ledgerBase: Omit = { schema: "ReconstructionObligationLedger", ledger_id: `rol_${digest}`, - closure_digest: digest, - status: "ready", + status: "unknown", coverage: { status: "complete" }, summary: { - total: 0, - required: 0, - verified: 0, - required_open: 0, - by_status: [], - by_application_layer: [], - by_evidence_authority: [], + ...summarizeReconstructionObligations([]), }, reports: { missing_owner_obligation_ids: [], @@ -62,6 +60,10 @@ describe("reconstruction obligation ledger schemas", () => { ), limitations: [], }; + const ledger = { + ...ledgerBase, + closure_digest: reconstructionObligationLedgerClosureDigest(ledgerBase), + }; expect( reconstructionObligationLedgerSchema.parse(ledger).evidence_links, diff --git a/src/domain/reconstructionObligationLedgerSchemas.ts b/src/domain/reconstructionObligationLedgerSchemas.ts index bc61969a8..6d3403c8d 100644 --- a/src/domain/reconstructionObligationLedgerSchemas.ts +++ b/src/domain/reconstructionObligationLedgerSchemas.ts @@ -5,11 +5,42 @@ import { STABLE_IDENTIFIER_PATTERN } from "./stringPatterns.js"; import { evidenceBundleSchema } from "./evidenceBundle.js"; import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; +import { canonicalJson } from "./comparisonSemantics.js"; +import { + reconstructionObligationCoverage, + reconstructionObligationDependencyGraph, + reconstructionObligationLedgerClosureDigest, + reconstructionObligationLedgerStatus, + reconstructionObligationEvidenceIds, + reconstructionObligationOwnershipGraph, + reconstructionObligationReports, + summarizeReconstructionObligations, +} from "./reconstructionObligationLedgerIntegrity.js"; const evidenceIdSchema = prefixedDigestSchema("ev"); const stableIdSchema = z.string().regex(STABLE_IDENTIFIER_PATTERN); const textSchema = z.string().trim().min(1); +const sameProjection = ( + actual: readonly Value[], + expected: readonly Value[], +): boolean => { + if (actual.length !== expected.length) return false; + const counts = new Map(); + for (const value of actual) { + const key = canonicalJson(value, "Reconstruction obligation ledger"); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + for (const value of expected) { + const key = canonicalJson(value, "Reconstruction obligation ledger"); + const count = counts.get(key); + if (count === undefined) return false; + if (count === 1) counts.delete(key); + else counts.set(key, count - 1); + } + return counts.size === 0; +}; + export const obligationStatusSchema = z.enum([ "unowned", "characterized", @@ -236,48 +267,174 @@ const countBySchema = ( }), ); -export const reconstructionObligationLedgerSchema = z.strictObject({ - schema: z.literal("ReconstructionObligationLedger"), - ledger_id: prefixedDigestSchema("rol"), - closure_digest: digestSchema, - status: z.enum(["ready", "open", "failed", "unknown"]), - coverage: z.strictObject({ - status: z.enum(["complete", "partial"]), - }), - summary: z.strictObject({ - total: z.number().int().min(0), - required: z.number().int().min(0), - verified: z.number().int().min(0), - required_open: z.number().int().min(0), - by_status: countBySchema(obligationStatusSchema), - by_application_layer: countBySchema(obligationLayerSchema), - by_evidence_authority: countBySchema( - z.enum(["candidate", "observed", "reviewed", "unknown"]), +const countEntriesMatch = ( + actual: readonly { readonly key: string; readonly count: number }[], + expected: readonly { readonly key: string; readonly count: number }[], +): boolean => { + const actualCounts = new Map(actual.map(({ key, count }) => [key, count])); + return ( + actualCounts.size === actual.length && + actual.length === expected.length && + expected.every(({ key, count }) => actualCounts.get(key) === count) + ); +}; + +const obligationSummariesMatch = ( + actual: ReconstructionObligationLedger["summary"], + expected: ReconstructionObligationLedger["summary"], +): boolean => + actual.total === expected.total && + actual.required === expected.required && + actual.verified === expected.verified && + actual.required_open === expected.required_open && + countEntriesMatch(actual.by_status, expected.by_status) && + countEntriesMatch( + actual.by_application_layer, + expected.by_application_layer, + ) && + countEntriesMatch( + actual.by_evidence_authority, + expected.by_evidence_authority, + ); + +export const reconstructionObligationLedgerSchema = z + .strictObject({ + schema: z.literal("ReconstructionObligationLedger"), + ledger_id: prefixedDigestSchema("rol"), + closure_digest: digestSchema, + status: z.enum(["ready", "open", "failed", "unknown"]), + coverage: z.strictObject({ + status: z.enum(["complete", "partial"]), + }), + summary: z.strictObject({ + total: z.number().int().min(0), + required: z.number().int().min(0), + verified: z.number().int().min(0), + required_open: z.number().int().min(0), + by_status: countBySchema(obligationStatusSchema), + by_application_layer: countBySchema(obligationLayerSchema), + by_evidence_authority: countBySchema( + z.enum(["candidate", "observed", "reviewed", "unknown"]), + ), + }), + reports: z.strictObject({ + missing_owner_obligation_ids: z.array(stableIdSchema), + missing_verifier_obligation_ids: z.array(stableIdSchema), + contradicted_obligation_ids: z.array(stableIdSchema), + residual_unknown_ids: z.array(stableIdSchema), + }), + ownership_graph: z.array( + z.strictObject({ + obligation_id: stableIdSchema, + module_path: textSchema, + symbol: textSchema, + }), ), - }), - reports: z.strictObject({ - missing_owner_obligation_ids: z.array(stableIdSchema), - missing_verifier_obligation_ids: z.array(stableIdSchema), - contradicted_obligation_ids: z.array(stableIdSchema), - residual_unknown_ids: z.array(stableIdSchema), - }), - ownership_graph: z.array( - z.strictObject({ - obligation_id: stableIdSchema, - module_path: textSchema, - symbol: textSchema, - }), - ), - dependency_graph: z.array( - z.strictObject({ - obligation_id: stableIdSchema, - depends_on_obligation_id: stableIdSchema, - }), - ), - obligations: z.array(reconstructionObligationSchema), - evidence_links: z.array(evidenceIdSchema), - limitations: z.array(textSchema), -}); + dependency_graph: z.array( + z.strictObject({ + obligation_id: stableIdSchema, + depends_on_obligation_id: stableIdSchema, + }), + ), + obligations: z.array(reconstructionObligationSchema), + evidence_links: z.array(evidenceIdSchema), + limitations: z.array(textSchema), + }) + .superRefine((ledger, context) => { + if ( + ledger.status !== + reconstructionObligationLedgerStatus( + ledger.obligations, + ledger.limitations, + ) + ) + context.addIssue({ + code: "custom", + path: ["status"], + message: "Ledger status does not match its obligations", + }); + if ( + !obligationSummariesMatch( + ledger.summary, + summarizeReconstructionObligations(ledger.obligations), + ) + ) + context.addIssue({ + code: "custom", + path: ["summary"], + message: "Ledger summary does not match its obligations", + }); + if ( + ledger.closure_digest !== + reconstructionObligationLedgerClosureDigest(ledger) + ) + context.addIssue({ + code: "custom", + path: ["closure_digest"], + message: "Ledger closure digest does not match its contents", + }); + if ( + ledger.coverage.status !== + reconstructionObligationCoverage(ledger.limitations).status + ) + context.addIssue({ + code: "custom", + path: ["coverage"], + message: "Ledger coverage does not match its limitations", + }); + const expectedReports = reconstructionObligationReports(ledger.obligations); + for (const key of [ + "missing_owner_obligation_ids", + "missing_verifier_obligation_ids", + "contradicted_obligation_ids", + "residual_unknown_ids", + ] as const) + if (!sameProjection(ledger.reports[key], expectedReports[key])) + context.addIssue({ + code: "custom", + path: ["reports", key], + message: "Ledger report does not match its obligations", + }); + if ( + !sameProjection( + ledger.ownership_graph, + reconstructionObligationOwnershipGraph(ledger.obligations), + ) + ) + context.addIssue({ + code: "custom", + path: ["ownership_graph"], + message: "Ownership graph does not match its obligations", + }); + if ( + !sameProjection( + ledger.dependency_graph, + reconstructionObligationDependencyGraph(ledger.obligations), + ) + ) + context.addIssue({ + code: "custom", + path: ["dependency_graph"], + message: "Dependency graph does not match its obligations", + }); + const ledgerEvidenceIds = new Set(ledger.evidence_links); + if (ledgerEvidenceIds.size !== ledger.evidence_links.length) + context.addIssue({ + code: "custom", + path: ["evidence_links"], + message: "Evidence links must be unique", + }); + if ( + reconstructionObligationEvidenceIds(ledger.obligations).some( + (evidenceId) => !ledgerEvidenceIds.has(evidenceId), + ) + ) + context.addIssue({ + code: "custom", + path: ["evidence_links"], + message: "Evidence links omit Evidence referenced by an obligation", + }); + }); export const reconstructionObligationLedgerInputSchema = z.strictObject({ evidence_bundle: evidenceBundleSchema, diff --git a/src/domain/reconstructionReadiness.test.ts b/src/domain/reconstructionReadiness.test.ts index df0d5e2a0..9418f0dd1 100644 --- a/src/domain/reconstructionReadiness.test.ts +++ b/src/domain/reconstructionReadiness.test.ts @@ -2,6 +2,12 @@ import { describe, expect, it } from "vitest"; import { RECONSTRUCTION_READINESS_EXAMPLE } from "../contracts/reconstructionReadinessExample.js"; import { createReconstructionReadinessReport } from "./reconstructionReadiness.js"; +import { + reconstructionObligationLedgerClosureDigest, + reconstructionObligationLedgerStatus, + summarizeReconstructionObligations, +} from "./reconstructionObligationLedgerIntegrity.js"; +import { reconstructionObligationLedgerSchema } from "./reconstructionObligationLedgerSchemas.js"; import { reconstructionReadinessInputSchema, type ReconstructionReadinessInput, @@ -86,7 +92,96 @@ describe("reconstruction readiness report", () => { nondeterministic_replay_count: 0, }); }); +}); +describe("readiness obligation ledger integrity", () => { + it("rejects a required obligation changed behind the old ledger commitment", () => { + const value = input(); + expect( + reconstructionObligationLedgerSchema.parse(value.obligation_ledger), + ).toEqual(value.obligation_ledger); + const obligation = value.obligation_ledger.obligations[0]; + if (obligation === undefined) + throw new TypeError("Missing reconstruction obligation"); + value.obligation_ledger.obligations[0] = { + ...obligation, + status: "blocked", + }; + + expect(() => createReconstructionReadinessReport(value)).toThrow( + /Ledger status does not match its obligations/u, + ); + }); + + it("rejects a recomputed commitment over a false obligation summary", () => { + const ledger = structuredClone(input().obligation_ledger); + ledger.summary.required_open = 1; + ledger.closure_digest = reconstructionObligationLedgerClosureDigest(ledger); + + expect(() => reconstructionObligationLedgerSchema.parse(ledger)).toThrow( + /Ledger summary does not match its obligations/u, + ); + }); + + it("accepts reordered summary count entries with the same keyed values", () => { + const ledger = structuredClone(input().obligation_ledger); + ledger.summary.by_status.reverse(); + ledger.closure_digest = reconstructionObligationLedgerClosureDigest(ledger); + + expect(reconstructionObligationLedgerSchema.parse(ledger)).toEqual(ledger); + }); + + it.each([ + [ + "reports", + (ledger: ReconstructionReadinessInput["obligation_ledger"]) => { + ledger.reports.missing_owner_obligation_ids.push("obl.extra"); + }, + /Ledger report does not match its obligations/u, + ], + [ + "ownership graph", + (ledger: ReconstructionReadinessInput["obligation_ledger"]) => { + const owner = ledger.ownership_graph[0]; + if (owner === undefined) + throw new TypeError("Missing owner graph edge"); + owner.symbol = "tamperedOwner"; + }, + /Ownership graph does not match its obligations/u, + ], + [ + "dependency graph", + (ledger: ReconstructionReadinessInput["obligation_ledger"]) => { + ledger.dependency_graph.push({ + obligation_id: "obl.readiness", + depends_on_obligation_id: "obl.extra", + }); + }, + /Dependency graph does not match its obligations/u, + ], + [ + "evidence links", + (ledger: ReconstructionReadinessInput["obligation_ledger"]) => { + ledger.evidence_links = []; + }, + /Evidence links omit Evidence referenced by an obligation/u, + ], + ] as const)( + "rejects recomputed commitments over tampered %s", + (_, mutate, expected) => { + const ledger = structuredClone(input().obligation_ledger); + mutate(ledger); + ledger.closure_digest = + reconstructionObligationLedgerClosureDigest(ledger); + + expect(() => reconstructionObligationLedgerSchema.parse(ledger)).toThrow( + expected, + ); + }, + ); +}); + +describe("readiness failure semantics", () => { it("rejects equivalence over truncated concurrent evidence", () => { const value = input(); const comparison = value.comparisons[0]; @@ -110,14 +205,28 @@ describe("reconstruction readiness report", () => { it("keeps green stages unknown while one reconstruction obligation is open", () => { const value = input(); - value.obligation_ledger = { + const obligation = value.obligation_ledger.obligations[0]; + if (obligation === undefined) + throw new TypeError("Missing reconstruction obligation"); + const obligations = [{ ...obligation, status: "blocked" as const }]; + const uncommitted = { ...value.obligation_ledger, + obligations, + status: reconstructionObligationLedgerStatus(obligations, []), + summary: summarizeReconstructionObligations(obligations), + }; + value.obligation_ledger = { + ...uncommitted, + closure_digest: reconstructionObligationLedgerClosureDigest(uncommitted), + }; + const latest = value.closure_history.at(-1); + if (latest === undefined) throw new TypeError("Missing closure history"); + value.closure_history[value.closure_history.length - 1] = { + ...latest, + ledger_digest: value.obligation_ledger.closure_digest, status: "open", - summary: { - ...value.obligation_ledger.summary, - verified: 0, - required_open: 1, - }, + required_open: 1, + newly_verified_obligation_ids: [], }; const report = createReconstructionReadinessReport(value); diff --git a/src/domain/recordedTransportUrl.ts b/src/domain/recordedTransportUrl.ts new file mode 100644 index 000000000..c524ae1a8 --- /dev/null +++ b/src/domain/recordedTransportUrl.ts @@ -0,0 +1,75 @@ +/** Raw coordinates are independent of WHATWG's normalized origin and pathname. */ +export interface RecordedTransportUrl { + readonly origin: string | null; + readonly path: string | null; + readonly credentials: + | { readonly start: number; readonly end: number } + | "ambiguous" + | null; +} + +/** + * Interpret transport URL syntax once for credential exclusion and recorded-path + * projection. Never use URL.pathname: it removes observed dot segments. + * Noncanonical spellings may have parsed credentials but no safe raw span. + */ +export const parseRecordedTransportUrl = ( + value: string, +): RecordedTransportUrl => { + const prefix = /^[\x00-\x20]*(?:([A-Za-z][A-Za-z0-9+.-]*):)?\/\//u.exec( + value, + ); + const absolute = URL.parse(value); + // Resolve references only to recognize credentials (including WHATWG's + // backslash/control-character spellings), never to supply a projected origin. + const parsed = absolute ?? URL.parse(value, "https://rea.invalid"); + const hasCredentials = + parsed !== null && (parsed.username !== "" || parsed.password !== ""); + const origin = + absolute !== null && absolute.origin !== "null" ? absolute.origin : null; + if (prefix === null) + return { + origin, + path: null, + credentials: hasCredentials ? "ambiguous" : null, + }; + + const authorityStart = prefix[0].length; + const protocol = + parsed?.protocol ?? `${prefix[1]?.toLowerCase() ?? "https"}:`; + const special = ["ftp:", "file:", "http:", "https:", "ws:", "wss:"].includes( + protocol, + ); + let authorityEnd = authorityStart; + while ( + authorityEnd < value.length && + !"/?#".includes(value[authorityEnd]!) && + !(special && value[authorityEnd] === "\\") + ) + authorityEnd += 1; + const authority = value.slice(authorityStart, authorityEnd); + const stableAuthority = authority.length > 0 && !/[\t\n\r]/u.test(authority); + const at = authority.lastIndexOf("@"); + const credentials = + !stableAuthority && hasCredentials + ? "ambiguous" + : at >= 0 + ? { start: authorityStart, end: authorityStart + at + 1 } + : hasCredentials + ? "ambiguous" + : null; + // Only an explicit, unpadded scheme://authority has a recorded absolute path. + // Backslashes are authority/path separators for special schemes in WHATWG. + if ( + origin === null || + !/^[A-Za-z][A-Za-z0-9+.-]*:\/\//u.test(value) || + !stableAuthority || + (special && value[authorityEnd] === "\\") + ) + return { origin, path: null, credentials }; + if (value[authorityEnd] !== "/") return { origin, path: "/", credentials }; + let pathEnd = authorityEnd; + while (pathEnd < value.length && !"?#".includes(value[pathEnd]!)) + pathEnd += 1; + return { origin, path: value.slice(authorityEnd, pathEnd), credentials }; +}; diff --git a/src/domain/relationshipCoverage.ts b/src/domain/relationshipCoverage.ts new file mode 100644 index 000000000..1de973669 --- /dev/null +++ b/src/domain/relationshipCoverage.ts @@ -0,0 +1,24 @@ +import { z } from "zod"; + +const diagnostics = z.array(z.string().min(1)).min(1); +const completeCoverage = z.strictObject({ status: z.literal("complete") }); +const partialCoverage = z.strictObject({ + status: z.literal("partial"), + limitations: z.array(z.string().min(1)).min(1), +}); +const unknownCoverage = z.strictObject({ + status: z.literal("unknown"), + reason: z.string().min(1), +}); + +/** Explicit coverage state for a provider's observed caller/callee set. */ +export const relationshipCoverageSchema = z.union([ + completeCoverage, + completeCoverage.extend({ diagnostics }), + partialCoverage, + partialCoverage.extend({ diagnostics }), + unknownCoverage, + unknownCoverage.extend({ diagnostics }), +]); + +export type RelationshipCoverage = z.infer; diff --git a/src/domain/sourceMapContents.test.ts b/src/domain/sourceMapContents.test.ts new file mode 100644 index 000000000..06f07197b --- /dev/null +++ b/src/domain/sourceMapContents.test.ts @@ -0,0 +1,11 @@ +import { expect, it } from "vitest"; + +import { hasValidSourceMapContents } from "./sourceMapContents.js"; + +it("accepts absent map-level source contents and aligned nullable slots", () => { + expect(hasValidSourceMapContents(2, undefined)).toBe(true); + expect(hasValidSourceMapContents(2, null)).toBe(true); + expect(hasValidSourceMapContents(2, [null, "retained source"])).toBe(true); + expect(hasValidSourceMapContents(2, ["retained source"])).toBe(false); + expect(hasValidSourceMapContents(2, [null, 42])).toBe(false); +}); diff --git a/src/domain/sourceMapContents.ts b/src/domain/sourceMapContents.ts index 254ad72b5..5d55810a9 100644 --- a/src/domain/sourceMapContents.ts +++ b/src/domain/sourceMapContents.ts @@ -4,6 +4,7 @@ export const hasValidSourceMapContents = ( contents: unknown, ): boolean => contents === undefined || + contents === null || (Array.isArray(contents) && contents.length === sourceCount && contents.every( diff --git a/src/domain/sourceMapEnvelope.test.ts b/src/domain/sourceMapEnvelope.test.ts index 6359ef889..4f8ea5c0e 100644 --- a/src/domain/sourceMapEnvelope.test.ts +++ b/src/domain/sourceMapEnvelope.test.ts @@ -14,11 +14,31 @@ describe("source-map envelope walk", () => { const root = { version: 3, sections: [ - { offset: { line: 0, column: 0 }, map: leaf }, - { offset: { line: 10, column: 0 }, map: { ...leaf, sources: [] } }, + { + offset: { line: 0, column: 0 }, + map: { + version: 3, + sections: [ + { + offset: { line: 0, column: 0 }, + map: { ...leaf, sources: ["b.ts"] }, + }, + { + offset: { line: 1, column: 0 }, + map: { ...leaf, sources: ["c.ts"] }, + }, + ], + }, + }, + { + offset: { line: 10, column: 0 }, + map: { ...leaf, sources: ["d.ts"] }, + }, ], }; - expect(flattenSourceMapLeaves(root)).toHaveLength(2); + expect(flattenSourceMapLeaves(root)?.map(({ sources }) => sources)).toEqual( + [["b.ts"], ["c.ts"], ["d.ts"]], + ); }); it("rejects sections without embedded maps", () => { diff --git a/src/domain/sourceMapEnvelope.ts b/src/domain/sourceMapEnvelope.ts index 7be21703e..b1cc1ff49 100644 --- a/src/domain/sourceMapEnvelope.ts +++ b/src/domain/sourceMapEnvelope.ts @@ -37,7 +37,8 @@ export const flattenSourceMapLeaves = ( const map = pending.pop(); if (!isVersion3Map(map)) return undefined; if (Array.isArray(map.sections)) { - for (const section of map.sections) { + for (let index = map.sections.length - 1; index >= 0; index -= 1) { + const section = map.sections[index]; if (!isRecord(section) || !isRecord(section.map)) return undefined; pending.push(section.map); } diff --git a/src/domain/wasm/wasmArtifact.ts b/src/domain/wasm/wasmArtifact.ts index 70a86b79a..ed2a120e0 100644 --- a/src/domain/wasm/wasmArtifact.ts +++ b/src/domain/wasm/wasmArtifact.ts @@ -1,5 +1,7 @@ import { z } from "zod"; +import { localPathStringSchema } from "../localPath.js"; + const digest = z.string().regex(/^[a-f0-9]{64}$/); const artifact = z.strictObject({ path: z.string().min(1), @@ -8,12 +10,11 @@ const artifact = z.strictObject({ }); /** All artifacts are explicitly selected local files; references never trigger acquisition. */ export const inspectWasmArtifactInputSchema = z.strictObject({ - path: z - .string() - .min(1) - .describe("Absolute selected local WASM artifact path"), - glue_paths: z.array(z.string().min(1)).max(16).default([]), - candidate_paths: z.array(z.string().min(1)).max(16).default([]), + path: localPathStringSchema.describe( + "Absolute selected local WASM artifact path", + ), + glue_paths: z.array(localPathStringSchema).max(16).default([]), + candidate_paths: z.array(localPathStringSchema).max(16).default([]), }); /** Upstream output stays intact; import/export names are never split on ambiguous punctuation. */ export const wasmArtifactSchema = z diff --git a/src/domain/webBundleAnalysis.ts b/src/domain/webBundleAnalysis.ts index 240016c54..27c79d126 100644 --- a/src/domain/webBundleAnalysis.ts +++ b/src/domain/webBundleAnalysis.ts @@ -32,7 +32,9 @@ const findingSchema = z.object({ }); const originalSourceSchema = z.object({ - source: z.string(), + source_index: z.number().int().min(0).optional(), + declaration_index: z.number().int().min(0).optional(), + source: z.string().nullable(), artifact: webTextArtifactSchema.nullable(), }); const originalModuleEdgeSchema = z.object({ @@ -42,9 +44,11 @@ const originalModuleEdgeSchema = z.object({ resolved_source: z.string().nullable(), }); const sourceMapMappingSchema = z.object({ + source_index: z.number().int().min(0).optional(), + declaration_index: z.number().int().min(0).optional(), generated_line: z.number().int().min(1), generated_column: z.number().int().min(0), - source: z.string(), + source: z.string().nullable(), original_line: z.number().int().min(1), original_column: z.number().int().min(0), name: z.string().nullable(), diff --git a/src/domain/webNetworkCapture.ts b/src/domain/webNetworkCapture.ts index 78309d1dc..ed31d5c70 100644 --- a/src/domain/webNetworkCapture.ts +++ b/src/domain/webNetworkCapture.ts @@ -97,40 +97,69 @@ export const webNetworkCaptureRecordSchema = z.strictObject({ limitations: z.array(z.string()), }); +/** Decoded records retain unique producer ordinals within their declared coverage. */ +export const webNetworkCapturePayloadSchema = z + .strictObject({ + decoder: z.strictObject({ + id: z.string().min(1), + name: z.string().min(1), + version: z.string().min(1), + }), + container: z.strictObject({ + reported: jsonValueSchema, + numeric_literals: z.array(numberSchema), + redactions: z.array(redactionSchema), + records_pointer: pointerSchema.nullable(), + }), + total_records: z.number().int().nonnegative(), + records: z.array(webNetworkCaptureRecordSchema), + }) + .superRefine((capture, context) => { + if (capture.records.length > capture.total_records) { + context.addIssue({ + code: "custom", + path: ["records"], + message: "Retained record count exceeds the declared capture total.", + }); + return; + } + const seen = new Set(); + for (const [index, record] of capture.records.entries()) { + if (record.ordinal >= capture.total_records || seen.has(record.ordinal)) { + context.addIssue({ + code: "custom", + path: ["records", index, "ordinal"], + message: + "Retained ordinals must be unique and within the declared capture total.", + }); + return; + } + seen.add(record.ordinal); + } + }); + /** Historical capture inspection has file identity, producer provenance and explicit unknowns. */ -export const webNetworkCaptureSchema = z.strictObject({ - artifact: z.strictObject({ - path: z - .string() - .describe( - "Selected artifact path; empty when explicitly excluded from Evidence.", - ), - sha256: z - .string() - .regex(/^[0-9a-f]{64}$/) - .nullable() - .describe( - "Observed artifact SHA-256; null only when explicitly excluded by a sensitive declaration.", - ), - bytes: z.number().int().nonnegative(), - }), - format: z.enum(["har", "mitmproxy"]), - decoder: z.strictObject({ - id: z.string().min(1), - name: z.string().min(1), - version: z.string().min(1), - }), - container: z.strictObject({ - reported: jsonValueSchema, - numeric_literals: z.array(numberSchema), - redactions: z.array(redactionSchema), - records_pointer: pointerSchema.nullable(), - }), - total_records: z.number().int().nonnegative(), - records: z.array(webNetworkCaptureRecordSchema), - runtime_attribution: z.literal("unknown"), - limitations: z.array(z.string()), -}); +export const webNetworkCaptureSchema = + webNetworkCapturePayloadSchema.safeExtend({ + artifact: z.strictObject({ + path: z + .string() + .describe( + "Selected artifact path; empty when explicitly excluded from Evidence.", + ), + sha256: z + .string() + .regex(/^[0-9a-f]{64}$/) + .nullable() + .describe( + "Observed artifact SHA-256; null only when explicitly excluded by a sensitive declaration.", + ), + bytes: z.number().int().nonnegative(), + }), + format: z.enum(["har", "mitmproxy"]), + runtime_attribution: z.literal("unknown"), + limitations: z.array(z.string()), + }); export type InspectWebNetworkCaptureInput = z.output< typeof inspectWebNetworkCaptureInputSchema diff --git a/src/domain/webRuntime.ts b/src/domain/webRuntime.ts index 289d52325..57abb4835 100644 --- a/src/domain/webRuntime.ts +++ b/src/domain/webRuntime.ts @@ -3,7 +3,6 @@ import { browserAllowedOriginsSchema, browserEndpointSchema, } from "./browserObservation.js"; -import { browserVersionSchema } from "./browserObservationSchemas.js"; import { jsonObjectSchema } from "./jsonValue.js"; // CDP positions are zero-based and relative to the resource named by url. diff --git a/src/dotnet/ManagedArtifactFailures.test.ts b/src/dotnet/ManagedArtifactFailures.test.ts index 1647157b5..2f953bee2 100644 --- a/src/dotnet/ManagedArtifactFailures.test.ts +++ b/src/dotnet/ManagedArtifactFailures.test.ts @@ -46,6 +46,14 @@ describe("managed artifact failure classification", () => { ]); }); + it("rejects a ReadyToRun OS-override machine without a CLI directory", () => { + const bytes = buildNativePeFixture(); + bytes.writeUInt16LE(0x014c ^ 0x7b79, bytes.readUInt32LE(0x3c) + 4); + expect(() => + inspectManagedArtifactBytes(bytes, managedPeFixtureTarget(bytes)), + ).toThrow("Unsupported PE machine 0x7a35"); + }); + it("keeps malformed reference and attribute issues across inventory callers", () => { const bytes = buildManagedPeFixture({ references: [ @@ -66,6 +74,7 @@ describe("managed artifact failure classification", () => { "System.Runtime", "System.Runtime", ]); + expect(artifact.attributes).toEqual([]); expect(artifact.coverage.issues.map(({ code }) => code)).toEqual( issueCodes, ); diff --git a/src/dotnet/ManagedArtifactInspector.test.ts b/src/dotnet/ManagedArtifactInspector.test.ts index 5de925254..3fc74337e 100644 --- a/src/dotnet/ManagedArtifactInspector.test.ts +++ b/src/dotnet/ManagedArtifactInspector.test.ts @@ -177,47 +177,54 @@ describe("managed artifact coded indexes", () => { }); }); - it("reports an invalid CustomAttribute constructor coded index", () => { - const bytes = buildManagedPeFixture({ customAttributeTypeRaw: 19 }); - const result = inspectManagedArtifactBytes( - bytes, - managedPeFixtureTarget(bytes), - ); + it.each([ + [ + 0, + "CustomAttribute constructor coded index 0x0 is null, but Type must reference a MethodDef or MemberRef row", + ], + [ + 19, + "coded index 0x13 is invalid: coded index selects row 2 in table 10, which has 1 rows", + ], + ] as const)( + "preserves readable attribute facts for constructor index %s", + (rawType, detail) => { + const bytes = buildManagedPeFixture({ customAttributeTypeRaw: rawType }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + const text = Buffer.from(".NETCoreApp,Version=v8.0", "utf8"); + const attributeValue = Buffer.concat([ + Buffer.from([1, 0, text.length]), + text, + Buffer.alloc(2), + ]); - expect(result.attributes).toEqual([]); - expect(result.coverage).toMatchObject({ - state: "partial", - issues: [ - expect.objectContaining({ - code: "invalid-row", - scope: "metadata.CustomAttribute:0x0c000001", - detail: expect.stringContaining( - "coded index 0x13 is invalid: coded index selects row 2 in table 10, which has 1 rows", - ), - }), - ], - }); - }); - - it("reports null for the required CustomAttribute constructor coded index", () => { - const bytes = buildManagedPeFixture({ customAttributeTypeRaw: 0 }); - const result = inspectManagedArtifactBytes( - bytes, - managedPeFixtureTarget(bytes), - ); - - expect(result.attributes).toEqual([]); - expect(result.coverage).toMatchObject({ - state: "partial", - issues: [ - expect.objectContaining({ - code: "invalid-row", - detail: - "CustomAttribute constructor coded index 0x0 is null, but Type must reference a MethodDef or MemberRef row", - }), - ], - }); - }); + expect(result.attributes).toMatchObject([ + { + parent_token: "0x20000001", + constructor_token: null, + type_name: null, + value_length: attributeValue.length, + value_sha256: createHash("sha256") + .update(attributeValue) + .digest("hex"), + token: "0x0c000001", + }, + ]); + expect(result.coverage).toMatchObject({ + state: "partial", + issues: [ + expect.objectContaining({ + code: "invalid-row", + scope: "metadata.CustomAttribute:0x0c000001", + detail: expect.stringContaining(detail), + }), + ], + }); + }, + ); it("reports coded-index column offsets after widened string and coded-index columns", () => { const references = Array.from( @@ -227,7 +234,7 @@ describe("managed artifact coded indexes", () => { const bytes = buildManagedPeFixture({ references, extendsRaw: 9, - customAttributeTypeRaw: 19, + customAttributeTypeRaw: 0, }); const layout = metadataLayoutFor(bytes); const artifact = inspectManagedArtifactBytes( diff --git a/src/dotnet/ManagedMemberInspector.test.ts b/src/dotnet/ManagedMemberInspector.test.ts index 0b342332e..81d64f4a1 100644 --- a/src/dotnet/ManagedMemberInspector.test.ts +++ b/src/dotnet/ManagedMemberInspector.test.ts @@ -859,3 +859,68 @@ it.each([ } }, ); + +describe("managed member type selection", () => { + const bytes = buildManagedPeFixture(); + const target = managedPeFixtureTarget(bytes); + const everything = inspectManagedMembersBytes(bytes, target); + + it("reports no selection when every type is inspected", () => { + expect(everything.selection).toBeNull(); + }); + + it("decodes the selected type's members and reports unmatched names", () => { + const result = inspectManagedMembersBytes(bytes, target, { + typeNames: ["Fixture.Program", "Missing.Type"], + }); + + expect(result.selection).toEqual({ + type_names: ["Fixture.Program", "Missing.Type"], + unmatched_type_names: ["Missing.Type"], + }); + expect(result.types).toEqual(everything.types); + expect(result.fields).toEqual(everything.fields); + expect(result.methods).toEqual(everything.methods); + expect(result.call_edges).toEqual(everything.call_edges); + expect(result.field_accesses).toEqual(everything.field_accesses); + expect(result.coverage).toEqual(everything.coverage); + expect(result.limitations).toContain( + "Types, fields, methods, call edges, and field accesses cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", + ); + }); + + it("does not treat a name as absent when a TypeDef name is unreadable", () => { + const corrupt = Buffer.from(bytes); + const row = everything.types[0]?.row_offset ?? 0; + // Point the TypeDef Name past the #Strings heap. + corrupt.writeUInt16LE(0xffff, row + 4); + const result = inspectManagedMembersBytes( + corrupt, + managedPeFixtureTarget(corrupt), + { typeNames: ["Fixture.Program"] }, + ); + + expect(result.selection?.unmatched_type_names).toEqual(["Fixture.Program"]); + expect(result.coverage.state).toBe("partial"); + expect(result.limitations).toContain( + "1 TypeDef name(s) could not be read; a name in selection.unmatched_type_names may belong to one of them, so its absence is not observed.", + ); + }); + + it("omits members of unselected types without dropping member references", () => { + const result = inspectManagedMembersBytes(bytes, target, { + typeNames: ["Missing.Type"], + }); + + expect(result.selection).toEqual({ + type_names: ["Missing.Type"], + unmatched_type_names: ["Missing.Type"], + }); + expect(result.types).toEqual([]); + expect(result.fields).toEqual([]); + expect(result.methods).toEqual([]); + expect(result.call_edges).toEqual([]); + expect(result.field_accesses).toEqual([]); + expect(result.member_refs).toEqual(everything.member_refs); + }); +}); diff --git a/src/dotnet/ManagedMemberInspector.ts b/src/dotnet/ManagedMemberInspector.ts index 6c27ac802..24b97f9eb 100644 --- a/src/dotnet/ManagedMemberInspector.ts +++ b/src/dotnet/ManagedMemberInspector.ts @@ -22,19 +22,25 @@ import { ManagedReaderFailure, uniqueManagedParseIssues, } from "./ManagedReaderFailure.js"; -import { parseTypes, typeRanges } from "./ManagedMemberInspectorCore.js"; +import { + parseTypes, + typeRanges, + type TypeRange, +} from "./ManagedMemberInspectorCore.js"; import { edges, parseExportedTypes, parseFields, parseMemberRefs, parseMethods, + type SelectedTypeTokens, } from "./ManagedMemberRows.js"; const unavailable = ( target: BinaryTarget, bytes: Buffer, issue: ManagedParseIssue | null, + typeNames: readonly string[] | undefined, ): ManagedMemberInspection => managedMemberInspectionSchema.parse( admitManagedProjection({ @@ -67,6 +73,11 @@ const unavailable = ( member_refs: [], call_edges: [], field_accesses: [], + // Nothing was examined, so no requested name is known to be unmatched. + selection: + typeNames === undefined + ? null + : { type_names: [...typeNames], unmatched_type_names: [] }, coverage: { state: "unavailable", issues: issue === null ? [] : [issue], @@ -105,23 +116,61 @@ const readMemberInventory = ( return { layout, inventory }; }; -/** Inspect metadata members and method bodies without loading target code. */ +const selectTypes = ( + ranges: readonly TypeRange[], + typeNames: readonly string[] | undefined, +): { + readonly tokens: SelectedTypeTokens; + readonly selection: ManagedMemberInspection["selection"]; + readonly unreadableNames: number; +} => { + if (typeNames === undefined) + return { tokens: null, selection: null, unreadableNames: 0 }; + const requested = new Set(typeNames); + const tokens = new Set(); + const matched = new Set(); + for (const range of ranges) { + if (range.fullName === null || !requested.has(range.fullName)) continue; + tokens.add(range.token); + matched.add(range.fullName); + } + return { + tokens, + unreadableNames: ranges.filter(({ fullName }) => fullName === null).length, + selection: { + type_names: [...typeNames], + unmatched_type_names: typeNames.filter((name) => !matched.has(name)), + }, + }; +}; + +/** + * Inspect metadata members and method bodies without loading target code. + * Selected type names restrict full decoding to members of those exact types. + */ export const inspectManagedMembersBytes = ( bytes: Buffer, target: BinaryTarget, + { typeNames }: { readonly typeNames?: readonly string[] } = {}, ): ManagedMemberInspection => { const pe = readManagedPeLayout(bytes); - if (pe.cli === null) return unavailable(target, bytes, pe.cliIssue); + if (pe.cli === null) + return unavailable(target, bytes, pe.cliIssue, typeNames); const issues: ManagedParseIssue[] = []; try { const { layout, inventory } = readMemberInventory(bytes, pe, issues); issues.push(...inventory.issues); const ranges = typeRanges(bytes, layout, issues); + const { + tokens: selected, + selection, + unreadableNames, + } = selectTypes(ranges, typeNames); const parsedTypes = parseTypes(bytes, layout, ranges); issues.push(...parsedTypes.issues); const exportedTypes = parseExportedTypes(bytes, layout); issues.push(...exportedTypes.issues); - const fields = parseFields(bytes, layout, ranges); + const fields = parseFields(bytes, layout, ranges, selected); const memberRefs = parseMemberRefs(bytes, layout); issues.push(...memberRefs.issues); const methods = parseMethods({ @@ -129,6 +178,7 @@ export const inspectManagedMembersBytes = ( layout, pe, ranges, + selected, }); issues.push(...fields.issues, ...methods.issues); const coverageIssues = uniqueManagedParseIssues(issues); @@ -168,13 +218,17 @@ export const inspectManagedMembersBytes = ( requires_artifact_sha256: target.sha256, requires_mvid: inventory.module?.mvid ?? null, }, - types: parsedTypes.types, + types: + selected === null + ? parsedTypes.types + : parsedTypes.types.filter((type) => selected.has(type.token)), exported_types: exportedTypes.exportedTypes, fields: fields.fields, methods: methods.methods, member_refs: memberRefs.refs, call_edges: related.callEdges, field_accesses: related.fieldAccesses, + selection, coverage: { state: issues.length === 0 && !incompleteMembers @@ -187,6 +241,16 @@ export const inspectManagedMembersBytes = ( "CIL instruction anchors are decoded from file-backed method bodies only; no target assembly is loaded or executed.", "Exported types are reported from ExportedType rows as declared; forwarded types are not resolved in their implementing assemblies.", "Member signatures decode ECMA-335 primitive, class, valuetype, generic instantiation, generic variable, pointer, byref, single-dimension and general array, function pointer, custom modifier, and typed reference forms; class and valuetype operands keep their encoded TypeDefOrRefOrSpec values, and unsupported forms such as vararg sentinels retain raw signature hashes.", + ...(selection === null + ? [] + : [ + "Types, fields, methods, call edges, and field accesses cover only the selected declaring types; coverage describes that selection. Edge targets outside it keep their names but are not described.", + ]), + ...(unreadableNames === 0 + ? [] + : [ + `${String(unreadableNames)} TypeDef name(s) could not be read; a name in selection.unmatched_type_names may belong to one of them, so its absence is not observed.`, + ]), ], }, managedDecodeBudget(layout), @@ -194,7 +258,7 @@ export const inspectManagedMembersBytes = ( ); } catch (cause: unknown) { if (cause instanceof ManagedReaderFailure) - return unavailable(target, bytes, cause.issue); + return unavailable(target, bytes, cause.issue, typeNames); throw cause; } }; diff --git a/src/dotnet/ManagedMemberRows.ts b/src/dotnet/ManagedMemberRows.ts index 559201f13..f4a63db5a 100644 --- a/src/dotnet/ManagedMemberRows.ts +++ b/src/dotnet/ManagedMemberRows.ts @@ -32,10 +32,23 @@ import { readManagedValue } from "./ManagedReaderFailure.js"; import { methodBody } from "./ManagedMethodBodyReader.js"; import { memberDeclaringTypeRowLookup } from "./ManagedMemberOwnership.js"; +/** + * Declaring-type tokens whose members are decoded in full; null selects every + * type. Unselected members keep only their name so edges stay labelled. + */ +export type SelectedTypeTokens = ReadonlySet | null; + +const isSelected = ( + selected: SelectedTypeTokens, + declared: { readonly token: string } | undefined, +): boolean => + selected === null || (declared !== undefined && selected.has(declared.token)); + export const parseFields = ( bytes: Buffer, layout: ManagedMetadataLayout, ranges: readonly TypeRange[], + selected: SelectedTypeTokens = null, ): { readonly fields: readonly ManagedField[]; readonly core: ReadonlyMap; @@ -59,18 +72,20 @@ export const parseFields = ( cursor.readIndex(layout.stringIndexSize), layout.strings.size, ); - const sig = readMetadataBlob( - bytes, - layout, - cursor.readIndex(layout.blobIndexSize), - layout.blob.size, - ); const owner = ownerRow === undefined ? 0 : (readManagedValue(() => ownerRow(row), issues) ?? 0); const declared = ranges[owner - 1]; const token = metadataToken(4, row); + core.set(token, { token, name }); + if (!isSelected(selected, declared)) return; + const sig = readMetadataBlob( + bytes, + layout, + cursor.readIndex(layout.blobIndexSize), + layout.blob.size, + ); fields.push({ token, row_offset: cursor.start, @@ -80,7 +95,6 @@ export const parseFields = ( flags, signature: budgetedSignature(sig, layout), }); - core.set(token, { token, name }); }, issues); } return { fields, core, issues }; @@ -227,6 +241,7 @@ interface ParseMethodsInput { readonly layout: ManagedMetadataLayout; readonly pe: ManagedPeLayout; readonly ranges: readonly TypeRange[]; + readonly selected?: SelectedTypeTokens; } export const parseMethods = ({ @@ -234,6 +249,7 @@ export const parseMethods = ({ layout, pe, ranges, + selected = null, }: ParseMethodsInput): { readonly methods: readonly ManagedMethod[]; readonly core: ReadonlyMap; @@ -259,6 +275,18 @@ export const parseMethods = ({ cursor.readIndex(layout.stringIndexSize), layout.strings.size, ); + const owner = + ownerRow === undefined + ? 0 + : (readManagedValue(() => ownerRow(row), issues) ?? 0); + const declared = ranges[owner - 1]; + const token = metadataToken(6, row); + core.set(token, { + token, + name, + declaringType: declared?.fullName ?? null, + }); + if (!isSelected(selected, declared)) return; const sig = readMetadataBlob( bytes, layout, @@ -266,12 +294,6 @@ export const parseMethods = ({ layout.blob.size, ); cursor.readIndex(layout.tableIndexSize(8)); - const owner = - ownerRow === undefined - ? 0 - : (readManagedValue(() => ownerRow(row), issues) ?? 0); - const declared = ranges[owner - 1]; - const token = metadataToken(6, row); methods.push({ token, row_offset: cursor.start, @@ -290,11 +312,6 @@ export const parseMethods = ({ managedDecodeBudget(layout), ), }); - core.set(token, { - token, - name, - declaringType: declared?.fullName ?? null, - }); }, issues); } return { methods, core, issues }; diff --git a/src/dotnet/ManagedMetadataInventoryRows.ts b/src/dotnet/ManagedMetadataInventoryRows.ts index 4689b72eb..30f9e8e5a 100644 --- a/src/dotnet/ManagedMetadataInventoryRows.ts +++ b/src/dotnet/ManagedMetadataInventoryRows.ts @@ -27,6 +27,13 @@ type AssemblyReference = ManagedArtifactInspection["references"][number]; type ManagedResource = ManagedArtifactInspection["resources"][number]; type CustomAttribute = ManagedArtifactInspection["attributes"][number]; +const HAS_CUSTOM_ATTRIBUTE_TABLES = [ + 6, 4, 1, 2, 8, 9, 10, 0, 14, 23, 20, 17, 26, 27, 32, 35, 38, 39, 40, 42, 44, + 43, +] as const; + +const CUSTOM_ATTRIBUTE_TYPE_TABLES = [undefined, undefined, 6, 10] as const; + const publicKeyIdentity = ( bytes: Buffer, kind: "public-key" | "public-key-token", @@ -360,56 +367,56 @@ export const readCustomAttribute = ( const parent = metadataCodedToken( parentRaw, 5, - [ - 6, 4, 1, 2, 8, 9, 10, 0, 14, 23, 20, 17, 26, 27, 32, 35, 38, 39, 40, 42, - 44, 43, - ], + HAS_CUSTOM_ATTRIBUTE_TABLES, layout.rowCounts, ); - const parentReason = metadataCodedTokenInvalidReason( - parentRaw, - 5, - [ - 6, 4, 1, 2, 8, 9, 10, 0, 14, 23, 20, 17, 26, 27, 32, 35, 38, 39, 40, 42, - 44, 43, - ], - layout.rowCounts, - ); - if (parent === null) + if (parent === null) { + const parentReason = metadataCodedTokenInvalidReason( + parentRaw, + 5, + HAS_CUSTOM_ATTRIBUTE_TABLES, + layout.rowCounts, + ); throw managedFailure( "invalid-row", `metadata.CustomAttribute:${metadataToken(12, row)}`, `CustomAttribute parent coded index 0x${parentRaw.toString(16)} is invalid${parentReason === null ? " because the required parent is null" : `: ${parentReason}`}`, cursor.start, ); - const typeReason = metadataCodedTokenInvalidReason( + } + const constructor = metadataCodedToken( typeRaw, 3, - [undefined, undefined, 6, 10], + CUSTOM_ATTRIBUTE_TYPE_TABLES, layout.rowCounts, ); - if (typeReason !== null || typeRaw === 0) - throw managedFailure( - "invalid-row", - `metadata.CustomAttribute:${metadataToken(12, row)}`, - typeReason === null - ? "CustomAttribute constructor coded index 0x0 is null, but Type must reference a MethodDef or MemberRef row" - : `CustomAttribute constructor coded index 0x${typeRaw.toString(16)} is invalid: ${typeReason}`, - typeOffset, - ); - const typeName = - readManagedValue( - () => attributeTypeName(bytes, layout, typeRaw, heapExtent), - issues, - ) ?? null; - return { - parent_token: parent, - constructor_token: metadataCodedToken( + if (constructor === null) { + const typeReason = metadataCodedTokenInvalidReason( typeRaw, 3, - [undefined, undefined, 6, 10], + CUSTOM_ATTRIBUTE_TYPE_TABLES, layout.rowCounts, - ), + ); + issues.push({ + code: "invalid-row", + scope: `metadata.CustomAttribute:${metadataToken(12, row)}`, + offset: typeOffset, + detail: + typeReason === null + ? "CustomAttribute constructor coded index 0x0 is null, but Type must reference a MethodDef or MemberRef row" + : `CustomAttribute constructor coded index 0x${typeRaw.toString(16)} is invalid: ${typeReason}`, + }); + } + let typeName: string | null = null; + if (constructor !== null) + typeName = + readManagedValue( + () => attributeTypeName(bytes, layout, typeRaw, heapExtent), + issues, + ) ?? null; + return { + parent_token: parent, + constructor_token: constructor, type_name: typeName, value_length: value.length, value_sha256: managedDecodeBudget(layout).digest(value, "sha256"), diff --git a/src/dotnet/ManagedPeReader.ts b/src/dotnet/ManagedPeReader.ts index 5f345d402..6f11cc491 100644 --- a/src/dotnet/ManagedPeReader.ts +++ b/src/dotnet/ManagedPeReader.ts @@ -1,4 +1,5 @@ import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; +import { decodePeMachine } from "../domain/peMachine.js"; import { managedFailure, ManagedReaderFailure, @@ -100,25 +101,18 @@ const add = (left: number, right: number, scope: string): number => { return result; }; -const architectureFor = (machine: number): ManagedPeLayout["architecture"] => { - switch (machine) { - case 0x014c: - return "x86"; - case 0x8664: - return "x86_64"; - case 0x01c0: - case 0x01c2: - case 0x01c4: - return "arm"; - case 0xaa64: - return "arm64"; - default: - throw managedFailure( - "invalid-directory", - "pe.machine", - `Unsupported PE machine 0x${machine.toString(16).padStart(4, "0")}`, - ); - } +const architectureFor = ( + machine: number, + managed: boolean, +): ManagedPeLayout["architecture"] => { + const decoded = decodePeMachine(machine); + if (decoded === undefined || (decoded.nativeOsOverride && !managed)) + throw managedFailure( + "invalid-directory", + "pe.machine", + `Unsupported PE machine 0x${machine.toString(16).padStart(4, "0")}`, + ); + return decoded.architecture; }; /** Resolve one section identity for both exact reads and available-byte queries. */ @@ -398,7 +392,7 @@ export const readManagedPeLayout = (bytes: Buffer): ManagedPeLayout => { } return { machine: coff.machine, - architecture: architectureFor(coff.machine), + architecture: architectureFor(coff.machine, cliDirectoryPresent), optionalHeader: optional.optionalHeader, sectionCount: coff.sectionCount, characteristics: coff.characteristics, diff --git a/src/dotnet/ManagedStaticProvider.ts b/src/dotnet/ManagedStaticProvider.ts index 116f93382..e3827c346 100644 --- a/src/dotnet/ManagedStaticProvider.ts +++ b/src/dotnet/ManagedStaticProvider.ts @@ -13,6 +13,7 @@ import { managedStaticCapabilities } from "./ManagedStaticProviderMetadata.js"; import { MANAGED_STATIC_PROVIDER } from "../application/InvestigationProviders.js"; import { MANAGED_TOOL_CONTRACTS, + managedMembersInputSchema, managedTargetInputSchema, type ManagedToolName, } from "../contracts/managed/managedToolContracts.js"; @@ -235,14 +236,20 @@ const inspectManagedOperation = ( | ManagedArtifactInspection | ManagedMemberInspection | ManagedNativeBoundaryInspection => { + if (operation === "inspect_managed_members") { + const { type_names: typeNames } = + managedMembersInputSchema.parse(parameters); + return inspectManagedMembersBytes( + bytes, + target, + typeNames === undefined ? {} : { typeNames }, + ); + } managedTargetInputSchema.parse(parameters); if (operation === "inspect_managed_artifact") { return inspectManagedArtifactBytes(bytes, target); } - if (operation === "inspect_managed_native_boundaries") { - return inspectManagedNativeBoundariesBytes(bytes, target); - } - return inspectManagedMembersBytes(bytes, target); + return inspectManagedNativeBoundariesBytes(bytes, target); }; const managedLocations = ( diff --git a/src/filesystem/OwnedStagedOutput.ts b/src/filesystem/OwnedStagedOutput.ts new file mode 100644 index 000000000..2a1cfd9f3 --- /dev/null +++ b/src/filesystem/OwnedStagedOutput.ts @@ -0,0 +1,169 @@ +import { lstat, readdir, rmdir, unlink } from "node:fs/promises"; +import type { Stats } from "node:fs"; + +import { OwnedFileHandle } from "./OwnedFileHandle.js"; + +interface Identity { + readonly dev: number; + readonly ino: number; +} + +const identityOf = (stats: Stats): Identity => ({ + dev: stats.dev, + ino: stats.ino, +}); + +const sameIdentity = (stats: Stats, identity: Identity): boolean => + stats.dev === identity.dev && stats.ino === identity.ino; + +/** One atomically staged file and directory retained through verified cleanup. */ +export class OwnedStagedOutput { + #file: OwnedFileHandle | undefined; + #fileIdentity: Identity | undefined; + #fileRemovedByOwner = false; + #publishedRename = false; + #closeAttempt: Promise | undefined; + #closed = false; + #identityFailure: unknown; + + private constructor( + readonly directory: string, + private readonly directoryIdentity: Identity | undefined, + ) {} + + static async create(directory: string): Promise { + try { + const stats = await lstat(directory); + if (!stats.isDirectory() || stats.isSymbolicLink()) + return new OwnedStagedOutput(directory, undefined); + return new OwnedStagedOutput(directory, identityOf(stats)); + } catch (cause: unknown) { + // Keep an owner that refuses deletion when creation identity is unknown. + const owner = new OwnedStagedOutput(directory, undefined); + owner.#identityFailure = cause; + return owner; + } + } + + async setFile(file: OwnedFileHandle): Promise { + if (this.#file !== undefined) + throw new Error("Staged output already owns a file handle"); + this.#file = file; + try { + this.#fileIdentity = identityOf(await file.handle.stat()); + } catch (cause: unknown) { + // The handle remains owned; cleanup will close it and refuse pathname removal. + this.#identityFailure = cause; + } + } + + /** Verify that the path still names the exact closed file before publication. */ + async assertReadyForPublication(content: string): Promise { + await this.#verifiedDirectory(); + if (this.#fileIdentity === undefined) + throw new Error("Cannot verify staged file identity", { + cause: this.#identityFailure, + }); + const stagedFile = await lstat(content); + if ( + !stagedFile.isFile() || + stagedFile.isSymbolicLink() || + !sameIdentity(stagedFile, this.#fileIdentity) + ) + throw new Error(`Staged file identity changed: ${content}`); + } + + closeFile(): Promise { + return this.#file?.close() ?? Promise.resolve(); + } + + /** Record that the staged file moved to the published destination by rename. */ + recordPublishedRename(): void { + this.#publishedRename = true; + } + + /** Close the owned descriptor and remove only the exact staged entries. */ + close(): Promise { + if (this.#closed) return Promise.resolve(); + if (this.#closeAttempt !== undefined) return this.#closeAttempt; + const attempt = Promise.resolve().then(() => this.#cleanup()); + this.#closeAttempt = attempt; + void attempt.then( + () => { + this.#closed = true; + this.#closeAttempt = undefined; + }, + () => { + this.#closeAttempt = undefined; + }, + ); + return attempt; + } + + async #cleanup(): Promise { + await this.closeFile(); + await this.#verifiedDirectory(); + + const content = `${this.directory}/content`; + let stagedFile: Stats | undefined; + try { + stagedFile = await lstat(content); + } catch (cause: unknown) { + if ( + errorCode(cause) === "ENOENT" && + (this.#file === undefined || + this.#fileRemovedByOwner || + this.#publishedRename) + ) { + stagedFile = undefined; + } else if (errorCode(cause) === "ENOENT") { + throw new Error(`Owned staged file disappeared: ${content}`, { + cause, + }); + } else { + throw cause; + } + } + if (stagedFile !== undefined) { + if ( + !stagedFile.isFile() || + stagedFile.isSymbolicLink() || + this.#fileIdentity === undefined || + !sameIdentity(stagedFile, this.#fileIdentity) + ) + throw new Error(`Staged file identity changed: ${content}`); + await unlink(content); + this.#fileRemovedByOwner = true; + } + + const remaining = await readdir(this.directory); + if (remaining.length > 0) + throw new Error( + `Unexpected staging entries remain in ${this.directory}: ${remaining.join(", ")}`, + ); + await this.#verifiedDirectory(); + await rmdir(this.directory); + } + + async #verifiedDirectory(): Promise { + if (this.directoryIdentity === undefined) + throw new Error( + `Cannot verify staging directory ownership: ${this.directory}`, + { + cause: this.#identityFailure, + }, + ); + const directory = await lstat(this.directory); + if ( + !directory.isDirectory() || + directory.isSymbolicLink() || + !sameIdentity(directory, this.directoryIdentity) + ) + throw new Error(`Staging directory identity changed: ${this.directory}`); + } +} + +const errorCode = (cause: unknown): unknown => + typeof cause === "object" && cause !== null && "code" in cause + ? cause.code + : undefined; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index 6e1b1e7cc..a736cd0bf 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -13,7 +13,10 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { err, ok, type Result } from "../domain/result.js"; import type { GhidraInstallationInspection } from "./GhidraInstallation.js"; -import { ghidraMipsProfileParameters } from "./GhidraMipsProfile.js"; +import { + ghidraMipsProfileParameters, + ghidraMipsSharedLanguageOverride, +} from "./GhidraMipsProfile.js"; import { ghidraRawBinaryLanguage } from "./GhidraRawBinaryLanguage.js"; import { @@ -102,6 +105,7 @@ export const resolveGhidraAnalysisProfile = async ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", + call_relationship_coverage: "resolved-call-reference-diagnostics-v1", string_inventory_evidence: "defined-data-coverage-v1", location_resolution: "explicit-address-exact-entry-symbol-first-v3", instruction_flow_evidence: "decoded-return-pcode-v1", @@ -132,6 +136,22 @@ export const resolveGhidraAnalysisProfile = async ( const dosMz = target.format === "dos-mz"; const dosCom = target.format === "dos-com"; const dos = dosMz || dosCom; + const mipsLanguage = ghidraMipsSharedLanguageOverride(target); + if ( + mipsLanguage !== undefined && + languageOverride !== undefined && + (languageOverride.languageId !== mipsLanguage.languageId || + (languageOverride.compilerSpecId !== undefined && + languageOverride.compilerSpecId !== mipsLanguage.compilerSpecId)) + ) + return err( + new AnalysisUnsupportedTargetError( + "resolve_analysis_profile", + target.path, + "The caller-selected processor or compiler specification is outside the verified endian-matched MIPS32 shared-image profile. The requested override has not been changed.", + ), + ); + const selectedLanguage = languageOverride ?? mipsLanguage; // DOS imports already force an admitted loader, language, and context. if (dos && languageOverride !== undefined) return Promise.resolve( @@ -170,6 +190,7 @@ export const resolveGhidraAnalysisProfile = async ( load_image_observations: "source-mappings-entry-context-v2", function_body_evidence: "complete-inclusive-ranges-v1", function_references: "complete-body-and-entry-reference-manager-v2", + call_relationship_coverage: "resolved-call-reference-diagnostics-v1", string_inventory_evidence: "defined-data-coverage-v1", location_resolution: "explicit-address-exact-entry-symbol-first-v3", instruction_flow_evidence: "decoded-return-pcode-v1", @@ -199,14 +220,16 @@ export const resolveGhidraAnalysisProfile = async ( : "auto-from-header", language_id: dos ? "x86:LE:16:Real Mode" - : (languageOverride?.languageId ?? "auto-from-header"), + : (selectedLanguage?.languageId ?? "auto-from-header"), compiler_spec_id: dos ? "default" - : languageOverride === undefined + : selectedLanguage === undefined ? "auto-default" - : (languageOverride.compilerSpecId ?? "language-default"), + : (selectedLanguage.compilerSpecId ?? "language-default"), ...(languageOverride === undefined - ? {} + ? mipsLanguage === undefined + ? {} + : { language_selection: "provider-pinned-v1" } : { language_selection: "configured-v1" }), ...(dos ? { @@ -246,7 +269,11 @@ export const resolveGhidraAnalysisProfile = async ( export const ghidraProfileLanguageOverride = ( parameters: Readonly>, ): GhidraLanguageOverride | undefined => { - if (parameters.language_selection !== "configured-v1") return undefined; + if ( + parameters.language_selection !== "configured-v1" && + parameters.language_selection !== "provider-pinned-v1" + ) + return undefined; const languageId = parameters.language_id; const compilerSpecId = parameters.compiler_spec_id; if (typeof languageId !== "string" || typeof compilerSpecId !== "string") diff --git a/src/ghidra/GhidraInstallation.test.ts b/src/ghidra/GhidraInstallation.test.ts index 18f029758..059915d99 100644 --- a/src/ghidra/GhidraInstallation.test.ts +++ b/src/ghidra/GhidraInstallation.test.ts @@ -664,6 +664,30 @@ describe("Ghidra native decompiler admission", () => { }); }); + it("accepts a one-slice universal-format Darwin native decompiler", () => { + const bytes = fat([0x0100000c]); + const result = inspectGhidraInstallation( + { + environment: {}, + installDir: INSTALL, + platform: "darwin", + architecture: "arm64", + }, + host({ + executableHeader: (path: string) => + path === MAC_DECOMPILER ? { bytes, size: bytes.length } : undefined, + }), + ); + expect(result.status).toBe("available"); + if (result.status !== "available") return; + expect( + result.checks.find(({ name }) => name === "native_decompiler"), + ).toMatchObject({ + status: "passed", + detail: `${MAC_DECOMPILER} reports mach-o-universal arm64, which matches darwin/arm64`, + }); + }); + it("names the observed and the required executable in the failure", () => { const result = inspectGhidraInstallation( { diff --git a/src/ghidra/GhidraInstallation.ts b/src/ghidra/GhidraInstallation.ts index 4ff7a6868..8d2043e6b 100644 --- a/src/ghidra/GhidraInstallation.ts +++ b/src/ghidra/GhidraInstallation.ts @@ -17,7 +17,10 @@ import { parseExecutableHeader, type ExecutableMetadata, } from "../domain/binaryTarget.js"; -import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryArchitecture, +} from "../domain/binaryTargetTypes.js"; import type { JsonValue } from "../domain/jsonValue.js"; import { ghidraApplicationProperties, @@ -28,7 +31,6 @@ import { ghidraReleaseCompatibility, ghidraReleaseLine, ghidraVersionDetail, - SUPPORTED_GHIDRA_JAVA_MAJOR, SUPPORTED_GHIDRA_VERSION, type GhidraApplicationProperties, type GhidraJavaRequirement, @@ -132,7 +134,10 @@ const assessNativeDecompiler = ( }; const observed = `${parsed.value.format} ${parsed.value.architecture}`; const expected = `${expectedFormat} ${expectedArchitecture}`; - if (observed === expected) + const formatMatches = isMachOFormat(expectedFormat) + ? isMachOFormat(parsed.value.format) + : parsed.value.format === expectedFormat; + if (formatMatches && parsed.value.architecture === expectedArchitecture) return { status: "compatible", detail: `${path} reports ${observed}, which matches ${platform}/${architecture}`, diff --git a/src/ghidra/GhidraLanguageOverride.test.ts b/src/ghidra/GhidraLanguageOverride.test.ts index f29294d4f..a355b39fa 100644 --- a/src/ghidra/GhidraLanguageOverride.test.ts +++ b/src/ghidra/GhidraLanguageOverride.test.ts @@ -158,8 +158,14 @@ describe("Ghidra language catalog", () => { { languageId: "x86:LE:32:default", compilerSpecId: "removed" }, catalog, "12.1.4", - ), - ).not.toBeNull(); + )?.settings, + ).toEqual([ + { + setting: "REA_GHIDRA_COMPILER_SPEC_ID", + constraint: + "must be one of the selected language's compiler specs: borlandcpp", + }, + ]); }); it.each([ diff --git a/src/ghidra/GhidraMipsProfile.test.ts b/src/ghidra/GhidraMipsProfile.test.ts index 42b9a3e7e..31a121cd1 100644 --- a/src/ghidra/GhidraMipsProfile.test.ts +++ b/src/ghidra/GhidraMipsProfile.test.ts @@ -13,6 +13,7 @@ import { GhidraProvider } from "./GhidraProvider.js"; import type { GhidraInstallationHost } from "./GhidraInstallation.js"; import { GHIDRA_SESSION_CAPABILITIES } from "./GhidraSessionValues.js"; import type { GhidraMipsSoftFloatReport } from "./GhidraMipsProfile.js"; +import { ghidraProfileLanguageOverride } from "./GhidraAnalysisProfile.js"; // Fake installation inspection tests admission/profile contracts, not Ghidra. const host: GhidraInstallationHost = { @@ -217,6 +218,130 @@ const session = async ( }; }; +describe("MIPS16 shared-image profile", () => { + const shared = (change: Partial = {}) => + target({ + type: 3, + flags: 0x74001005, + abiFlags: null, + sharedLoad: { + status: "complete", + relocationCount: 14, + relocationTypes: [0, 3], + symbolRelativeCount: 0, + }, + ...change, + }); + it.each(["little", "big"] as const)( + "pins a generic 32-bit %s language and retains the source ABI uncertainty", + async (byteOrder) => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + const value = shared({ byteOrder }); + expect(ghidra.inspectTargetSupport(value).status).toBe("supported"); + const resolved = await ghidra.resolveAnalysisProfile(value); + if (!resolved.ok || resolved.value.profile === null) + throw new Error("expected shared profile"); + const parameters = resolved.value.profile.parameters; + expect(parameters).toMatchObject({ + language_id: `MIPS:${byteOrder === "little" ? "LE" : "BE"}:32:default`, + compiler_spec_id: "default", + language_selection: "provider-pinned-v1", + mips_support_lane: "elf32-dyn-o32-arch32r2-mips16-local-rel32-v1", + mips_elf: { abi_flags: null }, + }); + expect(ghidraProfileLanguageOverride(parameters)).toEqual({ + languageId: parameters.language_id, + compilerSpecId: "default", + }); + }, + ); + it("retains shared-image and unknown floating-point limitations on the session", async () => { + const { client } = await session(shared(), undefined); + const health = await client.execute("health", {}); + if (!health.ok) throw health.error; + expect(health.value.limitations).toEqual( + expect.arrayContaining([ + expect.stringContaining("External GOT bindings"), + expect.stringContaining("does not establish the floating-point ABI"), + ]), + ); + }); + it("continues to admit a declared FPXX record only when its MIPS16 ASE agrees", () => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + expect( + ghidra.inspectTargetSupport(shared(abi({ ases: 0x400 }))).status, + ).toBe("supported"); + expect(ghidra.inspectTargetSupport(shared(abi({ ases: 0 }))).status).toBe( + "unsupported", + ); + }); + it.each([ + { flags: 0x74001045 }, + { limitations: ["Unexamined section declarations"] }, + { + sharedLoad: { + status: "complete", + relocationCount: 1, + relocationTypes: [3], + symbolRelativeCount: 1, + }, + }, + ] satisfies Partial[])( + "refuses incomplete or unverified source declarations %j", + (change) => { + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + expect(ghidra.inspectTargetSupport(shared(change)).status).toBe( + "unsupported", + ); + }, + ); + it.each(["sharedLoad", "abiFlags"] as const)( + "refuses an uninspected %s declaration", + (declaration) => { + const value = shared(); + if (value.kind !== "executable" || value.architecture !== "mips") + throw new Error("expected MIPS target"); + const facts = { ...value.mips }; + delete facts[declaration]; + const ghidra = new GhidraProvider(config, silentLogger, {}, host); + expect( + ghidra.inspectTargetSupport({ ...value, mips: facts }), + ).toMatchObject({ + status: "unsupported", + reason: expect.stringContaining( + declaration === "abiFlags" + ? "inspected ABI flags" + : "have not been inspected", + ), + }); + }, + ); + it("refuses an endian pin committed for a different target", async () => { + const { counts, client } = await session( + shared({ byteOrder: "big" }), + undefined, + shared(), + ); + const health = await client.execute("health", {}); + expect(health.ok).toBe(false); + expect(counts.starts).toBe(0); + }); + it("rejects a caller-selected global MIPS16 processor without replacing the request", async () => { + const parsed = parseConfig({ + GHIDRA_INSTALL_DIR: "/ghidra", + REA_GHIDRA_LANGUAGE_ID: "MIPS:LE:32:16e", + }); + if (!parsed.ok) throw parsed.error; + const ghidra = new GhidraProvider(parsed.value, silentLogger, {}, host); + const resolved = await ghidra.resolveAnalysisProfile(shared()); + expect(resolved.ok).toBe(false); + if (!resolved.ok) + expect(resolved.error.message).toContain( + "requested override has not been changed", + ); + }); +}); + describe("soft-float MIPS storage", () => { it("admits soft-float o32 and commits its storage policy only for it", async () => { const soft = target(softFloat); diff --git a/src/ghidra/GhidraMipsProfile.ts b/src/ghidra/GhidraMipsProfile.ts index accc773fd..031e273f1 100644 --- a/src/ghidra/GhidraMipsProfile.ts +++ b/src/ghidra/GhidraMipsProfile.ts @@ -1,9 +1,30 @@ import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; const MIPS_FP_ABI_SOFT = 3; const MIPS_ET_REL = 1; const MIPS_ET_EXEC = 2; +const MIPS_ET_DYN = 3; +const MIPS16_HEADER_ASE = 0x04000000; + +/** A distinct compressed shared-image shape; admission still checks every declaration. */ +const isGhidraMips16SharedTarget = (target: BinaryTarget): boolean => + target.kind === "executable" && + target.architecture === "mips" && + target.mips.type === MIPS_ET_DYN && + (target.mips.flags & 0x0f000000) === MIPS16_HEADER_ASE; + +/** Generic MIPS32 permits per-function ISA_MODE; a global 16e processor does not. */ +export const ghidraMipsSharedLanguageOverride = ( + target: BinaryTarget, +): GhidraLanguageOverride | undefined => + isGhidraMips16SharedTarget(target) && target.architecture === "mips" + ? { + languageId: `MIPS:${target.mips.byteOrder === "little" ? "LE" : "BE"}:32:default`, + compilerSpecId: "default", + } + : undefined; /** Keep family recognition separate from the deliberately bounded Ghidra lane. */ export const ghidraMipsUnsupportedReason = ( @@ -14,11 +35,16 @@ export const ghidraMipsUnsupportedReason = ( const metadata = target.mips; if (target.format !== "elf" || metadata === undefined) return "MIPS analysis requires ELF header metadata; a family name alone does not select an ISA or ABI."; + const shared = isGhidraMips16SharedTarget(target); if (metadata.elfClass !== 32) return "This Ghidra MIPS lane supports ELF32 only; MIPS64 and n64 remain unsupported."; - if (metadata.type !== MIPS_ET_EXEC && metadata.type !== MIPS_ET_REL) + if ( + metadata.type !== MIPS_ET_EXEC && + metadata.type !== MIPS_ET_REL && + !shared + ) return "This Ghidra MIPS lane supports ET_EXEC and ET_REL only; shared images and PSP PRX need separate load verification."; - if ((metadata.flags & 0x0f000000) !== 0) + if ((metadata.flags & 0x0f000000) !== 0 && !shared) return "MIPS16, microMIPS and other ASE encodings are outside this standard-instruction MIPS lane."; if ((metadata.flags & 0x00ff0000) !== 0) return "Machine-specific MIPS variants, including PSP Allegrex, need a separately supported processor profile."; @@ -33,7 +59,20 @@ export const ghidraMipsUnsupportedReason = ( return "EF_MIPS_FP64 requires separate verification of 64-bit floating-point register semantics in this Ghidra lane."; if ((metadata.flags & 0x00000400) !== 0) return "EF_MIPS_NAN2008 requires separate verification of NaN encoding semantics in this Ghidra lane."; + if (shared) { + if ((metadata.flags & 0x40) !== 0) + return "EF_MIPS_ABI_ON32 declares an unverified ABI for the MIPS16 shared-image lane."; + const load = metadata.sharedLoad; + if (load === undefined || load.status !== "complete") + return `MIPS16 shared images require complete dynamic relocation inspection. ${load?.reason ?? "The source declarations have not been inspected."}`; + if (load.relocationTypes.some((type) => type !== 0 && type !== 3)) + return "This MIPS16 shared-image lane verifies R_MIPS_NONE and R_MIPS_REL32 only; other dynamic relocation types require separate verification."; + if (load.symbolRelativeCount !== 0) + return "Symbol-bearing R_MIPS_REL32 is unsupported: Ghidra normalizes MIPS16 function addresses and can lose the source function-pointer mode tag. Symbol-index-zero REL32 is verified separately."; + } const abi = metadata.abiFlags; + if (shared && abi === null && (metadata.limitations?.length ?? 0) === 0) + return null; // Inspected absence leaves FP semantics unknown, never inferred. if (abi === undefined || abi === null) return [ "This Ghidra MIPS lane requires an inspected ABI flags record; header flags alone do not establish the floating-point ABI.", @@ -49,8 +88,8 @@ export const ghidraMipsUnsupportedReason = ( const softFloat = abi.cpr1Size === 0 && abi.fpAbi === MIPS_FP_ABI_SOFT; if (!hardFloat && !softFloat) return "This Ghidra MIPS lane admits double-precision or FPXX ABI declarations with 32-bit CPR1, or soft-float without CPR1; single-float, FP64 and unspecified modes need separate verification."; - if (abi.isaExtension !== 0 || abi.ases !== 0) - return "MIPS ABI flags declare an ISA extension or ASE outside this standard-instruction lane."; + if (abi.isaExtension !== 0 || abi.ases !== (shared ? 0x400 : 0)) + return "MIPS ABI flags declare an ISA extension or ASE outside this profile's verified instruction declarations."; // MIPS_AFL_FLAGS1_ODDSPREG (bit 0) is the GCC and LLVM default for MIPS32 o32 // whatever the FP ABI; soft-float objects from both carry it. With no CPR1 it // selects nothing, so it is accepted for soft-float as well as hard-float. @@ -116,6 +155,22 @@ export const ghidraMipsRelocatableLimitations = ( ] : []; +/** Shared-image results are analysis observations, not dynamic-loader execution. */ +export const ghidraMipsSharedLimitations = ( + target: BinaryTarget, +): readonly string[] => + isGhidraMips16SharedTarget(target) && target.architecture === "mips" + ? [ + "MIPS16 shared ELF: the endian-matched generic MIPS32 language uses the provider's per-address ISA_MODE context. The program language variant is not the instruction's encoding mode.", + "Shared-image addresses are Ghidra analysis coordinates, not runtime addresses. The source bytes remain unchanged; inspect_native_load_image retains original relocation bytes, loaded memory bytes and provider relocation status. External GOT bindings remain unverified without their defining images.", + ...(target.mips.abiFlags === null + ? [ + "No MIPS ABI flags record was found in the completely examined declaration tables. The header declares o32 but does not establish the floating-point ABI; the default compiler specification does not verify floating-point argument or return storage.", + ] + : []), + ] + : []; + /** Commit source ELF interpretation without confusing target ISA with host CPU. */ export const ghidraMipsProfileParameters = ( target: BinaryTarget, @@ -152,12 +207,33 @@ export const ghidraMipsProfileParameters = ( flags1: abi.flags1, flags2: abi.flags2, }, + ...(target.mips.sharedLoad === undefined + ? {} + : { + shared_load: + target.mips.sharedLoad.status === "complete" + ? { + status: "complete", + relocation_count: target.mips.sharedLoad.relocationCount, + relocation_types: [ + ...target.mips.sharedLoad.relocationTypes, + ], + symbol_relative_count: + target.mips.sharedLoad.symbolRelativeCount, + } + : { + status: "unavailable", + reason: target.mips.sharedLoad.reason, + }, + }), }, ...(isGhidraMipsSoftFloatTarget(target) ? { mips_soft_float_storage: GHIDRA_MIPS_SOFT_FLOAT_STORAGE } : {}), - mips_support_lane: - target.mips.type === MIPS_ET_REL + instruction_context_evidence: "per-address-isa-mode-v1", + mips_support_lane: isGhidraMips16SharedTarget(target) + ? "elf32-dyn-o32-arch32r2-mips16-local-rel32-v1" + : target.mips.type === MIPS_ET_REL ? "elf32-rel-o32-arch32r2-standard-v1" : "elf32-exec-o32-arch32r2-standard-v3", }; diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 57fd8aa8a..eb3ca1171 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -64,6 +64,15 @@ const provider = ( ); }; +const createElfClient = async (ghidra: GhidraProvider) => { + const target = executableTarget("elf", "x86_64"); + const resolved = await ghidra.resolveAnalysisProfile(target); + if (!resolved.ok) throw resolved.error; + if (resolved.value.profile === null) + throw new Error("Expected a bound Ghidra profile"); + return ghidra.createClient(target, resolved.value.profile); +}; + describe("Ghidra provider", () => { it("commits DOS loader and real-mode language while retaining the CPU family", async () => { const ghidra = provider(); @@ -230,7 +239,7 @@ describe("Ghidra Mach-O slice support", () => { path: "/tmp/fixture", sha256: "a".repeat(64), kind: "executable", - format: "mach-o", + format: "mach-o-universal", architecture: "arm64", availableArchitectures: ["x86_64", "arm64"], }; @@ -247,6 +256,13 @@ describe("Ghidra Mach-O slice support", () => { expect( ghidra.inspectTargetSupport(executableTarget("mach-o", "arm64")), ).toMatchObject({ status: "supported" }); + + expect( + ghidra.inspectTargetSupport({ + ...universal, + availableArchitectures: ["arm64"] as const, + }), + ).toMatchObject({ status: "supported" }); }); }); @@ -570,6 +586,7 @@ describe("Ghidra client projection", () => { expect(resolved.value.profile.parameters).toMatchObject({ language_id: "auto-from-header", compiler_spec_id: "auto-default", + call_relationship_coverage: "resolved-call-reference-diagnostics-v1", }); const result = await ghidra @@ -608,15 +625,6 @@ describe("Ghidra client projection", () => { }); describe("Ghidra result projection", () => { - const createElfClient = async (ghidra: GhidraProvider) => { - const target = executableTarget("elf", "x86_64"); - const resolved = await ghidra.resolveAnalysisProfile(target); - if (!resolved.ok) throw resolved.error; - if (resolved.value.profile === null) - throw new Error("Expected a bound Ghidra profile"); - return ghidra.createClient(target, resolved.value.profile); - }; - it("rejects malformed inventory output before Evidence creation", async () => { const ghidra = provider(installationHost(), () => ({ start: () => Promise.resolve(ok(sessionInfo())), @@ -763,6 +771,35 @@ describe("Ghidra result projection", () => { }); }); +describe("Ghidra relationship coverage", () => { + it("marks the full direct call-reference response complete while keeping static caveats", async () => { + const ghidra = provider(installationHost(), () => ({ + start: () => Promise.resolve(ok(sessionInfo())), + callTool: (operation) => + Promise.resolve( + ok(operation === "procedure_callees" ? ["0x2000"] : []), + ), + close: () => Promise.resolve(ok(null)), + })); + const client = await createElfClient(ghidra); + const execution = await client.execute("procedure_callees", { + procedure: "0x1000", + document: "fixture", + }); + if (!execution.ok) throw execution.error; + expect(execution.value.relationshipCoverage).toEqual({ + status: "complete", + diagnostics: [ + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals.", + ], + }); + expect(execution.value.limitations).not.toContain( + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals.", + ); + await client.close(); + }); +}); + describe("Ghidra measured load-image projection", () => { it.each(["verified", "changed-snapshot", "cancelled"] as const)( "streams raw snapshot identity and preserves %s outcomes", diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index 3936d5c99..aa5044079 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -14,7 +14,10 @@ import { createAnalysisProfile, type AnalysisProfileCommitment, } from "../domain/analysisProfile.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { jsonObjectSchema, jsonValueSchema, @@ -46,7 +49,6 @@ import { CAPABILITIES, windowsP0Capabilities, GHIDRA_PROVIDER_IDENTITY, - GHIDRA_OPERATIONS, } from "./GhidraProviderCapabilities.js"; import { createGhidraProviderClient, @@ -165,7 +167,10 @@ export class GhidraProvider implements AnalysisProviderCandidate { reason: mipsReason, diagnostics, }; - if (target.format === "mach-o" && target.availableArchitectures.length > 1) + if ( + isMachOFormat(target.format) && + target.availableArchitectures.length > 1 + ) return { status: "unsupported", code: "architecture_unsupported", diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index b64b49d7f..f3f94c645 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -2,6 +2,7 @@ import type { CapabilityDescriptor, ProviderIdentity, } from "../application/AnalysisProvider.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; import { hasWindowsNativeAuthority, windowsNativeAuthorityUnavailableReason, @@ -43,6 +44,20 @@ const stringInventoryLimitation = const searchLimitation = "Regex mode uses Java Pattern semantics. Matcher or compiler stack exhaustion returns a resource constraint without closing the session; literal mode avoids regex recursion and returns complete matching values."; +const resolvedCallReferenceLimitation = + "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals."; + +/** Relationship-only caveats belong to each observed caller/callee result. */ +export const relationshipCoverageFor = ( + operation: string, +): RelationshipCoverage | undefined => + operation === "procedure_callers" || operation === "procedure_callees" + ? { + status: "complete", + diagnostics: [resolvedCallReferenceLimitation], + } + : undefined; + /** Limitation text for one admitted operation, including the common base. */ export const limitationsFor = (operation: string): readonly string[] => { const common = [ @@ -135,10 +150,7 @@ export const limitationsFor = (operation: string): readonly string[] => { ]; case "procedure_callers": case "procedure_callees": - return [ - ...common, - "Only resolved Ghidra call references are returned; unresolved computed or indirect calls remain unknown, while function classifications distinguish thunks and externals.", - ]; + return common; case "xrefs": return [ ...common, diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 742ef79f7..b7ba24e0a 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -60,6 +60,8 @@ import { GHIDRA_MIPS_SOFT_FLOAT_STORAGE, ghidraMipsSoftFloatFailure, ghidraMipsRelocatableLimitations, + ghidraMipsSharedLimitations, + ghidraMipsSharedLanguageOverride, ghidraMipsSoftFloatLimitations, isGhidraMipsSoftFloatTarget, } from "./GhidraMipsProfile.js"; @@ -83,6 +85,7 @@ import { healthLimitations, windowsP0Limitations, limitationsFor, + relationshipCoverageFor, } from "./GhidraProviderCapabilities.js"; import type { GhidraSessionError } from "./GhidraSessionError.js"; import type { GhidraSessionInfo } from "./GhidraSessionValues.js"; @@ -444,7 +447,10 @@ export const createGhidraProviderClient = (input: { committedProfile.parameters, ); if ( - !sameGhidraLanguageOverride(languageOverride, config.ghidraLanguageOverride) + !sameGhidraLanguageOverride( + languageOverride, + config.ghidraLanguageOverride ?? ghidraMipsSharedLanguageOverride(target), + ) ) return unavailableClient( new ProviderAdapterError("ghidra", "open_binary", { @@ -512,6 +518,7 @@ export const createGhidraProviderClient = (input: { ? (target.mips.limitations ?? []) : []), ...ghidraMipsRelocatableLimitations(target), + ...ghidraMipsSharedLimitations(target), ]; const providerLimitations = installation.platform === "win32" @@ -817,10 +824,14 @@ export const createGhidraProviderClient = (input: { if (!attested.ok) return attested; normalized = attested.value; } + const relationshipCoverage = relationshipCoverageFor(operation); return ok( createAnalysisExecution(normalized, committedProfile.provider, { rawResult: called.value, analysisProfile: committedProfile, + ...(relationshipCoverage === undefined + ? {} + : { relationshipCoverage }), limitations: [ ...limitationsFor(operation), ...observationLimitations, diff --git a/src/ghidra/GhidraSessionError.test.ts b/src/ghidra/GhidraSessionError.test.ts new file mode 100644 index 000000000..874ef6a11 --- /dev/null +++ b/src/ghidra/GhidraSessionError.test.ts @@ -0,0 +1,117 @@ +import { expect, it } from "vitest"; + +import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; +import { bindGhidraSessionFailure } from "./GhidraSessionError.js"; +import { GhidraLaunchError } from "./GhidraLauncher.js"; + +it("reports truncated cyclic causes in serializable diagnostics", () => { + const cause = new Error("cyclic launcher failure"); + cause.cause = cause; + const projectFailure = bindGhidraSessionFailure(() => ({})); + + const diagnostics = projectFailure( + "start", + "Ghidra launcher failed", + cause, + ).diagnostics; + + expect(JSON.stringify(diagnostics)).toContain('"cause_truncated":true'); +}); + +it("keeps bounded compiler and nested launcher outcomes in diagnostics", () => { + const compiler = new DarwinProcessOwnershipInspectionError( + "macOS process ownership helper compilation failed via xcrun (compiler result: 7)", + { + cause: Object.assign(new Error("compiler exited"), { code: 7 }), + compilerOutcome: { + outcome: "exited", + code: 7, + signal: null, + stderr: "SDK mismatch", + stderrBytes: 12, + stderrTruncated: false, + outputTruncated: false, + killed: false, + }, + }, + ); + const launch = new GhidraLaunchError("Ghidra headless launch failed", { + cause: compiler, + }); + const projectFailure = bindGhidraSessionFailure(() => ({})); + + expect( + projectFailure("start", "Ghidra launcher failed", launch).diagnostics, + ).toMatchObject({ + failure_cause: { + name: "GhidraLaunchError", + cause: { + name: "DarwinProcessOwnershipInspectionError", + compiler_outcome: { + outcome: "exited", + code: 7, + stderr: "SDK mismatch", + }, + cause: { code: 7 }, + }, + }, + }); + + const spawnFailure = Object.assign( + new Error("spawn analyzeHeadless ENOENT"), + { code: "ENOENT" }, + ); + const launchFailure = new GhidraLaunchError("Ghidra headless launch failed", { + cause: spawnFailure, + }); + expect( + projectFailure("start", "Ghidra launcher failed", launchFailure) + .diagnostics, + ).toMatchObject({ + failure_cause: { + name: "GhidraLaunchError", + cause: { code: "ENOENT" }, + }, + }); +}); + +it("bounds compiler stderr after explicit redaction at UTF-8 boundaries", () => { + const sensitiveValue = "caller-selected-secret"; + const projectFailure = bindGhidraSessionFailure( + () => ({}), + (value) => value.replaceAll(sensitiveValue, "replacement".repeat(100)), + ); + const stderr = `${"é".repeat(3900)}${sensitiveValue}`; + const compiler = new DarwinProcessOwnershipInspectionError( + "compiler failed", + { + compilerOutcome: { + outcome: "exited", + code: 1, + signal: null, + stderr, + stderrBytes: Buffer.byteLength(stderr, "utf8"), + stderrTruncated: false, + outputTruncated: false, + killed: false, + }, + }, + ); + const failureCause = projectFailure( + "start", + "Ghidra launcher failed", + compiler, + ).diagnostics.failure_cause as { + readonly compiler_outcome: { + readonly stderr: string; + readonly stderrTruncated: boolean; + }; + }; + + expect( + Buffer.byteLength(failureCause.compiler_outcome.stderr, "utf8"), + ).toBeLessThanOrEqual(8 * 1024); + expect(failureCause.compiler_outcome.stderrTruncated).toBe(true); + expect(failureCause.compiler_outcome.stderr).not.toContain(sensitiveValue); + expect(failureCause.compiler_outcome.stderr).not.toContain("\uFFFD"); +}); diff --git a/src/ghidra/GhidraSessionError.ts b/src/ghidra/GhidraSessionError.ts index feaad5c50..4b8bead7a 100644 --- a/src/ghidra/GhidraSessionError.ts +++ b/src/ghidra/GhidraSessionError.ts @@ -1,5 +1,12 @@ import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import { DarwinProcessOwnershipInspectionError } from "../process/DarwinProcessRunTokenReader.js"; +import { boundedUtf8Text } from "../process/boundedUtf8Text.js"; + +const MAX_CAUSE_DEPTH = 3; +const MAX_CAUSE_MESSAGE_BYTES = 2 * 1024; +const MAX_CAUSE_FIELD_BYTES = 1024; +const MAX_COMPILER_STDERR_BYTES = 8 * 1024; /** Stable failure phase for one private Ghidra headless session. */ export type GhidraSessionFailureKind = @@ -104,24 +111,121 @@ export const bindGhidraSessionFailure = const describeCause = ( cause: unknown, redact: (value: string) => string, + depth = 0, ): JsonValue => { if (cause instanceof Error) { - const code = "code" in cause ? cause.code : undefined; + return describeErrorCause(cause, redact, depth); + } + if (typeof cause === "string") { + const message = boundedUtf8Text(redact(cause), MAX_CAUSE_MESSAGE_BYTES); return { - name: redact(cause.name), - message: redact(cause.message), - ...(typeof code === "string" - ? { code: redact(code) } - : typeof code === "number" && Number.isFinite(code) - ? { code } - : {}), + type: "string", + message: message.text, + ...(message.truncated ? { message_truncated: true } : {}), }; } - if (typeof cause === "string") - return { type: "string", message: redact(cause) }; if (cause === null || typeof cause === "boolean") return { type: cause === null ? "null" : "boolean", value: cause }; if (typeof cause === "number" && Number.isFinite(cause)) return { type: "number", value: cause }; return { type: typeof cause }; }; + +const describeErrorCause = ( + cause: Error, + redact: (value: string) => string, + depth: number, +): JsonValue => { + const message = boundedUtf8Text( + redact(cause.message), + MAX_CAUSE_MESSAGE_BYTES, + ); + const name = boundedUtf8Text(redact(cause.name), MAX_CAUSE_FIELD_BYTES); + return { + name: name.text, + ...(name.truncated ? { name_truncated: true } : {}), + message: message.text, + ...(message.truncated ? { message_truncated: true } : {}), + ...causeCode(cause, redact), + ...causeSignal(cause, redact), + ...causeCompilerOutcome(cause, redact), + ...nestedErrorCause(cause, redact, depth), + }; +}; + +const causeCode = ( + cause: Error, + redact: (value: string) => string, +): Record => { + const code = "code" in cause ? cause.code : undefined; + if (typeof code === "string") { + const bounded = boundedUtf8Text(redact(code), MAX_CAUSE_FIELD_BYTES); + return { + code: bounded.text, + ...(bounded.truncated ? { code_truncated: true } : {}), + }; + } + if (typeof code === "number" && Number.isFinite(code)) return { code }; + return {}; +}; + +const causeSignal = ( + cause: Error, + redact: (value: string) => string, +): Record => { + const signal = "signal" in cause ? cause.signal : undefined; + if (typeof signal === "string") { + const bounded = boundedUtf8Text(redact(signal), MAX_CAUSE_FIELD_BYTES); + return { + signal: bounded.text, + ...(bounded.truncated ? { signal_truncated: true } : {}), + }; + } + if (signal === null) return { signal: null }; + return {}; +}; + +const causeCompilerOutcome = ( + cause: Error, + redact: (value: string) => string, +): Record => { + if (!(cause instanceof DarwinProcessOwnershipInspectionError)) return {}; + const outcome = cause.compilerOutcome; + if (outcome === undefined) return {}; + const stderr = boundedUtf8Text( + redact(outcome.stderr), + MAX_COMPILER_STDERR_BYTES, + ); + const code = boundedCompilerField(outcome.code, redact); + const signal = boundedCompilerField(outcome.signal, redact); + return { + compiler_outcome: { + ...outcome, + code: code.value, + signal: signal.value, + stderr: stderr.text, + stderrTruncated: outcome.stderrTruncated || stderr.truncated, + ...(code.truncated ? { codeTruncated: true } : {}), + ...(signal.truncated ? { signalTruncated: true } : {}), + }, + }; +}; + +const boundedCompilerField = ( + value: string | number | null, + redact: (value: string) => string, +): { readonly value: string | number | null; readonly truncated: boolean } => { + if (typeof value !== "string") return { value, truncated: false }; + const bounded = boundedUtf8Text(redact(value), MAX_CAUSE_FIELD_BYTES); + return { value: bounded.text, truncated: bounded.truncated }; +}; + +const nestedErrorCause = ( + cause: Error, + redact: (value: string) => string, + depth: number, +): Record => { + if (!("cause" in cause)) return {}; + if (depth >= MAX_CAUSE_DEPTH) return { cause_truncated: true }; + return { cause: describeCause(cause.cause, redact, depth + 1) }; +}; diff --git a/src/ghidra/extensions/GhidraExtensions.test.ts b/src/ghidra/extensions/GhidraExtensions.test.ts index 72bd6cf3d..1fbe884cf 100644 --- a/src/ghidra/extensions/GhidraExtensions.test.ts +++ b/src/ghidra/extensions/GhidraExtensions.test.ts @@ -149,7 +149,7 @@ describe("optional Ghidra extension boundary", () => { ); await expect( snapshotGhidraExtensions([extension], runtime), - ).rejects.toThrow(); + ).rejects.toMatchObject({ code: "EEXIST" }); }); it("rejects artifact changes after the profile commitment", async () => { const { extension, path } = await artifact(); @@ -174,15 +174,21 @@ describe("optional Ghidra extension boundary", () => { diagnostics: { path, extension: "nativeaot" }, }); } - expect( - ( - await resolveGhidraExtensions( - config(join(directory, "absent.jar")), - target, - "linux", - ) - ).ok, - ).toBe(false); + const absentPath = join(directory, "absent.jar"); + const absent = await resolveGhidraExtensions( + config(absentPath), + target, + "linux", + ); + expect(absent.ok).toBe(false); + if (!absent.ok) + expect(absent.error).toMatchObject({ + diagnostics: { + path: absentPath, + extension: "nativeaot", + reason: expect.stringContaining("ENOENT"), + }, + }); }); it.skipIf(process.platform === "win32")( "rejects a FIFO without requiring another process to open its writer", @@ -250,24 +256,30 @@ describe("Ghidra extension profile and producer validation", () => { it("requires exact inventory, artifact digest and consistent producer status", async () => { const { extension } = await artifact(); const valid = result(extension.sha256); + const inventoryMismatch = + "Ghidra extension result inventory differs from its committed profile."; expect(validateGhidraExtensionResults([extension], [valid])).toBeNull(); - expect(validateGhidraExtensionResults([extension], [])).not.toBeNull(); - expect(validateGhidraExtensionResults([], [valid])).not.toBeNull(); - expect( - validateGhidraExtensionResults([extension], [valid, valid]), - ).not.toBeNull(); + expect(validateGhidraExtensionResults([extension], [])).toBe( + inventoryMismatch, + ); + expect(validateGhidraExtensionResults([], [valid])).toBe(inventoryMismatch); + expect(validateGhidraExtensionResults([extension], [valid, valid])).toBe( + inventoryMismatch, + ); expect( validateGhidraExtensionResults( [extension], [{ ...valid, sha256: "b".repeat(64) }], ), - ).not.toBeNull(); + ).toBe("Ghidra extension identity mismatch: nativeaot"); expect( validateGhidraExtensionResults( [extension], [{ ...valid, status: "complete" }], ), - ).not.toBeNull(); + ).toBe( + "NativeAOT extension returned a malformed or inconsistent producer report.", + ); }); it("requires matching explicit configuration for imported executable profiles", async () => { const { extension, path } = await artifact(); @@ -281,7 +293,9 @@ describe("Ghidra extension profile and producer validation", () => { ).toBeNull(); expect( validateGhidraExtensionProfile([extension], config(), target, "linux"), - ).not.toBeNull(); + ).toBe( + "Configured Ghidra extension inventory differs from the resolved analysis profile.", + ); expect( validateGhidraExtensionProfile( [extension], @@ -289,7 +303,9 @@ describe("Ghidra extension profile and producer validation", () => { target, "darwin", ), - ).not.toBeNull(); + ).toBe( + "NativeAOT recovery is verified on Linux only; omit REA_GHIDRA_NATIVEAOT_JAR for ordinary native analysis.", + ); expect( validateGhidraExtensionProfile( [{ ...extension, entry_class: "arbitrary.Code" }], @@ -297,7 +313,9 @@ describe("Ghidra extension profile and producer validation", () => { target, "linux", ), - ).not.toBeNull(); + ).toBe( + "Ghidra extension profile lacks matching explicit local configuration: nativeaot", + ); expect( validateGhidraExtensionProfile( [{ ...extension, configured_path: "/another.jar" }], @@ -305,7 +323,9 @@ describe("Ghidra extension profile and producer validation", () => { target, "linux", ), - ).not.toBeNull(); + ).toBe( + "Ghidra extension profile lacks matching explicit local configuration: nativeaot", + ); }); it("does not classify PE/CLI or ReadyToRun as NativeAOT", async () => { const resolved = await resolveGhidraExtensions( @@ -389,9 +409,9 @@ describe("Ghidra extension recovery coverage", () => { }, }, }; - expect( - validateGhidraExtensionResults([extension], [incomplete]), - ).not.toBeNull(); + expect(validateGhidraExtensionResults([extension], [incomplete])).toBe( + "NativeAOT recovery coverage contradicts its reported completion status.", + ); expect( validateGhidraExtensionResults( [extension], @@ -409,7 +429,9 @@ describe("Ghidra extension recovery coverage", () => { [extension], [{ ...positive, result: { ...positive.result, discovery: null } }], ), - ).not.toBeNull(); + ).toBe( + "NativeAOT extension returned a malformed or inconsistent producer report.", + ); }); it("preserves real loader failures without requiring a successful analyzer report", async () => { const { extension } = await artifact(); @@ -439,7 +461,9 @@ describe("Ghidra extension recovery coverage", () => { [extension], [{ ...incomplete, status: "complete", reason: null }], ), - ).not.toBeNull(); + ).toBe( + "NativeAOT recovery omitted its supported format, method-table inventory or derived-memory identity.", + ); }); it.each(["not_applicable", "unsupported", "failed"] as const)( "rejects contradictory %s reports without discarding discovery context", diff --git a/src/hopper/HopperAnalysisProfile.test.ts b/src/hopper/HopperAnalysisProfile.test.ts new file mode 100644 index 000000000..b71e0d09e --- /dev/null +++ b/src/hopper/HopperAnalysisProfile.test.ts @@ -0,0 +1,22 @@ +import { expect, it } from "vitest"; + +import { hopperLoaderArgsForTarget } from "./HopperAnalysisProfile.js"; + +it("routes physical universal Mach-O containers through Hopper's FAT loader", () => { + const result = hopperLoaderArgsForTarget( + { + path: "/tmp/universal", + sha256: "a".repeat(64), + kind: "executable", + format: "mach-o-universal", + architecture: "arm64", + availableArchitectures: ["x86_64", "arm64"], + }, + "fat64", + ); + + expect(result).toEqual({ + ok: true, + value: ["-l", "FAT", "--aarch64", "-l", "Mach-O"], + }); +}); diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index 9dfef9e07..927f7289b 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -6,8 +6,11 @@ import type { ProviderIdentity, } from "../application/AnalysisProvider.js"; import { createAnalysisProfile } from "../domain/analysisProfile.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; -import type { BinaryArchitecture } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, + type BinaryArchitecture, +} from "../domain/binaryTargetTypes.js"; import { AnalysisCancelledError, AnalysisCapabilityUnavailableError, @@ -37,7 +40,7 @@ export const resolveHopperAnalysisProfile = async ( if (target.kind !== "executable" && target.kind !== "database") return ok({ profile: null }); let machoContainer: "thin" | "fat32" | "fat64" | undefined; - if (target.kind === "executable" && target.format === "mach-o") { + if (target.kind === "executable" && isMachOFormat(target.format)) { const header = await readMachoMagic(target.path, options.signal); if (!header.ok) return header; machoContainer = [0xcafebabf, 0xbfbafeca].includes(header.value) @@ -119,21 +122,22 @@ export const hopperLoaderArgsForTarget = ( ), ); const flag = hopperArchitectureFlag(architecture); - switch (target.format) { - case "mach-o": - if (machoContainer === undefined) - return err( - new AnalysisCapabilityUnavailableError( - "hopper", - "resolve_analysis_profile", - "Mach-O container kind is unknown; resolve the Hopper analysis profile before starting its client.", - ), - ); - return ok( - machoContainer === "fat32" || machoContainer === "fat64" - ? ["-l", "FAT", flag, "-l", "Mach-O"] - : ["-l", "Mach-O", flag], + if (isMachOFormat(target.format)) { + if (machoContainer === undefined) + return err( + new AnalysisCapabilityUnavailableError( + "hopper", + "resolve_analysis_profile", + "Mach-O container kind is unknown; resolve the Hopper analysis profile before starting its client.", + ), ); + return ok( + machoContainer === "fat32" || machoContainer === "fat64" + ? ["-l", "FAT", flag, "-l", "Mach-O"] + : ["-l", "Mach-O", flag], + ); + } + switch (target.format) { case "elf": return ok(["-l", "ELF", flag]); case "pe": diff --git a/src/hopper/HopperMachOImage.ts b/src/hopper/HopperMachOImage.ts index 451289d92..499533316 100644 --- a/src/hopper/HopperMachOImage.ts +++ b/src/hopper/HopperMachOImage.ts @@ -2,7 +2,10 @@ import { createHash } from "node:crypto"; import { open, type FileHandle } from "node:fs/promises"; import { z } from "zod"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { @@ -44,7 +47,7 @@ export const resolveHopperMachOImage = async ( ), ); if (signal?.aborted) return err(new AnalysisCancelledError("open_binary")); - if (target.format !== "mach-o") + if (!isMachOFormat(target.format) || target.architecture === "mips") return invalid("Hopper FAT64 image preparation requires a Mach-O target"); let file: FileHandle; try { diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index d8b2dafa6..0b7316a19 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -271,6 +271,10 @@ export class HopperProvider implements AnalysisProviderCandidate { // Only the file-offset mapping differs from Hopper's reply. rawResult: mapped === result ? null : result.value, ...(profile === undefined ? {} : { analysisProfile: profile }), + ...(operation === "procedure_callers" || + operation === "procedure_callees" + ? { relationshipCoverage: { status: "complete" as const } } + : {}), limitations: [ ...(preparedImage === undefined ? [] diff --git a/src/ida/IdaCallGraph.ts b/src/ida/IdaCallGraph.ts index 53152d138..f37d26af5 100644 --- a/src/ida/IdaCallGraph.ts +++ b/src/ida/IdaCallGraph.ts @@ -5,6 +5,7 @@ import { } from "../domain/analysisErrorCore.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; import { ZodError } from "zod"; import { idaAddressSchema, @@ -48,6 +49,7 @@ export interface CallGraphCallees { readonly name: string; }[]; readonly limitations: readonly string[]; + readonly coverage: RelationshipCoverage; } /** No function edges can be observed when the server lacks the graph tool. */ @@ -57,6 +59,11 @@ export const unavailableCallGraph = (reason: string): CallGraphCallees => ({ UNRECOGNIZED_CALLEE_LIMITATION, `Upstream callgraph was unavailable (${reason}); callee edges remain unknown.`, ], + coverage: { + status: "unknown", + reason: `Upstream callgraph was unavailable (${reason}); callee edges remain unknown.`, + diagnostics: [UNRECOGNIZED_CALLEE_LIMITATION], + }, }); /** @@ -95,6 +102,8 @@ export const readCallGraphCallees = async ( BigInt(left) < BigInt(right) ? -1 : BigInt(left) > BigInt(right) ? 1 : 0, ); const limitations = [UNRECOGNIZED_CALLEE_LIMITATION]; + const diagnostics = [UNRECOGNIZED_CALLEE_LIMITATION]; + const coverageLimitations: string[] = []; const truncated = row.truncated === true || row.per_func_capped === true || @@ -104,19 +113,25 @@ export const readCallGraphCallees = async ( const saturated = row.max_edges_per_func !== undefined && edges.length >= row.max_edges_per_func; - if (truncated) limitations.push(TRUNCATED_LIMITATION); - else if (saturated) limitations.push(POSSIBLY_TRUNCATED_LIMITATION); + if (truncated) { + limitations.push(TRUNCATED_LIMITATION); + coverageLimitations.push(TRUNCATED_LIMITATION); + } else if (saturated) { + limitations.push(POSSIBLY_TRUNCATED_LIMITATION); + coverageLimitations.push(POSSIBLY_TRUNCATED_LIMITATION); + } const names = new Map(); if (targets.length > 0) { try { const raw = await call("lookup_funcs", { queries: targets }); const parsed = modernLookupBatchSchema.safeParse(raw); - if (!parsed.success) - limitations.push( - "IDA observed callee addresses but returned an unreadable name lookup result; the addresses are retained.", - ); - else { + if (!parsed.success) { + const limitation = + "IDA observed callee addresses but returned an unreadable name lookup result; the addresses are retained."; + limitations.push(limitation); + diagnostics.push(limitation); + } else { const rows = new Map(); for (const item of parsed.data) { const query = idaAddressSchema.safeParse(item.query); @@ -127,18 +142,19 @@ export const readCallGraphCallees = async ( const item = rows.get(target); if (item?.fn?.address === target && item.error === null) names.set(target, item.fn.name); - else - limitations.push( - `IDA observed callee address ${target} but could not resolve its name${item?.error == null ? "" : `: ${item.error}`}; the address is retained.`, - ); + else { + const limitation = `IDA observed callee address ${target} but could not resolve its name${item?.error == null ? "" : `: ${item.error}`}; the address is retained.`; + limitations.push(limitation); + diagnostics.push(limitation); + } } } } catch (cause: unknown) { if (cause instanceof AnalysisCancelledError) throw cause; if (!expectedReadFailure(cause)) throw cause; - limitations.push( - `IDA observed callee addresses but the name lookup failed (${errorReason(cause)}); the addresses are retained.`, - ); + const limitation = `IDA observed callee addresses but the name lookup failed (${errorReason(cause)}); the addresses are retained.`; + limitations.push(limitation); + diagnostics.push(limitation); } } return { @@ -149,5 +165,14 @@ export const readCallGraphCallees = async ( }; }), limitations, + coverage: { + ...(coverageLimitations.length === 0 + ? { status: "complete" as const } + : { + status: "partial" as const, + limitations: coverageLimitations, + }), + diagnostics, + }, }; }; diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index 6099f379e..fea479c19 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -10,6 +10,7 @@ import { functionInstructionWindowSchema, } from "../domain/hopperValues.js"; import type { JsonValue } from "../domain/jsonValue.js"; +import type { RelationshipCoverage } from "../domain/relationshipCoverage.js"; import { readCallGraphCallees, unavailableCallGraph } from "./IdaCallGraph.js"; import type { IdaMcpConnection } from "./IdaMcpConnection.js"; import type { IdaOperation } from "./IdaProviderCapabilities.js"; @@ -65,6 +66,7 @@ export class IdaOperationRunner { readonly raw: IdaRawObservation[] = []; readonly locations: EvidenceLocation[] = []; readonly limitations: string[] = []; + relationshipCoverage: RelationshipCoverage | undefined; constructor( private readonly connection: IdaMcpConnection, private readonly protocol: "legacy" | "modern", @@ -203,6 +205,7 @@ export class IdaOperationRunner { (tool, args) => this.call(tool, args), fn.address, ); + this.relationshipCoverage = graph.coverage; this.limitations.push(...graph.limitations); for (const callee of graph.callees) if (!result.has(callee.address)) @@ -366,6 +369,11 @@ export class IdaOperationRunner { this.limitations.push( "Modern upstream xrefs classify references as code or data, without proving call edges. Direct callers are unavailable in this profile.", ); + this.relationshipCoverage = { + status: "unknown", + reason: + "Modern upstream xrefs do not establish direct caller relationships.", + }; return []; } else { const page = modernCalleesSchema.parse( @@ -379,6 +387,15 @@ export class IdaOperationRunner { this.limitations.push( "Upstream callees capped results at 500; additional callees remain unknown.", ); + this.relationshipCoverage = + page.more === true + ? { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], + } + : { status: "complete" }; for (const item of page.callees) { if ("type" in item && item.type === "external") result.set( @@ -392,6 +409,11 @@ export class IdaOperationRunner { const resolved = identity(await this.resolve(query)); result.set(resolved.address, resolved); } + if (this.relationshipCoverage === undefined) + this.relationshipCoverage = { + status: "unknown", + reason: "The legacy relationship response does not report coverage.", + }; return [...result.values()]; } diff --git a/src/ida/IdaSessionClient.ts b/src/ida/IdaSessionClient.ts index ff2257752..1d6aa599d 100644 --- a/src/ida/IdaSessionClient.ts +++ b/src/ida/IdaSessionClient.ts @@ -211,6 +211,9 @@ export class IdaSessionClient implements AnalysisClient { observations: jsonValueSchema.parse(runner.raw), }, limitations: [...IDA_LIMITATIONS, ...runner.limitations], + ...(runner.relationshipCoverage === undefined + ? {} + : { relationshipCoverage: runner.relationshipCoverage }), locations: [ { kind: "artifact-path", path: this.target.path }, ...runner.locations, diff --git a/src/javascript/analysis/JavaScriptAnalysisTransfer.ts b/src/javascript/analysis/JavaScriptAnalysisTransfer.ts index 75a3f1692..64882e4ab 100644 --- a/src/javascript/analysis/JavaScriptAnalysisTransfer.ts +++ b/src/javascript/analysis/JavaScriptAnalysisTransfer.ts @@ -20,6 +20,7 @@ import { export const JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES = 8 * 1024 * 1024; const FRAME_BYTES = JAVASCRIPT_ANALYSIS_TRANSFER_RECORD_BYTES; const TRANSFER_BYTES = 512 * 1024 * 1024; +const MAX_TRANSFER_BYTES = 2 * 1024 * 1024 * 1024; const BATCH_BYTES = 1024 * 1024; const recordSchema = z.discriminatedUnion("kind", [ z.strictObject({ @@ -36,7 +37,7 @@ const recordSchema = z.discriminatedUnion("kind", [ /** Complete-file coordinates authenticated before adopting any worker facts. */ export const javaScriptAnalysisTransferDescriptorSchema = z.strictObject({ - bytes: z.number().int().nonnegative().max(TRANSFER_BYTES), + bytes: z.number().int().nonnegative().max(MAX_TRANSFER_BYTES), sha256: digestSchema, records: z.number().int().nonnegative(), module_bytes: z.number().int().nonnegative(), @@ -52,11 +53,35 @@ export const javaScriptTransferRetentionBytes = ( records: number, ): number => bytes * 8 + records * 64; -const encodeTransferRecord = (input: unknown, previousBytes: number) => { +/** Larger streamed files require corresponding parent admission capacity. */ +export const javaScriptTransferByteBudget = ( + resultBudgetBytes: number, +): number => + Math.min( + MAX_TRANSFER_BYTES, + Math.max(TRANSFER_BYTES, Math.floor(resultBudgetBytes / 8)), + ); + +const assertTransferByteBudget = (transferBytes: number): void => { + if ( + !Number.isSafeInteger(transferBytes) || + transferBytes < 1 || + transferBytes > MAX_TRANSFER_BYTES + ) + throw new RangeError( + "JavaScript transfer requires a valid complete-file byte budget", + ); +}; + +const encodeTransferRecord = ( + input: unknown, + previousBytes: number, + transferBytes = TRANSFER_BYTES, +) => { const record = recordSchema.parse(input); const line = `${JSON.stringify(record)}\n`; const size = Buffer.byteLength(line); - if (size > FRAME_BYTES || previousBytes + size > TRANSFER_BYTES) + if (size > FRAME_BYTES || previousBytes + size > transferBytes) throw new AnalysisResourceConstraintError( "analyze_javascript_application", "transport", @@ -65,7 +90,7 @@ const encodeTransferRecord = (input: unknown, previousBytes: number) => { : "Complete JavaScript file facts exceed the transfer byte budget", { transfer_record_bytes: FRAME_BYTES, - transfer_file_bytes: TRANSFER_BYTES, + transfer_file_bytes: transferBytes, record_bytes: size, transfer_bytes_at_least: previousBytes + size, }, @@ -89,11 +114,13 @@ function* moduleTransferRecords( /** Measure actual validated module rows before allocating an unadmittable graph. */ export const measureJavaScriptModuleTransfer = ( module: JavaScriptModuleSemanticIr, + transferBytes = TRANSFER_BYTES, ) => { + assertTransferByteBudget(transferBytes); let bytes = 0; let records = 0; for (const record of moduleTransferRecords(module)) { - bytes += encodeTransferRecord(record, bytes).size; + bytes += encodeTransferRecord(record, bytes, transferBytes).size; records += 1; } return { @@ -108,7 +135,9 @@ export const writeJavaScriptAnalysisTransfer = async ( path: string, module: JavaScriptModuleSemanticIr, projection: JavaScriptSemanticFileProjection, + transferBytes = TRANSFER_BYTES, ): Promise => { + assertTransferByteBudget(transferBytes); const output = await open(path, "wx", 0o600); const digest = createHash("sha256"); let bytes = 0; @@ -129,7 +158,11 @@ export const writeJavaScriptAnalysisTransfer = async ( batchBytes = 0; }; const append = async (input: unknown): Promise => { - const { line, size, kind } = encodeTransferRecord(input, bytes); + const { line, size, kind } = encodeTransferRecord( + input, + bytes, + transferBytes, + ); bytes += size; records += 1; if ( diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts index 110dfe3b7..0cf42176e 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerProtocol.ts @@ -1,7 +1,10 @@ import { z } from "zod"; import { jsonValueSchema } from "../../domain/jsonValue.js"; import { digestSchema } from "../../domain/digests.js"; -import { JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS } from "../../domain/javascript/javascriptAnalysisResourceControls.js"; +import { + JAVASCRIPT_ANALYSIS_RESOURCE_DEFAULTS, + JAVASCRIPT_SEMANTIC_NODE_MAXIMUM, +} from "../../domain/javascript/javascriptAnalysisResourceControls.js"; import { javaScriptAnalysisTransferDescriptorSchema } from "./JavaScriptAnalysisTransfer.js"; /** Independent worker execution budget; source length is not an admission rule. */ @@ -14,7 +17,7 @@ export const JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS = { /** Caller-selected artifact identity, separate from the decoded source-text digest. */ export const javaScriptWorkerRequestSchema = z.strictObject({ id: z.uuid(), - node_budget: z.number().int().min(0).max(20_000), + node_budget: z.number().int().min(0).max(JAVASCRIPT_SEMANTIC_NODE_MAXIMUM), text_sha256: digestSchema, file: z.strictObject({ path: z.string(), diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts index 92bfaec35..196c8b7b9 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.ts @@ -23,13 +23,19 @@ const findingContext = { module_runtime: z.string().optional(), location: range, }; -const withRuntime = ( +const withRuntime = < + Value extends { + module_runtime?: string | undefined; + resolution_basis?: "wrapper-substitution" | undefined; + }, +>( value: Value, ) => { - const { module_runtime, ...rest } = value; + const { module_runtime, resolution_basis, ...rest } = value; return { ...rest, ...(module_runtime === undefined ? {} : { module_runtime }), + ...(resolution_basis === undefined ? {} : { resolution_basis }), }; }; const staticValue = z.discriminatedUnion("status", [ @@ -62,12 +68,16 @@ const ipcChannel = (shape: Shape) => z.union([ z.strictObject({ ...findingContext, + invocation_location: range.optional(), + resolution_basis: z.literal("wrapper-substitution").optional(), ...shape, channel: z.string(), channel_expression: z.null(), }), z.strictObject({ ...findingContext, + invocation_location: range.optional(), + resolution_basis: z.literal("wrapper-substitution").optional(), ...shape, channel: z.null(), channel_expression: z.string(), @@ -113,10 +123,13 @@ const ipc = z }), ]) .transform((value) => { - const { module_runtime, ...rest } = value; + const { module_runtime, resolution_basis, invocation_location, ...rest } = + value; return { ...rest, ...(module_runtime === undefined ? {} : { module_runtime }), + ...(resolution_basis === undefined ? {} : { resolution_basis }), + ...(invocation_location === undefined ? {} : { invocation_location }), }; }); const windowShape = { @@ -128,6 +141,7 @@ const windowShape = { ), }; const bridgeShape = { + resolution_basis: z.literal("wrapper-substitution").optional(), ...findingContext, world: z.enum(["main", "isolated"]), world_id: staticValue.nullable(), @@ -136,99 +150,134 @@ const bridgeShape = { unknown_members: count, }; const utilityShape = { ...findingContext, service_name: z.string().nullable() }; -const electron = z.strictObject({ - browser_windows: z.array( - z.union([ - z - .strictObject({ - ...windowShape, - preload_path: z.string(), - preload_resolution_context: pathContext, - }) - .transform(withRuntime), - z - .strictObject({ - ...windowShape, - preload_path: z.null(), - preload_resolution_context: z.null(), - }) - .transform(withRuntime), - ]), - ), - context_bridge_apis: z.array( - z.union([ - z - .strictObject({ - ...bridgeShape, - api_key: z.string(), - api_key_expression: z.null(), - }) - .transform(withRuntime), - z - .strictObject({ - ...bridgeShape, - api_key: z.null(), - api_key_expression: z.string(), - }) - .transform(withRuntime), - z - .strictObject({ - ...bridgeShape, - api_key: z.null(), - api_key_expression: z.null(), - }) - .transform(withRuntime), - ]), - ), - ipc: z.array(ipc), - sender_validations: z.array( - z.strictObject({ - ...findingContext, - subject: z.enum([ - "sender-url", - "sender-origin", - "sender-frame", - "sender-id", - "frame-id", - "process-id", +const electron = z + .strictObject({ + call_paths: z + .array( + z.strictObject({ + entry_kind: z.enum(["bridge-member", "ipc-handler"]), + entry: z.string(), + channel: z.string().nullable(), + terminal: z.string(), + resolution: z.literal("candidate"), + steps: z.array( + z.strictObject({ + kind: z.enum([ + "exposure", + "factory", + "function", + "call", + "ipc", + "terminal", + ]), + label: z.string(), + location: range, + module_key: z.string().nullable(), + module_runtime: z.string().nullable(), + }), + ), + conditions: z.array( + z.strictObject({ expression: z.string(), location: range }), + ), + }), + ) + .optional(), + browser_windows: z.array( + z.union([ + z + .strictObject({ + ...windowShape, + preload_path: z.string(), + preload_resolution_context: pathContext, + }) + .transform(withRuntime), + z + .strictObject({ + ...windowShape, + preload_path: z.null(), + preload_resolution_context: z.null(), + }) + .transform(withRuntime), ]), - mechanism: z.string(), - expected: staticValue, - enforcement: z.literal("unknown"), - }), - ), - utility_processes: z.array( - z.union([ - z - .strictObject({ - ...utilityShape, - module_path: z.string(), - module_resolution_context: pathContext, - module_expression: z.null(), - }) - .transform(withRuntime), - z - .strictObject({ - ...utilityShape, - module_path: z.null(), - module_resolution_context: z.null(), - module_expression: z.string(), - }) - .transform(withRuntime), - ]), - ), - native_addon_bindings: z.array( - z.strictObject({ - ...findingContext, - specifier: z.string(), - binding_kind: z.enum(["import", "require", "re-export"]), - module_kind: z.enum(["import", "require"]), - members: strings, - namespace_access: z.boolean().optional(), - dynamic_member_access: z.boolean().optional(), - }), - ), -}); + ), + context_bridge_apis: z.array( + z.union([ + z + .strictObject({ + ...bridgeShape, + api_key: z.string(), + api_key_expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...bridgeShape, + api_key: z.null(), + api_key_expression: z.string(), + }) + .transform(withRuntime), + z + .strictObject({ + ...bridgeShape, + api_key: z.null(), + api_key_expression: z.null(), + }) + .transform(withRuntime), + ]), + ), + ipc: z.array(ipc), + sender_validations: z.array( + z.strictObject({ + ...findingContext, + subject: z.enum([ + "sender-url", + "sender-origin", + "sender-frame", + "sender-id", + "frame-id", + "process-id", + ]), + mechanism: z.string(), + expected: staticValue, + enforcement: z.literal("unknown"), + }), + ), + utility_processes: z.array( + z.union([ + z + .strictObject({ + ...utilityShape, + module_path: z.string(), + module_resolution_context: pathContext, + module_expression: z.null(), + }) + .transform(withRuntime), + z + .strictObject({ + ...utilityShape, + module_path: z.null(), + module_resolution_context: z.null(), + module_expression: z.string(), + }) + .transform(withRuntime), + ]), + ), + native_addon_bindings: z.array( + z.strictObject({ + ...findingContext, + specifier: z.string(), + binding_kind: z.enum(["import", "require", "re-export"]), + module_kind: z.enum(["import", "require"]), + members: strings, + namespace_access: z.boolean().optional(), + dynamic_member_access: z.boolean().optional(), + }), + ), + }) + .transform(({ call_paths, ...rest }) => ({ + ...rest, + ...(call_paths === undefined ? {} : { call_paths }), + })); const referenceShape = { ...findingContext, kind: z.enum([ diff --git a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts index 25ab14e64..0e9f15b59 100644 --- a/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts +++ b/src/javascript/analysis/JavaScriptAnalysisWorkerServer.ts @@ -9,7 +9,10 @@ import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptA import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js"; import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js"; import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js"; -import { writeJavaScriptAnalysisTransfer } from "./JavaScriptAnalysisTransfer.js"; +import { + javaScriptTransferByteBudget, + writeJavaScriptAnalysisTransfer, +} from "./JavaScriptAnalysisTransfer.js"; import { javaScriptWorkerStaticAnalysisSchema } from "./JavaScriptAnalysisWorkerSchemas.js"; import { JAVASCRIPT_ANALYSIS_WORKER_DEFAULTS as DEFAULTS, @@ -95,6 +98,7 @@ const analyzeJob = async ( requests: AsyncIterator, ): Promise => { try { + let resultBudgetBytes = 0; const source = await readFile(join(root, `${request.id}.source`), "utf8"); if ( createHash("sha256").update(source).digest("hex") !== request.text_sha256 @@ -148,7 +152,8 @@ const analyzeJob = async ( throw new TypeError( "JavaScript worker admission names a different source job", ); - return admission.semantic_result_budget_bytes; + resultBudgetBytes = admission.semantic_result_budget_bytes; + return resultBudgetBytes; }, }, ); @@ -160,6 +165,7 @@ const analyzeJob = async ( join(root, `${request.id}.records`), result.module, result.projection, + javaScriptTransferByteBudget(resultBudgetBytes), ); await send({ kind: "complete", id: request.id, descriptor }); } catch (cause: unknown) { diff --git a/src/native/AppleDispatchMetadata.ts b/src/native/AppleDispatchMetadata.ts index 1b54c42b8..26ff8caaf 100644 --- a/src/native/AppleDispatchMetadata.ts +++ b/src/native/AppleDispatchMetadata.ts @@ -4,7 +4,10 @@ import { nativeDispatchMetadataResultSchema, type ObjcSwiftMetadata, } from "../domain/native/objcSwiftMetadata.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { AnalysisCancelledError, AnalysisResourceConstraintError, @@ -174,7 +177,7 @@ export const inspectAppleDispatchMetadata = async ( maxRecords: number, signal?: AbortSignal, ) => { - if (target.kind !== "executable" || target.format !== "mach-o") + if (target.kind !== "executable" || !isMachOFormat(target.format)) throw new TypeError("Apple dispatch metadata requires a Mach-O target"); const maximumBytes = 64 * 1024 * 1024; const capacityError = () => diff --git a/src/native/NativeCallObservation.ts b/src/native/NativeCallObservation.ts index b9ad41620..bb3960b56 100644 --- a/src/native/NativeCallObservation.ts +++ b/src/native/NativeCallObservation.ts @@ -7,7 +7,10 @@ import { AnalysisInputError, } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { EvidenceIntegrityError } from "../domain/evidenceErrors.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -40,7 +43,7 @@ export const observeNativeCalls = async ( signal?: AbortSignal, readTargetDigest: typeof fileSha256 = fileSha256, ): Promise> => { - if (target.kind !== "executable" || target.format !== "mach-o") + if (target.kind !== "executable" || !isMachOFormat(target.format)) return err( new AnalysisCapabilityUnavailableError( "native-macos", diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index 75227b02d..2ae6a7cb6 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -24,7 +24,10 @@ import { NATIVE_TOOL_CONTRACTS, type NativeToolName, } from "../contracts/native/nativeToolContracts.js"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { toolContract } from "../contracts/toolContracts.js"; import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; import type { EvidenceLocation } from "../domain/evidence.js"; @@ -36,7 +39,6 @@ import { AnalysisResourceConstraintError, AnalysisTimeoutError, } from "../domain/analysisErrorCore.js"; -import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; @@ -168,7 +170,10 @@ class NativeMacOSClient implements AnalysisClient { }, ), ); - if (this.target.kind !== "executable" || this.target.format !== "mach-o") + if ( + this.target.kind !== "executable" || + !isMachOFormat(this.target.format) + ) return err( new AnalysisCapabilityUnavailableError( IDENTITY.id, @@ -455,7 +460,7 @@ class NativeMacOSClient implements AnalysisClient { ): Promise> { // lipo reads only Mach-O; its refusal of a PE, ELF, or plist target is not // a tool failure that a retry or `rea doctor` could repair. - if (this.target.format !== "mach-o") + if (!isMachOFormat(this.target.format)) return err( new AnalysisCapabilityUnavailableError( IDENTITY.id, @@ -783,7 +788,7 @@ class NativeMacOSClient implements AnalysisClient { async #inspectMacho( signal?: AbortSignal, ): Promise> { - if (this.target.format !== "mach-o") + if (!isMachOFormat(this.target.format)) return err( new AnalysisCapabilityUnavailableError( IDENTITY.id, @@ -996,7 +1001,7 @@ const codeSignArchitectures = ( ); if (architectures.length > 0) return [...new Set(architectures)]; } - if (target.kind !== "executable" || target.format !== "mach-o") return []; + if (target.kind !== "executable" || !isMachOFormat(target.format)) return []; return target.availableArchitectures.map((architecture) => architecture === "x86" ? "i386" : architecture, ); diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index d2619f8bd..9980fc241 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -67,9 +67,8 @@ export const inspectNativeMacho = async ( } const lipo = captures.find(({ tool }) => tool === "lipo"); if (lipo === undefined) throw new TypeError("Missing lipo capture"); - const availableArchitectures = parseLipoArchitectures(lipo.stdout).map( - ({ name }) => name, - ); + const architectures = parseLipoArchitectures(lipo.stdout); + const availableArchitectures = architectures.map(({ name }) => name); const selection = selectArchitecture(context.target, availableArchitectures); if (selection.status === "absent") return err( @@ -133,31 +132,43 @@ export const inspectNativeMacho = async ( ), ); } + const selectedArchitectureName = + selectedArchitecture ?? + (architectures.length === 1 ? (architectures[0]?.name ?? null) : null); + const containerOffset = + selectedArchitectureName === null + ? undefined + : /^Non-fat file:/mu.test(lipo.stdout) + ? 0 + : (architectures.find(({ name }) => name === selectedArchitectureName) + ?.file_offset ?? undefined); const normalized = normalizeMacho( captures, context.invocation, limitations, load, - ); - const segmentOffset = segmentContainerOffset( - normalized, - selectedArchitecture, - /^Non-fat file:/mu.test(lipo.stdout), + architectures, + { + name: selectedArchitectureName, + container_file_offset: containerOffset ?? null, + }, ); const result = - segmentOffset === undefined + containerOffset === undefined ? inspectMachoSchema.parse({ ...normalized, limitations: [ ...normalized.limitations, - "Segment evidence file offsets are unavailable because the selected Mach-O slice's container offset was not observed.", + selectedArchitectureName === null + ? "The selected Mach-O slice is unknown; slice-based evidence locations are unavailable." + : "The selected Mach-O slice's container file offset was not observed; slice-based evidence locations are unavailable.", ], }) : normalized; return ok({ result: jsonValueSchema.parse(result), limitations: result.limitations, - locations: fileOffsetLocations(result, segmentOffset), + locations: fileOffsetLocations(result, containerOffset), }); }; @@ -166,9 +177,13 @@ const normalizeMacho = ( toInvocation: (capture: NativeCommandCapture) => NativeCommandInvocation, additionalLimitations: readonly string[], load: ReturnType, + architectures: ReturnType, + selectedArchitecture: { + readonly name: string | null; + readonly container_file_offset: number | null; + }, ) => { const byTool = captureLookup(captures); - const architectures = parseLipoArchitectures(byTool("lipo").stdout); const imports = parseDyldSymbols( sliceOutput(byTool("dyld_info", 0)), "imports", @@ -217,6 +232,7 @@ const normalizeMacho = ( ]; return inspectMachoSchema.parse({ format: "mach-o", + selected_architecture: selectedArchitecture, endian: parseEndian(byTool("file").stdout), word_size: parseWordSize(byTool("file").stdout), file_type: load.fileType, @@ -307,45 +323,31 @@ const uniqueSymbols = ( const fileOffsetLocations = ( macho: ReturnType, - segmentOffset: number | undefined, + containerOffset: number | undefined, ): EvidenceLocation[] => { const locations = architectureLocations(macho.architectures.items); - if (segmentOffset === undefined) return locations; + if (containerOffset === undefined) return locations; + for (const entrypoint of macho.entrypoints.items) + locations.push({ + kind: "file-offset", + offset: containerOffset + entrypoint.file_offset, + }); for (const segment of macho.segments.items) { - if (segment.file_offset === null || segment.file_size === null) continue; + if ( + segment.file_offset === null || + segment.file_size === null || + segment.file_size === 0 + ) + continue; locations.push({ kind: "file-offset-range", - start: segmentOffset + segment.file_offset, - end: segmentOffset + segment.file_offset + segment.file_size, + start: containerOffset + segment.file_offset, + end: containerOffset + segment.file_offset + segment.file_size, }); } return locations; }; -/** otool segment offsets are slice-relative; evidence addresses the input file. */ -const segmentContainerOffset = ( - macho: ReturnType, - selectedArchitecture: string | null, - thinInput: boolean, -): number | undefined => { - const architectures = macho.architectures.items; - const slice = - selectedArchitecture === null - ? architectures.length === 1 - ? architectures[0] - : undefined - : architectures.find(({ name }) => name === selectedArchitecture); - if (slice?.file_offset !== null && slice?.file_offset !== undefined) - return slice.file_offset; - // A thin lipo observation has no separate slice offset. Unknown FAT slice - // offsets must not silently become zero and identify unrelated container bytes. - return thinInput && - selectedArchitecture === null && - architectures.length === 1 - ? 0 - : undefined; -}; - /** Project architecture slices into evidence file-offset locations. */ export const architectureLocations = ( architectures: readonly { diff --git a/src/native/NativeUiObservation.ts b/src/native/NativeUiObservation.ts index 021e0d132..1256f5949 100644 --- a/src/native/NativeUiObservation.ts +++ b/src/native/NativeUiObservation.ts @@ -1,6 +1,9 @@ import { fileURLToPath } from "node:url"; import { z } from "zod"; -import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; +import { + isMachOFormat, + type BinaryTarget, +} from "../domain/binaryTargetTypes.js"; import { nativeUiObservationInputSchema, nativeUiScenarioInputSchema, @@ -78,7 +81,7 @@ const observeWithHelper = async ( ).safeParse(parameters); if (!parsed.success) return err(new AnalysisInputError(operation, { cause: parsed.error })); - if (target.kind !== "executable" || target.format !== "mach-o") + if (target.kind !== "executable" || !isMachOFormat(target.format)) return err( new AnalysisCapabilityUnavailableError( "native-macos", diff --git a/src/native/parsers/lipo.test.ts b/src/native/parsers/lipo.test.ts index 4de28398f..43f08d8b8 100644 --- a/src/native/parsers/lipo.test.ts +++ b/src/native/parsers/lipo.test.ts @@ -62,4 +62,31 @@ architecture arm64e }, ]); }); + + it.each([ + ["CPU_SUBTYPE_LIB64", 0x80000003], + ["0x80", 0x80000003], + ["0x0", 3], + ])("applies printed x86_64 capabilities %s", (capabilities, code) => { + expect( + parseLipoArchitectures( + `architecture x86_64\n cpusubtype CPU_SUBTYPE_X86_64_ALL\n capabilities ${capabilities}\n`, + )[0]?.cpu_subtype_code, + ).toBe(code); + }); + + it("leaves the subtype code unknown when lipo prints no capabilities", () => { + // macOS 26 lipo prints this blank line for LIB64 and plain x86_64 slices. + expect( + parseLipoArchitectures( + "architecture x86_64\n cputype CPU_TYPE_X86_64\n cpusubtype CPU_SUBTYPE_X86_ALL\n capabilities \n offset 16384\n", + ), + ).toMatchObject([ + { + cpu_subtype: "CPU_SUBTYPE_X86_ALL", + cpu_subtype_code: null, + capabilities: null, + }, + ]); + }); }); diff --git a/src/native/parsers/lipo.ts b/src/native/parsers/lipo.ts index 4fb13cda7..4f56d8b6b 100644 --- a/src/native/parsers/lipo.ts +++ b/src/native/parsers/lipo.ts @@ -128,7 +128,7 @@ const cpuTypeCode = (value: string | undefined): number | null => { return numeric ?? CPU_TYPES[value] ?? null; }; -/** Normalize the subtype and its separately printed ptrauth capability bits. */ +/** Normalize the subtype and its separately printed capability bits. */ const cpuSubtypeCode = ( value: string | undefined, capabilities: string | undefined, @@ -138,7 +138,11 @@ const cpuSubtypeCode = ( if (numeric !== null) return numeric; const subtype = CPU_SUBTYPES[value]; if (subtype === undefined) return null; - if (value !== "CPU_SUBTYPE_ARM64E") return subtype; + if (value !== "CPU_SUBTYPE_ARM64E") { + // Recent lipo prints an empty capabilities line, even for LIB64 slices. + const bits = capabilityBits(capabilities); + return bits === null ? null : ((bits << 24) | subtype) >>> 0; + } const version = /PTR_AUTH_VERSION\s+USERSPACE\s+(\d+)/u.exec( capabilities ?? "", )?.[1]; @@ -153,6 +157,20 @@ const cpuSubtypeCode = ( return (0x80000000 | (parsedVersion << 24) | subtype) >>> 0; }; +/** The subtype bits a lipo record names, without its capability byte. */ +export const lipoSubtypeBits = (value: string | null): number | null => { + if (value === null) return null; + const subtype = unsigned32(value) ?? CPU_SUBTYPES[value]; + return subtype === undefined ? null : subtype & 0x00ffffff; +}; + +/** lipo names the LIB64 capability and prints other capability bits in hex. */ +const capabilityBits = (capabilities: string | undefined): number | null => { + if (capabilities === "CPU_SUBTYPE_LIB64") return 0x80; + const bits = /^0x([\da-f]{1,2})$/iu.exec(capabilities ?? "")?.[1]; + return bits === undefined ? null : Number.parseInt(bits, 16); +}; + const unsigned32 = (value: string): number | null => { if (!/^(?:\d+|0x[\da-f]+)$/iu.test(value)) return null; const parsed = Number(value); diff --git a/src/native/parsers/otool.ts b/src/native/parsers/otool.ts index 85c54785a..3e1c3a90f 100644 --- a/src/native/parsers/otool.ts +++ b/src/native/parsers/otool.ts @@ -61,6 +61,7 @@ const DYLIB_FIELDS = [ ]; // otool prints options for dylib_use_command and a timestamp for dylib_command. const DYLIB_ENCODING_FIELDS = ["time stamp", "options"]; +const EMPTY_VALUE_FIELDS = new Set(["options", "segname", "sectname"]); /** * otool emits LF, but a capture replayed from another host or toolchain can @@ -201,11 +202,12 @@ const parseLoadCommand = (block: string, state: ParsedLoadCommands): void => { if (!Number.isSafeInteger(index)) throw new TypeError("otool load-command index is unsafe"); const header = block.split(/\n\s*Section\n/u)[0] ?? block; - const fields = parseFields(header); + const parsed = parseFields(header); + const fields = parsed.values; const kind = stringField(fields, "cmd"); if (kind === null) throw new TypeError("otool load command is missing cmd"); - for (const key of scalarFields(kind)) requireUniqueField(header, key); - if (DYLIB_COMMANDS.has(kind)) validateDylibCommandLines(header, fields); + for (const key of scalarFields(kind)) requireUniqueField(parsed, key); + if (DYLIB_COMMANDS.has(kind)) validateDylibCommandLines(parsed); state.commands.push({ index, kind, @@ -218,23 +220,18 @@ const parseLoadCommand = (block: string, state: ParsedLoadCommands): void => { collectSegment(kind, block, fields, state); }; -const validateDylibCommandLines = ( - block: string, - fields: Readonly>, -): void => { +const validateDylibCommandLines = (parsed: ParsedFields): void => { const encodingFields = DYLIB_ENCODING_FIELDS.filter((key) => - Object.hasOwn(fields, key), + Object.hasOwn(parsed.values, key), ); if (encodingFields.length !== 1) throw new TypeError( "otool dylib command must contain timestamp or options", ); - for (const key of encodingFields) requireUniqueField(block, key); + for (const key of encodingFields) requireUniqueField(parsed, key); const allowed = new Set([...DYLIB_FIELDS, ...DYLIB_ENCODING_FIELDS]); - for (const rawLine of block.split("\n").slice(1)) { - const line = rawLine.trim(); - if (line.length === 0) continue; - const key = parseField(line)?.[0]; + for (const field of parsed.rows) { + const key = field?.[0]; if (key === undefined || !allowed.has(key)) throw new TypeError( "otool dylib command contains ambiguous multiline text", @@ -343,10 +340,18 @@ const collectSegment = ( state.segments.push(segment); }; -const parseFields = (block: string): Record => { +interface ParsedFields { + readonly values: Record; + readonly rows: readonly (readonly [string, string] | undefined)[]; +} + +const parseFields = (block: string): ParsedFields => { const fields: Record = {}; + const rows: Array = []; for (const rawLine of block.split(/\r?\n/u).slice(1)) { + if (rawLine.trim().length === 0) continue; const field = parseField(rawLine); + rows.push(field); if (field === undefined) continue; const [key, value] = field; fields[key] = @@ -354,34 +359,33 @@ const parseFields = (block: string): Record => { ? value : (numeric(value) ?? value); } - return fields; + return { values: fields, rows }; }; const parseField = (rawLine: string): readonly [string, string] | undefined => { const line = rawLine.trim(); - // A dylib-use command with no flags still has an empty options row. - if (line === "options") return ["options", ""]; // Match time stamp as a whole without swallowing `name stamp plugin.dylib`. - const match = /^((?:time stamp)|\S+(?:\s+version)?)\s+(.+)$/u.exec(line); - return match?.[1] === undefined || match[2] === undefined + const match = /^((?:time stamp)|\S+(?:\s+version)?)(?:\s+(.+))?$/u.exec(line); + return match?.[1] === undefined ? undefined - : [match[1], match[2].trim()]; + : [match[1], match[2]?.trim() ?? ""]; }; -const requireUniqueField = (block: string, key: string): void => { - const escaped = key.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); - const pattern = new RegExp(`^\\s*${escaped}(?:\\s|$)`, "mu"); - let count = 0; - for (const line of block.split("\n")) if (pattern.test(line)) count += 1; - if (count !== 1) +const requireUniqueField = (parsed: ParsedFields, key: string): void => { + const matches = parsed.rows.filter((field) => field?.[0] === key); + const field = matches[0]; + if (matches.length !== 1 || field === undefined) throw new TypeError(`otool field ${key} is missing or repeated`); + // Only options and fixed-width Mach-O names may legitimately be empty. + if (field[1] === "" && !EMPTY_VALUE_FIELDS.has(key)) + throw new TypeError(`otool field ${key} has no value`); }; const parseSegment = ( block: string, fields: Readonly>, ): ParsedSegment => ({ - name: stringField(fields, "segname") ?? "unknown", + name: nonEmptyName(stringField(fields, "segname")), vm_address: hexField(fields, "vmaddr"), vm_size: numberField(fields, "vmsize"), file_offset: numberField(fields, "fileoff"), @@ -396,7 +400,8 @@ const parseSections = (block: string): ParsedSection[] => .split(/(?=\n\s*Section\n)/u) .slice(1) .map((sectionBlock) => { - const fields = parseFields(`Section${sectionBlock}`); + const parsed = parseFields(`Section${sectionBlock}`); + const fields = parsed.values; for (const key of [ "sectname", "segname", @@ -406,10 +411,10 @@ const parseSections = (block: string): ParsedSection[] => "align", "flags", ]) - requireUniqueField(sectionBlock, key); + requireUniqueField(parsed, key); return { - segment: stringField(fields, "segname") ?? "unknown", - name: stringField(fields, "sectname") ?? "unknown", + segment: nonEmptyName(stringField(fields, "segname")), + name: nonEmptyName(stringField(fields, "sectname")), address: hexField(fields, "addr"), size: numberField(fields, "size"), file_offset: numberField(fields, "offset"), @@ -464,6 +469,10 @@ const numeric = (value: string): number | null => { return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null; }; +/** Project an absent or empty Mach-O name as before: the "unknown" label. */ +const nonEmptyName = (name: string | null): string => + name === null || name.length === 0 ? "unknown" : name; + const stringField = ( fields: Readonly>, name: string, diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts index 4c1add09c..fef52bff9 100644 --- a/src/native/pwntools/PwntoolsDecoder.ts +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -27,7 +27,6 @@ import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; import { runOwnedCommand } from "../../process/OwnedCommand.js"; -import { OwnedCommandFailure } from "../../process/OwnedCommand.js"; import { pwntoolsResourceLimitsSchema, readPwntoolsFailureEvidence, diff --git a/src/process/DarwinProcessRunTokenReader.ts b/src/process/DarwinProcessRunTokenReader.ts index cc837b028..3fbb44a6a 100644 --- a/src/process/DarwinProcessRunTokenReader.ts +++ b/src/process/DarwinProcessRunTokenReader.ts @@ -6,7 +6,8 @@ import { fileURLToPath } from "node:url"; import { z } from "zod"; import { safeParseJson } from "../domain/safeJson.js"; -import { execFileOutput } from "./ExecFileOutput.js"; +import { boundedUtf8Text } from "./boundedUtf8Text.js"; +import { execFileOutput, execFileOutputFailure } from "./ExecFileOutput.js"; import type { ProcessIdentityObservation, ProcessRunTokenObservation, @@ -72,14 +73,124 @@ const sourceFiles = [ */ export const SWIFTC_INTERRUPT = "SIGINT"; +/** Deadline for the one-shot helper compilation. */ +const COMPILER_TIMEOUT_MS = 60_000; + +/** Bound on compiler diagnostics embedded in the inspection failure text. */ +const STDERR_EXCERPT_LIMIT = 512; + /** The native Darwin process-ownership reader could not be prepared. */ export class DarwinProcessOwnershipInspectionError extends Error { - constructor(message: string, options?: ErrorOptions) { - super(message, options); + constructor( + message: string, + options?: ErrorOptions & { + readonly compilerOutcome?: DarwinCompilerOutcome; + }, + ) { + const { compilerOutcome, ...errorOptions } = options ?? {}; + super(message, errorOptions); this.name = "DarwinProcessOwnershipInspectionError"; + this.compilerOutcome = compilerOutcome; } + + readonly compilerOutcome: DarwinCompilerOutcome | undefined; } +/** Explicit, bounded outcome from compiling REA's Darwin ownership helper. */ +export interface DarwinCompilerOutcome { + readonly outcome: + | "exited" + | "signaled" + | "timed_out" + | "spawn_failed" + | "failed"; + readonly code: string | number | null; + readonly signal: string | null; + readonly stderr: string; + readonly stderrBytes: number; + readonly stderrTruncated: boolean; + readonly outputTruncated: boolean; + readonly killed: boolean; + readonly timeoutMs?: number; +} + +const MAX_COMPILER_STDERR_BYTES = 8 * 1024; + +const compilerOutcome = ( + cause: unknown, + timeoutMs: number, +): DarwinCompilerOutcome | undefined => { + const failure = execFileOutputFailure(cause); + if (failure === undefined) return undefined; + const stderr = boundedUtf8Text(failure.stderr, MAX_COMPILER_STDERR_BYTES); + const outcome = + failure.code === "ETIMEDOUT" + ? "timed_out" + : typeof failure.code === "number" + ? "exited" + : failure.signal !== null + ? "signaled" + : failure.code === "ENOENT" + ? "spawn_failed" + : "failed"; + return { + outcome, + code: failure.code, + signal: failure.signal, + stderr: stderr.text, + stderrBytes: Buffer.byteLength(failure.stderr, "utf8"), + stderrTruncated: stderr.truncated, + outputTruncated: failure.outputTruncated, + killed: failure.killed, + ...(outcome === "timed_out" ? { timeoutMs } : {}), + }; +}; + +const compilerOutcomeDescription = ( + outcome: DarwinCompilerOutcome | undefined, + fallbackCode: string | number, +): string => { + if (outcome === undefined) return `compiler result: ${String(fallbackCode)}`; + if (outcome.outcome === "exited") + return `compiler result: exit code ${String(outcome.code)}`; + if (outcome.outcome === "timed_out") + return `compiler result: timed out after ${String(outcome.timeoutMs)} ms`; + if (outcome.outcome === "signaled") + return `compiler result: stopped by signal ${outcome.signal ?? "an unknown signal"}`; + return `compiler result: ${String(outcome.code)}`; +}; + +const compilerInspectionError = ( + cause: unknown, + timeoutMs: number, +): DarwinProcessOwnershipInspectionError => { + const outcome = compilerOutcome(cause, timeoutMs); + const code = outcome?.code ?? "unknown"; + const description = compilerOutcomeDescription(outcome, code); + const prefix = + code === "ENOENT" + ? "macOS process ownership inspection requires the Apple Swift compiler via xcrun" + : "macOS process ownership helper compilation failed via xcrun"; + const stderr = boundedUtf8Text(outcome?.stderr ?? "", STDERR_EXCERPT_LIMIT); + const diagnostics = stderr.text.trim(); + const truncated = + stderr.truncated || + outcome?.stderrTruncated === true || + outcome?.outputTruncated === true; + const suffix = diagnostics + ? `: ${diagnostics}${truncated ? "… [diagnostics truncated]" : ""}` + : outcome === undefined + ? `: ${cause instanceof Error ? cause.message : String(cause)}` + : ": no compiler diagnostics were captured"; + return new DarwinProcessOwnershipInspectionError( + `${prefix} (${description})${suffix}`, + { + cause, + ...(outcome === undefined ? {} : { compilerOutcome: outcome }), + }, + ); +}; + interface ProcessReaderCompilation { readonly controller: AbortController; promise: Promise; @@ -126,8 +237,13 @@ const waitForCompilation = async ( /** Compile and retain the narrow Darwin sysctl reader for this REA process. */ export const createDarwinProcessRunTokenReader = ( - options: { readonly xcrun?: string } = {}, + options: { + readonly xcrun?: string; + /** Deadline for the helper compilation; the default keeps verification fast. */ + readonly compileTimeoutMs?: number; + } = {}, ) => { + const compilerTimeoutMs = options.compileTimeoutMs ?? COMPILER_TIMEOUT_MS; let root: string | undefined; let executable: string | undefined; let compilation: ProcessReaderCompilation | undefined; @@ -171,7 +287,7 @@ export const createDarwinProcessRunTokenReader = ( options.xcrun ?? "/usr/bin/xcrun", ["swiftc", ...sourceFiles, "-o", output], { - timeout: 60_000, + timeout: compilerTimeoutMs, maxBuffer: 1024 * 1024, signal, stopSignal: SWIFTC_INTERRUPT, @@ -180,16 +296,7 @@ export const createDarwinProcessRunTokenReader = ( ); } catch (cause: unknown) { if (isExpectedAbort(cause, signal)) throw cause; - const code = - cause instanceof Error && "code" in cause - ? String(cause.code) - : "unknown"; - throw new DarwinProcessOwnershipInspectionError( - code === "ENOENT" - ? `macOS process ownership inspection requires the Apple Swift compiler via xcrun (compiler result: ${code})` - : `macOS process ownership helper compilation failed via xcrun (compiler result: ${code}): ${cause instanceof Error ? cause.message : String(cause)}`, - { cause }, - ); + throw compilerInspectionError(cause, compilerTimeoutMs); } if (signal.aborted || closed) throw abortReason(signal); executable = output; diff --git a/src/process/boundedUtf8Text.ts b/src/process/boundedUtf8Text.ts new file mode 100644 index 000000000..5915beb65 --- /dev/null +++ b/src/process/boundedUtf8Text.ts @@ -0,0 +1,16 @@ +/** Keep a UTF-8 string within a byte budget without cutting a code point. */ +export const boundedUtf8Text = ( + value: string, + maximumBytes: number, +): { readonly text: string; readonly truncated: boolean } => { + let bytes = 0; + let end = 0; + for (const codePoint of value) { + const codePointBytes = Buffer.byteLength(codePoint, "utf8"); + if (bytes + codePointBytes > maximumBytes) + return { text: value.slice(0, end), truncated: true }; + bytes += codePointBytes; + end += codePoint.length; + } + return { text: value, truncated: false }; +}; diff --git a/src/process/capture/ProcessCaptureError.ts b/src/process/capture/ProcessCaptureError.ts index f618b8969..7cf6f2705 100644 --- a/src/process/capture/ProcessCaptureError.ts +++ b/src/process/capture/ProcessCaptureError.ts @@ -1,11 +1,14 @@ import { DarwinProcessOwnershipInspectionError } from "../DarwinProcessRunTokenReader.js"; -import { AnalysisError } from "../../domain/analysisErrorBase.js"; +import { + AnalysisError, + type AnalysisErrorOptions, +} from "../../domain/analysisErrorBase.js"; import type { PartialProcessCaptureObservation, ProcessCaptureCleanupReport, } from "../../domain/process/processCapture.js"; -interface ProcessCaptureErrorOptions extends ErrorOptions { +interface ProcessCaptureErrorOptions extends AnalysisErrorOptions { readonly userMessage?: string; readonly userCategory?: "cancelled"; readonly reason?: "capture_failed" | "cleanup_incomplete" | "cancelled"; @@ -69,8 +72,28 @@ export const normalizeCaptureFailure = ( signal: AbortSignal | undefined, ): unknown => { if (cause instanceof ProcessCaptureError) return cause; - if (cause instanceof DarwinProcessOwnershipInspectionError) - return new ProcessCaptureError(cause.message, { cause }); + if (cause instanceof DarwinProcessOwnershipInspectionError) { + const compiler = cause.compilerOutcome; + return new ProcessCaptureError(cause.message, { + cause, + ...(compiler === undefined + ? {} + : { + capturedOutput: { + stdout: "", + stderr: compiler.stderr, + truncated: compiler.stderrTruncated || compiler.outputTruncated, + stderr_bytes: compiler.stderrBytes, + exit_code: + compiler.outcome === "exited" && + typeof compiler.code === "number" + ? compiler.code + : null, + signal: compiler.signal, + }, + }), + }); + } if ( signal?.aborted === true && (cause === signal.reason || diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 8318a2f60..e7b6a880c 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -348,6 +348,7 @@ export const createServer = ( if (session !== undefined) { registerSessionTools(server, session, toolLogger, { ...selectedOptions, + artifactResources, ...(availability === undefined ? {} : { availabilityPolicy: availability.policy }), diff --git a/src/server/registerBrowserTools.ts b/src/server/registerBrowserTools.ts index 1a3e691d3..6e7a98449 100644 --- a/src/server/registerBrowserTools.ts +++ b/src/server/registerBrowserTools.ts @@ -24,8 +24,6 @@ import { toolContract } from "../contracts/toolContracts.js"; import type { ToolContract } from "../contracts/toolContractTypes.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { Evidence } from "../domain/evidence.js"; -import { analyzeWebBundleInputSchema } from "../domain/webBundleAnalysis.js"; -import { inspectWebPageInputSchema } from "../domain/browserObservation.js"; import type { Result } from "../domain/result.js"; import type { Logger } from "pino"; import { mcpProgressReporter } from "./mcpProgress.js"; @@ -86,11 +84,7 @@ export const registerBrowserTools = ( inspectContract, { input, context }, (parsed, { signal, progress }) => - inspectWebPage( - options.browser, - inspectWebPageInputSchema.parse(parsed), - { signal, progress }, - ), + inspectWebPage(options.browser, parsed, { signal, progress }), ), ); server.registerTool( @@ -102,14 +96,10 @@ export const registerBrowserTools = ( analyzeContract, { input, context }, (parsed, { signal, progress }) => - analyzeWebBundle( - options.browser, - analyzeWebBundleInputSchema.parse(parsed), - { - signal, - progress, - }, - ), + analyzeWebBundle(options.browser, parsed, { + signal, + progress, + }), ), ); server.registerTool( diff --git a/src/server/registerElectronTools.ts b/src/server/registerElectronTools.ts index fbb19f37c..567718f8b 100644 --- a/src/server/registerElectronTools.ts +++ b/src/server/registerElectronTools.ts @@ -26,7 +26,6 @@ import type { ToolContract } from "../contracts/toolContractTypes.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { Evidence } from "../domain/evidence.js"; import type { Result } from "../domain/result.js"; -import { inspectElectronPageInputSchema } from "../domain/javascript/electronObservation.js"; import type { Logger } from "pino"; import { mcpProgressReporter } from "./mcpProgress.js"; import { logToolExecution } from "./toolLogging.js"; @@ -86,13 +85,11 @@ export const registerElectronTools = ( registration, inspectContract, { input, context }, - async (parsed, { signal, progress }) => { - const request = inspectElectronPageInputSchema.parse(parsed); - return inspectElectronPage(options.electron, request, { + (parsed, { signal, progress }) => + inspectElectronPage(options.electron, parsed, { signal, progress, - }); - }, + }), ), ); server.registerTool( diff --git a/src/server/registerFirmwareTools.ts b/src/server/registerFirmwareTools.ts index 7a1a95800..84aa1b394 100644 --- a/src/server/registerFirmwareTools.ts +++ b/src/server/registerFirmwareTools.ts @@ -1,7 +1,6 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { FirmwareAnalysisService } from "../application/firmware/FirmwareAnalysisService.js"; import { toolContract } from "../contracts/toolContracts.js"; -import type { FirmwareOperation } from "../domain/firmware/firmwareAnalysis.js"; import type { Logger } from "pino"; import type { EvidenceWriter } from "../application/investigation/InvestigationRecordPort.js"; import type { WithAdmittedAnalysis } from "./analysisAdmission.js"; diff --git a/src/server/registerJavaScriptRuntimeObservationTools.ts b/src/server/registerJavaScriptRuntimeObservationTools.ts index 3b6bd8b70..1594222c9 100644 --- a/src/server/registerJavaScriptRuntimeObservationTools.ts +++ b/src/server/registerJavaScriptRuntimeObservationTools.ts @@ -18,7 +18,6 @@ import type { ToolContract } from "../contracts/toolContractTypes.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { Evidence } from "../domain/evidence.js"; import type { Result } from "../domain/result.js"; -import { observeJavaScriptRuntimeInputSchema } from "../domain/javascript/javascriptRuntimeObservation.js"; import type { Logger } from "pino"; import { logToolExecution } from "./toolLogging.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; @@ -61,10 +60,8 @@ export const registerJavaScriptRuntimeObservationTools = ( registration, observeContract, { input, context }, - async (parsed, signal) => { - const request = observeJavaScriptRuntimeInputSchema.parse(parsed); - return observeJavaScriptRuntime(options.runtime, request, { signal }); - }, + (parsed, signal) => + observeJavaScriptRuntime(options.runtime, parsed, { signal }), ), ); }; diff --git a/src/server/registerManagedTools.ts b/src/server/registerManagedTools.ts index f54f2981f..8d1125c52 100644 --- a/src/server/registerManagedTools.ts +++ b/src/server/registerManagedTools.ts @@ -52,6 +52,7 @@ export const registerManagedTools = ( path, operation, executionOptions?.signal, + parameters, ); if (execution.ok && typeof requestedPath === "string") selectedManagedPath = requestedPath; diff --git a/src/server/registerOfficialTools.ts b/src/server/registerOfficialTools.ts index dcb5b10ab..8cc603b8f 100644 --- a/src/server/registerOfficialTools.ts +++ b/src/server/registerOfficialTools.ts @@ -89,6 +89,11 @@ const registerOfficialTool = ( : { analysisProfile: result.value.analysisProfile }), rawResult: result.value.rawResult, limitations: result.value.limitations, + ...(result.value.relationshipCoverage === undefined + ? {} + : { + relationshipCoverage: result.value.relationshipCoverage, + }), locations: result.value.locations, }, ); diff --git a/src/server/registerSessionRecordTools.ts b/src/server/registerSessionRecordTools.ts index a1558fed4..5854aac9d 100644 --- a/src/server/registerSessionRecordTools.ts +++ b/src/server/registerSessionRecordTools.ts @@ -7,6 +7,7 @@ import { } from "../application/EvidenceBundleFiles.js"; import { toolContract } from "../contracts/toolContracts.js"; import type { EvidenceBundle } from "../domain/evidenceBundle.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { err, ok } from "../domain/result.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { inspectEvidenceBundle } from "../application/investigation/InspectEvidenceBundle.js"; @@ -15,6 +16,7 @@ import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; interface EvidenceToolRegistration { readonly server: EvidenceMcpServer; readonly session: BinarySessionPort; + readonly resources: ArtifactResourceScope; readonly exportContract: ReturnType< typeof toolContract<"export_evidence_bundle"> >; @@ -38,6 +40,7 @@ export const registerEvidenceTools = ( const registerExportEvidenceTool = ({ server, session, + resources, exportContract, }: EvidenceToolRegistration): void => { server.registerTool( @@ -49,6 +52,7 @@ const registerExportEvidenceTool = ({ bundle, input.path, input.overwrite, + resources, context.mcpReq.signal, ); return written.ok diff --git a/src/server/registerSessionStatusTool.ts b/src/server/registerSessionStatusTool.ts index c20170647..563e7528b 100644 --- a/src/server/registerSessionStatusTool.ts +++ b/src/server/registerSessionStatusTool.ts @@ -3,9 +3,15 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; import type { ProviderAvailability } from "../application/AnalysisProvider.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; -import { buildCapabilityInventory } from "../application/CapabilityInventory.js"; +import { + buildCapabilityInventory, + compactToolAvailability, +} from "../application/CapabilityInventory.js"; import { toolContract } from "../contracts/toolContracts.js"; -import type { ClientFeatureAvailability } from "../contracts/toolOutputSchemaPrimitives.js"; +import type { + ClientFeatureAvailability, + ToolAvailability, +} from "../contracts/toolOutputSchemaPrimitives.js"; import { jsonObjectSchema } from "../domain/jsonValue.js"; import { createServerIdentity } from "../serverIdentity.js"; import { mcpClientMetadata } from "./mcpClientMetadata.js"; @@ -13,8 +19,6 @@ import type { AvailabilityPolicy } from "../application/CapabilityInventory.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { err } from "../domain/result.js"; -type ToolAvailability = ReturnType[number]; - /** Inputs required to register the binary session status tool. */ export interface SessionStatusToolOptions { readonly server: EvidenceMcpServer; @@ -111,7 +115,7 @@ const projectSessionStatus = (options: { }) => jsonObjectSchema.parse({ ...options.status, - tool_availability: options.toolAvailability, + tool_availability: compactToolAvailability(options.toolAvailability), client_features: options.clientFeatures, server_identity: options.serverIdentity, }); diff --git a/src/server/registerSessionTools.ts b/src/server/registerSessionTools.ts index 1c4b89a54..4db8031ce 100644 --- a/src/server/registerSessionTools.ts +++ b/src/server/registerSessionTools.ts @@ -1,4 +1,5 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { getNavigationContext, @@ -231,6 +232,7 @@ const registerOpenLifecycleTool = ({ /** Register MCP-only target lifecycle operations on a long-lived session. */ export interface SessionToolOptions { + readonly artifactResources: ArtifactResourceScope; readonly startedAt?: string; readonly availabilityPolicy?: () => AvailabilityPolicy; readonly androidAnalysisAvailability?: ( @@ -286,7 +288,7 @@ export const registerSessionTools = ( server: EvidenceMcpServer, session: BinarySessionPort, logger: Logger, - options: SessionToolOptions = {}, + options: SessionToolOptions, ): void => { const openContract = toolContract("open_binary"); const closeContract = toolContract("close_binary"); @@ -339,6 +341,7 @@ export const registerSessionTools = ( registerEvidenceTools({ server, session, + resources: options.artifactResources, exportContract, importContract, snapshotContract, diff --git a/src/wasm/WabtOutput.test.ts b/src/wasm/WabtOutput.test.ts index fa65230e3..655dd5dab 100644 --- a/src/wasm/WabtOutput.test.ts +++ b/src/wasm/WabtOutput.test.ts @@ -160,6 +160,19 @@ it.each([ ])("rejects incomplete decoded module-name metadata: %s", (text) => { expect(() => parseWabtWat(text)).toThrow(/metadata/u); }); +it("frames custom section names at their NUL-terminated objdump display", async () => { + const text = await golden(); + expect(parseWabtOutput(text, 76, ["test\0hidden"]).sections.at(-1)).toEqual({ + kind: "Custom", + start: 64, + end: 76, + bytes: 12, + description: '"test"', + }); + expect(() => parseWabtOutput(text, 76, ["other\0test"])).toThrow( + /custom-section header/u, + ); +}); it("extracts escaped WAT import/export forms while retaining multiline producer text", async () => { const text = await readFile( new URL("../../tests/fixtures/wasm/decoded.wat", import.meta.url), diff --git a/src/wasm/WabtOutput.ts b/src/wasm/WabtOutput.ts index e13e94f31..23883510d 100644 --- a/src/wasm/WabtOutput.ts +++ b/src/wasm/WabtOutput.ts @@ -1,3 +1,7 @@ +/** WABT prints names with %.*s, so an embedded NUL ends their display. */ +const wabtDisplayedName = (name: string | undefined): string | undefined => + name?.split("\0", 1)[0]; + /** Frame WABT's raw custom names using the ordered names from its escaped WAT output. */ export const parseWabtOutput = ( text: string, @@ -5,14 +9,10 @@ export const parseWabtOutput = ( customSectionNames: readonly string[], debugModuleNames: readonly string[] = [], ) => { - // WABT prints module names with printf's %.*s: embedded NUL ends the display. const preamble = "\nmodule.wasm:\tfile format wasm 0x1\n" + debugModuleNames - .map((name) => { - const displayedName = name.split("\0", 1)[0]; - return `module name: <${displayedName}>\n`; - }) + .map((name) => `module name: <${wabtDisplayedName(name)}>\n`) .join(""); const prefix = `${preamble}\nSections:\n\n`; if (!text.startsWith(prefix)) @@ -29,7 +29,7 @@ export const parseWabtOutput = ( offset += match[0].length; let description: string; if (match[1] === "Custom") { - const name = customSectionNames[customIndex++]; + const name = wabtDisplayedName(customSectionNames[customIndex++]); if (name === undefined || !text.startsWith(`"${name}"\n`, offset)) throw new Error( "WABT custom-section header does not match decoded WAT.", diff --git a/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts b/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts new file mode 100644 index 000000000..e57f96adb --- /dev/null +++ b/tests/acceptance/applications/JavaScriptModuleLookupCli.test.ts @@ -0,0 +1,168 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; + +const execute = promisify(execFile); + +const createModuleLookupFixture = async () => { + const root = await createTestTempDirectory("rea-module-lookup-"); + await writeFixtureFiles(root, { + "package.json": JSON.stringify({ name: "module-lookup-fixture" }), + "esm/entry.mjs": [ + 'import "./extensionless";', + 'import "./directory";', + 'import "./explicit.js";', + 'import "./exact.js";', + 'import fallback from "empty-main";', + 'import "nested-main";', + "void fallback;", + ].join("\n"), + "esm/extensionless.js": "module.exports = 1;", + "esm/directory/index.js": "module.exports = 2;", + "esm/explicit.js.js": "module.exports = 3;", + "esm/exact.js": "module.exports = 4;", + "cjs/entry.cjs": [ + 'require("./extensionless");', + 'require("./feature");', + 'require("./feature.cjs");', + 'require("./default-js");', + 'require("./default-json");', + 'require("./default-node");', + 'require("empty-main");', + 'require("nested-main");', + ].join("\n"), + "cjs/extensionless.cjs": "module.exports = 1;", + "cjs/feature.cjs": "module.exports = 2;", + "cjs/default-js.js": "module.exports = 3;", + "cjs/default-json.json": JSON.stringify({ value: 4 }), + // Inventory-only placeholder: never pass this path to Node's runtime. + "cjs/default-node.node": "source-owned opaque native-module placeholder", + "node_modules/empty-main/package.json": JSON.stringify({ main: "" }), + "node_modules/empty-main/index.js": "module.exports = 42;", + "node_modules/nested-main/package.json": JSON.stringify({ main: "sub" }), + "node_modules/nested-main/sub/package.json": JSON.stringify({ + main: "other", + }), + "node_modules/nested-main/sub/other.js": "module.exports = 99;", + "runtime-control.mjs": [ + 'const specifiers = ["./esm/extensionless", "./esm/directory", "./esm/explicit.js", "./esm/exact.js", "empty-main", "nested-main"];', + "const results = [];", + "for (const specifier of specifiers) {", + " try { await import(specifier); results.push({ specifier, ok: true }); }", + " catch (error) { results.push({ specifier, ok: false, code: error.code }); }", + "}", + "console.log(JSON.stringify(results));", + ].join("\n"), + "runtime-control.cjs": [ + 'const specifiers = ["./cjs/extensionless", "./cjs/feature", "./cjs/feature.cjs", "./cjs/default-js", "./cjs/default-json", "empty-main", "nested-main"];', + "const results = [];", + "for (const specifier of specifiers) {", + " try { require(specifier); results.push({ specifier, ok: true }); }", + " catch (error) { results.push({ specifier, ok: false, code: error.code }); }", + "}", + "console.log(JSON.stringify(results));", + ].join("\n"), + }); + return root; +}; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "matches Node module lookup for ESM and CommonJS through %s", + async (surface, { cli }) => { + const root = await createModuleLookupFixture(); + const [esmControl, cjsControl] = await Promise.all([ + execute(process.execPath, ["runtime-control.mjs"], { + cwd: root, + timeout: 5_000, + }), + execute(process.execPath, ["runtime-control.cjs"], { + cwd: root, + timeout: 5_000, + }), + ]); + const esmResults = JSON.parse(esmControl.stdout) as Array<{ + readonly specifier: string; + readonly ok: boolean; + readonly code?: string; + }>; + const cjsResults = JSON.parse(cjsControl.stdout) as Array<{ + readonly specifier: string; + readonly ok: boolean; + readonly code?: string; + }>; + expect(esmResults).toEqual([ + { + specifier: "./esm/extensionless", + ok: false, + code: "ERR_MODULE_NOT_FOUND", + }, + { + specifier: "./esm/directory", + ok: false, + code: "ERR_UNSUPPORTED_DIR_IMPORT", + }, + { + specifier: "./esm/explicit.js", + ok: false, + code: "ERR_MODULE_NOT_FOUND", + }, + { specifier: "./esm/exact.js", ok: true }, + { specifier: "empty-main", ok: true }, + { specifier: "nested-main", ok: false, code: "ERR_MODULE_NOT_FOUND" }, + ]); + expect(cjsResults).toEqual([ + { + specifier: "./cjs/extensionless", + ok: false, + code: "MODULE_NOT_FOUND", + }, + { + specifier: "./cjs/feature", + ok: false, + code: "MODULE_NOT_FOUND", + }, + { specifier: "./cjs/feature.cjs", ok: true }, + { specifier: "./cjs/default-js", ok: true }, + { specifier: "./cjs/default-json", ok: true }, + { specifier: "empty-main", ok: true }, + { specifier: "nested-main", ok: false, code: "MODULE_NOT_FOUND" }, + ]); + + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const expectations = [ + ["import", "./extensionless", null, "not-found"], + ["import", "./directory", null, "not-found"], + ["import", "./explicit.js", null, "not-found"], + ["import", "./exact.js", "esm/exact.js", "resolved"], + ["import", "empty-main", "node_modules/empty-main/index.js", "resolved"], + ["import", "nested-main", null, "not-found"], + ["require", "./extensionless", null, "not-found"], + ["require", "./feature", null, "not-found"], + ["require", "./feature.cjs", "cjs/feature.cjs", "resolved"], + ["require", "./default-js", "cjs/default-js.js", "resolved"], + ["require", "./default-json", "cjs/default-json.json", "resolved"], + ["require", "./default-node", "cjs/default-node.node", "resolved"], + ["require", "empty-main", "node_modules/empty-main/index.js", "resolved"], + ["require", "nested-main", null, "not-found"], + ] as const; + + for (const [kind, specifier, resolvedPath, status] of expectations) + expect(graph.edges).toContainEqual( + expect.objectContaining({ + relation: "imports", + properties: expect.objectContaining({ + module_link_kind: kind, + specifier, + resolved_path: resolvedPath, + resolution_status: status, + }), + }), + ); + }, +); diff --git a/tests/acceptance/applications/runtime.test.ts b/tests/acceptance/applications/runtime.test.ts index 3c454258f..cd9f2c312 100644 --- a/tests/acceptance/applications/runtime.test.ts +++ b/tests/acceptance/applications/runtime.test.ts @@ -7,7 +7,11 @@ import { z } from "zod"; import { CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; -import { evidenceResultOf } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { + evidenceResultOf, + expandToolAvailability, + toolAvailabilityInventory, +} from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; const mainPath = fileURLToPath( @@ -24,15 +28,14 @@ const expectAvailableToolInventory = async (client: Client) => { arguments: {}, }); expect(status.isError).not.toBe(true); - const availability = z + const inventory = z .object({ result: z.object({ - tool_availability: z.array( - z.object({ name: z.string(), available: z.boolean() }), - ), + tool_availability: toolAvailabilityInventory, }), }) .parse(status.structuredContent).result.tool_availability; + const availability = expandToolAvailability(inventory); // This asserts the compiled entrypoint matches the source catalog. When it // fails, check that `dist/` is current (`npm run build:cached`) before // suspecting a registration bug -- a stale build is by far the likeliest diff --git a/tests/acceptance/applications/sourceMapPrefixCli.test.ts b/tests/acceptance/applications/sourceMapPrefixCli.test.ts index 4865b0289..55d94283f 100644 --- a/tests/acceptance/applications/sourceMapPrefixCli.test.ts +++ b/tests/acceptance/applications/sourceMapPrefixCli.test.ts @@ -3,12 +3,10 @@ import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { expect } from "vitest"; -import { z } from "zod"; - -import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; -import { connectLocalToolsMcp } from "../../fixtures/localToolsMcp.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; cliTest.for(["CLI", "stdio MCP"] as const)( "recovers retained prefixed source-map originals and byte identity through %s", @@ -29,27 +27,9 @@ cliTest.for(["CLI", "stdio MCP"] as const)( ); await writeFile(mapPath, mapText); - let document: unknown; - if (surface === "CLI") { - const response = await cli.run({ - arguments: ["analyze-javascript-application", root, "--format", "json"], - environment: { REA_LOG_LEVEL: "silent" }, - }); - expect(response.exitCode, response.stderr).toBe(0); - document = response.json; - } else { - const { call } = await connectLocalToolsMcp(); - const response = await call("analyze_javascript_application", { - input_path: root, - format: "directory", - }); - expect(response.isError, JSON.stringify(response)).not.toBe(true); - document = response.structuredContent; - } - const { graph } = z - .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) - .parse(document).normalized_result; + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); const mapDigest = createHash("sha256").update(mapText).digest("hex"); + const originalDigest = createHash("sha256").update(original).digest("hex"); expect(graph.nodes).toContainEqual( expect.objectContaining({ kind: "source-module", @@ -57,7 +37,7 @@ cliTest.for(["CLI", "stdio MCP"] as const)( strategy: "source-map-original", source_map_sha256: mapDigest, original_source: "original.ts", - source_sha256: createHash("sha256").update(original).digest("hex"), + source_sha256: originalDigest, }), observations: expect.arrayContaining([ expect.objectContaining({ @@ -65,7 +45,13 @@ cliTest.for(["CLI", "stdio MCP"] as const)( source_name: "original.ts", map_path: "bundle.js.map", }), - properties: { content_available: true }, + properties: { + source_index: 0, + declaration_index: 0, + source_name_available: true, + content_available: true, + content_sha256: originalDigest, + }, evidence: expect.objectContaining({ artifact: expect.objectContaining({ sha256: mapDigest }), }), @@ -77,3 +63,91 @@ cliTest.for(["CLI", "stdio MCP"] as const)( expect(await readFile(mapPath, "utf8")).toBe(mapText); }, ); + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains nullable source-map declarations and aligned content through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-nullable-map-"); + const original = "export const answer: number = 42;"; + const unnamed = "retained source text without a declared name"; + const maps = { + "absent.js.map": JSON.stringify({ + version: 3, + sources: ["unembedded.ts"], + sourcesContent: null, + names: [], + mappings: "AAAA", + }), + "mixed.js.map": JSON.stringify({ + version: 3, + sources: [null, "original.ts"], + sourcesContent: [unnamed, original], + names: [], + mappings: "ACAA", + }), + }; + await writeFixtureFiles(root, { + ...maps, + "absent.js": + "export const absent = 1;\n//# sourceMappingURL=absent.js.map\n", + "mixed.js": + "export const answer = 42;\n//# sourceMappingURL=mixed.js.map\n", + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const digest = (text: string) => + createHash("sha256").update(text).digest("hex"); + for (const [name, mapPath, content] of [ + ["unembedded.ts", "absent.js.map", null], + ["original.ts", "mixed.js.map", original], + ] as const) { + const mapDigest = digest(maps[mapPath]); + expect(graph.nodes).toContainEqual( + expect.objectContaining({ + kind: "source-module", + identity: expect.objectContaining({ + strategy: "source-map-original", + source_map_sha256: mapDigest, + original_source: name, + source_sha256: content === null ? null : digest(content), + }), + observations: expect.arrayContaining([ + expect.objectContaining({ + source_map_reference: expect.objectContaining({ + source_name: name, + map_path: mapPath, + }), + properties: expect.objectContaining({ + content_available: content !== null, + }), + evidence: expect.objectContaining({ + artifact: expect.objectContaining({ sha256: mapDigest }), + }), + }), + ]), + }), + ); + } + expect(graph.nodes).toContainEqual( + expect.objectContaining({ + kind: "unknown", + observations: expect.arrayContaining([ + expect.objectContaining({ + properties: expect.objectContaining({ + source_name_available: false, + content_available: true, + content_sha256: digest(unnamed), + }), + evidence: expect.objectContaining({ + artifact: expect.objectContaining({ + sha256: digest(maps["mixed.js.map"]), + }), + }), + }), + ]), + }), + ); + expect(graph.coverage.status).toBe("complete"); + for (const [path, text] of Object.entries(maps)) + expect(await readFile(join(root, path), "utf8")).toBe(text); + }, +); diff --git a/tests/acceptance/setup/dshClientSetup.test.ts b/tests/acceptance/setup/dshClientSetup.test.ts new file mode 100644 index 000000000..1ee935da6 --- /dev/null +++ b/tests/acceptance/setup/dshClientSetup.test.ts @@ -0,0 +1,232 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { expect } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "DSH CLI previews, approves, repairs, launches MCP, repeats and uninstalls while preserving caller YAML", + async ({ cli }) => { + const home = await createTestTempDirectory("rea-dsh-lifecycle-"); + const profile = join(home, "selected harness"); + const config = join(profile, "cordis.patch.yml"); + const skill = join( + home, + ".agents/skills/reverse-engineer-anything/SKILL.md", + ); + await mkdir(profile); + const original = `# caller home patch +- insert: + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: {serverName: rea, transport: stdio, command: stale, args: [mcp]} +- holder: &op + insert: + - id: unrelated + name: caller-plugin + disabled: true + config: {cwd: !!js process.cwd()} + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: + serverName: rea + transport: stdio + command: stale + args: [mcp] + cwd: !!js process.cwd() + env: + REA_LOG_LEVEL: !!js process.env.REA_LOG_LEVEL +- *op +`; + await writeFile(config, original); + const environment = { HOME: home, USERPROFILE: home, DSH_HOME: profile }; + const run = (args: readonly string[]) => + cli.run({ + arguments: [...args, "--json"], + cwd: home, + environment, + timeoutMs: 20_000, + }); + const plan = await run(["setup", "--client", "dsh", "--dry-run"]); + expect(plan.exitCode).toBe(0); + expect(plan.json).toMatchObject({ + status: "planned", + appliedActions: [], + plannedActions: expect.arrayContaining([ + expect.objectContaining({ id: "configure_client:dsh", target: config }), + expect.objectContaining({ id: "install_skill" }), + ]), + }); + expect(await readFile(config, "utf8")).toBe(original); + await expect(readFile(`${config}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + const unapproved = await run(["setup", "--client", "dsh"]); + expect(unapproved.json).toMatchObject({ + status: "needs_confirmation", + appliedActions: [], + }); + expect(await readFile(config, "utf8")).toBe(original); + const configured = await run(["setup", "--client", "dsh", "--yes"]); + expect(configured.exitCode).toBe(0); + expect(configured.json).toMatchObject({ + status: "ready", + clients: { dsh: { status: "configured" } }, + }); + const text = await readFile(config, "utf8"); + expect(text).toContain("# caller home patch"); + const patches = parse(text, { + customTags: [ + { + tag: "tag:yaml.org,2002:js", + resolve: (source: string) => ({ __jsExpr: source }), + }, + ], + }) as { insert?: { id: string; config: unknown }[] }[]; + const rows = patches.flatMap((patch) => patch.insert ?? []); + expect(rows.filter(({ id }) => id === "unrelated")).toEqual([ + { + id: "unrelated", + name: "caller-plugin", + disabled: true, + config: { cwd: { __jsExpr: "process.cwd()" } }, + }, + ]); + expect(rows.filter(({ id }) => id === "rea-mcp")).toHaveLength(1); + const registration = z + .object({ + command: z.string(), + args: z.array(z.string()), + cwd: z.unknown(), + env: z.unknown(), + }) + .parse(rows.find(({ id }) => id === "rea-mcp")?.config); + expect(registration.cwd).toEqual({ __jsExpr: "process.cwd()" }); + expect(registration.env).toMatchObject({ + REA_LOG_LEVEL: { __jsExpr: "process.env.REA_LOG_LEVEL" }, + }); + const doctor = await run(["doctor", "--client", "dsh", "--skill"]); + expect(doctor.exitCode).toBe(0); + expect(doctor.json).toMatchObject({ + healthy: true, + identity: { + registrations: expect.arrayContaining([ + expect.objectContaining({ client: "dsh", state: "aligned" }), + ]), + }, + }); + await verifyGeneratedLauncher(registration, home, environment); + const repeated = await run(["setup", "--client", "dsh", "--yes"]); + expect(repeated.json).toMatchObject({ + status: "ready", + appliedActions: [], + plannedActions: [], + }); + expect(await readFile(config, "utf8")).toBe(text); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + const removed = await run(["uninstall"]); + expect(removed.exitCode).toBe(0); + expect(removed.json).toMatchObject({ + status: "complete", + items: expect.arrayContaining([ + expect.objectContaining({ name: "dsh", status: "removed" }), + ]), + }); + const remaining = parse(await readFile(config, "utf8"), { + logLevel: "silent", + }) as { insert?: { id: string }[] }[]; + expect( + remaining.flatMap((patch) => patch.insert ?? []).map(({ id }) => id), + ).toEqual(["unrelated"]); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + +async function verifyGeneratedLauncher( + registration: { command: string; args: string[] }, + home: string, + environment: Record, +): Promise { + // Exercise the generated launcher. Native Cordis expression evaluation belongs + // to verify:deepseek-client, not this SDK transport journey. + const transport = new StdioClientTransport({ + command: registration.command, + args: registration.args, + cwd: home, + env: { PATH: process.env.PATH ?? "", ...environment }, + stderr: "pipe", + }); + const client = new Client({ name: "dsh-setup-lifecycle", version: "1" }); + try { + await client.connect(transport); + expect((await client.listTools()).tools.map(({ name }) => name)).toContain( + "binary_session", + ); + const session = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + expect(session.isError).not.toBe(true); + expect(session.structuredContent).toMatchObject({ + result: { open: false }, + }); + } finally { + await client.close(); + await transport.close(); + } +} + +cliTest.for(["null", "caller-value", "[caller-value]"])( + "DSH CLI refuses to edit config %s and plain uninstall removes its owned row", + async (configValue, { cli }) => { + const home = await createTestTempDirectory("rea-dsh-malformed-config-"); + const profile = join(home, ".dsh"); + const config = join(profile, "cordis.patch.yml"); + await mkdir(profile); + const initial = `# caller patch +- insert: + - id: other + name: caller-plugin + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: ${configValue} +`; + await writeFile(config, initial); + const run = (args: readonly string[]) => + cli.run({ + arguments: [...args, "--json"], + cwd: home, + environment: { HOME: home, USERPROFILE: home, DSH_HOME: profile }, + }); + const configured = await run(["setup", "--client", "dsh", "--yes"]); + expect(configured.exitCode).not.toBe(0); + expect(configured.json).toMatchObject({ + status: "needs_human", + appliedActions: [], + remediation: expect.stringContaining("config: Use a literal mapping"), + }); + expect(await readFile(config, "utf8")).toBe(initial); + await expect(readFile(`${config}.rea.backup`)).rejects.toMatchObject({ + code: "ENOENT", + }); + const removed = await run(["uninstall"]); + expect(removed.exitCode).toBe(0); + expect(removed.json).toMatchObject({ + status: "complete", + items: expect.arrayContaining([ + expect.objectContaining({ name: "dsh", status: "removed" }), + ]), + }); + const remaining = await readFile(config, "utf8"); + expect(remaining).toContain("# caller patch"); + expect(parse(remaining)).toEqual([ + { insert: [{ id: "other", name: "caller-plugin" }] }, + ]); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(initial); + }, +); diff --git a/tests/acceptance/tools/localToolContracts.test.ts b/tests/acceptance/tools/localToolContracts.test.ts index 638240296..dfd38344b 100644 --- a/tests/acceptance/tools/localToolContracts.test.ts +++ b/tests/acceptance/tools/localToolContracts.test.ts @@ -96,6 +96,16 @@ const pathRequests: readonly [string, Record][] = [ "export_web_scripts", { capture_path: nul, output_directory: "/tmp/exported-scripts" }, ], + ["inspect_pe_resources", { path: nul }], + ["inspect_binary_layout", { path: nul }], + ["inspect_recorded_crash", { path: nul }], + ["inspect_wasm_artifact", { path: nul }], + ["inspect_wasm_artifact", { path: "/tmp/sample.wasm", glue_paths: [nul] }], + [ + "inspect_wasm_artifact", + { path: "/tmp/sample.wasm", candidate_paths: [nul] }, + ], + ["inspect_evm_interface", { path: nul, encoding: "raw" }], [ "export_web_scripts", { capture_path: "/tmp/capture.json", output_directory: nul }, diff --git a/tests/boundary/browser/cdpBrowserProvider.network.test.ts b/tests/boundary/browser/cdpBrowserProvider.network.test.ts index a1e967122..65ed14180 100644 --- a/tests/boundary/browser/cdpBrowserProvider.network.test.ts +++ b/tests/boundary/browser/cdpBrowserProvider.network.test.ts @@ -131,6 +131,178 @@ it("preserves unavailable response observations and captures completed peers", a ).toEqual(["missing-body", "available"]); }); +it("marks malformed JSON body shapes unavailable while preserving partial request shapes", async () => { + const browser = await startFakeCdpBrowser({ + commandEvents(command, origin) { + if (command.method !== "Network.enable") return undefined; + const session = + command.sessionId === undefined ? {} : { sessionId: command.sessionId }; + return ["unavailable", "partial"].flatMap((requestId) => { + const url = `${origin}/${requestId}`; + return [ + { + ...session, + method: "Network.requestWillBeSent", + params: { + requestId, + request: { + url, + method: requestId === "partial" ? "POST" : "GET", + ...(requestId === "partial" + ? { + headers: { "Content-Type": "application/json" }, + postData: '{"request":true}', + } + : {}), + }, + initiator: { type: "other" }, + }, + }, + { + ...session, + method: "Network.responseReceived", + params: { + requestId, + response: { url, status: 200, mimeType: "application/json" }, + }, + }, + { + ...session, + method: "Network.loadingFinished", + params: { requestId, encodedDataLength: 0 }, + }, + ]; + }); + }, + commandResult(command) { + return command.method === "Network.getResponseBody" + ? { body: "{", base64Encoded: false } + : undefined; + }, + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 20, + include_json_body_shapes: true, + }), + ); + + if (!result.ok) throw result.error; + expect(result.value.network.requests).toEqual([ + expect.objectContaining({ + request_id: "unavailable", + body_shapes: { status: "unavailable", request: null, response: null }, + }), + expect.objectContaining({ + request_id: "partial", + body_shapes: { + status: "partial", + request: expect.any(Object), + response: null, + }, + }), + ]); + expect(result.value.completeness.unavailable_sections).toContain( + "json_body_shapes", + ); + expect(result.value.completeness.excluded).not.toContainEqual( + expect.objectContaining({ + section: "json_body_shapes", + reason: "invalid_protocol_value", + }), + ); +}); + +it("rejects malformed response-body envelopes and preserves request shapes", async () => { + const browser = await startFakeCdpBrowser({ + commandEvents(command, origin) { + if (command.method !== "Network.enable") return undefined; + const session = + command.sessionId === undefined ? {} : { sessionId: command.sessionId }; + return ["missing-encoding-flag", "invalid-encoding-flag"].flatMap( + (requestId) => { + const url = `${origin}/${requestId}`; + return [ + { + ...session, + method: "Network.requestWillBeSent", + params: { + requestId, + request: { + url, + method: + requestId === "invalid-encoding-flag" ? "POST" : "GET", + ...(requestId === "invalid-encoding-flag" + ? { + headers: { "Content-Type": "application/json" }, + postData: '{"request":true}', + } + : {}), + }, + initiator: { type: "other" }, + }, + }, + { + ...session, + method: "Network.responseReceived", + params: { + requestId, + response: { url, status: 200, mimeType: "application/json" }, + }, + }, + { + ...session, + method: "Network.loadingFinished", + params: { requestId, encodedDataLength: 0 }, + }, + ]; + }, + ); + }, + commandResult(command) { + if (command.method !== "Network.getResponseBody") return undefined; + return command.params.requestId === "missing-encoding-flag" + ? { body: '{"response":true}' } + : { body: '{"response":true}', base64Encoded: "false" }; + }, + }); + trackBrowser(browser); + const result = await new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + allowed_origins: [browser.allowedOrigin], + target_id: "allowed-page", + observation_ms: 20, + include_json_body_shapes: true, + }), + ); + + if (!result.ok) throw result.error; + expect(result.value.network.requests).toEqual([ + expect.objectContaining({ + request_id: "missing-encoding-flag", + body_shapes: { status: "unavailable", request: null, response: null }, + }), + expect.objectContaining({ + request_id: "invalid-encoding-flag", + body_shapes: { + status: "partial", + request: expect.any(Object), + response: null, + }, + }), + ]); + expect(result.value.completeness.excluded).toContainEqual({ + section: "json_body_shapes", + reason: "invalid_protocol_value", + count: 2, + }); +}); + it.each([ { options: { diff --git a/tests/boundary/browser/webSourceMapFetcher.test.ts b/tests/boundary/browser/webSourceMapFetcher.test.ts index f3bb5fada..35fc34b6f 100644 --- a/tests/boundary/browser/webSourceMapFetcher.test.ts +++ b/tests/boundary/browser/webSourceMapFetcher.test.ts @@ -751,7 +751,7 @@ describe("web source-map sourcesContent validation", () => { JSON.stringify({ version: 3, sources: ["source.ts"], - sourcesContent: [null], + sourcesContent: null, names: [], mappings: "", }), @@ -763,10 +763,61 @@ describe("web source-map sourcesContent validation", () => { expect(result.items[0]).toMatchObject({ status: "included", original_sources: [ - { source: `${origin}/assets/source.ts`, artifact: null }, + { + source_index: 0, + declaration_index: 0, + source: `${origin}/assets/source.ts`, + artifact: null, + }, ], }); }); + + it("keeps unnamed source slots and mapping indices", async () => { + const result = await fetchWebSourceMaps([request], input(), undefined, { + fetch: () => + Promise.resolve( + new Response( + JSON.stringify({ + version: 3, + sources: [null, "source.ts"], + sourcesContent: ["unknown source text", "named source text"], + names: [], + mappings: "ACAA", + }), + { status: 200 }, + ), + ), + }); + + expect(result.items[0]).toMatchObject({ + status: "partial", + original_sources: [ + { + source_index: 0, + declaration_index: 0, + source: null, + artifact: { text: "unknown source text" }, + }, + { + source_index: 1, + declaration_index: 1, + source: `${origin}/assets/source.ts`, + artifact: { text: "named source text" }, + }, + ], + mappings: [ + { + source_index: 1, + declaration_index: 1, + source: `${origin}/assets/source.ts`, + }, + ], + }); + expect(result.items[0]?.limitation).toContain( + "unnamed source at declaration 0", + ); + }); }); describe("source-map original dependency syntax", () => { diff --git a/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts b/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts index ac6bc1a1d..3ed94f72d 100644 --- a/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts +++ b/tests/boundary/browser/webSourceMapNestedCoordinates.test.ts @@ -345,7 +345,7 @@ it.each(cases)( item.mappings.map((entry) => [ entry.generated_line, entry.generated_column, - entry.source.replace(origin + "/", ""), + entry.source?.replace(origin + "/", "") ?? null, ]), ).toEqual(expected); expect(item.artifact.sha256).toBe( diff --git a/tests/boundary/cli/adbCli.test.ts b/tests/boundary/cli/adbCli.test.ts index 88d20af9b..71309e878 100644 --- a/tests/boundary/cli/adbCli.test.ts +++ b/tests/boundary/cli/adbCli.test.ts @@ -1,9 +1,14 @@ -import { expect, it } from "vitest"; +import { chmod, lstat, readFile } from "node:fs/promises"; +import { expect, it, onTestFinished } from "vitest"; import { expectConfigurationRefusal, serveCli as serve, } from "./cliServeHarness.js"; +import { + createTestWorkspace, + removeTestWorkspace, +} from "../../support/workspace/workspaceFixture.js"; it.each([ [ @@ -46,3 +51,40 @@ it("requires the output directory option on pull", async () => { ]); expect(result.exitCode).not.toBe(0); }); + +it.skipIf(process.platform === "win32")( + "refuses a trailing device separator through the CLI before adb or output effects", + async () => { + const workspace = await createTestWorkspace("rea-adb-cli-pull-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const adb = await workspace.write( + "adb", + `#!/bin/sh +if [ "$1" = "version" ]; then + printf 'Android Debug Bridge version 1.0.41\\nVersion 34.0.5-cli\\nInstalled as %s\\n' "$0" + exit 0 +fi +: > '${workspace.path("invoked")}' +exit 0 +`, + ); + await chmod(adb, 0o755); + const output = workspace.path("pulls"); + const result = await serve( + [ + "pull-adb-file", + "emulator-5554", + "/data/local/tmp/", + "--output-directory", + output, + "--json", + ], + { REA_ADB_PATH: adb }, + ); + expect(JSON.parse(result.stdout)).toMatchObject({ + code: "invalid_request", + }); + await expect(readFile(workspace.path("invoked"))).rejects.toThrow(); + await expect(lstat(output)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); diff --git a/tests/boundary/cli/analysisViewCli.test.ts b/tests/boundary/cli/analysisViewCli.test.ts index d8de7df53..0ba1a3ecf 100644 --- a/tests/boundary/cli/analysisViewCli.test.ts +++ b/tests/boundary/cli/analysisViewCli.test.ts @@ -1,3 +1,6 @@ +import { apiSurfaceHar } from "../../fixtures/historicalHar.js"; +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { createHash } from "node:crypto"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -6,12 +9,89 @@ import { expect } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; -import { analysisViewLayoutEvidence } from "../../fixtures/analysisView.js"; +import { + analysisViewLayoutEvidence, + analysisViewCaptureFixture, + analysisViewCaptureEvidence, + analysisViewObjectDenseCapture, +} from "../../fixtures/analysisView.js"; import { createEvidence } from "../../../src/domain/evidence.js"; import { ghidraFunctionDossier } from "../../../src/domain/ghidraValues.fixture.js"; import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js"; import { inspectEvidenceBundle } from "../../../src/application/investigation/InspectEvidenceBundle.js"; +cliTest( + "admits portable captures before parsing and preserves capture caveats", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-inline-capture-cli-"); + const path = join(root, "view.json"); + const environment = { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }; + await writeFile( + path, + JSON.stringify({ + source: { + kind: "inline", + evidence: analysisViewCaptureEvidence( + analysisViewObjectDenseCapture(), + ), + }, + view: { kind: "api-surface" }, + }), + ); + const refused = await cli.run({ + arguments: ["inspect-analysis-view", path, "--json"], + environment, + }); + expect(refused.exitCode).toBe(1); + expect(refused.json).toMatchObject({ + code: "resource_constraint", + details: { resource: "memory" }, + }); + + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 1); + capture.total_records = 1; + const caveat = "Producer omitted exchanges before capture startup."; + capture.limitations = [caveat, caveat]; + capture.records[0]!.reported = { + request: { method: "GET", url: "https://api.test/a" }, + response: { + status: 200, + content: { mimeType: "\u000btext/plain\u000b" }, + }, + }; + const parent = analysisViewCaptureEvidence(capture); + expect(parent.limitations).not.toContain(caveat); + await writeFile( + path, + JSON.stringify({ + source: { kind: "inline", evidence: parent }, + view: { kind: "api-surface" }, + }), + ); + const response = await cli.run({ + arguments: ["inspect-analysis-view", path, "--json"], + environment, + }); + expect(response.exitCode, response.stdout).toBe(0); + const result = parseEvidence(response.json); + expect(result.limitations.filter((value) => value === caveat)).toEqual([ + caveat, + ]); + expect(result.normalized_result).toMatchObject({ + groups: [{ content_types: [], flags: ["single-sample"] }], + limitations: expect.arrayContaining([caveat]), + unknowns: [ + "Some retained records have unavailable status or content type values.", + ], + }); + }, +); + cliTest( "discovers native records from a complete portable bundle using the shared workflow", async ({ cli }) => { @@ -216,3 +296,80 @@ for (const [view, reason] of [ }, ); } + +cliTest( + "decodes HAR credentials and projects recorded API paths through the compiled workflows", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-surface-cli-"); + const fixture = apiSurfaceHar(); + const entry = fixture.log.entries[0]!; + const urls = [ + "https://user:password@api.example.test/a/%2e%2e/admin?token=ordinary#fragment", + "https://api.example.test/admin", + "https:user:password@api.example.test/path", + "https:///user:password@api.example.test/path", + ]; + fixture.log.entries.push( + ...urls.map((url) => ({ + ...entry, + request: { ...entry.request, url }, + })), + ); + const bytes = JSON.stringify(fixture); + const capturePath = join(root, "capture.har"); + const input = join(root, "view.json"); + const environment = { + HOME: root, + XDG_CONFIG_HOME: root, + XDG_CACHE_HOME: root, + }; + await writeFile(capturePath, bytes); + const capture = await cli.run({ + arguments: ["inspect-web-network-capture", capturePath, "har", "--json"], + environment, + }); + expect(capture.exitCode, capture.stdout).toBe(0); + const parent = parseEvidence(capture.json); + expect(parent.subject?.digest.sha256).toBe( + createHash("sha256").update(bytes).digest("hex"), + ); + expect(JSON.stringify(parent)).not.toContain("password"); + await writeFile( + input, + JSON.stringify({ + source: { kind: "inline", evidence: parent }, + view: { kind: "api-surface" }, + }), + ); + const result = await cli.run({ + arguments: ["inspect-analysis-view", input, "--json"], + environment, + }); + expect(result.exitCode, result.stdout).toBe(0); + expect(result.json).toMatchObject({ + confidence: "inferred", + evidence_links: [parent.evidence_id], + normalized_result: { + kind: "api-surface", + records: { examined: 17, total: 17, excluded: 2 }, + coverage: { total: 14, exhausted: true }, + groups: expect.arrayContaining([ + expect.objectContaining({ + path_template: "/a/%2e%2e/admin", + record_ordinals: [13], + }), + expect.objectContaining({ + path_template: "/admin", + record_ordinals: [14], + }), + ]), + excluded_records: [ + { ordinal: 15, reason: "url-redacted" }, + { ordinal: 16, reason: "url-redacted" }, + ], + }, + }); + expect(JSON.stringify(result.json)).not.toContain("password"); + expect(JSON.stringify(result.json)).not.toContain("ordinary"); + }, +); diff --git a/tests/boundary/cli/cliStartupImports.test.ts b/tests/boundary/cli/cliStartupImports.test.ts index a792c82b4..cc1232606 100644 --- a/tests/boundary/cli/cliStartupImports.test.ts +++ b/tests/boundary/cli/cliStartupImports.test.ts @@ -123,7 +123,9 @@ registerHooks({ resolve(specifier, context, next) { const resolved = next(specifier, context); const name = imports.find((item) => - resolved.url.includes(item.path), + resolved.url.includes(item.path) || + (item.path.startsWith("/dist/") && item.path.endsWith(".js") && + resolved.url.includes("/dist/cli-runtime/" + item.path.slice(6, -3).split("/").join("__") + "-")), ); if (name !== undefined) process.stderr.write(${JSON.stringify(MARKER)} + name.name + "\\n"); diff --git a/tests/boundary/cli/dispatcher.test.ts b/tests/boundary/cli/dispatcher.test.ts index f002fcd51..583bd601d 100644 --- a/tests/boundary/cli/dispatcher.test.ts +++ b/tests/boundary/cli/dispatcher.test.ts @@ -11,7 +11,7 @@ describe("executable dispatcher", () => { async ({ processes, workspace }) => { const scripts = await workspace.mkdir("scripts"); await workspace.mkdir("dist"); - await workspace.mkdir("dist/cli"); + await workspace.mkdir("dist/cli-runtime"); await workspace.mkdir("dist/mcp"); await Promise.all([ copyFile("scripts/rea.mjs", join(scripts, "rea.mjs")), @@ -24,19 +24,19 @@ describe("executable dispatcher", () => { 'export const runEntrypoint = () => process.stdout.write(JSON.stringify({ adapter: "mcp" }));\n', ), workspace.write( - "dist/cli.js", + "dist/cli-runtime/cli.js", 'export const createCli = () => ({ serve: (args) => process.stdout.write(JSON.stringify({ adapter: "cli", args })) });\n', ), workspace.write( - "dist/cliOutput.js", + "dist/cli-runtime/cliOutput.js", "export const validateCliOutputArguments = () => ({ ok: true }); export const renderCliOutputArgumentError = () => ''; export const renderEmptyFilteredCliOutput = () => undefined;\n", ), workspace.write( - "dist/cli/streamedJsonOutput.js", + "dist/cli-runtime/streamedJsonOutput.js", "export const createStreamedCliJsonOutput = () => undefined;\n", ), workspace.write( - "dist/mcpDoctor.js", + "dist/cli-runtime/mcpDoctor.js", 'export const runProductionMcpDoctorCli = (args) => Promise.resolve({ output: JSON.stringify({ adapter: "mcp-doctor", args }), exitCode: 0 });\n', ), ]); diff --git a/tests/boundary/cli/electronWrappedCallPaths.test.ts b/tests/boundary/cli/electronWrappedCallPaths.test.ts new file mode 100644 index 000000000..af61f0885 --- /dev/null +++ b/tests/boundary/cli/electronWrappedCallPaths.test.ts @@ -0,0 +1,227 @@ +import { expect } from "vitest"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { z } from "zod"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { writeFixtureFiles } from "../../support/javascriptApplicationFixture.js"; +import { cliTest, type TestCli } from "../../support/cli/cliFixture.js"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; +import { applicationFeatureTraceResultSchema } from "../../../src/domain/javascript/javascriptFeatureTraceSchemas.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; + +const analyze = async (cli: TestCli, extra = "") => { + const root = await createTestTempDirectory("rea-electron-wrapped-paths-"); + await writeFixtureFiles(root, { + "preload.js": `const e=require("electron"); + const sender=channel=>(...args)=>e.ipcRenderer.send(channel,...args); + e.contextBridge.exposeInMainWorld("desktop",{copy:sender("test:copy"),cut:sender("test:cut")});`, + "main.js": `const e=require("electron"); + function register(channel,handler){e.ipcMain.addListener(channel,event=>allowed(event)&&handler(event))} + register("test:copy",event=>event.sender.copy()); + register("test:cut",event=>event.sender.cut());${extra}`, + }); + const output = await cli.run({ + arguments: ["analyze-javascript-application", root, "--json"], + }); + expect(output.exitCode).toBe(0); + const { normalized_result: result } = z + .object({ + normalized_result: javascriptApplicationAnalysisResultSchema, + }) + .parse(output.json); + return { root, application: output.json, ...result }; +}; + +cliTest( + "traces a wrapped bridge member through a unique IPC channel using the public CLI", + async ({ cli }) => { + const { root, application, graph } = await analyze(cli); + const byId = new Map(graph.nodes.map((n) => [n.node_id, n])); + const joins = graph.edges.filter( + (e) => e.evidence.extractor.operation === "join-electron-wrapper-paths", + ); + expect(joins).toHaveLength(2); + for (const edge of joins) { + expect(edge.evidence).toMatchObject({ + state: "inferred", + authority: "static-relationship-inference", + }); + expect(edge.properties.pairing_basis).toBe( + "unique-exact-literal-channel", + ); + expect( + byId.get(edge.source_node_id)?.observations[0]?.properties.channel, + ).toBe(edge.properties.channel); + expect( + byId.get(edge.target_node_id)?.observations[0]?.properties.channel, + ).toBe(edge.properties.channel); + expect(edge.properties.conditions).toEqual([ + expect.objectContaining({ expression: "allowed(event)" }), + ]); + } + const start = graph.nodes.find((n) => + n.observations.some( + (o) => + o.properties.entry === "desktop.copy" && + o.properties.step_kind === "exposure", + ), + ); + expect(start).toBeDefined(); + const visited = new Set(), + pending = start ? [start.node_id] : []; + while (pending.length) { + const id = pending.pop(); + if (!id || visited.has(id)) continue; + visited.add(id); + for (const edge of graph.edges) + if (edge.source_node_id === id) pending.push(edge.target_node_id); + } + expect( + [...visited].some((id) => + byId + .get(id) + ?.observations.some((o) => o.label === "ipc-event.sender.copy"), + ), + ).toBe(true); + expect( + [...visited].some((id) => + byId + .get(id) + ?.observations.some((o) => o.label === "ipc-event.sender.cut"), + ), + ).toBe(false); + const input = join(root, "trace-input.json"); + await writeFile( + input, + JSON.stringify({ + application, + seed: { kind: "api", value: "desktop.copy", match: "exact" }, + direction: "outgoing", + }), + ); + const traced = await cli.run({ + arguments: ["trace-application-feature", input, "--json"], + }); + expect(traced.exitCode).toBe(0); + const { normalized_result: trace } = z + .object({ + normalized_result: applicationFeatureTraceResultSchema, + }) + .parse(traced.json); + expect(trace.seed_matches).toHaveLength(1); + expect( + trace.paths.some( + (path) => + path.contains_inference && + path.node_ids.length > 8 && + byId + .get(path.end_node_id) + ?.observations.some((o) => o.label === "ipc-event.sender.copy"), + ), + ).toBe(true); + }, +); + +for (const surface of ["CLI", "stdio MCP"] as const) { + cliTest( + `preserves handler ambiguity and short-circuit factory paths through ${surface}`, + async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-branch-paths-"); + await writeFixtureFiles(root, { + "preload.js": `const e=require("electron"); + e.contextBridge.exposeInMainWorld("empty", {}); + function choose(){ + if(flag)return ()=>e.ipcRenderer.send("test:copy"); + else return ()=>e.ipcRenderer.send("test:cut"); + return ()=>e.ipcRenderer.send("test:dead"); + } + e.contextBridge.exposeInMainWorld("desktop", { + choose:choose(), + ambiguous:()=>e.ipcRenderer.send("test:ambiguous"), + competing:()=>e.ipcRenderer.send("test:competing"), + conditional:()=>e.ipcRenderer.send("test:conditional"), + fallback:()=>e.ipcRenderer.send("test:fallback"), + });`, + "main.js": `const e=require("electron"); + const copy=event=>event.sender.copy(), cut=event=>event.sender.cut(); + function register(channel,handler){e.ipcMain.on(channel,handler)} + register("test:copy",copy);register("test:cut",cut); + register("test:ambiguous",flag?copy:cut); + register("test:competing",copy); + register("test:competing",flag?copy:unknownHandler); + if(flag) register("test:conditional",copy); + register("test:fallback",event=>{false ?? event.sender.copy()});`, + }); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + const empty = graph.nodes + .filter((node) => node.kind === "context-bridge-api") + .flatMap((node) => node.observations) + .find((observation) => observation.properties.api_key === "empty"); + expect(empty).toMatchObject({ + properties: { + api_status: "object-literal", + members: [], + unknown_members: 0, + resolution_basis: "direct-syntax", + }, + evidence: { state: "observed" }, + }); + const joins = graph.edges.filter( + (edge) => + edge.evidence.extractor.operation === "join-electron-wrapper-paths", + ); + expect(joins.map((edge) => edge.properties.channel).sort()).toEqual([ + "test:copy", + "test:cut", + ]); + const steps = graph.nodes + .flatMap((node) => node.observations) + .filter( + (observation) => observation.properties.step_kind === "terminal", + ); + expect( + [ + ...new Set( + steps + .filter((step) => step.properties.entry === "desktop.choose") + .map((step) => step.properties.channel), + ), + ].sort(), + ).toEqual(["test:copy", "test:cut"]); + for (const [channel, expression] of [ + ["test:copy", "flag"], + ["test:cut", "!(flag)"], + ]) + expect( + steps.find( + (step) => + step.properties.entry === "desktop.choose" && + step.properties.channel === channel, + )?.properties.conditions, + ).toEqual([expect.objectContaining({ expression })]); + expect( + steps.some( + (step) => + step.properties.entry_kind === "ipc-handler" && + ["test:ambiguous", "test:fallback"].includes( + String(step.properties.channel), + ), + ), + ).toBe(false); + }, + ); +} + +cliTest( + "does not join a channel with another untraced compatible handler", + async ({ cli }) => { + const { graph } = await analyze( + cli, + 'e.ipcMain.on("test:copy",event=>unresolved(event));', + ); + const joins = graph.edges.filter( + (e) => e.evidence.extractor.operation === "join-electron-wrapper-paths", + ); + expect(joins.map((e) => e.properties.channel)).toEqual(["test:cut"]); + }, +); diff --git a/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts index af41425ed..a70eb463b 100644 --- a/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts +++ b/tests/boundary/cli/javascriptApplicationJsonOutput.test.ts @@ -1,15 +1,15 @@ import { writeFile } from "node:fs/promises"; import { join } from "node:path"; -import { Filter } from "incur"; import { expect } from "vitest"; import { z } from "zod"; +import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js"; import { analyzeJavaScriptApplication } from "../../support/javascriptApplicationScope.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; -const filter = "evidence_id,normalized_result.graph.nodes[0:1].node_id"; +const filter = "evidence_id,normalized_result.graph.nodes[0,1].node_id"; for (const scenario of [ { @@ -34,6 +34,9 @@ for (const scenario of [ await writeFile(join(root, "main.js"), "fetch('');\n"); const direct = await analyzeJavaScriptApplication({ input_path: root }); if (!direct.ok) throw direct.error; + const normalized = javascriptApplicationAnalysisResultSchema.parse( + direct.value.normalized_result, + ); const output = await cli.run({ arguments: [ scenario.command, @@ -61,9 +64,20 @@ for (const scenario of [ } expect(data).toEqual( scenario.filtered - ? Filter.apply(direct.value, Filter.parse(filter)) + ? { + evidence_id: direct.value.evidence_id, + normalized_result: { + graph: { + nodes: normalized.graph.nodes + .slice(0, 1) + .map(({ node_id }) => ({ node_id })), + }, + }, + } : direct.value, ); + if (scenario.filtered) + expect(normalized.graph.nodes.length).toBeGreaterThan(0); }, ); } @@ -123,3 +137,67 @@ cliTest( expect(output.json).toEqual({}); }, ); + +cliTest( + "honors caller-selected semantic node limits across files and records them in Evidence", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-js-node-budgets-"); + await writeFile(join(root, "first.js"), "external;\n".repeat(40)); + await writeFile(join(root, "second.js"), "another;\n".repeat(40)); + const output = await cli.run({ + arguments: [ + "analyze-javascript-application", + root, + "--format", + "json", + "--max-semantic-nodes", + "50", + "--max-semantic-nodes-per-file", + "30", + ], + }); + expect(output.exitCode).toBe(0); + const result = z + .object({ + parameters: z.object({ + max_semantic_nodes: z.number(), + max_semantic_nodes_per_file: z.number(), + }), + normalized_result: z.object({ + semantic_graph: z.object({ + nodes: z.array( + z.object({ identity: z.object({ module_path: z.string() }) }), + ), + coverage: z.object({ + truncated: z.boolean(), + omitted_nodes: z.number().nullable(), + limits: z.array( + z.object({ name: z.string(), value: z.number() }), + ), + }), + }), + }), + }) + .parse(output.json); + expect(result.parameters).toEqual({ + max_semantic_nodes: 50, + max_semantic_nodes_per_file: 30, + }); + const graph = result.normalized_result.semantic_graph; + expect(graph.nodes).toHaveLength(50); + expect( + graph.nodes.filter((node) => node.identity.module_path === "first.js"), + ).toHaveLength(30); + expect( + graph.nodes.filter((node) => node.identity.module_path === "second.js"), + ).toHaveLength(20); + expect(graph.coverage.truncated).toBe(true); + expect(graph.coverage.omitted_nodes).toBeNull(); + expect(graph.coverage.limits).toEqual( + expect.arrayContaining([ + { name: "semantic_graph_node_ceiling", value: 50 }, + { name: "semantic_graph_file_node_ceiling", value: 30 }, + ]), + ); + }, +); diff --git a/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts new file mode 100644 index 000000000..79716b6cd --- /dev/null +++ b/tests/boundary/cli/keyedArchivePrototypeKeyCli.test.ts @@ -0,0 +1,24 @@ +import { expect } from "vitest"; + +import { parseEvidence } from "../../../src/domain/evidence.js"; +import { + expectKeyedArchivePrototypeKeys, + keyedArchivePrototypeKeyFixture, +} from "../../fixtures/keyedArchivePrototypeKeys.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "compiled CLI decodes formatted XML prototype keys", + async ({ cli }) => { + const fixture = await keyedArchivePrototypeKeyFixture(); + const output = await cli.run({ + arguments: ["inspect-keyed-archive", fixture.path, "--json"], + environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" }, + }); + expect(output.exitCode, output.stderr).toBe(0); + expectKeyedArchivePrototypeKeys( + parseEvidence(output.json).normalized_result, + fixture.digest, + ); + }, +); diff --git a/tests/boundary/cli/rawBinaryCli.test.ts b/tests/boundary/cli/rawBinaryCli.test.ts index b16d296ce..1cff4cac6 100644 --- a/tests/boundary/cli/rawBinaryCli.test.ts +++ b/tests/boundary/cli/rawBinaryCli.test.ts @@ -32,7 +32,7 @@ cliTest( }); expect(missing.exitCode).toBe(1); expect(missing.json).toMatchObject({ - code: "target_unavailable", + code: "invalid_request", details: { reason: expect.stringContaining("architecture") }, }); }, @@ -62,7 +62,7 @@ cliTest( }); expect(result.exitCode).toBe(1); expect(result.json).toMatchObject({ - code: "target_unavailable", + code: "invalid_request", details: { reason: expect.stringContaining("must be 4-byte aligned") }, }); }, diff --git a/tests/boundary/filesystem/analysisSnapshot.test.ts b/tests/boundary/filesystem/analysisSnapshot.test.ts index 84cd14e03..a235fb4e6 100644 --- a/tests/boundary/filesystem/analysisSnapshot.test.ts +++ b/tests/boundary/filesystem/analysisSnapshot.test.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { describe, expect, it } from "vitest"; +import { withArtifactResourceScope } from "../../fixtures/artifactInventory.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; @@ -76,11 +77,14 @@ describe("analysis snapshots: persistence", () => { const directory = await createTestTempDirectory("rea-snapshot-"); const path = join(directory, "analysis.json"); const snapshot = snapshotFixture(); - const written = await writeAnalysisSnapshot(snapshot, path, false); - expect(written.ok).toBe(true); + const written = await withArtifactResourceScope((resources) => + writeAnalysisSnapshot(snapshot, path, false, resources), + ); + if (!written.ok) throw written.error; expect((await readFile(path, "utf8")).endsWith("\n")).toBe(true); expect((await stat(path)).mode & 0o777).toBe(0o600); - expect((await readAnalysisSnapshot(path)).ok).toBe(true); + const roundTrip = await readAnalysisSnapshot(path); + if (!roundTrip.ok) throw roundTrip.error; const altered: unknown = JSON.parse(await readFile(path, "utf8")); if ( @@ -102,8 +106,9 @@ describe("analysis snapshots: persistence", () => { Reflect.set(first.parameters, "address", "0x2000"); await writeFile(path, JSON.stringify(altered)); const loaded = await readAnalysisSnapshot(path); - expect(loaded.ok).toBe(false); - if (!loaded.ok) expect(loaded.error._tag).toBe("EvidenceIntegrityError"); + if (loaded.ok) + throw new Error("Expected changed snapshot identity rejection"); + expect(loaded.error._tag).toBe("EvidenceIntegrityError"); expect(() => parseAnalysisSnapshot(altered)).toThrow(/identifier/u); }); }); diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index 9dd776ce9..6f84a1782 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -5,6 +5,7 @@ import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; import { expect, it } from "vitest"; import { writeOrderedZip } from "../../fixtures/artifactEntryOrder.js"; +import { inventoryArtifact } from "../../fixtures/artifactInventory.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; @@ -12,6 +13,108 @@ import { ZipArtifactReader } from "../../../src/artifacts/ZipArtifactReader.js"; import { withArtifactClient } from "../../support/artifactClientScope.js"; import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js"; +it.each([ + ["legacy CP437", Buffer.from([0x82, 0x2e, 0x74, 0x78, 0x74]), false, "é.txt"], + [ + "EFS with leading U+FEFF", + Buffer.from([0xef, 0xbb, 0xbf, 0x61, 0x62]), + true, + "\uFEFFab", + ], +] as const)( + "preserves %s ZIP member names", + async (_label, rawName, utf8Flag, logicalPath) => { + const root = await createTestTempDirectory("rea-zip-decoded-name-"); + const path = join(root, "names.zip"); + await writeSingleEntryZip(path, "x.txt"); + const archive = await readFile(path); + await writeFile( + path, + rewriteSingleEntryName(archive, "x.txt", rawName, utf8Flag), + ); + + const inventory = await inventoryArtifact(path); + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: logicalPath, + hash_status: "verified", + }), + ); + }, +); + +it("preserves valid Info-ZIP Unicode Path overrides", async () => { + const root = await createTestTempDirectory("rea-zip-unicode-path-"); + const path = join(root, "unicode-path.zip"); + const rawName = Buffer.from("legacy.txt"); + const unicodeName = Buffer.from("café.txt", "utf8"); + const field = Buffer.alloc(5 + unicodeName.length); + field[0] = 1; + field.writeUInt32LE(zipCrc32(rawName), 1); + unicodeName.copy(field, 5); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("legacy.txt", new TextReader("unicode path"), { + extraField: new Map([[0x7075, field]]), + }); + const archive = Buffer.from(await writer.close()); + await writeFile( + path, + rewriteSingleEntryName(archive, "legacy.txt", rawName, false), + ); + + const inventory = await inventoryArtifact(path); + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: "café.txt", + hash_status: "verified", + }), + ); +}); + +it("rejects malformed UTF-8 in flagged ZIP member names", async () => { + const root = await createTestTempDirectory("rea-zip-invalid-utf8-name-"); + const path = join(root, "invalid-utf8.zip"); + await writeSingleEntryZip(path, "x.txt"); + const archive = await readFile(path); + const malformedName = Buffer.from([0xc3, 0x28, 0x2e, 0x74, 0x78]); + await writeFile( + path, + rewriteSingleEntryName(archive, "x.txt", malformedName, true), + ); + + await expect(inventoryArtifact(path)).rejects.toMatchObject({ + name: "ArtifactReaderFailure", + reason: "format", + message: "ZIP filename is not valid UTF-8", + }); +}); + +it("rejects malformed UTF-8 in a CRC-valid Unicode Path extra field", async () => { + const root = await createTestTempDirectory("rea-zip-invalid-unicode-extra-"); + const path = join(root, "invalid-unicode-extra.zip"); + const rawName = Buffer.from("legacy.txt"); + const malformedName = Buffer.from([0xc3, 0x28, 0x2e, 0x74, 0x78]); + const field = Buffer.alloc(5 + malformedName.length); + field[0] = 1; + field.writeUInt32LE(zipCrc32(rawName), 1); + malformedName.copy(field, 5); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("legacy.txt", new TextReader("malformed extra"), { + extraField: new Map([[0x7075, field]]), + }); + const archive = Buffer.from(await writer.close()); + await writeFile( + path, + rewriteSingleEntryName(archive, "legacy.txt", rawName, false), + ); + + await expect(inventoryArtifact(path)).rejects.toMatchObject({ + name: "ArtifactReaderFailure", + reason: "format", + message: "ZIP Unicode Path extra field is not valid UTF-8", + }); +}); + it("bounds a requested metadata read without rejecting the underlying valid ZIP", async () => { const root = await createTestTempDirectory("rea-zip-metadata-budget-"); const path = join(root, "large-directory.zip"); @@ -295,3 +398,66 @@ it("rejects a truncated ZIP through artifact inventory", async () => { error: { _tag: "ArtifactOperationError", reason: "io" }, }); }); + +const writeSingleEntryZip = async ( + path: string, + name: string, +): Promise => { + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add(name, new TextReader("member bytes")); + await writeFile(path, await writer.close()); +}; + +const rewriteSingleEntryName = ( + archive: Buffer, + originalName: string, + rawName: Buffer, + utf8Flag: boolean, +): Buffer => { + const rewritten = Buffer.from(archive); + const expectedName = Buffer.from(originalName, "utf8"); + for (const { signature, nameOffset, flagsOffset, lengthOffset } of [ + { + signature: Buffer.from([0x50, 0x4b, 0x03, 0x04]), + nameOffset: 30, + flagsOffset: 6, + lengthOffset: 26, + }, + { + signature: Buffer.from([0x50, 0x4b, 0x01, 0x02]), + nameOffset: 46, + flagsOffset: 8, + lengthOffset: 28, + }, + ]) { + const header = rewritten.indexOf(signature); + if (header < 0) throw new Error("Expected ZIP member header"); + const nameLength = rewritten.readUInt16LE(header + lengthOffset); + if ( + nameLength !== expectedName.length || + nameLength !== rawName.length || + !rewritten + .subarray(header + nameOffset, header + nameOffset + nameLength) + .equals(expectedName) + ) + throw new Error("Expected the original ZIP member name"); + rawName.copy(rewritten, header + nameOffset); + const flags = rewritten.readUInt16LE(header + flagsOffset); + rewritten.writeUInt16LE( + utf8Flag ? flags | 0x0800 : flags & ~0x0800, + header + flagsOffset, + ); + } + return rewritten; +}; + +const zipCrc32 = (bytes: Uint8Array): number => { + let checksum = 0xffffffff; + for (const byte of bytes) { + checksum ^= byte; + for (let bit = 0; bit < 8; bit += 1) + checksum = + (checksum & 1) === 1 ? (checksum >>> 1) ^ 0xedb88320 : checksum >>> 1; + } + return (checksum ^ 0xffffffff) >>> 0; +}; diff --git a/tests/boundary/filesystem/binaryTarget.test.ts b/tests/boundary/filesystem/binaryTarget.test.ts index 070a50f46..3ab9bdd27 100644 --- a/tests/boundary/filesystem/binaryTarget.test.ts +++ b/tests/boundary/filesystem/binaryTarget.test.ts @@ -20,12 +20,47 @@ import { } from "../../../src/application/AppBundleExecutable.js"; import { dosMz, + fat, pe, thinMach, } from "../../../src/domain/binaryTarget.fixture.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { createEvidence, parseEvidence } from "../../../src/domain/evidence.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +it.each([{ cpus: [0x0100000c] }, { cpus: [0x01000007, 0x0100000c] }])( + "preserves a FAT container's format in the resolved target and Evidence ($cpus)", + async ({ cpus }) => { + const directory = await createTestTempDirectory("rea-universal-target-"); + const path = join(directory, "image"); + await writeFile(path, fat(cpus)); + const resolved = await parseBinaryTarget(path, { + hostArchitecture: "arm64", + }); + if (!resolved.ok) throw resolved.error; + expect(resolved.value).toMatchObject({ + kind: "executable", + format: "mach-o-universal", + architecture: "arm64", + }); + const evidence = createEvidence( + resolved.value, + { id: "fixture", name: "Fixture", version: "1" }, + { + operation: "inspect_macho", + parameters: {}, + result: null, + }, + ); + expect( + parseEvidence(JSON.parse(JSON.stringify(evidence))).subject, + ).toMatchObject({ + format: "mach-o-universal", + architecture: "arm64", + }); + }, +); + describe("binary target I/O: app plist permissions and decoding", () => { it.each([ ["EACCES", "permission denied reading app Info.plist"], diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 9b3e8f5b9..db0b517e7 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -67,6 +67,7 @@ describe("client configuration filesystem lifecycle", () => { "omp", "pi", "hermes", + "dsh", "grok_bot", ]); expect( diff --git a/tests/boundary/filesystem/directAnalysisPermissions.test.ts b/tests/boundary/filesystem/directAnalysisPermissions.test.ts index e3faf63c4..26e308c44 100644 --- a/tests/boundary/filesystem/directAnalysisPermissions.test.ts +++ b/tests/boundary/filesystem/directAnalysisPermissions.test.ts @@ -3,6 +3,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { withArtifactResourceScope } from "../../fixtures/artifactInventory.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { writeAnalysisSnapshot } from "../../../src/application/binary/AnalysisSnapshotFiles.js"; @@ -84,9 +85,10 @@ describe("direct analysis snapshot files", () => { workflow_entries: [], evidence_bundle: createEvidenceBundle([evidence]), }; - expect( - (await writeAnalysisSnapshot(snapshot, snapshotPath, false)).ok, - ).toBe(true); + const written = await withArtifactResourceScope((resources) => + writeAnalysisSnapshot(snapshot, snapshotPath, false, resources), + ); + if (!written.ok) throw written.error; vi.stubEnv("HOPPER_LAUNCHER_PATH", launcherPath); await expect( diff --git a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts index 6827340eb..6687b0ef7 100644 --- a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts +++ b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts @@ -2,7 +2,7 @@ import { parseConfig } from "../../../src/config/parseConfig.js"; import { writeFile } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { z } from "zod"; import { runDirectAnalysis } from "../../../src/application/DirectAnalysis.js"; @@ -33,11 +33,13 @@ import { createAnalysisSnapshotWorkflowEntry, type AnalysisSnapshot, } from "../../../src/domain/analysisSnapshot.js"; +import { withArtifactResourceScope } from "../../fixtures/artifactInventory.js"; import { createTestBinarySession, createTestProviderRouter, } from "../../fixtures/binarySession.js"; import { BinarySession } from "../../../src/application/binary/BinarySession.js"; +import { OwnedStagedOutput } from "../../../src/filesystem/OwnedStagedOutput.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { CAPABILITIES as GHIDRA_CAPABILITIES, @@ -342,6 +344,58 @@ const withEarlierHistoricalEvidence = ( }; }; +describe("snapshot output cleanup lifecycle", () => { + it("returns completed analysis after retrying cleanup of a published snapshot", async () => { + const fixture = await createSnapshotReplayFixture( + "rea-snapshot-cleanup-retry-", + ); + const cleanup = vi + .spyOn(OwnedStagedOutput.prototype, "close") + .mockRejectedValueOnce(new Error("transient staging cleanup failure")); + try { + const output = await runDirectAnalysis( + fixture.dependencies(fixture.createProvider()), + fixture.path, + "binary_overview", + {}, + { snapshotPath: fixture.snapshotPath }, + ); + expect(parseEvidence(output).operation).toBe("binary_overview"); + const saved = await readAnalysisSnapshot(fixture.snapshotPath); + if (!saved.ok) throw saved.error; + expect(saved.value.workflow_entries).toHaveLength(1); + expect(cleanup).toHaveBeenCalledTimes(2); + } finally { + cleanup.mockRestore(); + } + }); + + it("saves snapshots after closing and reopening the same session", async () => { + const fixture = await createSnapshotReplayFixture("rea-snapshot-reopen-"); + const session = createTestBinarySession(fixture.createProvider()); + try { + for (const index of [1, 2]) { + const opened = await session.open(fixture.path); + if (!opened.ok) throw opened.error; + const analyzed = await session.execute("list_segments", {}); + if (!analyzed.ok) throw analyzed.error; + const path = join( + dirname(fixture.snapshotPath), + `snapshot-${index}.json`, + ); + const saved = await session.closeWithSnapshot(path, false); + if (!saved.ok) throw saved.error; + const loaded = await readAnalysisSnapshot(path); + if (!loaded.ok) throw loaded.error; + expect(loaded.value.target.sha256).toBe(opened.value.sha256); + } + } finally { + const closed = await session.close(); + if (!closed.ok) throw closed.error; + } + }); +}); + describe("direct analysis composed snapshot replay", () => { it.each([ { @@ -530,15 +584,15 @@ describe("binary overview snapshot replay", () => { ); if (current === undefined) throw new Error("composed Evidence was not saved"); - expect( - ( - await writeAnalysisSnapshot( - withEarlierHistoricalEvidence(loaded.value, fixture.path, current), - fixture.snapshotPath, - true, - ) - ).ok, - ).toBe(true); + const written = await withArtifactResourceScope((resources) => + writeAnalysisSnapshot( + withEarlierHistoricalEvidence(loaded.value, fixture.path, current), + fixture.snapshotPath, + true, + resources, + ), + ); + if (!written.ok) throw written.error; const second = await runDirectAnalysis( dependencies, @@ -735,15 +789,15 @@ describe("workflow snapshot profile and cancellation binding", () => { ); if (current === undefined) throw new Error("composed Evidence was not saved"); - expect( - ( - await writeAnalysisSnapshot( - withAlternateWorkflowProfile(loaded.value, fixture.path, current), - fixture.snapshotPath, - true, - ) - ).ok, - ).toBe(true); + const written = await withArtifactResourceScope((resources) => + writeAnalysisSnapshot( + withAlternateWorkflowProfile(loaded.value, fixture.path, current), + fixture.snapshotPath, + true, + resources, + ), + ); + if (!written.ok) throw written.error; const refreshed = await runDirectAnalysis( dependencies, diff --git a/tests/boundary/filesystem/dshClientConfiguration.test.ts b/tests/boundary/filesystem/dshClientConfiguration.test.ts new file mode 100644 index 000000000..c33354b40 --- /dev/null +++ b/tests/boundary/filesystem/dshClientConfiguration.test.ts @@ -0,0 +1,2062 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + isNode, + isSeq, + parseDocument, + parse as parseYaml, + stringify as stringifyYaml, +} from "yaml"; + +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { npxRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; +import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { + clientSkillDirectories, + supportedClients, +} from "../../../src/application/SupportedClients.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; + +/** + * The registration command setup writes on the host running the tests. Windows + * wraps it in `cmd.exe /d /c`, so the expected entry follows the host too. + */ +const command = npxRegistrationCommand(process.platform); + +beforeEach(clearClientLocationEnvironment); +afterEach(() => vi.unstubAllEnvs()); + +const dshClient = ( + env: Parameters[2], + platform: NodeJS.Platform = "linux", +) => + supportedClients(join("/home/a"), platform, env).find( + ({ name }) => name === "dsh", + ); + +describe("DeepSeek Harness configuration paths", () => { + it("resolves the home Cordis patch under the Harness home", () => { + expect(dshClient({})).toEqual({ + name: "dsh", + displayName: "DeepSeek Harness", + configPath: join("/home/a", ".dsh", "cordis.patch.yml"), + markerPath: join("/home/a", ".dsh"), + format: "dsh", + }); + // The home patch applies to every profile; DSH_HOME relocates it. + expect(dshClient({ DSH_HOME: "/custom/harness" })?.configPath).toBe( + join("/custom/harness", "cordis.patch.yml"), + ); + expect( + clientSkillDirectories(join("/home/a"), ["dsh"], { + DSH_AGENTS_HOME: "/custom/agents", + }), + ).toEqual([{ client: "dsh", directory: join("/custom/agents", "skills") }]); + }); +}); + +const harnessClient = async () => { + const home = await createTestTempDirectory("rea-dsh-"); + const client = supportedClients(home, process.platform, {}).find( + ({ name }) => name === "dsh", + ); + if (client?.markerPath === undefined) + throw new Error("missing DeepSeek Harness client"); + await mkdir(client.markerPath, { recursive: true }); + return { home, client }; +}; + +describe("DeepSeek Harness Cordis patch registration", () => { + it("appends the rea-mcp insert, preserves user patches, and reads back", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `# Harness home patch +- insert: + - id: user-tool + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: other + transport: stdio + command: other-server +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("# Harness home patch"); + expect(parseYaml(text)).toEqual([ + { + insert: [ + { + id: "user-tool", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "other", + transport: "stdio", + command: "other-server", + }, + }, + ], + }, + { + insert: [ + { + id: "rea-mcp", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "rea", + transport: "stdio", + command: command[0], + args: command.slice(1), + }, + }, + ], + }, + ]); + + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "aligned", command }); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual([ + { + insert: [ + { + id: "user-tool", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "other", + transport: "stdio", + command: "other-server", + }, + }, + ], + }, + ]); + }); + + it("replaces an existing rea-mcp entry without duplicating it", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml( + await readFile(client.configPath, "utf8"), + ) as unknown[]; + expect(patches).toHaveLength(1); + expect((patches[0] as { insert: unknown[] }).insert).toHaveLength(1); + expect(textOf(await readFile(client.configPath, "utf8"))).not.toContain( + "stale-launcher", + ); + }); + + it("rewrites a wrong-package rea-mcp entry into the canonical one", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: not-the-harness-client + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + GHIDRA_INSTALL_DIR: /caller/nondefault/ghidra +`, + ); + + // The non-canonical entry reads back as stale, so setup rewrites it with + // the canonical package name rather than reporting unchanged, and the + // row's configured environment survives the identity repair. + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: { id: string; name: string; config?: { env?: unknown } }[]; + }[]; + expect(patches).toHaveLength(1); + expect(patches[0]?.insert).toHaveLength(1); + expect(patches[0]?.insert[0]?.id).toBe("rea-mcp"); + expect(patches[0]?.insert[0]?.name).toBe("@deepseek-ai/dsh-mcp-client"); + expect(patches[0]?.insert[0]?.config?.env).toEqual({ + GHIDRA_INSTALL_DIR: "/caller/nondefault/ghidra", + }); + expect(textOf(await readFile(client.configPath, "utf8"))).not.toContain( + "not-the-harness-client", + ); + }); +}); + +describe("DeepSeek Harness registration identity", () => { + it("applies a direct config patch before judging the registration", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +- id: rea-mcp + config: + serverName: rea + transport: stdio + command: redirected-launcher + args: [mcp] +`, + ); + + // The later direct patch replaces the row's config, so the effective + // command is the redirect and the registration is stale, not aligned. + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "stale" }); + + // Setup rewrites one canonical entry and drops the superseded direct op. + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("redirected-launcher"); + expect(text).not.toMatch(/^- id: rea-mcp$/mu); + const patches = parseYaml(text) as unknown[]; + expect(patches).toHaveLength(1); + }); + + it("collapses duplicate rea entries into one canonical row", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: first-launcher + args: [mcp] +- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: second-launcher + args: [mcp] +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: unknown[]; + }[]; + const reaRows = patches.flatMap(({ insert }) => + insert.filter( + (entry) => + typeof entry === "object" && + entry !== null && + (entry as { id?: unknown }).id === "rea-mcp", + ), + ); + expect(reaRows).toHaveLength(1); + }); + + it("reports a foreign server name as stale instead of aligned", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: other + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +`, + ); + + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "stale" }); + }); + + it("creates the patch list for an absent file and empties it on uninstall", async () => { + const { home, client } = await harnessClient(); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const created = parseYaml( + await readFile(client.configPath, "utf8"), + ) as unknown[]; + expect(created).toEqual([ + { + insert: [ + { + id: "rea-mcp", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "rea", + transport: "stdio", + command: command[0], + args: command.slice(1), + }, + }, + ], + }, + ]); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(client.configPath, "utf8")).toBe("[]\n"); + }); +}); + +describe("DeepSeek Harness repair boundaries", () => { + it("repairs a direct-only registration by appending the insert", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +`, + ); + + // An incomplete manual registration has no insertion to rewrite, so the + // repair appends one canonical row instead of leaving the entry absent. + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual([ + { + insert: [ + { + id: "rea-mcp", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "rea", + transport: "stdio", + command: command[0], + args: command.slice(1), + }, + }, + ], + }, + ]); + }); + + it("refuses a group-targeted insertion whose applicability is unknown", async () => { + const { home, client } = await harnessClient(); + const initial = stringifyYaml([ + { id: "missing-group", insert: [canonicalEntry()] }, + ]); + await writeFile(client.configPath, initial); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("failed"); + expect(await dshStatus(home)).not.toMatchObject({ state: "aligned" }); + expect(await readFile(client.configPath, "utf8")).toBe(initial); + }); +}); + +describe("DeepSeek Harness registration state", () => { + it("diagnoses a disabled entry as stale and re-enables it on repair", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + disabled: true + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +`, + ); + + // The command matches, but Cordis never activates a disabled row, so the + // registration reads as stale rather than aligned. + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "stale" }); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: { disabled?: unknown }[]; + }[]; + expect(patches[0]?.insert[0]?.disabled).toBeUndefined(); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("retains the existing registration environment across a repair", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + GHIDRA_INSTALL_DIR: /caller/nondefault/ghidra +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: { config: { env?: Record } }[]; + }[]; + expect(patches[0]?.insert[0]?.config.env).toEqual({ + GHIDRA_INSTALL_DIR: "/caller/nondefault/ghidra", + }); + }); + + it("treats a direct-only registration as absent even when its config matches", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +`, + ); + + // A direct patch needs an inserted row to target; with none, Harness has + // no entry, so the registration is absent rather than aligned. + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "missing" }); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: unknown[]; + }[]; + expect(patches).toHaveLength(1); + expect(patches[0]?.insert).toHaveLength(1); + }); + + it("keeps caller-owned config fields across a refresh", async () => { + const { client } = await harnessClient(); + const withCwd = (text: string): string => + text.replace("args: [mcp]", "args: [mcp]\n cwd: /custom/workdir"); + await writeFile( + client.configPath, + withCwd(`- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +`), + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: { config: { cwd?: unknown; command?: unknown } }[]; + }[]; + expect(patches[0]?.insert[0]?.config.cwd).toBe("/custom/workdir"); + expect(patches[0]?.insert[0]?.config.command).toBe(command[0]); + + // An otherwise-matching row with caller fields is current, not stale. + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); +}); + +describe("DeepSeek Harness canonical row repair", () => { + it("normalizes two aligned insertions instead of reporting unchanged", async () => { + const { home, client } = await harnessClient(); + const row = `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +`; + await writeFile(client.configPath, `${row}${row}`); + + // Both rows already match, but Cordis still has two insertions. Doctor + // and setup must normalize to the single owned row. + const registration = async () => + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"); + expect(await registration()).toMatchObject({ state: "stale" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert: unknown[]; + }[]; + const reaRows = patches.flatMap(({ insert }) => + insert.filter( + (entry) => + typeof entry === "object" && + entry !== null && + (entry as { id?: unknown }).id === "rea-mcp", + ), + ); + expect(reaRows).toHaveLength(1); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect(await registration()).toMatchObject({ state: "aligned" }); + }); + + it("keeps Cordis expression tags on caller-owned config fields", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + cwd: !!js process.cwd() # caller launch directory +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.cwd()"); + expect(text).toContain("# caller launch directory"); + expect(text).not.toContain("stale-launcher"); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert: { config: { cwd?: unknown; command?: unknown } }[]; + }[]; + expect(patches[0]?.insert[0]?.config.command).toBe(command[0]); + expect(patches[0]?.insert[0]?.config.cwd).toBe("process.cwd()"); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect(await readFile(client.configPath, "utf8")).toContain( + "!!js process.cwd()", + ); + }); + + it("keeps a Cordis tag from the direct patch that owns the effective config", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: first-launcher + args: [mcp] +- id: rea-mcp + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + cwd: !!js process.cwd() +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.cwd()"); + expect(text).not.toContain("stale-launcher"); + expect(text).not.toContain("first-launcher"); + expect(text).not.toMatch(/^- id: rea-mcp$/mu); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert?: { config?: { cwd?: unknown } }[]; + }[]; + expect(patches).toHaveLength(1); + expect(patches[0]?.insert?.[0]?.config?.cwd).toBe("process.cwd()"); + }); +}); + +describe("DeepSeek Harness document edges", () => { + it("keeps Cordis tags on environment values that were not overridden", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + OTHER: !!js process.env.OTHER + TOKEN: !!js process.env.TOKEN # caller token +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.env.TOKEN"); + expect(text).toContain("# caller token"); + expect(text).toContain("!!js process.env.OTHER"); + expect(text).not.toContain("stale-launcher"); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert: { config: { env?: Record; command?: string } }[]; + }[]; + expect(patches[0]?.insert[0]?.config.command).toBe(command[0]); + expect(patches[0]?.insert[0]?.config.env).toEqual({ + OTHER: "process.env.OTHER", + TOKEN: "process.env.TOKEN", + }); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect(await readFile(client.configPath, "utf8")).toContain( + "!!js process.env.TOKEN", + ); + }); + + it("replaces only the environment entry a caller overrode", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + OTHER: !!js process.env.OTHER + TOKEN: !!js process.env.TOKEN +`, + ); + + expect( + ( + await configureClientConfiguration( + client, + { TOKEN: "literal-override" }, + command, + ) + ).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.env.OTHER"); + expect(text).not.toContain("!!js process.env.TOKEN"); + expect(text).toContain("literal-override"); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert: { config: { env?: Record } }[]; + }[]; + expect(patches[0]?.insert[0]?.config.env).toEqual({ + OTHER: "process.env.OTHER", + TOKEN: "literal-override", + }); + }); + + it("appends a registration to a comment-only patch file", async () => { + const { home, client } = await harnessClient(); + const notes = "# harness notes\n# keep me\n"; + await writeFile(client.configPath, notes); + + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "missing" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("# harness notes"); + expect(text).toContain("# keep me"); + expect(text).toContain("id: rea-mcp"); + expect(text).not.toMatch(/^null$/mu); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const removed = await readFile(client.configPath, "utf8"); + expect(removed).toContain("# harness notes"); + expect(removed).toContain("# keep me"); + expect(removed).not.toContain("rea-mcp"); + expect(removed).not.toMatch(/^null$/mu); + }); +}); + +describe("DeepSeek Harness entry ownership", () => { + const registration = async (home: string) => + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"); + + it("removes a stale launcher and a direct-only row by entry id", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: other-tool + name: other + config: + command: keep-me + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +`, + ); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const removed = await readFile(client.configPath, "utf8"); + expect(removed).toContain("other-tool"); + expect(removed).not.toContain("rea-mcp"); + expect(removed).not.toContain("stale-launcher"); + + await writeFile( + client.configPath, + `- id: other-tool + config: + command: keep-me +- id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: {} +`, + ); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const direct = await readFile(client.configPath, "utf8"); + expect(direct).toContain("other-tool"); + expect(direct).not.toContain("rea-mcp"); + }); + + it("treats an expression disabled state as stale and clears it", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + disabled: !!js process.env.DISABLE_REA === "1" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +`, + ); + + expect(await registration(home)).toMatchObject({ state: "stale" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("DISABLE_REA"); + expect(text).not.toContain("disabled:"); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect(await registration(home)).toMatchObject({ state: "aligned" }); + }); + + it("keeps a Cordis expression on a known environment setting", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + GHIDRA_INSTALL_DIR: !!js process.env.GHIDRA_INSTALL_DIR +`, + ); + + // The expression is not a literal path, so doctor can still classify the + // row instead of rejecting the file. + expect(await registration(home)).toMatchObject({ state: "stale" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.env.GHIDRA_INSTALL_DIR"); + expect(text).not.toContain("stale-launcher"); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert: { config: { env?: Record } }[]; + }[]; + expect(patches[0]?.insert[0]?.config.env).toEqual({ + GHIDRA_INSTALL_DIR: "process.env.GHIDRA_INSTALL_DIR", + }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect(await readFile(client.configPath, "utf8")).toContain( + "!!js process.env.GHIDRA_INSTALL_DIR", + ); + }); +}); + +describe("DeepSeek Harness alias collapse", () => { + it("collapses an aliased insertion and keeps an unrelated alias", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &other + insert: + - id: other-tool + name: other + config: + command: keep-me +- *other +- &row + insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- *row +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("*other"); + expect(text).not.toContain("*row"); + expect(text).not.toContain("stale-launcher"); + const patches = parseYaml(text) as { insert?: { id?: string }[] }[]; + const reaRows = patches.flatMap(({ insert }) => + (insert ?? []).filter((entry) => entry.id === "rea-mcp"), + ); + expect(reaRows).toHaveLength(1); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("collapses an aliased entry inside one insertion", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - &entry + id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + - *entry +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("*entry"); + const patches = parseYaml(text) as { insert?: { id?: string }[] }[]; + expect( + patches.flatMap(({ insert }) => + (insert ?? []).filter((entry) => entry.id === "rea-mcp"), + ), + ).toHaveLength(1); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); +}); + +describe("DeepSeek Harness alias targets", () => { + it("collapses an alias of the insert list", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: &rows + - id: other-tool + name: other + config: + command: keep-me + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- insert: *rows +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("other-tool"); + expect(text).not.toContain("*rows"); + expect(text).not.toContain("stale-launcher"); + const patches = parseYaml(text) as { insert?: { id?: string }[] }[]; + expect( + patches.flatMap(({ insert }) => + (insert ?? []).filter((entry) => entry.id === "rea-mcp"), + ), + ).toHaveLength(1); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("keeps a Cordis tag when the config map is an alias", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &cfg + cwd: !!js process.cwd() # caller launch directory + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: *cfg +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.cwd()"); + expect(text).toContain("# caller launch directory"); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert?: { config?: { cwd?: unknown; command?: unknown } }[]; + }[]; + const row = patches.find((patch) => patch.insert !== undefined); + expect(row?.insert?.[0]?.config?.cwd).toBe("process.cwd()"); + expect(row?.insert?.[0]?.config?.command).toBe(command[0]); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); +}); + +describe("DeepSeek Harness alias identity", () => { + it("keeps unrelated rows when the insert list is anchored outside an operation", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &rows + - id: other-tool + name: other + config: + command: keep-me + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- insert: *rows +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert?: { id?: string; config?: { command?: string } }[]; + }[]; + const inserted = patches.flatMap((patch) => patch.insert ?? []); + expect(inserted.filter((entry) => entry.id === "other-tool")).toHaveLength( + 1, + ); + expect(inserted.filter((entry) => entry.id === "rea-mcp")).toEqual([ + expect.objectContaining({ + config: expect.objectContaining({ command: command[0] }), + }), + ]); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("keeps an aliased Cordis tag on an environment value", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &expr !!js process.env.TOKEN +- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + TOKEN: *expr +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.env.TOKEN"); + expect(text).toContain("*expr"); + expect(text).not.toContain("stale-launcher"); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("recognizes an aliased entry id", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- &entryId rea-mcp +- insert: + - id: *entryId + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const patches = parseYaml(await readFile(client.configPath, "utf8")) as { + insert?: { id?: string }[]; + }[]; + expect( + patches.flatMap((patch) => + (patch.insert ?? []).filter((entry) => entry.id === "rea-mcp"), + ), + ).toHaveLength(1); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(client.configPath, "utf8")).not.toContain("*entryId"); + }); +}); + +describe("DeepSeek Harness alias cleanup", () => { + it("keeps tags when the environment map is an alias", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &reaEnv + TOKEN: !!js process.env.TOKEN +- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: *reaEnv +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.env.TOKEN"); + expect(text).not.toContain("stale-launcher"); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("does not validate an aliased Cordis path expression as a literal", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- &expr !!js process.env.GHIDRA_INSTALL_DIR +- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + GHIDRA_INSTALL_DIR: *expr +`, + ); + + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "stale" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect(await readFile(client.configPath, "utf8")).toContain( + "!!js process.env.GHIDRA_INSTALL_DIR", + ); + }); + + it("removes an aliased insert list on uninstall", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: &rows + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +- insert: *rows +`, + ); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("rea-mcp"); + expect(text).not.toContain("*rows"); + }); +}); + +describe("DeepSeek Harness root aliases", () => { + it("uses a direct patch replayed by a root alias as the winning config", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: first-launcher + args: [mcp] + env: + GHIDRA_INSTALL_DIR: /absolute/ghidra +- &patch + id: rea-mcp + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + GHIDRA_INSTALL_DIR: !!js process.env.GHIDRA_INSTALL_DIR +- *patch +`, + ); + + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "dsh"), + ).toMatchObject({ state: "stale" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect(await readFile(client.configPath, "utf8")).toContain( + "!!js process.env.GHIDRA_INSTALL_DIR", + ); + }); + + it("removes a root alias of the owned insertion on uninstall", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- &row + insert: + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +- *row +`, + ); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("rea-mcp"); + expect(text).not.toContain("*row"); + }); + + it("keeps a Cordis tag when the only owned row is an alias", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &entry + id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + cwd: !!js process.cwd() +- insert: + - *entry +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).toContain("!!js process.cwd()"); + expect(text).not.toContain("stale-launcher"); + const patches = parseYaml(text, { logLevel: "error" }) as { + insert?: { config?: { cwd?: unknown; command?: unknown } }[]; + }[]; + const row = patches.find((patch) => patch.insert !== undefined); + expect(row?.insert?.[0]?.config?.cwd).toBe("process.cwd()"); + expect(row?.insert?.[0]?.config?.command).toBe(command[0]); + }); +}); + +const loadedInserts = ( + text: string, +): { id?: string; config?: { command?: string } }[] => { + const patches = parseYaml(text) as { + insert?: { id?: string; config?: { command?: string } }[]; + }[]; + return patches.flatMap((patch) => patch.insert ?? []); +}; + +describe("DeepSeek Harness inert anchors", () => { + it("keeps unrelated rows when the only active insertion is a root alias", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- holder: &op + insert: + - id: other-tool + name: other + config: + command: keep-me + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- *op +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const inserted = loadedInserts(await readFile(client.configPath, "utf8")); + expect(inserted.filter((entry) => entry.id === "other-tool")).toHaveLength( + 1, + ); + expect(inserted.filter((entry) => entry.id === "rea-mcp")).toEqual([ + expect.objectContaining({ + config: expect.objectContaining({ command: command[0] }), + }), + ]); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("keeps unrelated rows when uninstall removes that root alias", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- holder: &op + insert: + - id: other-tool + name: other + config: + command: keep-me + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +- *op +`, + ); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const text = await readFile(client.configPath, "utf8"); + expect(parseYaml(text)).toBeTruthy(); + const inserted = loadedInserts(text); + expect(inserted.filter((entry) => entry.id === "other-tool")).toHaveLength( + 1, + ); + expect(inserted.filter((entry) => entry.id === "rea-mcp")).toHaveLength(0); + }); +}); + +describe("DeepSeek Harness alias anchors", () => { + it("rewrites an entry id alias before removing its anchor", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- id: &entryId rea-mcp + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- insert: + - id: *entryId + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("*entryId"); + expect( + loadedInserts(text).filter((entry) => entry.id === "rea-mcp"), + ).toHaveLength(1); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("edits a root alias whose insert list is itself an alias", async () => { + const { client } = await harnessClient(); + await writeFile( + client.configPath, + `- &rows + - id: other-tool + name: other + config: + command: keep-me + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] +- holder: &op + insert: *rows +- *op +`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const inserted = loadedInserts(await readFile(client.configPath, "utf8")); + expect(inserted.filter((entry) => entry.id === "other-tool")).toHaveLength( + 1, + ); + expect(inserted.filter((entry) => entry.id === "rea-mcp")).toHaveLength(1); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("drops a replayed root alias after its insert alias is emptied", async () => { + const { home, client } = await harnessClient(); + await writeFile( + client.configPath, + `- &rows + - id: rea-mcp + name: "@deepseek-ai/dsh-mcp-client" + config: + serverName: rea + transport: stdio + command: ${command[0]} + args: [${command.slice(1).join(", ")}] +- &op + insert: *rows +- *op +`, + ); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const text = await readFile(client.configPath, "utf8"); + expect(text).not.toContain("*op"); + expect(() => parseYaml(text)).not.toThrow(); + }); +}); + +const textOf = (value: unknown): string => String(value); + +/** Same tagged expression representation as Harness's entry-list schema. */ +const cordisPatches = (text: string) => + parseYaml(text, { + customTags: [ + { + tag: "tag:yaml.org,2002:js", + resolve: (source: string) => ({ __jsExpr: source }), + }, + ], + }) as { insert?: { id?: string; config?: Record }[] }[]; + +const canonicalEntry = () => ({ + id: "rea-mcp", + name: "@deepseek-ai/dsh-mcp-client", + config: { + serverName: "rea", + transport: "stdio", + command: command[0], + args: command.slice(1), + }, +}); + +const dshStatus = async (home: string) => + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client }) => client === "dsh"); + +describe("DeepSeek Harness source ownership", () => { + it.each(["setup", "uninstall"])( + "preserves unrelated rows in a later externally anchored operation during %s", + async (operation) => { + const { home, client } = await harnessClient(); + const initial = `${stringifyYaml([{ insert: [canonicalEntry()] }])}- holder: &op + insert: + - id: other-tool + name: other + config: {command: keep-me} + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: {serverName: rea, transport: stdio, command: stale, args: [mcp]} +- *op +`; + await writeFile(client.configPath, initial); + const result = + operation === "setup" + ? await configureClientConfiguration(client, {}, command) + : await systemUninstallHost(home).removeClient(client); + expect(result.status).toBe( + operation === "setup" ? "configured" : "removed", + ); + const entries = loadedInserts(await readFile(client.configPath, "utf8")); + expect(entries.filter(({ id }) => id === "other-tool")).toEqual([ + { id: "other-tool", name: "other", config: { command: "keep-me" } }, + ]); + expect(entries.filter(({ id }) => id === "rea-mcp")).toHaveLength( + operation === "setup" ? 1 : 0, + ); + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + initial, + ); + }, + ); + + it.each(["inline", "aliased config"])( + "snapshots environment aliases before replacing their anchor in %s", + async (shape) => { + const { client } = await harnessClient(); + const config = `serverName: rea + transport: stdio + command: stale-launcher + args: [mcp] + env: + A: &expr !!js process.env.TOKEN # caller expression + B: *expr +`; + const initial = + shape === "inline" + ? `- insert:\n - id: rea-mcp\n name: '@deepseek-ai/dsh-mcp-client'\n config:\n ${config.replaceAll("\n", "\n ").trimEnd()}\n` + : `- holder: &cfg\n ${config}- insert:\n - id: rea-mcp\n name: '@deepseek-ai/dsh-mcp-client'\n config: *cfg\n`; + await writeFile(client.configPath, initial); + expect( + ( + await configureClientConfiguration( + client, + { A: "literal-override" }, + command, + ) + ).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + const row = cordisPatches(text) + .flatMap((op) => op.insert ?? []) + .find(({ id }) => id === "rea-mcp"); + expect(row?.config?.env).toEqual({ + A: "literal-override", + B: { __jsExpr: "process.env.TOKEN" }, + }); + expect(text).toContain("# caller expression"); + expect( + ( + await configureClientConfiguration( + client, + { A: "literal-override" }, + command, + ) + ).status, + ).toBe("unchanged"); + }, + ); + + it("keeps expression aliases used by unrelated rows when removing their source entry", async () => { + const { home, client } = await harnessClient(); + const initial = `- insert: + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: + command: stale + cwd: &cwd !!js process.cwd() + - id: other-tool + name: other + config: + cwd: *cwd +`; + await writeFile(client.configPath, initial); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(cordisPatches(await readFile(client.configPath, "utf8"))).toEqual([ + { + insert: [ + { + id: "other-tool", + name: "other", + config: { cwd: { __jsExpr: "process.cwd()" } }, + }, + ], + }, + ]); + }); +}); + +describe("DeepSeek Harness registration applicability", () => { + it("preserves literal scalar keys in unrelated and caller-owned config", async () => { + const { client } = await harnessClient(); + const initial = `- insert: + - id: other + name: caller-plugin + config: {1: one, true: yes} + - id: rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + config: {command: stale, extra: {2: two}} +`; + await writeFile(client.configPath, initial); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const rows = cordisPatches(await readFile(client.configPath, "utf8"))[0] + ?.insert; + expect(rows?.[0]?.config).toEqual({ "1": "one", true: "yes" }); + expect(rows?.[1]?.config?.extra).toEqual({ "2": "two" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + }); + + it("uses direct-patch names as selectors and preserves the actual effective config", async () => { + const { home, client } = await harnessClient(); + const entry = canonicalEntry(); + const initial = stringifyYaml([ + { + insert: [ + { + ...entry, + name: "other-package", + config: { ...entry.config, env: { FROM_INSERT: "keep-me" } }, + }, + ], + }, + { + id: "rea-mcp", + name: entry.name, + config: { ...entry.config, env: { FROM_SKIPPED_PATCH: "discard-me" } }, + }, + ]); + await writeFile(client.configPath, initial); + expect(await dshStatus(home)).toMatchObject({ state: "stale" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect( + cordisPatches(await readFile(client.configPath, "utf8"))[0]?.insert?.[0] + ?.config?.env, + ).toEqual({ FROM_INSERT: "keep-me" }); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + }); + + it.each(["serverName", "transport", "command", "args"] as const)( + "distinguishes expression-backed %s from literal launcher values", + async (key) => { + const { home, client } = await harnessClient(); + const entry = canonicalEntry(); + const doc = stringifyYaml([{ insert: [entry] }]); + const { parseDocument } = await import("yaml"); + const yaml = parseDocument(doc); + const expression = yaml.createNode( + key === "args" ? "[]" : String(entry.config[key]), + ); + expression.tag = "tag:yaml.org,2002:js"; + yaml.setIn([0, "insert", 0, "config", key], expression); + await writeFile(client.configPath, yaml.toString()); + expect((await dshStatus(home))?.state).not.toBe("aligned"); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect(cordisPatches(await readFile(client.configPath, "utf8"))).toEqual([ + { insert: [entry] }, + ]); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + }, + ); + + it("keeps caller config separate from package, duplicate and activation diagnostics", async () => { + const { home, client } = await harnessClient(); + const entry = canonicalEntry(); + const initial = stringifyYaml([ + { + insert: [ + { + ...entry, + config: { + ...entry.config, + package: "caller-package", + duplicate: "caller-state", + disabled: "caller-value", + enabled: false, + }, + }, + ], + }, + ]); + await writeFile(client.configPath, initial); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + expect(await readFile(client.configPath, "utf8")).toBe(initial); + }); + + it.each([ + "- &cycle {holder: *cycle}\n", + "- insert: [\n", + "- insert: *missing\n", + [ + "- holder: &a [value, value]", + "- holder: &b [*a, *a, *a, *a, *a, *a, *a, *a, *a, *a]", + "- holder: &c [*b, *b, *b, *b, *b, *b, *b, *b, *b, *b]", + "- insert: [{id: rea-mcp, config: {extra: *c}}]", + ].join("\n"), + "- insert: [{id: rea-mcp, name: '@deepseek-ai/dsh-mcp-client', config: !!js process.env.CONFIG}]\n", + ])("refuses unsafe YAML without mutation: %s", async (initial) => { + const { client } = await harnessClient(); + await writeFile(client.configPath, initial); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("failed"); + expect(await readFile(client.configPath, "utf8")).toBe(initial); + await expect( + readFile(`${client.configPath}.rea.backup`), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + }); +}); + +describe("DeepSeek Harness ownership and editability", () => { + it.each([ + `- insert: [{id: !!js '"rea-mcp"', name: caller-plugin}]\n`, + `- id: !!js '"rea-mcp"'\n config: {command: caller-launcher}\n`, + `- holder: &id !!js '"rea-mcp"'\n- insert: [{id: *id}]\n`, + ])("retains registrations with unknown ID ownership: %s", async (initial) => { + const { home, client } = await harnessClient(); + await writeFile(client.configPath, initial); + expect(await dshStatus(home)).toMatchObject({ state: "invalid" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("failed"); + const host = systemUninstallHost(home); + expect(await host.inspectClient(client)).toMatchObject({ + status: "failed", + }); + expect((await host.removeClient(client)).status).toBe("failed"); + expect(await readFile(client.configPath, "utf8")).toBe(initial); + await expect( + readFile(`${client.configPath}.rea.backup`), + ).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + + it.each(["selector", "insertion"] as const)( + "preserves unknown %s name applicability while allowing owned removal", + async (expressionPosition) => { + const { home, client } = await harnessClient(); + const { parseDocument } = await import("yaml"); + const yaml = parseDocument( + stringifyYaml([ + { insert: [canonicalEntry()] }, + { + id: "rea-mcp", + name: "@deepseek-ai/dsh-mcp-client", + config: { command: "patched-launcher" }, + }, + ]), + ); + const expression = yaml.createNode('"@deepseek-ai/dsh-mcp-client"'); + expression.tag = "tag:yaml.org,2002:js"; + yaml.setIn( + expressionPosition === "selector" + ? [1, "name"] + : [0, "insert", 0, "name"], + expression, + ); + const initial = yaml.toString(); + await writeFile(client.configPath, initial); + expect(await dshStatus(home)).toMatchObject({ state: "invalid" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("failed"); + expect(await readFile(client.configPath, "utf8")).toBe(initial); + const host = systemUninstallHost(home); + expect(await host.inspectClient(client)).toBeUndefined(); + expect((await host.removeClient(client)).status).toBe("removed"); + expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual([]); + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + initial, + ); + }, + ); +}); + +describe("DeepSeek Harness aliased mapping keys", () => { + it.each(["setup", "uninstall"])( + "resolves owned fields and preserves caller config during %s", + async (operation) => { + const { home, client } = await harnessClient(); + const initial = `- holder: &insertKey insert +- holder: &idKey id +- holder: &configKey config +- holder: &envKey env +- ? *insertKey + : - ? *idKey + : rea-mcp + name: '@deepseek-ai/dsh-mcp-client' + ? *configKey + : command: stale + ? *envKey + : TOKEN: keep-me + - id: other + name: caller-plugin +`; + await writeFile(client.configPath, initial); + const result = + operation === "setup" + ? await configureClientConfiguration(client, {}, command) + : await systemUninstallHost(home).removeClient(client); + expect(result.status).toBe( + operation === "setup" ? "configured" : "removed", + ); + const entries = loadedInserts(await readFile(client.configPath, "utf8")); + expect(entries.filter(({ id }) => id === "other")).toEqual([ + { id: "other", name: "caller-plugin" }, + ]); + const owned = entries.filter(({ id }) => id === "rea-mcp"); + expect(owned).toHaveLength(operation === "setup" ? 1 : 0); + if (operation === "setup") { + expect(owned[0]?.config).toMatchObject({ env: { TOKEN: "keep-me" } }); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("unchanged"); + } + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + initial, + ); + }, + ); +}); + +describe("DeepSeek Harness ordered selector applicability", () => { + it.each(["before insertion", "before replacement"])( + "discards selector uncertainty %s", + async (position) => { + const { home, client } = await harnessClient(); + const selector = `- id: rea-mcp + name: !!js '"@deepseek-ai/dsh-mcp-client"' + config: {command: ignored-launcher, env: {IGNORED: discarded}} +`; + const insertion = stringifyYaml([{ insert: [canonicalEntry()] }]); + const initial = + position === "before insertion" + ? selector + insertion + : insertion + selector + insertion; + await writeFile(client.configPath, initial); + expect((await dshStatus(home))?.state).toBe( + position === "before insertion" ? "aligned" : "stale", + ); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe(position === "before insertion" ? "unchanged" : "configured"); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + const text = await readFile(client.configPath, "utf8"); + expect( + cordisPatches(text) + .flatMap((patch) => patch.insert ?? []) + .filter(({ id }) => id === "rea-mcp"), + ).toEqual([canonicalEntry()]); + if (position === "before insertion") expect(text).toBe(initial); + expect( + (await systemUninstallHost(home).removeClient(client)).status, + ).toBe("removed"); + expect(loadedInserts(await readFile(client.configPath, "utf8"))).toEqual( + [], + ); + }, + ); +}); + +describe("DeepSeek Harness caller node metadata", () => { + it.each(["environment", "launcher"])( + "preserves scalar and argument comments during %s repairs", + async (repair) => { + const { home, client } = await harnessClient(); + const entry = canonicalEntry(); + const yaml = parseDocument( + stringifyYaml([ + { + insert: [ + { + ...entry, + config: { + ...entry.config, + ...(repair === "launcher" ? { command: "stale" } : {}), + env: { TOKEN: "original" }, + }, + }, + ], + }, + ]), + ); + for (const path of [ + [0, "insert", 0, "id"], + [0, "insert", 0, "name"], + ...["serverName", "transport", "command", "args"].map((key) => [ + 0, + "insert", + 0, + "config", + key, + ]), + [0, "insert", 0, "config", "env", "TOKEN"], + ]) { + const node = yaml.getIn(path, true); + if (!isNode(node)) throw new Error("missing caller node"); + node.comment = ` caller ${path.at(-1)}`; + node.commentBefore = ` before ${path.at(-1)}`; + } + const args = yaml.getIn([0, "insert", 0, "config", "args"], true); + if (!isSeq(args) || !isNode(args.items[0])) + throw new Error("missing launcher arguments"); + args.items[0].comment = " caller argument"; + const initial = yaml.toString(); + await writeFile(client.configPath, initial); + expect( + ( + await configureClientConfiguration( + client, + { TOKEN: "updated" }, + command, + ) + ).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + for (const key of [ + "id", + "name", + "serverName", + "transport", + "command", + "args", + "TOKEN", + ]) { + expect(text).toContain(`# caller ${key}`); + expect(text).toContain(`# before ${key}`); + } + expect(text).toContain("# caller argument"); + expect(loadedInserts(text)).toEqual([ + { + ...entry, + config: { ...entry.config, env: { TOKEN: "updated" } }, + }, + ]); + expect(await dshStatus(home)).toMatchObject({ state: "aligned" }); + expect( + ( + await configureClientConfiguration( + client, + { TOKEN: "updated" }, + command, + ) + ).status, + ).toBe("unchanged"); + expect(await readFile(client.configPath, "utf8")).toBe(text); + expect(await readFile(`${client.configPath}.rea.backup`, "utf8")).toBe( + initial, + ); + }, + ); +}); diff --git a/tests/boundary/filesystem/evidenceBundleFiles.test.ts b/tests/boundary/filesystem/evidenceBundleFiles.test.ts index 267d381ce..d93afe620 100644 --- a/tests/boundary/filesystem/evidenceBundleFiles.test.ts +++ b/tests/boundary/filesystem/evidenceBundleFiles.test.ts @@ -11,6 +11,7 @@ import { join } from "node:path"; import { describe, expect, it, vi } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { withArtifactResourceScope } from "../../fixtures/artifactInventory.js"; import { readEvidenceBundle, @@ -99,15 +100,10 @@ describe("evidence bundle export cancellation", () => { yield "unused"; } expect( - await writeTextParts( - parts(), - join(root, "missing", "bundle.json"), - false, - { - signal: controller.signal, - operation: "export_evidence_bundle", - }, - ), + await writeParts(parts(), join(root, "missing", "bundle.json"), false, { + signal: controller.signal, + operation: "export_evidence_bundle", + }), ).toMatchObject({ ok: false, error: { @@ -118,7 +114,7 @@ describe("evidence bundle export cancellation", () => { expect(advanced).toBe(false); expect(await readdir(root)).toEqual([]); expect( - await writeEvidenceBundle( + await writeBundle( bundle(), join(root, "bundle.json"), false, @@ -149,7 +145,7 @@ describe("evidence bundle export cancellation", () => { } } expect( - await writeTextParts(parts(), path, overwrite, { + await writeParts(parts(), path, overwrite, { signal: controller.signal, operation: "export_evidence_bundle", }), @@ -164,9 +160,9 @@ describe("evidence bundle export cancellation", () => { expect(continued).toBe(false); expect(await readdir(root)).toEqual(overwrite ? ["bundle.json"] : []); if (overwrite) expect(await readFile(path, "utf8")).toBe("original"); - expect( - await writeEvidenceBundle(bundle(), path, overwrite), - ).toMatchObject({ ok: true }); + expect(await writeBundle(bundle(), path, overwrite)).toMatchObject({ + ok: true, + }); expect(await readEvidenceBundle(path)).toEqual({ ok: true, value: bundle(), @@ -184,7 +180,7 @@ describe("evidence bundle export cancellation", () => { controller.abort(); } expect( - await writeTextParts(parts(), path, true, { + await writeParts(parts(), path, true, { signal: controller.signal, operation: "export_evidence_bundle", }), @@ -194,6 +190,29 @@ describe("evidence bundle export cancellation", () => { }); }); +const writeBundle = ( + value: ReturnType, + path: string, + overwrite: boolean, + signal?: AbortSignal, +) => + withArtifactResourceScope((resources) => + writeEvidenceBundle(value, path, overwrite, resources, signal), + ); + +const writeParts = ( + parts: Iterable, + path: string, + overwrite: boolean, + cancellation?: { + readonly signal: AbortSignal; + readonly operation: string; + }, +) => + withArtifactResourceScope((resources) => + writeTextParts(parts, path, overwrite, resources, cancellation), + ); + describe("evidence bundle publication", () => { it.each([false, true])( "never publishes a partial streamed file (overwrite: %s)", @@ -205,9 +224,7 @@ describe("evidence bundle publication", () => { yield "partial"; throw new Error("Source stream failed"); } - expect( - await writeTextParts(brokenParts(), path, overwrite), - ).toMatchObject({ + expect(await writeParts(brokenParts(), path, overwrite)).toMatchObject({ ok: false, error: { _tag: "EvidenceFileError", reason: "io" }, }); @@ -220,7 +237,7 @@ describe("evidence bundle publication", () => { const root = await createTestTempDirectory("rea-stream-bytes-"); const path = join(root, "bundle.json"); const parts = ["雪", "😀", "\n"]; - expect(await writeTextParts(parts, path, false)).toEqual({ + expect(await writeParts(parts, path, false)).toEqual({ ok: true, value: { path, bytes: Buffer.byteLength(parts.join("")) }, }); @@ -233,7 +250,7 @@ describe("evidence bundle publication", () => { const path = join(root, "bundle.json"); const candidates = [bundle(), bundle(false)]; const results = await Promise.all( - candidates.map((value) => writeEvidenceBundle(value, path, false)), + candidates.map((value) => writeBundle(value, path, false)), ); expect(results.filter(({ ok }) => ok)).toHaveLength(1); expect(results.filter(({ ok }) => !ok)).toMatchObject([ @@ -253,7 +270,7 @@ describe("evidence bundle publication", () => { await mkdir(directory); const path = join(directory, "bundle.json"); const value = bundle(); - expect(await writeEvidenceBundle(value, path, false)).toMatchObject({ + expect(await writeBundle(value, path, false)).toMatchObject({ ok: true, }); expect(await readEvidenceBundle(path)).toEqual({ @@ -308,7 +325,7 @@ describe("evidence bundle filesystem adapter", () => { const directory = await createTestTempDirectory("rea-evidence-"); const path = join(directory, "bundle.json"); const evidenceBundle = bundle(); - const first = await writeEvidenceBundle(evidenceBundle, path, false); + const first = await writeBundle(evidenceBundle, path, false); expect(first).toMatchObject({ ok: true, value: { path } }); expect(await readFile(path, "utf8")).toBe( serializeEvidenceBundle(evidenceBundle), @@ -329,15 +346,13 @@ describe("evidence bundle filesystem adapter", () => { summary: { records_unchanged: 1 }, }, }); - expect( - await writeEvidenceBundle(evidenceBundle, path, false), - ).toMatchObject({ + expect(await writeBundle(evidenceBundle, path, false)).toMatchObject({ ok: false, error: { _tag: "EvidenceFileError", reason: "exists" }, }); - expect(await writeEvidenceBundle(evidenceBundle, path, true)).toMatchObject( - { ok: true }, - ); + expect(await writeBundle(evidenceBundle, path, true)).toMatchObject({ + ok: true, + }); }); it("reads the caller-selected symlink target and refuses to replace a symlink", async () => { @@ -352,7 +367,7 @@ describe("evidence bundle filesystem adapter", () => { ok: true, value: bundle(), }); - expect(await writeEvidenceBundle(bundle(), link, true)).toMatchObject({ + expect(await writeBundle(bundle(), link, true)).toMatchObject({ ok: false, error: { _tag: "EvidenceFileError", reason: "not-file" }, }); @@ -373,7 +388,7 @@ describe("evidence bundle filesystem adapter", () => { }); const orphan = join(directory, "absent", "bundle.json"); - const written = await writeEvidenceBundle(bundle(), orphan, false); + const written = await writeBundle(bundle(), orphan, false); expect(written).toMatchObject({ ok: false, error: { _tag: "EvidenceFileError", reason: "missing", path: orphan }, diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 20db28d9a..f6b3815ee 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -185,4 +185,28 @@ mcp_servers: }, }); }); + + it.each(["", "\n"])( + "configures an empty config.yaml %j as a new document", + async (original) => { + const { home, client } = await hermesClient(); + await writeFile(client.configPath, original); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + expect(parseYaml(await readFile(client.configPath, "utf8"))).toEqual({ + mcp_servers: { + rea: { command: command[0], args: command.slice(1), enabled: true }, + }, + }); + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client: name }) => name === "hermes"), + ).toMatchObject({ state: "aligned", command }); + }, + ); }); diff --git a/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts index 59525b439..4dbf49ae0 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptApplicationService.test.ts @@ -11,7 +11,78 @@ import { ArtifactReaderFailure } from "../../../../src/artifacts/ArtifactReader. import { scanCanonicalArtifactInventory } from "../../../fixtures/artifactInventory.js"; import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; -import { analyzeJavaScriptApplication } from "../../../support/javascriptApplicationScope.js"; +import { AnalysisCancelledError } from "../../../../src/domain/analysisErrorCore.js"; +import { + analyzeJavaScriptApplication, + readJavaScriptArtifactFiles, +} from "../../../support/javascriptApplicationScope.js"; +import { createJavaScriptArtifactReader } from "../../../../src/artifacts/javascript/JavaScriptArtifactReader.js"; +import { createJavaScriptSourceAnalysis } from "../../../../src/artifacts/javascript/JavaScriptSourceAnalysis.js"; +import { analyzeAndAdoptJavaScriptArtifactFiles } from "../../../../src/application/javascript/JavaScriptArtifactAnalysis.js"; +import { createJavaScriptSemanticGraphProjection } from "../../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; + +describe("JavaScript source selection cancellation boundary", () => { + it("keeps unselected files outside scope when the real worker cancels after the last selected file", async () => { + const root = await createTestTempDirectory( + "rea-js-selected-last-cancellation-", + ); + await writeJavaScriptSourceSelectionFixture(root); + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + const controller = new AbortController(); + const sourcePaths = ["main.mjs"]; + try { + const files = await readJavaScriptArtifactFiles( + reader, + snapshot, + controller.signal, + sourcePaths, + ); + const worker = createJavaScriptSourceAnalysis( + { heapMb: 512, timeoutMs: 30_000 }, + 1, + ); + let outcome: Awaited< + ReturnType + >; + try { + const realWorker = worker.analyze.bind(worker); + outcome = await analyzeAndAdoptJavaScriptArtifactFiles( + files, + { + analyze: async (...args) => { + const result = await realWorker(...args); + controller.abort(); + return result; + }, + }, + createJavaScriptSemanticGraphProjection(), + async () => {}, + controller.signal, + ); + } finally { + await worker.close(); + } + expect(outcome.interruption).toBeInstanceOf(AnalysisCancelledError); + const unselected = outcome.analysis.files.filter( + ({ file }) => + !file.text.included && file.text.reason === "outside-analysis-scope", + ); + expect(unselected).toHaveLength(5); + expect( + unselected.every( + ({ analysis_failure }) => analysis_failure === undefined, + ), + ).toBe(true); + expect( + outcome.analysis.files.find(({ file }) => file.path === "main.mjs") + ?.javascript, + ).toMatchObject({ parse_status: "complete" }); + } finally { + await reader.close(); + } + }); +}); describe("Selected JavaScript source failure diagnostics", () => { it("analyzes selected files in first-request order and reports only selected progress", async () => { diff --git a/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts index b1c6952ba..7f0a44c1d 100644 --- a/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascriptArtifactReconstruction.test.ts @@ -1,6 +1,7 @@ -import { Readable } from "node:stream"; +import { createHash } from "node:crypto"; import { symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { Readable } from "node:stream"; import { expect, it } from "vitest"; @@ -135,6 +136,7 @@ it.each([ ["a non-array sourcesContent value", { sourcesContent: "source" }], ["a non-string, non-null source entry", { sourcesContent: [42] }], ["a source-content array with the wrong length", { sourcesContent: [] }], + ["a non-string source name", { sources: [42] }], ])( "reports %s as an invalid local source map", async (_description, content) => { @@ -187,7 +189,7 @@ it("accepts omitted and explicitly null source-map source contents", async () => JSON.stringify({ version: 3, sources: ["null.ts"], - sourcesContent: [null], + sourcesContent: null, names: [], mappings: "", }), @@ -209,6 +211,77 @@ it("accepts omitted and explicitly null source-map source contents", async () => expect(result.graph.coverage.status).toBe("complete"); }); +it("retains unnamed source declarations without shifting later source evidence", async () => { + const root = await createTestTempDirectory("rea-javascript-unnamed-source-"); + const text = JSON.stringify({ + version: 3, + sections: [ + { + offset: { line: 0, column: 0 }, + map: { + version: 3, + sources: ["prior.ts"], + sourcesContent: ["export const prior = 1;"], + names: [], + mappings: "", + }, + }, + { + offset: { line: 1, column: 0 }, + map: { + version: 3, + sources: [null, "original.ts"], + sourcesContent: [null, "export const answer: number = 42;"], + names: [], + mappings: "ACAA", + }, + }, + ], + }); + await writeFile(join(root, "bundle.js.map"), text); + + const result = await reconstructJavaScriptArtifact({ input_path: root }); + const nodes = result.graph.nodes; + const original = nodes.find( + ({ kind, identity }) => + kind === "source-module" && + identity.strategy === "source-map-original" && + identity.original_source === "original.ts", + ); + const unnamed = nodes.find( + ({ kind, observations }) => + kind === "unknown" && + observations.some( + ({ properties }) => properties.source_name_available === false, + ), + ); + + expect(original?.identity).toMatchObject({ + strategy: "source-map-original", + source_sha256: createHash("sha256") + .update("export const answer: number = 42;") + .digest("hex"), + }); + expect(original?.observations[0]?.properties).toMatchObject({ + source_index: 1, + declaration_index: 2, + content_available: true, + }); + expect(unnamed?.observations[0]?.properties).toMatchObject({ + source_index: 0, + declaration_index: 1, + source_name_available: false, + content_available: false, + content_sha256: null, + }); + expect(unnamed?.identity).toMatchObject({ + strategy: "artifact-local-key", + namespace: "source-map-unnamed-declaration", + key: "1", + }); + expect(result.graph.coverage.status).toBe("complete"); +}); + it("retains every repeated content observation and containment path", async () => { const root = await createTestTempDirectory("rea-javascript-observations-"); await Promise.all( diff --git a/tests/boundary/filesystem/jsonOutputOwnership.test.ts b/tests/boundary/filesystem/jsonOutputOwnership.test.ts new file mode 100644 index 000000000..daac89b9d --- /dev/null +++ b/tests/boundary/filesystem/jsonOutputOwnership.test.ts @@ -0,0 +1,273 @@ +import { readdirSync, renameSync, writeFileSync } from "node:fs"; +import { mkdir, open, rm } from "node:fs/promises"; +import * as fs from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, vi } from "vitest"; + +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; +import { EvidenceFileError } from "../../../src/domain/evidenceErrors.js"; +import { OwnedFileHandle } from "../../../src/filesystem/OwnedFileHandle.js"; +import { OwnedStagedOutput } from "../../../src/filesystem/OwnedStagedOutput.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { writeTextParts } from "../../../src/application/JsonFiles.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("reports committed output when post-publication staging cleanup remains owned", async () => { + const root = await createTestTempDirectory("rea-json-write-"); + const resources = new ArtifactResourceScope(); + try { + const result = await writeTextParts( + stagingPollutingParts(root), + join(root, "bundle.json"), + false, + resources, + ); + if (result.ok) throw new Error("Expected staged cleanup failure"); + const cleanupResource = result.error.cleanup?.resources[0]; + if (cleanupResource === undefined) + throw new Error("Expected retained staging resource"); + expect(result.error).toBeInstanceOf(EvidenceFileError); + expect(result.error.published).toMatchObject({ + path: join(root, "bundle.json"), + bytes: 16, + }); + expect(cleanupResource).toContain(".rea-write-"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "cleanup_incomplete", + details: { published: true, published_path: join(root, "bundle.json") }, + }); + expect(await fs.readFile(join(root, "bundle.json"), "utf8")).toBe( + "canonical output", + ); + await fs.rm(join(cleanupResource, "unexpected")); + } finally { + await resources.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("preserves cancellation when staging cleanup also fails", async () => { + const root = await createTestTempDirectory("rea-json-cancel-"); + const resources = new ArtifactResourceScope(); + const controller = new AbortController(); + try { + function* cancelledParts(): Generator { + const directories = requireStages(root); + if (directories.length !== 1) + throw new Error("Expected one staging directory"); + writeFileSync(join(root, directories[0]!, "unexpected"), "retain this"); + controller.abort(); + yield "must not publish"; + } + const result = await writeTextParts( + cancelledParts(), + join(root, "cancelled.json"), + false, + resources, + { signal: controller.signal, operation: "export_evidence_bundle" }, + ); + if (result.ok) throw new Error("Expected cancellation"); + expect(result.error).toBeInstanceOf(AnalysisCancelledError); + expect(result.error.cleanupIncomplete).toBe(true); + expect(result.error.cause).toMatchObject({ + cause: expect.any(AnalysisCancelledError), + owner: { directory: expect.stringContaining(".rea-write-") }, + }); + await expect(fs.access(join(root, "cancelled.json"))).rejects.toThrow(); + const stage = requireStages(root)[0]; + if (stage !== undefined) await fs.rm(join(root, stage, "unexpected")); + } finally { + await resources.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("refuses to publish a staged path replaced after its descriptor opened", async () => { + const root = await createTestTempDirectory("rea-staged-swap-"); + const resources = new ArtifactResourceScope(); + const destination = join(root, "swapped.json"); + try { + const result = await writeTextParts( + replacingStagedParts(root), + destination, + false, + resources, + ); + if (result.ok) throw new Error("Expected staged identity rejection"); + expect( + await fs.readFile(join(root, requireStages(root)[0]!, "content"), "utf8"), + ).toBe("foreign bytes"); + await expect(fs.access(destination)).rejects.toThrow(); + const stage = requireStages(root)[0]; + if (stage !== undefined) { + const directory = join(root, stage); + await fs.rm(join(directory, "content")); + await fs.rename(join(directory, "original"), join(directory, "content")); + } + } finally { + await resources.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("does not invent a staged owner when the supplied scope is closed", async () => { + const root = await createTestTempDirectory("rea-closed-output-scope-"); + const resources = new ArtifactResourceScope(); + await resources.close(); + const destination = join(root, "closed.json"); + const result = await writeTextParts( + ["unused"], + destination, + false, + resources, + ); + if (result.ok) throw new Error("Expected closed-scope admission failure"); + expect(result.error).toBeInstanceOf(EvidenceFileError); + expect(result.error.cleanupIncomplete).toBe(false); + expect(result.error.cleanupResources).toEqual([]); + expect(await fs.readdir(root)).toEqual([]); +}); + +function stagingPollutingParts(root: string): Iterable { + return { + *[Symbol.iterator](): Generator { + const directories = readdirSync(root); + const stage = directories.find((entry) => + entry.startsWith(".rea-write-"), + ); + if (stage === undefined) throw new Error("Expected staging directory"); + writeFileSync(join(root, stage, "unexpected"), "retain this"); + yield "canonical output"; + }, + }; +} + +function replacingStagedParts(root: string): Iterable { + return { + *[Symbol.iterator](): Generator { + const stage = requireStages(root)[0]; + if (stage === undefined) throw new Error("Expected staging directory"); + const content = join(root, stage, "content"); + renameSync(content, join(root, stage, "original")); + writeFileSync(content, "foreign bytes"); + yield "owned bytes"; + }, + }; +} + +const requireStages = (root: string): string[] => + readdirSync(root).filter((entry) => entry.startsWith(".rea-write-")); + +it("retains a failed file close owner and retries before removing its stage", async () => { + const root = await createTestTempDirectory("rea-staged-close-"); + const directory = join(root, ".rea-write-owned"); + await mkdir(directory); + const path = join(directory, "content"); + const handle = await open(path, "wx", 0o600); + const owner = await OwnedStagedOutput.create(directory); + let closeAttempts = 0; + const actualClose = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementation(async () => { + closeAttempts += 1; + if (closeAttempts === 1) throw new Error("injected close failure"); + await actualClose(); + }); + await owner.setFile(new OwnedFileHandle(handle)); + const resources = new ArtifactResourceScope(); + const stagedOwner = { + kind: "staged-output" as const, + resource: directory, + output: owner, + }; + try { + expect(await resources.release(stagedOwner)).toMatchObject({ + kind: "failed", + }); + await resources.close(); + expect(closeAttempts).toBe(2); + await expect(fs.access(directory)).rejects.toThrow(); + } finally { + await resources.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("refuses to remove unexpected files from an owned staging directory", async () => { + const root = await createTestTempDirectory("rea-staged-replaced-"); + const directory = join(root, ".rea-write-owned"); + await mkdir(directory); + const owner = await OwnedStagedOutput.create(directory); + const unexpected = join(directory, "unexpected"); + writeFileSync(unexpected, "unowned"); + try { + await expect(owner.close()).rejects.toThrow(/Unexpected staging entries/u); + expect(await fs.readFile(unexpected, "utf8")).toBe("unowned"); + } finally { + await fs.rm(unexpected, { force: true }); + await owner.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("retains cleanup when its staging directory was renamed", async () => { + const root = await createTestTempDirectory("rea-staged-renamed-dir-"); + const directory = join(root, ".rea-write-owned"); + const moved = join(root, ".rea-write-moved"); + await mkdir(directory); + const path = join(directory, "content"); + const handle = await open(path, "wx", 0o600); + const owner = await OwnedStagedOutput.create(directory); + await owner.setFile(new OwnedFileHandle(handle)); + const resources = new ArtifactResourceScope(); + const stagedOwner = { + kind: "staged-output" as const, + resource: directory, + output: owner, + }; + try { + writeFileSync(join(directory, "unexpected"), "retain this"); + expect(await resources.release(stagedOwner)).toMatchObject({ + kind: "failed", + }); + await fs.rename(directory, moved); + await expect(resources.close()).rejects.toThrow(); + await fs.rename(moved, directory); + await rm(join(directory, "unexpected")); + await expect(resources.close()).resolves.toBeUndefined(); + } finally { + await resources.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +it("retains cleanup when owned staged content was renamed", async () => { + const root = await createTestTempDirectory("rea-staged-renamed-file-"); + const directory = join(root, ".rea-write-owned"); + const content = join(directory, "content"); + const moved = join(directory, "moved-content"); + await mkdir(directory); + const handle = await open(content, "wx", 0o600); + const owner = await OwnedStagedOutput.create(directory); + await owner.setFile(new OwnedFileHandle(handle)); + const resources = new ArtifactResourceScope(); + const stagedOwner = { + kind: "staged-output" as const, + resource: directory, + output: owner, + }; + try { + await fs.rename(content, moved); + expect(await resources.release(stagedOwner)).toMatchObject({ + kind: "failed", + }); + await expect(resources.close()).rejects.toThrow( + /cleanup remains incomplete/u, + ); + await fs.rename(moved, content); + await expect(resources.close()).resolves.toBeUndefined(); + } finally { + await resources.close(); + await rm(root, { recursive: true, force: true }); + } +}); diff --git a/tests/boundary/filesystem/mipsAbiFlags.test.ts b/tests/boundary/filesystem/mipsAbiFlags.test.ts index 4cab9b702..1b724d31b 100644 --- a/tests/boundary/filesystem/mipsAbiFlags.test.ts +++ b/tests/boundary/filesystem/mipsAbiFlags.test.ts @@ -7,7 +7,7 @@ import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import type { MipsElfMetadata } from "../../../src/domain/binaryTargetTypes.js"; -import { readMipsElfAbiFlags } from "../../../src/application/MipsElfAbiFlags.js"; +import { readMipsElfMetadata } from "../../../src/application/MipsElfMetadata.js"; import { ok } from "../../../src/domain/result.js"; import { ghidraMipsProfileParameters, @@ -85,7 +85,7 @@ const inspect = async ( const path = await writeFixture(bytes); const handle = await open(path, "r"); try { - const result = await readMipsElfAbiFlags( + const result = await readMipsElfMetadata( handle, metadata(little), checkCancelled, @@ -131,6 +131,8 @@ for (const little of [true, false]) { const fp64 = await inspect(bytes, little); assert.ok(fp64.ok); const changed = target({ ...metadata(little), abiFlags: fp64.value }); + assert.equal(fp64.value?.cpr1Size, 2); + assert.equal(fp64.value?.fpAbi, 6); assert.match(ghidraMipsUnsupportedReason(changed) ?? "", /FP64/u); assert.notDeepEqual(ghidraMipsProfileParameters(changed), before); }); @@ -271,6 +273,13 @@ const unsupported: [string, (bytes: Buffer) => void, RegExp][] = [ }, /MIPS32r2/u, ], + [ + "soft-float with declared CPR1", + (b) => { + b[abiOffset + 7] = 3; + }, + /soft-float without CPR1/u, + ], [ "unspecified FP ABI", (b) => { @@ -301,19 +310,22 @@ for (const [name, change, reason] of unsupported) { ); }); } -it("resolves soft-float ABI metadata and commits its integer-register storage policy", async () => { - const bytes = fixture(); - bytes[abiOffset + 5] = 0; - bytes[abiOffset + 7] = 3; - const resolved = await resolveFixture(bytes); - assert.equal(resolved.mips.abiFlags?.cpr1Size, 0); - assert.equal(resolved.mips.abiFlags?.fpAbi, 3); - assert.equal(ghidraMipsUnsupportedReason(resolved), null); - assert.equal( - ghidraMipsProfileParameters(resolved).mips_soft_float_storage, - "o32-soft-float-prototype-v1", - ); -}); +it.each([true, false])( + "resolves soft-float ABI metadata and commits its integer-register storage policy (little endian: %s)", + async (little) => { + const bytes = fixture(little); + bytes[abiOffset + 5] = 0; + bytes[abiOffset + 7] = 3; + const resolved = await resolveFixture(bytes); + assert.equal(resolved.mips.abiFlags?.cpr1Size, 0); + assert.equal(resolved.mips.abiFlags?.fpAbi, 3); + assert.equal(ghidraMipsUnsupportedReason(resolved), null); + assert.equal( + ghidraMipsProfileParameters(resolved).mips_soft_float_storage, + "o32-soft-float-prototype-v1", + ); + }, +); it("preserves the caller's cancellation error during table inspection", async () => { const cancelled = new Error("caller cancellation sentinel"); let checks = 0; diff --git a/tests/boundary/filesystem/mipsSharedLoad.test.ts b/tests/boundary/filesystem/mipsSharedLoad.test.ts new file mode 100644 index 000000000..6ad6a38c2 --- /dev/null +++ b/tests/boundary/filesystem/mipsSharedLoad.test.ts @@ -0,0 +1,260 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { open, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { it } from "vitest"; +import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; +import { readMipsElfMetadata } from "../../../src/application/MipsElfMetadata.js"; +import { AnalysisCancelledError } from "../../../src/domain/analysisErrorCore.js"; +import { + ghidraMipsProfileParameters, + ghidraMipsUnsupportedReason, +} from "../../../src/ghidra/GhidraMipsProfile.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const relOffset = 4096; +const fixture = (little: boolean, count = 2): Buffer => { + const bytes = Buffer.alloc(relOffset + count * 8 + 16); + bytes.set([0x7f, 0x45, 0x4c, 0x46, 1, little ? 1 : 2, 1]); + const u16 = (value: number, offset: number) => + little + ? bytes.writeUInt16LE(value, offset) + : bytes.writeUInt16BE(value, offset); + const u32 = (value: number, offset: number) => + little + ? bytes.writeUInt32LE(value, offset) + : bytes.writeUInt32BE(value, offset); + u16(3, 16); + u16(8, 18); + u32(1, 20); + u32(52, 28); + u32(0x74001005, 36); + u16(52, 40); + u16(32, 42); + u16(2, 44); + u32(1, 52); + u32(bytes.length, 68); + u32(bytes.length, 72); + u32(2, 84); + u32(256, 88); + u32(256, 92); + u32(32, 100); + u32(32, 104); + u32(17, 256); + u32(relOffset, 260); + u32(18, 264); + u32(count * 8, 268); + u32(19, 272); + u32(8, 276); + for (let i = 1; i < count; i += 1) { + u32(512, relOffset + 8 * i); + u32(3, relOffset + 8 * i + 4); + } + return bytes; +}; +const resolve = async (bytes: Buffer) => { + const directory = await createTestTempDirectory("rea-mips-shared-test-"); + const path = join(directory, "shared.elf"); + await writeFile(path, bytes); + const result = await parseBinaryTarget(path); + assert.ok(result.ok); + assert.equal(result.value.kind, "executable"); + assert.equal(result.value.architecture, "mips"); + if ( + result.value.kind !== "executable" || + result.value.architecture !== "mips" + ) + throw new Error("expected MIPS ELF"); + assert.equal( + result.value.sha256, + createHash("sha256").update(bytes).digest("hex"), + ); + assert.deepEqual(await readFile(path), bytes); + return result.value; +}; + +for (const little of [true, false]) { + it(`retains complete dynamic declarations beyond the initial probe (${little ? "LE" : "BE"})`, async () => { + const target = await resolve(fixture(little)); + assert.equal(target.mips.abiFlags, null); + assert.deepEqual(target.mips.sharedLoad, { + status: "complete", + relocationCount: 2, + relocationTypes: [0, 3], + symbolRelativeCount: 0, + }); + assert.equal(ghidraMipsUnsupportedReason(target), null); + assert.deepEqual(ghidraMipsProfileParameters(target).mips_elf, { + elf_class: 32, + byte_order: little ? "little" : "big", + type: 3, + flags: 0x74001005, + abi_flags: null, + shared_load: { + status: "complete", + relocation_count: 2, + relocation_types: [0, 3], + symbol_relative_count: 0, + }, + }); + }); + it(`retains and refuses symbol-bearing REL32 without repairing the mode tag (${little ? "LE" : "BE"})`, async () => { + const bytes = fixture(little); + if (little) bytes.writeUInt32LE(0x103, relOffset + 12); + else bytes.writeUInt32BE(0x103, relOffset + 12); + const target = await resolve(bytes); + assert.equal(target.mips.sharedLoad?.status, "complete"); + assert.match( + ghidraMipsUnsupportedReason(target) ?? "", + /Symbol-bearing R_MIPS_REL32/u, + ); + }); +} + +it("examines relocation records beyond a read chunk before deciding completeness", async () => { + const count = 8194; + const bytes = fixture(true, count); + bytes.writeUInt32LE(0x103, relOffset + (count - 1) * 8 + 4); + const target = await resolve(bytes); + assert.deepEqual(target.mips.sharedLoad, { + status: "complete", + relocationCount: count, + relocationTypes: [0, 3], + symbolRelativeCount: 1, + }); + assert.match(ghidraMipsUnsupportedReason(target) ?? "", /mode tag/u); +}); + +it("does not treat uncaptured ABI declaration headers as an inspected absence", async () => { + const bytes = fixture(true); + bytes.writeUInt32LE(bytes.length + 1, 32); + bytes.writeUInt16LE(40, 46); + bytes.writeUInt16LE(1, 48); + const target = await resolve(bytes); + assert.equal(target.mips.abiFlags, null); + assert.equal(target.mips.sharedLoad?.status, "unavailable"); + assert.match( + ghidraMipsUnsupportedReason(target) ?? "", + /unavailable headers remain unknown/u, + ); +}); + +const malformed: [string, (bytes: Buffer) => void, RegExp][] = [ + [ + "overlapping load ranges", + (b) => { + b.writeUInt16LE(3, 44); + b.writeUInt32LE(1, 116); + b.writeUInt32LE(256, 124); + b.writeUInt32LE(16, 132); + b.writeUInt32LE(16, 136); + }, + /overlapping MIPS PT_LOAD/u, + ], + [ + "inconsistent dynamic address", + (b) => b.writeUInt32LE(0x10000, 92), + /matching file-backed PT_LOAD/u, + ], + ["missing dynamic table", (b) => b.writeUInt32LE(0, 84), /no PT_DYNAMIC/u], + ["repeated dynamic tables", (b) => b.writeUInt32LE(2, 52), /duplicate/u], + [ + "declared dynamic tail outside file", + (b) => b.writeUInt32LE(b.length, 100), + /beyond the file/u, + ], + [ + "unavailable load bytes", + (b) => b.writeUInt32LE(b.length + 1, 68), + /PT_LOAD range/u, + ], + ["missing terminator", (b) => b.writeUInt32LE(1, 280), /DT_NULL/u], + [ + "repeated scalar tags", + (b) => b.writeUInt32LE(17, 272), + /duplicate MIPS dynamic tag/u, + ], + ["invalid REL entry size", (b) => b.writeUInt32LE(16, 276), /DT_RELENT/u], + ["truncated REL record", (b) => b.writeUInt32LE(17, 268), /DT_RELENT/u], + [ + "unmapped REL table", + (b) => b.writeUInt32LE(0x10000, 260), + /PT_LOAD mapping/u, + ], + [ + "unmapped REL destination", + (b) => b.writeUInt32LE(0x10000, relOffset + 8), + /PT_LOAD memory range/u, + ], + [ + "RELA declaration", + (b) => { + b.writeUInt32LE(7, 272); + b.writeUInt32LE(4096, 276); + }, + /RELA or PLT/u, + ], +]; +it.each(malformed)( + "preserves useful partial header facts for %s", + async (_name, mutate, reason) => { + const bytes = fixture(true); + mutate(bytes); + const target = await resolve(bytes); + assert.equal(target.mips.flags, 0x74001005); + assert.equal(target.mips.abiFlags, null); + assert.equal(target.mips.sharedLoad?.status, "unavailable"); + assert.match(ghidraMipsUnsupportedReason(target) ?? "", reason); + }, +); + +it("retains unverified relocation type instead of claiming every examined record is supported", async () => { + const bytes = fixture(true); + bytes.writeUInt32LE(5, relOffset + 12); + const target = await resolve(bytes); + assert.deepEqual(target.mips.sharedLoad, { + status: "complete", + relocationCount: 2, + relocationTypes: [0, 5], + symbolRelativeCount: 0, + }); + assert.match( + ghidraMipsUnsupportedReason(target) ?? "", + /other dynamic relocation types/u, + ); +}); + +it("does not equate complete dynamic REL coverage with coverage of another section relocation table", async () => { + const bytes = fixture(true); + bytes.writeUInt32LE(1024, 32); + bytes.writeUInt16LE(40, 46); + bytes.writeUInt16LE(1, 48); + bytes.writeUInt32LE(9, 1028); + bytes.writeUInt32LE(2048, 1040); + bytes.writeUInt32LE(8, 1044); + bytes.writeUInt32LE(8, 1060); + const target = await resolve(bytes); + assert.equal(target.mips.sharedLoad?.status, "unavailable"); + assert.match( + ghidraMipsUnsupportedReason(target) ?? "", + /section relocation declarations/u, + ); +}); + +it("preserves the caller cancellation error during a bounded relocation scan", async () => { + const target = await resolve(fixture(true, 8194)); + const handle = await open(target.path, "r"); + const cancellation = new AnalysisCancelledError("open_binary"); + let checks = 0; + try { + await assert.rejects( + readMipsElfMetadata(handle, target.mips, () => { + checks += 1; + if (checks === 100) throw cancellation; + }), + (error) => error === cancellation, + ); + } finally { + await handle.close(); + } +}); diff --git a/tests/boundary/filesystem/qwenClientConfiguration.test.ts b/tests/boundary/filesystem/qwenClientConfiguration.test.ts index 55e8d0ab3..b915288fd 100644 --- a/tests/boundary/filesystem/qwenClientConfiguration.test.ts +++ b/tests/boundary/filesystem/qwenClientConfiguration.test.ts @@ -150,7 +150,10 @@ describe("Qwen Code profile setup", () => { expect(await setup.configureClient(client, {}, command)).toEqual({ status: "configured", }); - expect(await setup.installSkill([client.name])).toBe("installed"); + expect(await setup.installSkill([client.name])).toEqual({ + ok: true, + value: "installed", + }); const skill = join(home, ".agents", "skills", PRODUCT_IDENTITY.skillName); await writeFile(join(skill, "user-notes.txt"), "user-authored"); expect(await readFile(join(skill, "SKILL.md"), "utf8")).toContain( @@ -158,7 +161,10 @@ describe("Qwen Code profile setup", () => { ); expect(await setup.clientNeedsConfigure(client, {}, command)).toBe(false); expect(await setup.skillNeedsInstall([client.name])).toBe(false); - expect(await setup.installSkill([client.name])).toBe("unchanged"); + expect(await setup.installSkill([client.name])).toEqual({ + ok: true, + value: "unchanged", + }); expect( ( await readClientRegistrationStatuses( diff --git a/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts b/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts new file mode 100644 index 000000000..18003b039 --- /dev/null +++ b/tests/boundary/filesystem/setupSkillCleanupOwnership.test.ts @@ -0,0 +1,237 @@ +import { + mkdir, + open as fsOpen, + writeFile, + type FileHandle, +} from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { ArtifactResourceScope } from "../../../src/artifacts/ArtifactResourceScope.js"; +import { PRODUCT_IDENTITY } from "../../../src/identity.js"; +import { createDoctorHostFixture } from "../../../src/application/Doctor.fixture.js"; +import { + runDoctorOwned, + type DoctorHost, +} from "../../../src/application/Doctor.js"; +import { + readSkillFile, + skillDestinations, +} from "../../../src/application/SetupSkill.js"; +import { systemSetupHost } from "../../../src/application/SetupHost.js"; +import { + runUninstall, + systemUninstallHost, +} from "../../../src/application/Uninstall.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, open: vi.fn(actual.open) }; +}); + +const openMock = vi.mocked(fsOpen); +let actualFs: typeof import("node:fs/promises"); + +beforeEach(async () => { + actualFs = + await vi.importActual( + "node:fs/promises", + ); + openMock.mockImplementation((...args) => actualFs.open(...args)); +}); + +afterEach(() => openMock.mockReset()); + +const failFirstCloses = (path: string) => { + const attempts: FileHandle[] = []; + let allowClose = false; + openMock.mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== path) return handle; + const nativeClose = handle.close.bind(handle); + vi.spyOn(handle, "close").mockImplementation(async () => { + attempts.push(handle); + if (!allowClose) throw new Error("test holds the skill handle open"); + await nativeClose(); + }); + return handle; + }); + return { + attempts, + release: () => { + allowClose = true; + }, + }; +}; + +describe("skill regular-file cleanup ownership", () => { + it("keeps setup skill reads owned until setup host disposal", async () => { + const home = await createTestTempDirectory("rea-setup-skill-close-"); + const [destination] = skillDestinations(home, []); + if (destination === undefined) throw new Error("Shared skill root missing"); + const manifest = join(destination.path, "SKILL.md"); + await mkdir(dirname(manifest), { recursive: true }); + await writeFile(manifest, "stale managed skill\n"); + const close = failFirstCloses(manifest); + const host = systemSetupHost( + { platform: "linux", homeDirectory: home } as DoctorHost, + { HOME: home }, + ); + + try { + expect(await host.skillNeedsInstall([])).toBe(true); + expect(close.attempts.length).toBeGreaterThan(0); + expect(new Set(close.attempts).size).toBe(1); + const attemptsBeforeClose = close.attempts.length; + close.release(); + expect(await host.close?.()).toBeUndefined(); + expect(close.attempts.length).toBeGreaterThan(attemptsBeforeClose); + expect(new Set(close.attempts).size).toBe(1); + expect(close.attempts[0]?.fd).toBe(-1); + } finally { + close.release(); + await host.close?.(); + } + }); + + it("waits for an admitted skill read before its resource scope closes", async () => { + const home = await createTestTempDirectory("rea-setup-skill-inflight-"); + const [destination] = skillDestinations(home, []); + if (destination === undefined) throw new Error("Shared skill root missing"); + const manifest = join(destination.path, "SKILL.md"); + await mkdir(dirname(manifest), { recursive: true }); + await writeFile(manifest, "stale managed skill\n"); + const resources = new ArtifactResourceScope(); + let beginRead!: () => void; + const readStarted = new Promise((resolve) => { + beginRead = resolve; + }); + let unblockRead!: () => void; + const readGate = new Promise((resolve) => { + unblockRead = resolve; + }); + let heldRead = false; + openMock.mockImplementation(async (...args) => { + const handle = await actualFs.open(...args); + if (String(args[0]) !== manifest || heldRead) return handle; + heldRead = true; + const nativeRead = handle.read.bind(handle); + vi.spyOn(handle, "read").mockImplementation(async (...readArgs) => { + beginRead(); + await readGate; + return nativeRead(...readArgs); + }); + return handle; + }); + + try { + const reading = readSkillFile(manifest, resources); + await readStarted; + let closed = false; + const closing = resources.close().then(() => { + closed = true; + }); + await Promise.resolve(); + expect(closed).toBe(false); + unblockRead(); + expect(await reading).toBe("stale managed skill\n"); + await closing; + expect(closed).toBe(true); + } finally { + unblockRead(); + await resources.close(); + } + }); +}); + +describe("skill cleanup diagnostics", () => { + it("preserves a doctor report and cleanup diagnostic when host close still fails", async () => { + const home = await createTestTempDirectory("rea-doctor-skill-close-"); + const [destination] = skillDestinations(home, []); + if (destination === undefined) throw new Error("Shared skill root missing"); + const manifest = join(destination.path, "SKILL.md"); + await mkdir(dirname(manifest), { recursive: true }); + await writeFile(manifest, "managed skill fixture\n"); + const close = failFirstCloses(manifest); + const resources = new ArtifactResourceScope(); + const host = createDoctorHostFixture({ + installedSkillIdentity: async () => { + await readSkillFile(manifest, resources); + return { version: null, toolCount: null, canonical: true }; + }, + close: async () => { + try { + await resources.close(); + return undefined; + } catch (cause: unknown) { + return cause instanceof Error ? cause.message : String(cause); + } + }, + }); + + try { + const report = await runDoctorOwned(undefined, host, { skill: true }); + + expect(report.healthy).toBe(false); + expect(report.identity?.skill.installed_version).toBeNull(); + expect(report.checks).toContainEqual( + expect.objectContaining({ + name: "doctor-resource-cleanup", + ok: false, + detail: expect.stringContaining(manifest), + }), + ); + expect(close.attempts.length).toBeGreaterThan(0); + expect(new Set(close.attempts).size).toBe(1); + const attemptsBeforeClose = close.attempts.length; + close.release(); + expect(await host.close?.()).toBeUndefined(); + expect(close.attempts.length).toBeGreaterThan(attemptsBeforeClose); + expect(new Set(close.attempts).size).toBe(1); + expect(close.attempts[0]?.fd).toBe(-1); + } finally { + close.release(); + await host.close?.(); + } + }); + + it("returns uninstall changes with a failed cleanup item for a retained handle", async () => { + const home = await createTestTempDirectory("rea-uninstall-skill-close-"); + const [destination] = skillDestinations(home, []); + if (destination === undefined) throw new Error("Shared skill root missing"); + const manifest = join(destination.path, "SKILL.md"); + await mkdir(dirname(manifest), { recursive: true }); + await writeFile( + manifest, + `---\nname: ${PRODUCT_IDENTITY.skillName}\n---\nmanaged fixture\n`, + ); + const close = failFirstCloses(manifest); + const host = systemUninstallHost(home, undefined, { HOME: home }); + + try { + const result = await runUninstall(false, host); + + expect(result.status).toBe("failed"); + expect(result.items).toContainEqual( + expect.objectContaining({ + name: "uninstall_cleanup", + status: "failed", + detail: expect.stringContaining(manifest), + }), + ); + expect(close.attempts.length).toBeGreaterThan(0); + expect(new Set(close.attempts).size).toBe(1); + const attemptsBeforeClose = close.attempts.length; + close.release(); + expect(await host.close?.()).toBeUndefined(); + expect(close.attempts.length).toBeGreaterThan(attemptsBeforeClose); + expect(new Set(close.attempts).size).toBe(1); + expect(close.attempts[0]?.fd).toBe(-1); + } finally { + close.release(); + await host.close?.(); + } + }); +}); diff --git a/tests/boundary/filesystem/targetDiagnostics.test.ts b/tests/boundary/filesystem/targetDiagnostics.test.ts index 645846fe5..5d1953760 100644 --- a/tests/boundary/filesystem/targetDiagnostics.test.ts +++ b/tests/boundary/filesystem/targetDiagnostics.test.ts @@ -6,7 +6,130 @@ import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +describe("executable format refusal diagnostics", () => { + it.each(["NE", "LE", "LX"] as const)( + "reports unsupported %s executables without path advice", + async (signature) => { + const directory = await createTestTempDirectory( + `rea-target-${signature.toLowerCase()}-`, + ); + const path = join(directory, "unsupported.exe"); + const bytes = Buffer.alloc(0x80 + 24); + bytes.write("MZ", 0, "ascii"); + bytes.writeUInt32LE(0x80, 0x3c); + bytes.write(signature, 0x80, "ascii"); + await writeFile(path, bytes); + + const result = await parseBinaryTarget(path); + if (result.ok) + throw new Error(`unsupported ${signature} target was admitted`); + const reason = `unsupported ${signature} executable in MZ image`; + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + retryable: false, + message: `Cannot open artifact: ${reason}.`, + details: { + reason, + constraint: "file_format_rejected", + }, + remediation: { + action: + "Review the reported file-format reason and select a supported, well-formed target.", + }, + }); + expect(projected.message).not.toContain("Check that the path"); + + expect(await systemDoctorHost().inspectTarget?.(path)).toMatchObject({ + classification: "unsupported_target", + details: { constraint: "file_format_rejected" }, + remediation: + "Review the reported file-format reason and select a supported, well-formed target.", + }); + }, + ); +}); + describe("target route diagnostics", () => { + it.each(["module.wasm", "module.bin", "misleading.js"])( + "identifies a WASM v1 header in %s without suggesting native provider installation", + async (name) => { + const directory = await createTestTempDirectory("rea-wasm-target-"); + const path = join(directory, name); + const bytes = Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0]); + await writeFile(path, bytes); + const result = await parseBinaryTarget(path); + if (result.ok) throw new Error("WASM was admitted to native opening"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + retryable: false, + details: { + path: await realpath(path), + constraint: "wasm_requires_static_inspection", + }, + }); + expect(projected.message).toContain("WebAssembly"); + expect(projected.remediation.action).toContain("inspect_wasm_artifact"); + expect(projected.remediation.action).toContain("REA_WABT_BIN_DIRECTORY"); + expect(projected.remediation.action).toContain("WABT 1.0.42"); + expect(projected.remediation.action).toContain("does not execute"); + expect(await systemDoctorHost().inspectTarget?.(path)).toMatchObject({ + ok: false, + classification: "unsupported_target", + details: { constraint: "wasm_requires_static_inspection" }, + remediation: projected.remediation.action, + }); + }, + ); + + it.each([ + [0, 0x61, 0x73, 0x6d], + [0, 0x61, 0x73, 0x6d, 2, 0, 0, 0], + [1, 2, 3, 4, 5, 6, 7, 8], + ])( + "does not infer supported WASM encoding from a suffix or incomplete header", + async (...bytes) => { + const directory = await createTestTempDirectory("rea-wasm-unknown-"); + const path = join(directory, "module.wasm"); + await writeFile(path, Buffer.from(bytes)); + const result = await parseBinaryTarget(path); + if (result.ok) throw new Error("unknown header was admitted"); + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ + category: "unsupported_target", + retryable: false, + details: { constraint: "file_format_rejected" }, + }); + expect(projected.remediation.action).not.toContain( + "inspect_wasm_artifact", + ); + }, + ); + + it("preserves an explicit raw-binary selection even when the bytes have a WASM header", async () => { + const directory = await createTestTempDirectory("rea-wasm-explicit-raw-"); + const path = join(directory, "module.wasm"); + await writeFile(path, Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])); + expect( + await parseBinaryTarget(path, { + formatHint: "raw-binary", + architecture: "x86", + entryPoint: "0x001000", + }), + ).toMatchObject({ + ok: true, + value: { + path: await realpath(path), + format: "raw-binary", + architecture: "x86", + entryPoint: "0x1000", + entryPointSource: "0x001000", + }, + }); + }); it("identifies the directory opening constraint and gives the direct analysis route", async () => { const directory = await createTestTempDirectory("rea-js-target-"); await writeFile(join(directory, "index.js"), "export const answer = 42;\n"); @@ -32,10 +155,14 @@ describe("target route diagnostics", () => { }); const missing = await parseBinaryTarget(join(directory, "missing")); if (missing.ok) throw new Error("missing target was admitted"); - expect(projectAnalysisError(missing.error)).toMatchObject({ + const missingProjection = projectAnalysisError(missing.error); + expect(missingProjection).toMatchObject({ details: { reason: expect.stringContaining("does not exist") }, message: expect.stringContaining("does not exist"), }); + expect(missingProjection.message).toContain( + "Check that the path exists, is readable, and points to a supported file.", + ); expect( await doctor.inspectTarget?.(join(directory, "missing")), ).toMatchObject({ @@ -54,10 +181,14 @@ describe("target route diagnostics", () => { try { const result = await parseBinaryTarget(path); if (result.ok) throw new Error("unreadable target was admitted"); - expect(projectAnalysisError(result.error)).toMatchObject({ + const projected = projectAnalysisError(result.error); + expect(projected).toMatchObject({ details: { reason: expect.stringContaining("permission denied") }, message: expect.stringContaining("permission denied"), }); + expect(projected.message).toContain( + "Check that the path exists, is readable, and points to a supported file.", + ); expect(await systemDoctorHost().inspectTarget?.(path)).toMatchObject({ ok: false, classification: "config_drift", @@ -69,3 +200,123 @@ describe("target route diagnostics", () => { }, ); }); + +describe("raw-binary load rejection diagnostics", () => { + const rawBinaryRejection = async ( + options: Parameters[1], + bytes = Buffer.alloc(64), + ) => { + const directory = await createTestTempDirectory("rea-raw-load-"); + const path = join(directory, "image.bin"); + await writeFile(path, bytes); + const result = await parseBinaryTarget(path, options); + if (result.ok) throw new Error("invalid raw-binary load was admitted"); + return projectAnalysisError(result.error); + }; + + it("projects a misaligned arm entry as caller input without path advice", async () => { + const projected = await rawBinaryRejection({ + formatHint: "raw-binary", + architecture: "arm", + entryPoint: "0x1", + }); + expect(projected).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + // invalid_request is retryable: correcting the load argument succeeds. + retryable: true, + message: + "Cannot open artifact: raw-binary arm entry_point 0x1 must be 4-byte aligned.", + details: { + reason: "raw-binary arm entry_point 0x1 must be 4-byte aligned", + constraint: "load_selection_rejected", + }, + remediation: { + action: + "Correct the reported load argument (--target-format, --architecture, or --entry-point on the CLI; format, architecture, or entry_point through MCP) and retry.", + }, + }); + expect(projected.message).not.toContain("Check that the path"); + }); + + it("projects an x86 entry beyond the address width as caller input", async () => { + const projected = await rawBinaryRejection({ + formatHint: "raw-binary", + architecture: "x86", + entryPoint: "0xffffffffff", + }); + expect(projected).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + details: { constraint: "load_selection_rejected" }, + }); + expect(projected.message).toContain("x86 address width"); + expect(projected.message).not.toContain("Check that the path"); + }); + + it("projects architecture and entry without raw-binary format as caller input", async () => { + const projected = await rawBinaryRejection({ + architecture: "arm", + entryPoint: "0x0", + }); + expect(projected).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + message: + "Cannot open artifact: architecture and entry_point apply only with format=raw-binary.", + details: { constraint: "load_selection_rejected" }, + }); + expect(projected.message).not.toContain("Check that the path"); + }); + + it("keeps an empty raw-binary file a target refusal", async () => { + const projected = await rawBinaryRejection( + { + formatHint: "raw-binary", + architecture: "arm", + entryPoint: "0x8000000", + }, + Buffer.alloc(0), + ); + expect(projected).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + details: { + reason: "raw-binary requires at least one file byte", + constraint: "file_format_rejected", + }, + }); + }); + + it("projects the raw-binary target-kind pairing as caller input", async () => { + const projected = await rawBinaryRejection({ + formatHint: "raw-binary", + targetKind: "database", + architecture: "x86", + entryPoint: "0x1000", + }); + expect(projected).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + details: { + reason: "raw-binary format requires an executable target kind", + constraint: "load_selection_rejected", + }, + }); + }); + + it("projects the DOS COM target-kind pairing as caller input too", async () => { + const projected = await rawBinaryRejection({ + formatHint: "dos-com", + targetKind: "database", + }); + expect(projected).toMatchObject({ + code: "invalid_request", + category: "invalid_input", + details: { + reason: "DOS COM format requires an executable target kind", + constraint: "load_selection_rejected", + }, + }); + }); +}); diff --git a/tests/boundary/javascript/javascriptAnalysisTransferBudgets.test.ts b/tests/boundary/javascript/javascriptAnalysisTransferBudgets.test.ts new file mode 100644 index 000000000..5089690b3 --- /dev/null +++ b/tests/boundary/javascript/javascriptAnalysisTransferBudgets.test.ts @@ -0,0 +1,100 @@ +import { createHash } from "node:crypto"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { analyzeJavaScriptSemantics } from "../../../src/domain/javascript/javascriptSemanticAnalysis.js"; +import { projectJavaScriptModuleSemantics } from "../../../src/domain/javascript/javascriptModuleSemanticIr.js"; +import type { JavaScriptArtifactFile } from "../../../src/domain/javascript/javascriptArtifactFiles.js"; +import { completeJavaScriptAnalysisSteps } from "../../../src/application/javascript/JavaScriptAnalysisControl.js"; +import { projectJavaScriptSemanticFileSteps } from "../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js"; +import { + javaScriptTransferByteBudget, + measureJavaScriptModuleTransfer, + writeJavaScriptAnalysisTransfer, + readJavaScriptAnalysisTransfer, +} from "../../../src/javascript/analysis/JavaScriptAnalysisTransfer.js"; + +it("allows larger streamed transfers only with corresponding parent capacity", () => { + const mib = 1024 * 1024; + expect(javaScriptTransferByteBudget(0)).toBe(512 * mib); + expect(javaScriptTransferByteBudget(4096 * mib)).toBe(512 * mib); + expect(javaScriptTransferByteBudget(12288 * mib)).toBe(1536 * mib); + expect(javaScriptTransferByteBudget(32768 * mib)).toBe(2048 * mib); +}); + +it("enforces the offered complete-file byte budget while preserving authenticated round trips", async () => { + const root = await createTestTempDirectory("rea-js-transfer-budget-"); + const text = "export function add(value) { return value + 1; }"; + const file: JavaScriptArtifactFile = { + path: "app.mjs", + sha256: createHash("sha256").update(text).digest("hex"), + container_sha256: "a".repeat(64), + bytes: Buffer.byteLength(text), + inventory_artifact_id: "fixture-source", + kind: "javascript", + unpacked: false, + text: { included: true, value: text }, + }; + const ir = analyzeJavaScriptSemantics(text); + const module = projectJavaScriptModuleSemantics(ir); + const projection = await completeJavaScriptAnalysisSteps( + projectJavaScriptSemanticFileSteps(file, ir, 100), + ); + const observed = await writeJavaScriptAnalysisTransfer( + join(root, "observed"), + module, + projection, + ); + const measured = measureJavaScriptModuleTransfer(module); + expect(measured).toMatchObject({ + bytes: observed.module_bytes, + records: observed.module_records, + }); + expect(measureJavaScriptModuleTransfer(module, measured.bytes)).toEqual( + measured, + ); + expect(() => + measureJavaScriptModuleTransfer(module, measured.bytes - 1), + ).toThrowError( + expect.objectContaining({ + resource: "transport", + reportedLimits: expect.objectContaining({ + transfer_file_bytes: measured.bytes - 1, + }), + }), + ); + expect(() => + measureJavaScriptModuleTransfer(module, 2 ** 31 + 1), + ).toThrowError(RangeError); + const exactPath = join(root, "exact"); + const exact = await writeJavaScriptAnalysisTransfer( + exactPath, + module, + projection, + observed.bytes, + ); + expect(exact).toEqual(observed); + expect(await readJavaScriptAnalysisTransfer(exactPath, exact, file)).toEqual({ + module, + projection, + }); + await expect( + writeJavaScriptAnalysisTransfer( + join(root, "short"), + module, + projection, + observed.bytes - 1, + ), + ).rejects.toMatchObject({ + resource: "transport", + reportedLimits: { transfer_file_bytes: observed.bytes - 1 }, + }); + await expect( + writeJavaScriptAnalysisTransfer( + join(root, "oversized"), + module, + projection, + 2 ** 31 + 1, + ), + ).rejects.toBeInstanceOf(RangeError); +}); diff --git a/tests/boundary/mcp/analysisViewMcp.test.ts b/tests/boundary/mcp/analysisViewMcp.test.ts index ca72d76e3..62ec7d20b 100644 --- a/tests/boundary/mcp/analysisViewMcp.test.ts +++ b/tests/boundary/mcp/analysisViewMcp.test.ts @@ -1,3 +1,11 @@ +import { API_SURFACE_EXAMPLE } from "../../../src/contracts/analysisViewApiSurfaceExample.js"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { WebNetworkCaptureService } from "../../../src/application/WebNetworkCaptureService.js"; +import { decodeHarCapture } from "../../../src/browser/history/HarCapture.js"; +import { join } from "node:path"; +import { apiSurfaceHar } from "../../fixtures/historicalHar.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { STDIO_DEFAULT_MAX_BUFFER_SIZE } from "@modelcontextprotocol/server"; @@ -25,17 +33,26 @@ import { analysisViewBindJavaScriptGraphs, analysisViewLayoutEvidence, analysisViewLayoutFixture, + analysisViewCaptureFixture, + analysisViewCaptureEvidence, + analysisViewObjectDenseCapture, } from "../../fixtures/analysisView.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { BINARY_LAYOUT_TEST_PROVIDER } from "../../fixtures/binaryDiagnostics/layout.js"; -const connect = async (binaryLayout?: BinaryLayoutService) => { +const connect = async ( + binaryLayout?: BinaryLayoutService, + webNetworkCapture?: WebNetworkCaptureService, +) => { const session = createTestBinarySession(() => { throw new Error("selected views must not start a deep provider"); }); const server = createServer( { kind: "session", session }, - binaryLayout === undefined ? {} : { binaryLayout }, + { + ...(binaryLayout === undefined ? {} : { binaryLayout }), + ...(webNetworkCapture === undefined ? {} : { webNetworkCapture }), + }, ); const client = new Client({ name: "analysis-view-mcp", version: "1" }); onTestFinished(async () => { @@ -176,6 +193,18 @@ it("advertises exact schemas and projects one layout section from retained Evide const outputSchema: Record = advertised.outputSchema; expect(ajv.validateSchema(inputSchema)).toBe(true); expect(ajv.validateSchema(outputSchema)).toBe(true); + expect( + ajv.validate(inputSchema, { + source: { kind: "inline", evidence: analysisViewLayoutEvidence() }, + view: { kind: "summary" }, + }), + ).toBe(true); + expect( + ajv.validate(inputSchema, { + source: { kind: "inline", evidence: {} }, + view: { kind: "summary" }, + }), + ).toBe(false); expect( ajv.validate(inputSchema, { source: { @@ -254,6 +283,100 @@ it("reports a tampered inline Evidence ID at its full MCP source path", async () }); }); +it("admits portable captures before SDK Evidence parsing and preserves capture caveats", async () => { + const { client } = await connect(); + const dense = analysisViewCaptureEvidence(analysisViewObjectDenseCapture()); + const refused = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "inline", evidence: dense }, + view: { kind: "api-surface" }, + }, + }); + expect(parseMcpToolError(refused).error).toMatchObject({ + code: "resource_constraint", + details: { resource: "memory" }, + }); + await client.ping(); + + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 1); + capture.total_records = 1; + const caveat = "Producer omitted exchanges before capture startup."; + capture.limitations = [caveat, caveat]; + capture.records[0]!.reported = { + request: { method: "GET", url: "https://api.test/a" }, + response: { status: 200, content: { mimeType: "\u00a0text/plain\u00a0" } }, + }; + const parent = analysisViewCaptureEvidence(capture); + expect(parent.limitations).not.toContain(caveat); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "inline", evidence: parent }, + view: { kind: "api-surface" }, + }, + }); + const result = parseEvidence(response.structuredContent); + expect(result.limitations.filter((value) => value === caveat)).toEqual([ + caveat, + ]); + expect(result.normalized_result).toMatchObject({ + groups: [{ content_types: [], flags: ["single-sample"] }], + limitations: expect.arrayContaining([caveat]), + unknowns: [ + "Some retained records have unavailable status or content type values.", + ], + }); +}); + +it("rejects inconsistent portable coverage and accepts caller-ordered ordinal subsets", async () => { + const { client } = await connect(); + for (const [total, ordinals] of [ + [0, [0]], + [2, [0, 0]], + [2, [2]], + ] as const) { + const capture = analysisViewCaptureFixture(); + capture.total_records = total; + capture.records = ordinals.map((ordinal) => ({ + ...capture.records[0]!, + ordinal, + })); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "inline", + evidence: analysisViewCaptureEvidence(capture), + }, + view: { kind: "api-surface" }, + }, + }); + expect(parseMcpToolError(response).error.code).toBe("invalid_request"); + } + const capture = analysisViewCaptureFixture(); + capture.records = [capture.records[8]!, capture.records[1]!]; + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { + kind: "inline", + evidence: analysisViewCaptureEvidence(capture), + }, + view: { kind: "api-surface" }, + }, + }); + expect( + parseEvidence(response.structuredContent).normalized_result, + ).toMatchObject({ + records: { examined: 2, total: 15 }, + limitations: expect.arrayContaining([ + "Parent contains an ordinal subset; the surface covers only retained records.", + ]), + }); +}); + it("delivers a bounded native dossier view over MCP without a Ghidra provider or lost connection", async () => { const { client, session } = await connect(); const original = functionDossierSchema.parse(ghidraFunctionDossier()); @@ -624,3 +747,87 @@ it("accepts a transport-constraint retained reference as the view source", async }, ]); }); + +it("projects an actual HAR capture retained by the public MCP workflow", async () => { + const root = await createTestTempDirectory("rea-surface-mcp-"); + const path = join(root, "surface.har"); + await writeFile(path, JSON.stringify(apiSurfaceHar())); + const service = new WebNetworkCaptureService({ + inspect: async (input) => { + const bytes = await readFile(input.capture_path); + return ok({ + ...decodeHarCapture(bytes.toString("utf8"), input.sensitive_values), + artifact: { + path: input.capture_path, + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + format: "har", + runtime_attribution: "unknown", + limitations: [], + }); + }, + }); + const { client } = await connect(undefined, service); + const captureResponse = await client.callTool({ + name: "inspect_web_network_capture", + arguments: { capture_path: path, format: "har" }, + }); + expect(captureResponse.isError).not.toBe(true); + const parent = parseEvidence(captureResponse.structuredContent); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: { + source: { kind: "retained-evidence", evidence_id: parent.evidence_id }, + view: { kind: "api-surface" }, + }, + }); + expect(response.isError).not.toBe(true); + const evidence = parseEvidence(response.structuredContent); + expect(evidence.confidence).toBe("inferred"); + expect(evidence.normalized_result).toMatchObject({ + records: { examined: 13, total: 13, excluded: 0 }, + coverage: { total: 12, exhausted: true }, + groups: expect.arrayContaining([ + expect.objectContaining({ + path_template: "/users/{numeric}", + record_ordinals: [0, 1], + }), + ]), + }); +}); + +it("reproduces the published API surface example from decoded fixture bytes", async () => { + const fixture = apiSurfaceHar(); + fixture.log.entries = fixture.log.entries.slice(0, 2); + const bytes = Buffer.from(JSON.stringify(fixture)); + const service = new WebNetworkCaptureService({ + inspect: async () => + ok({ + ...decodeHarCapture(bytes.toString("utf8"), []), + artifact: { + path: "/fixtures/api-surface.har", + sha256: createHash("sha256").update(bytes).digest("hex"), + bytes: bytes.length, + }, + format: "har", + runtime_attribution: "unknown", + limitations: [], + }), + }); + const { client } = await connect(undefined, service); + const capture = await client.callTool({ + name: "inspect_web_network_capture", + arguments: { capture_path: "/fixtures/api-surface.har", format: "har" }, + }); + expect(parseEvidence(capture.structuredContent).evidence_id).toBe( + API_SURFACE_EXAMPLE.input.source.evidence_id, + ); + const response = await client.callTool({ + name: "inspect_analysis_view", + arguments: API_SURFACE_EXAMPLE.input, + }); + expect(parseEvidence(response.structuredContent).normalized_result).toEqual( + API_SURFACE_EXAMPLE.output, + ); +}); diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index d97376c55..a8a780d61 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -10,7 +10,11 @@ import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js"; import { createWebScreenshotArtifact } from "../../../src/domain/webScreenshot.js"; import { JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE } from "../../../src/contracts/javascript/javascriptRuntimeReconciliationExample.js"; -import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { runCliJson } from "../../fixtures/cliJsonProcess.js"; @@ -59,16 +63,18 @@ it( name: "binary_session", arguments: {}, }); - expect(status.structuredContent).toMatchObject({ - result: { - tool_availability: expect.arrayContaining([ - expect.objectContaining({ - name: "inspect_web_page", - reason: expect.any(String), - }), - ]), - }, - }); + const sessionStatus = toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ); + const availability = expandToolAvailability( + sessionStatus.result.tool_availability, + ); + expect(availability).toContainEqual( + expect.objectContaining({ + name: "inspect_web_page", + reason: expect.any(String), + }), + ); const listed = await connected.client.callTool({ name: "list_browser_targets", arguments: { diff --git a/tests/boundary/mcp/clientCapabilities.test.ts b/tests/boundary/mcp/clientCapabilities.test.ts index a8acc0048..c741045b2 100644 --- a/tests/boundary/mcp/clientCapabilities.test.ts +++ b/tests/boundary/mcp/clientCapabilities.test.ts @@ -7,11 +7,25 @@ import { import { describe, expect, it } from "vitest"; import { createServer } from "../../../src/server/createServer.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { createCacheProvider, createTestBinarySession, } from "../../fixtures/binarySession.js"; +const assertCompleteToolAvailability = (structuredContent: unknown): void => { + const status = + toolContract("binary_session").outputSchema.parse(structuredContent); + const availability = expandToolAvailability(status.result.tool_availability); + expect(availability.map(({ name }) => name).sort()).toEqual( + TOOL_CONTRACTS.map(({ name }) => name).sort(), + ); +}; + describe("per-request client capability reporting", () => { it.each([ [{ elicitation: {} }, true, false], @@ -39,6 +53,7 @@ describe("per-request client capability reporting", () => { [CLIENT_CAPABILITIES_META_KEY]: capabilities, }, }); + assertCompleteToolAvailability(status.structuredContent); expect(status.structuredContent).toMatchObject({ result: { client_features: { elicitation_form: form, elicitation_url: url }, @@ -71,13 +86,15 @@ describe("initialize-scoped client metadata", () => { try { await server.connect(serverTransport); await client.connect(clientTransport); - return ( + const result = ( await client.callTool({ name: "binary_session", arguments: {}, ...(meta === undefined ? {} : { _meta: meta }), }) ).structuredContent; + assertCompleteToolAvailability(result); + return result; } finally { await Promise.allSettled([ client.close(), diff --git a/tests/boundary/mcp/compactInputSchemaMcp.test.ts b/tests/boundary/mcp/compactInputSchemaMcp.test.ts index 536b7afeb..5ca46c39b 100644 --- a/tests/boundary/mcp/compactInputSchemaMcp.test.ts +++ b/tests/boundary/mcp/compactInputSchemaMcp.test.ts @@ -4,7 +4,6 @@ import { Ajv2020 } from "ajv/dist/2020.js"; import { describe, expect, it } from "vitest"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; -import { compactAdvertisedInputSchema } from "../../../src/contracts/compactInputSchema.js"; import { COMPACT_INPUT_SCHEMA_BUDGET_BYTES, parseMcpInputSchemaProfile, @@ -121,29 +120,6 @@ describe("MCP compact input schema profile", () => { } }); - it("reduces at most the structurally oversized tools", async () => { - const full = fullServer(); - registerCatalog(full); - const tools = await listToolsThroughTransport(full); - const reduced: string[] = []; - for (const tool of tools) { - const { presentation } = compactAdvertisedInputSchema( - tool.inputSchema, - COMPACT_INPUT_SCHEMA_BUDGET_BYTES, - ); - if (presentation === "reduced") reduced.push(tool.name); - } - expect(reduced.sort()).toEqual([ - "build_reconstruction_obligation_ledger", - "capture_browser_scenario", - "compare_application_versions", - "compare_web_captures", - "evaluate_reconstruction_coverage", - "project_managed_application_graph", - ]); - expect(tools.length).toBe(TOOL_CONTRACTS.length); - }); - it("retains parameter types and avoids client-side reference expansion", async () => { const server = compactServer(); registerCatalog(server); diff --git a/tests/boundary/mcp/dylibResolutionMcp.test.ts b/tests/boundary/mcp/dylibResolutionMcp.test.ts index 4f2d70aab..4040935ae 100644 --- a/tests/boundary/mcp/dylibResolutionMcp.test.ts +++ b/tests/boundary/mcp/dylibResolutionMcp.test.ts @@ -8,11 +8,13 @@ import { z } from "zod"; import { ArtifactProvider } from "../../../src/artifacts/ArtifactProvider.js"; import { + CPU, FILE_TYPE, LC, buildVersionCommand, dylibCommand, dyldEnvironmentCommand, + fatImage, machoImage, rpathCommand, } from "../../../src/artifacts/apple/MachoImage.fixture.js"; @@ -112,6 +114,44 @@ it("traces dylib resolution for an opened app bundle and rejects a changed targe }); }); +it("traces architecture slices from a universal Mach-O target", async () => { + const root = await createTestTempDirectory("rea-dylib-universal-"); + const executable = join(root, "Universal"); + const cpu = process.arch === "arm64" ? CPU.arm64 : CPU.x86_64; + const architecture = process.arch === "arm64" ? "arm64" : "x86_64"; + await writeFile( + executable, + fatImage([ + { + cpu, + bytes: machoImage({ + cpu, + commands: [buildVersionCommand(1)], + }), + }, + ]), + ); + + await withClient(async (client) => { + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: executable }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const called = await client.callTool({ + name: "trace_dylib_resolution", + arguments: { architecture }, + }); + expect(called.isError, JSON.stringify(called.structuredContent)).not.toBe( + true, + ); + const trace = dylibResolutionResultSchema.parse( + structuredResult(called.structuredContent), + ); + expect(trace.roots).toEqual([{ image: "Universal", architecture }]); + }); +}); + it("rejects targets that are not Mach-O", async () => { const directory = await createTestTempDirectory("rea-dylib-mcp-plist-"); const path = join(directory, "Info.plist"); diff --git a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts index 5d031b940..62246a85c 100644 --- a/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts +++ b/tests/boundary/mcp/enhancedTools.01-lists-the-complete-target-open.test.ts @@ -189,7 +189,7 @@ const fixturePort = (): AnalysisOperationPort => ({ case "analyze_function": return Promise.resolve(targetObservation(functionDossier)); default: - return Promise.resolve(ok(null)); + throw new Error(`Unexpected analysis operation: ${name}`); } }, }); @@ -288,11 +288,43 @@ describe("enhanced MCP tools", () => { failed: 0, }); expect(results[3]).toEqual({ - "0": [{ address: "0x1", status: "ok", calls: ["0x2", "0x3"] }], - "1": [ - { address: "0x2", status: "ok", calls: ["0x1"] }, - { address: "0x3", status: "ok", calls: [] }, - ], + graph: { + "0": [ + { + address: "0x1", + status: "ok", + calls: ["0x2", "0x3"], + limitations: ["Provider did not report relationship coverage."], + relationship_coverage: { + status: "unknown", + reason: "Provider did not report relationship coverage.", + }, + }, + ], + "1": [ + { + address: "0x2", + status: "ok", + calls: ["0x1"], + limitations: ["Provider did not report relationship coverage."], + relationship_coverage: { + status: "unknown", + reason: "Provider did not report relationship coverage.", + }, + }, + { + address: "0x3", + status: "ok", + calls: [], + limitations: ["Provider did not report relationship coverage."], + relationship_coverage: { + status: "unknown", + reason: "Provider did not report relationship coverage.", + }, + }, + ], + }, + limitations: [], }); expect(results[4]).toMatchObject({ total: 1, @@ -354,7 +386,7 @@ describe("enhanced MCP tools", () => { { address: "0x1", depth: 0 }, { address: "0x2", depth: 1 }, ], - truncated: false, + truncated: true, }); expect(results[11]).toMatchObject({ start: "missing-selector", diff --git a/tests/boundary/mcp/enhancedTools.03-returns-typed-graph-failures-and.test.ts b/tests/boundary/mcp/enhancedTools.03-returns-typed-graph-failures-and.test.ts index 511dec47c..aaf1068ea 100644 --- a/tests/boundary/mcp/enhancedTools.03-returns-typed-graph-failures-and.test.ts +++ b/tests/boundary/mcp/enhancedTools.03-returns-typed-graph-failures-and.test.ts @@ -37,17 +37,20 @@ describe("enhanced MCP tools", () => { expect(graph).toMatchObject({ ok: true, value: { - "0": [ - { - address: "0x1", - status: "error", - error: { - category: "execution_failure", - message: - "Analysis returned an unreadable result. Run `rea doctor` and review the reported details; repeating the unchanged request is not expected to succeed.", + graph: { + "0": [ + { + address: "0x1", + status: "error", + error: { + category: "execution_failure", + message: + "Analysis returned an unreadable result. Run `rea doctor` and review the reported details; repeating the unchanged request is not expected to succeed.", + }, }, - }, - ], + ], + }, + limitations: [], }, }); expect(unresolved).toEqual({ diff --git a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts index 1666e129f..22d6cd771 100644 --- a/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts +++ b/tests/boundary/mcp/enhancedTools.04-uses-complete-inventories-inline.test.ts @@ -130,7 +130,7 @@ describe("enhanced MCP tools", () => { ); } default: - return Promise.resolve(ok(null)); + throw new Error(`Unexpected analysis operation: ${name}`); } }, }); @@ -174,26 +174,29 @@ describe("enhanced MCP tools", () => { it("rejects a segment with omitted coordinates instead of reporting zero length", async () => { const client = await connect({ - execute: (name) => - Promise.resolve( - name === "list_segments" - ? ok([ - { - name: "__TEXT", - end: "0x2000", - readable: null, - writable: null, - executable: null, - }, - ]) - : ok(name === "current_document" ? "fixture" : []), - ), + execute: (name) => { + if (name === "list_segments") + return Promise.resolve( + ok([ + { + name: "__TEXT", + end: "0x2000", + readable: null, + writable: null, + executable: null, + }, + ]), + ); + if (name === "current_document") return Promise.resolve(ok("fixture")); + if (name === "list_procedures" || name === "list_strings") + return Promise.resolve(ok([])); + throw new Error(`Unexpected analysis operation: ${name}`); + }, }); const result = await client.callTool({ name: "binary_overview", arguments: {}, }); - expect(result.isError).toBe(true); expect(result.content).toContainEqual({ type: "text", @@ -225,8 +228,11 @@ const overviewPort = ( }, ]), ); - default: + case "list_procedures": + case "list_strings": return Promise.resolve(ok([])); + default: + throw new Error(`Unexpected analysis operation: ${name}`); } }, }); diff --git a/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts b/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts index 625bc27fd..c13ea3c56 100644 --- a/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts +++ b/tests/boundary/mcp/enhancedTools.callPathsAndBatches.test.ts @@ -7,6 +7,7 @@ import { AnalysisCapabilityUnavailableError, AnalysisOutputError, } from "../../../src/domain/analysisErrorCore.js"; +import type { RelationshipCoverage } from "../../../src/domain/relationshipCoverage.js"; import { err, ok as resultOk } from "../../../src/domain/result.js"; import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; @@ -17,12 +18,16 @@ afterEach(closeEnhancedToolResources); const relationshipExecution = ( result: unknown, limitations: readonly string[] = [], + relationshipCoverage?: RelationshipCoverage, ) => resultOk( createAnalysisExecution( result, { id: "fixture", name: "Fixture analysis provider", version: "1" }, - { limitations }, + { + limitations, + ...(relationshipCoverage === undefined ? {} : { relationshipCoverage }), + }, ), ); @@ -38,6 +43,9 @@ describe("enhanced MCP tools", () => { return relationshipExecution( arguments_.procedure === "0x1" ? ["0x2"] : [], arguments_.procedure === "0x1" ? [limitation] : [], + arguments_.procedure === "0x1" + ? { status: "partial", limitations: [limitation] } + : { status: "complete" }, ); }, }); @@ -48,17 +56,23 @@ describe("enhanced MCP tools", () => { expect(graph.isError).not.toBe(true); expect(graph.structuredContent).toMatchObject({ normalized_result: { - "0": [ - { - address: "0x1", - status: "ok", - calls: ["0x2"], - limitations: [limitation], - }, - ], + graph: { + "0": [ + { + address: "0x1", + status: "ok", + calls: ["0x2"], + limitations: [limitation], + relationship_coverage: { + status: "partial", + limitations: [limitation], + }, + }, + ], + }, + limitations: [limitation], }, }); - const trace = await client.callTool({ name: "trace_call_path", arguments: { start: "0x1", goal: "0x2" }, @@ -77,6 +91,94 @@ describe("enhanced MCP tools", () => { }); }); + it("keeps generic provider caveats visible without truncating complete relationships", async () => { + const caveat = "Decompiler output may omit indirect calls."; + const client = await connect({ + execute: async (name, arguments_) => { + if (name === "procedure_address") + return relationshipExecution(String(arguments_.procedure)); + if (name !== "procedure_callees") + throw new Error(`Unexpected operation: ${name}`); + return relationshipExecution( + arguments_.procedure === "0x1" ? ["0x2"] : [], + [caveat], + { status: "complete" }, + ); + }, + }); + + const graph = await client.callTool({ + name: "get_call_graph", + arguments: { address: "0x1", direction: "forward" }, + }); + expect(graph.structuredContent).toMatchObject({ + normalized_result: { + graph: { + "0": [ + { status: "ok", relationship_coverage: { status: "complete" } }, + ], + "1": [ + { status: "ok", relationship_coverage: { status: "complete" } }, + ], + }, + limitations: [caveat], + }, + }); + expect(JSON.stringify(graph.structuredContent)?.split(caveat)).toHaveLength( + 2, + ); + + const trace = await client.callTool({ + name: "trace_call_path", + arguments: { start: "0x1", goal: "0x2" }, + }); + expect(trace.structuredContent).toMatchObject({ + normalized_result: { + truncated: false, + residual_unknowns: [], + limitations: expect.arrayContaining([expect.stringContaining(caveat)]), + relationship_coverage: { "0x1": { status: "complete" } }, + }, + }); + }); + + it("keeps unclassified legacy caveats on nodes with unknown coverage", async () => { + const caveat = "This legacy provider does not classify call limitations."; + const client = await connect({ + execute: async (name, arguments_) => { + if (name === "procedure_address") + return relationshipExecution(String(arguments_.procedure)); + if (name !== "procedure_callees") + throw new Error(`Unexpected operation: ${name}`); + return relationshipExecution([], [caveat]); + }, + }); + + const graph = await client.callTool({ + name: "get_call_graph", + arguments: { address: "0x1", direction: "forward" }, + }); + expect(graph.structuredContent).toMatchObject({ + normalized_result: { + graph: { + "0": [ + { + status: "ok", + relationship_coverage: { status: "unknown" }, + limitations: [ + "Provider did not report relationship coverage.", + caveat, + ], + }, + ], + }, + limitations: [caveat], + }, + }); + }); +}); + +describe("enhanced MCP batches", () => { it("returns every batch item in caller order with its own result", async () => { const addresses = Array.from( { length: 37 }, diff --git a/tests/boundary/mcp/ghidraRoutingMcp.test.ts b/tests/boundary/mcp/ghidraRoutingMcp.test.ts index f4ed96938..e9fa4d722 100644 --- a/tests/boundary/mcp/ghidraRoutingMcp.test.ts +++ b/tests/boundary/mcp/ghidraRoutingMcp.test.ts @@ -1,6 +1,8 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { describe, expect, it } from "vitest"; import { connectGhidraMcp } from "./ghidraMcpHarness.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; describe("Ghidra MCP capability routing", () => { it("routes composed and capability tools", async () => { @@ -41,44 +43,35 @@ describe("Ghidra MCP capability routing", () => { name: "binary_session", arguments: {}, }); - expect(status.structuredContent).toMatchObject({ - result: { - open: true, - analysis_provider_binding: { - provider: { id: "ghidra", version: "12.1.4" }, - }, - tool_availability: expect.arrayContaining([ - expect.objectContaining({ - name: "binary_overview", - available: true, - }), - expect.objectContaining({ - name: "get_objc_classes", - available: true, - }), - expect.objectContaining({ - name: "batch_decompile", - available: true, - }), - expect.objectContaining({ - name: "trace_feature", - available: true, - }), - expect.objectContaining({ - name: "get_call_graph", - available: true, - }), - expect.objectContaining({ - name: "find_xrefs_to_name", - available: true, - }), - expect.objectContaining({ - name: "set_comment", - available: false, - }), - ]), + const sessionStatus = toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ); + expect(sessionStatus.result).toMatchObject({ + open: true, + analysis_provider_binding: { + provider: { id: "ghidra", version: "12.1.4" }, }, }); + const availability = expandToolAvailability( + sessionStatus.result.tool_availability, + ); + expect(availability).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "binary_overview", available: true }), + expect.objectContaining({ + name: "get_objc_classes", + available: true, + }), + expect.objectContaining({ name: "batch_decompile", available: true }), + expect.objectContaining({ name: "trace_feature", available: true }), + expect.objectContaining({ name: "get_call_graph", available: true }), + expect.objectContaining({ + name: "find_xrefs_to_name", + available: true, + }), + expect.objectContaining({ name: "set_comment", available: false }), + ]), + ); } finally { await harness.close(); } diff --git a/tests/boundary/mcp/investigationAnalysisMcp.test.ts b/tests/boundary/mcp/investigationAnalysisMcp.test.ts index a1259e667..18e3166ac 100644 --- a/tests/boundary/mcp/investigationAnalysisMcp.test.ts +++ b/tests/boundary/mcp/investigationAnalysisMcp.test.ts @@ -3,7 +3,14 @@ import { ok as resultOk } from "../../../src/domain/result.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import { expect, it } from "vitest"; -import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { + createBinarySessionTargets, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; +import { + createAnalysisExecution, + type AnalysisOperationPort, +} from "../../../src/application/AnalysisProvider.js"; import type { BinarySession } from "../../../src/application/binary/BinarySession.js"; import { FUNCTION_COMPARISON_EXAMPLE } from "../../../src/contracts/functionComparisonExample.js"; import { INVESTIGATION_EXAMPLES } from "../../../src/contracts/investigationExamples.js"; @@ -20,6 +27,7 @@ import { } from "../../../src/domain/jsonValue.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; +import type { RelationshipCoverage } from "../../../src/domain/relationshipCoverage.js"; it("aggregates comparison Evidence and records an approved runtime gap", async () => { const { session, server, client } = await connected(); @@ -149,6 +157,183 @@ const dossierWithCallees = (addresses: readonly string[]) => { ); }; +it.each([ + { + label: "complete", + coverage: { status: "complete" } as RelationshipCoverage, + providerLimitations: ["The provider version identity is unavailable."], + dossierLimitations: ["The dossier has no native value-flow observation."], + status: "not_found", + exhaustive: true, + }, + { + label: "partial", + coverage: { + status: "partial", + limitations: ["Callee output was capped."], + } as RelationshipCoverage, + providerLimitations: [], + dossierLimitations: [], + status: "unknown", + exhaustive: false, + }, + { + label: "unknown", + coverage: { + status: "unknown", + reason: "Callee coverage was not established.", + } as RelationshipCoverage, + providerLimitations: [], + dossierLimitations: [], + status: "unknown", + exhaustive: false, + }, + { + label: "legacy limited", + coverage: undefined, + providerLimitations: [ + "Legacy relationship completeness was not established.", + ], + dossierLimitations: [], + status: "unknown", + exhaustive: false, + }, + { + label: "legacy unlimited", + coverage: undefined, + providerLimitations: [], + dossierLimitations: [], + status: "not_found", + exhaustive: true, + }, +])( + "preserves $label coverage through public analyze_function then build_call_path", + async (scenario) => { + const base = FUNCTION_COMPARISON_EXAMPLE.left; + if (base.subject === null) throw new Error("missing fixture subject"); + const provider = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }; + const rawResult = { fixture_reply: scenario.label }; + const execution = createAnalysisExecution( + { + ...jsonObjectSchema.parse(base.normalized_result), + callees: [], + limitations: scenario.dossierLimitations, + }, + provider, + { + rawResult, + limitations: scenario.providerLimitations, + locations: base.locations, + ...(scenario.coverage === undefined + ? {} + : { relationshipCoverage: scenario.coverage }), + }, + ); + const { session, server, client } = await connected({ + execute: async (operation, parameters) => { + if (operation === "health") return observed(null); + if (operation !== "analyze_function") + throw new Error(`Unexpected operation: ${operation}`); + expect(parameters.procedure).toBe("0x1000"); + return resultOk(execution); + }, + }); + try { + const [path] = await createBinarySessionTargets(); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const analyzed = await client.callTool({ + name: "analyze_function", + arguments: { procedure: "0x1000" }, + }); + expect(analyzed.isError, JSON.stringify(analyzed)).not.toBe(true); + const dossier = inlineEvidence(analyzed.structuredContent); + expect(dossier).toMatchObject({ + operation: "analyze_function", + parameters: { procedure: "0x1000" }, + provider, + subject: { + local_path: path, + digest: { sha256: session.activeTarget()?.sha256 }, + format: session.activeTarget()?.format, + }, + raw_result: rawResult, + limitations: scenario.providerLimitations, + locations: base.locations, + normalized_result: { + callees: [], + limitations: scenario.dossierLimitations, + }, + }); + const response = await client.callTool({ + name: "build_call_path", + arguments: { + functions: [dossier], + start: { address: "0x1000" }, + goal: { address: "0x4000" }, + }, + }); + expect(response.isError, JSON.stringify(response)).not.toBe(true); + const evidence = inlineEvidence(response.structuredContent); + const coverage = scenario.coverage; + const reasons = + coverage?.status === "partial" + ? coverage.limitations + : coverage?.status === "unknown" + ? [coverage.reason] + : []; + expect(evidence.normalized_result).toMatchObject({ + status: scenario.status, + shortest_hops: null, + paths: [], + search_scope: { exhaustive: scenario.exhaustive }, + limitations: expect.arrayContaining([ + ...scenario.providerLimitations, + ...scenario.dossierLimitations, + ...reasons, + ]), + }); + expect(evidence.evidence_links).toEqual([dossier.evidence_id]); + expect(dossier.relationship_coverage).toEqual(coverage ?? null); + if (coverage?.status === "partial") { + const tampered = { + ...dossier, + relationship_coverage: { status: "complete" }, + }; + expect(() => parseEvidence(tampered)).toThrow(); + const rejected = await client.callTool({ + name: "build_call_path", + arguments: { + functions: [tampered], + start: { address: "0x1000" }, + goal: { address: "0x4000" }, + }, + }); + expect(rejected.isError, JSON.stringify(rejected)).toBe(true); + } + if (scenario.exhaustive) + expect(session.listUnknowns({ domain: "call-path" })).toEqual([]); + else + expect(session.listUnknowns({ domain: "call-path" })).toMatchObject([ + { + question: + "Can the requested call path be established from complete analysis?", + supporting_evidence_ids: [evidence.evidence_id], + }, + ]); + } finally { + await close(session, server, client); + } + }, +); + it("records a safe unresolved call-path question", async () => { const { session, server, client } = await connected(); const dossier = dossierWithCallees(["0x2000"]); @@ -326,10 +511,12 @@ it("cannot omit a session-owned active unknown from reconstruction input", async } }); -const connected = async () => { +const connected = async (analysis?: AnalysisOperationPort) => { const session = createTestBinarySession(() => ({ health: () => Promise.resolve(), - execute: () => Promise.resolve(observed(null)), + execute: + analysis?.execute.bind(analysis) ?? + (() => Promise.resolve(observed(null))), close: () => Promise.resolve(resultOk(null)), })); const server = createServer({ kind: "session", session }); diff --git a/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts b/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts new file mode 100644 index 000000000..b7803f0bd --- /dev/null +++ b/tests/boundary/mcp/keyedArchivePrototypeKeyMcp.test.ts @@ -0,0 +1,49 @@ +import { expect } from "vitest"; +import { z } from "zod"; + +import { createBinarySession } from "../../../src/composition/binary.js"; +import { parseConfig } from "../../../src/config/parseConfig.js"; +import { keyedArchiveResultSchema } from "../../../src/domain/apple/keyedArchive.js"; +import { silentLogger } from "../../../src/logger.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { + expectKeyedArchivePrototypeKeys, + keyedArchivePrototypeKeyFixture, +} from "../../fixtures/keyedArchivePrototypeKeys.js"; +import { mcpTest } from "../../support/mcp/mcpFixture.js"; + +mcpTest( + "MCP retains ordinary evidence with formatted XML prototype keys", + async ({ mcp, onTestFinished }) => { + const fixture = await keyedArchivePrototypeKeyFixture(); + const configured = parseConfig({ REA_ANALYSIS_PROVIDER: "auto" }); + if (!configured.ok) throw configured.error; + const session = createBinarySession( + configured.value, + silentLogger, + process.env, + ); + onTestFinished(async () => { + await session.close(); + }); + const client = await mcp.connect( + createServer({ kind: "session", session }), + ); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: fixture.path }, + }); + expect(opened.isError, JSON.stringify(opened.content)).not.toBe(true); + const result = await client.callTool({ + name: "inspect_keyed_archive", + arguments: {}, + }); + expect(result.isError, JSON.stringify(result.content)).not.toBe(true); + expectKeyedArchivePrototypeKeys( + z + .object({ normalized_result: keyedArchiveResultSchema }) + .parse(result.structuredContent).normalized_result, + fixture.digest, + ); + }, +); diff --git a/tests/boundary/mcp/managedMcp.test.ts b/tests/boundary/mcp/managedMcp.test.ts index dc662eafe..d2e414a55 100644 --- a/tests/boundary/mcp/managedMcp.test.ts +++ b/tests/boundary/mcp/managedMcp.test.ts @@ -113,6 +113,7 @@ it("runs every managed static inspection independently of an active native targe operation: "inspect_managed_members", subject: { local_path: otherPath, format: "pe" }, }); + await verifyManagedTypeSelection(client, path, otherPath); const boundaries = structured( await client.callTool({ name: "inspect_managed_native_boundaries", @@ -445,6 +446,58 @@ const verifyImport = async ( }); }; +/** Selected type names reach the provider and unmatched names stay visible. */ +const verifyManagedTypeSelection = async ( + client: Client, + path: string, + otherPath: string, +): Promise => { + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_managed_members", + ); + if (advertised === undefined) throw new Error("missing managed member tool"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + expect(ajv.validateSchema(advertised.inputSchema)).toBe(true); + const validate = ajv.compile(advertised.inputSchema); + expect( + validate({ path, type_names: ["Fixture.Program", "Missing.Type"] }), + ).toBe(true); + expect( + validate({ path, type_names: ["Fixture.Program", "Fixture.Program"] }), + ).toBe(false); + const selected = structured( + await client.callTool({ + name: "inspect_managed_members", + arguments: { path, type_names: ["Fixture.Program", "Missing.Type"] }, + }), + ); + expect(selected).toMatchObject({ + parameters: { path, type_names: ["Fixture.Program", "Missing.Type"] }, + normalized_result: { + types: [expect.objectContaining({ full_name: "Fixture.Program" })], + methods: [expect.objectContaining({ declaring_type: "Fixture.Program" })], + selection: { + type_names: ["Fixture.Program", "Missing.Type"], + unmatched_type_names: ["Missing.Type"], + }, + }, + }); + const repeatedSelection = await client.callTool({ + name: "inspect_managed_members", + arguments: { path, type_names: ["Fixture.Program", "Fixture.Program"] }, + }); + expect(repeatedSelection.isError).toBe(true); + const reselectedOtherTarget = structured( + await client.callTool({ + name: "inspect_managed_artifact", + arguments: { path: otherPath }, + }), + ); + expect(reselectedOtherTarget).toMatchObject({ + subject: { local_path: otherPath, format: "pe" }, + }); +}; + const structured = (result: CallToolResult): Record => { if (result.isError === true) return parseMcpToolError(result); if ( diff --git a/tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts b/tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts index 5576fccef..5d8f9f828 100644 --- a/tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts +++ b/tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts @@ -7,57 +7,75 @@ import { NativeMacOSProvider } from "../../../src/native/NativeMacOSProvider.js" import { machoImage, rpathCommand, + CPU, + fatImage, } from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { createServer } from "../../../src/server/createServer.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { NativeFixtureRunner } from "../../fixtures/nativeCommands.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; -it("rejects changed signature targets through MCP and accepts reopening the new version", async () => { - const path = join( - await createTestTempDirectory("rea-signature-binding-mcp-"), - "program", - ); - await writeFile(path, machoImage({})); - const session = createTestBinarySession( - new NativeMacOSProvider({}, new NativeFixtureRunner(), "darwin"), - ); - const server = createServer({ kind: "session", session }); - const client = new Client({ name: "signature-binding-test", version: "1" }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - try { - await server.connect(serverTransport); - await client.connect(clientTransport); - const open = () => - client.callTool({ name: "open_binary", arguments: { path } }); - expect((await open()).isError).not.toBe(true); - await writeFile( - path, - machoImage({ commands: [rpathCommand("new-version")] }), +it.each(["mach-o", "mach-o-universal"] as const)( + "rejects changed %s signature targets through MCP and accepts reopening the new version", + async (format) => { + const path = join( + await createTestTempDirectory("rea-signature-binding-mcp-"), + "program", ); - const changed = await client.callTool({ - name: "inspect_signature", - arguments: {}, - }); - expect(changed.isError).toBe(true); - expect(parseMcpToolError(changed)).toMatchObject({ - error: { code: "artifact_changed", details: { path } }, - }); - expect((await open()).isError).not.toBe(true); - const inspected = await client.callTool({ - name: "inspect_signature", - arguments: {}, - }); - expect( - inspected.isError, - JSON.stringify(inspected.structuredContent), - ).not.toBe(true); - expect(inspected.structuredContent).toMatchObject({ - normalized_result: { signed: true, identifier: "com.example.fixture" }, - }); - } finally { - await client.close(); - await server.close(); - } -}); + const image = (commands: readonly Uint8Array[] = []) => + format === "mach-o" + ? machoImage({ commands }) + : fatImage( + [CPU.arm64, CPU.x86_64].map((cpu) => ({ + cpu, + bytes: machoImage({ cpu, commands }), + })), + ); + await writeFile(path, image()); + const session = createTestBinarySession( + new NativeMacOSProvider({}, new NativeFixtureRunner(), "darwin"), + ); + const server = createServer({ kind: "session", session }); + const client = new Client({ name: "signature-binding-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const open = () => + client.callTool({ name: "open_binary", arguments: { path } }); + const opened = await open(); + expect(opened.isError).not.toBe(true); + expect( + toolContract("open_binary").outputSchema.parse(opened.structuredContent) + .result, + ).toMatchObject({ format }); + await writeFile(path, image([rpathCommand("new-version")])); + const changed = await client.callTool({ + name: "inspect_signature", + arguments: {}, + }); + expect(changed.isError).toBe(true); + expect(parseMcpToolError(changed)).toMatchObject({ + error: { code: "artifact_changed", details: { path } }, + }); + expect((await open()).isError).not.toBe(true); + const inspected = await client.callTool({ + name: "inspect_signature", + arguments: {}, + }); + expect( + inspected.isError, + JSON.stringify(inspected.structuredContent), + ).not.toBe(true); + expect(inspected.structuredContent).toMatchObject({ + subject: { format }, + normalized_result: { signed: true, identifier: "com.example.fixture" }, + }); + } finally { + await client.close(); + await server.close(); + } + }, +); diff --git a/tests/boundary/mcp/optionalProviderIsolation.test.ts b/tests/boundary/mcp/optionalProviderIsolation.test.ts index eb8f64fd1..5986c7894 100644 --- a/tests/boundary/mcp/optionalProviderIsolation.test.ts +++ b/tests/boundary/mcp/optionalProviderIsolation.test.ts @@ -14,6 +14,7 @@ import { TOOL_CONTRACTS, toolContract, } from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { run } from "../../../src/main.js"; import { createServer } from "../../../src/server/createServer.js"; import { optionalObservationFactories } from "../../fixtures/optionalObservationProviders.js"; @@ -154,33 +155,37 @@ mcpTest.for(failedAdapters)( name: "binary_session", arguments: {}, }); - expect(status.structuredContent).toMatchObject({ - result: { - tool_availability: expect.arrayContaining([ - expect.objectContaining({ - name: operation, - available: false, - reason: "provider_unavailable", - remediation: expect.stringContaining(failure), - }), - expect.objectContaining({ - name: "analyze_javascript_application", - available: true, - }), - expect.objectContaining({ - name: "reconcile_javascript_runtime", - available: true, - }), - expect.objectContaining({ - name: - port === "electronActiveObservation" - ? "list_electron_targets" - : "capture_electron_scenario", - available: true, - }), - ]), - }, - }); + const sessionStatus = toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ); + const availability = expandToolAvailability( + sessionStatus.result.tool_availability, + ); + expect(availability).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: operation, + available: false, + reason: "provider_unavailable", + remediation: expect.stringContaining(failure), + }), + expect.objectContaining({ + name: "analyze_javascript_application", + available: true, + }), + expect.objectContaining({ + name: "reconcile_javascript_runtime", + available: true, + }), + expect.objectContaining({ + name: + port === "electronActiveObservation" + ? "list_electron_targets" + : "capture_electron_scenario", + available: true, + }), + ]), + ); const healthy = port === "javascriptRuntimeObservation" ? "list_browser_targets" diff --git a/tests/boundary/mcp/promptMcp.test.ts b/tests/boundary/mcp/promptMcp.test.ts index 0f9725c83..4e0072e8f 100644 --- a/tests/boundary/mcp/promptMcp.test.ts +++ b/tests/boundary/mcp/promptMcp.test.ts @@ -54,9 +54,11 @@ describe("guided prompts over MCP", () => { completions: {}, prompts: { listChanged: true }, }); - for (const [index, contract] of PROMPT_CONTRACTS.entries()) { - const advertised = prompts.prompts[index]; - expect(advertised?.arguments).toEqual( + const advertisedByName = new Map( + prompts.prompts.map((prompt) => [prompt.name, prompt]), + ); + for (const contract of PROMPT_CONTRACTS) { + expect(advertisedByName.get(contract.name)?.arguments).toEqual( Object.entries(contract.arguments).map( ([name, { description, required }]) => ({ name, @@ -271,15 +273,21 @@ describe("guided prompt completion registry", () => { }); await connectPair(server, client); resources.push(session); - expect((await client.listPrompts()).prompts[0]?.title).toBe( - "Investigate a feature", + const contract = PROMPT_CONTRACTS.find( + ({ name }) => name === "investigate_feature", ); + if (contract === undefined) + throw new Error("Missing investigate_feature prompt contract"); + const initialPrompt = (await client.listPrompts()).prompts.find( + ({ name }) => name === contract.name, + ); + expect(initialPrompt?.title).toBe(contract.title); registry.update({ - ...PROMPT_CONTRACTS[0], - title: "Investigate a feature (updated)", + ...contract, + title: `${contract.title} (updated)`, arguments: { - ...PROMPT_CONTRACTS[0].arguments, + ...contract.arguments, document: { description: "Updated provider-backed selector", required: false, @@ -289,7 +297,9 @@ describe("guided prompt completion registry", () => { }); await changed; const prompts = await client.listPrompts(); - expect(prompts.prompts[0]?.title).toBe("Investigate a feature (updated)"); + expect( + prompts.prompts.find(({ name }) => name === contract.name)?.title, + ).toBe(`${contract.title} (updated)`); expect(await complete(client, "document", "fixture")).toEqual(["fixture"]); }); }); diff --git a/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts b/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts index 572b18d63..653977a8f 100644 --- a/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts +++ b/tests/boundary/mcp/reconstructionObligationLedgerMcp.test.ts @@ -40,10 +40,6 @@ describe("reconstruction obligation ledger MCP parity", () => { summary: { total: 0 }, }, }); - expect(result.structuredContent).toHaveProperty( - "normalized_result.schema", - "ReconstructionObligationLedger", - ); } finally { await client.close(); await server.close(); diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 09ace062d..a570a5caf 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -15,6 +15,10 @@ import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js"; import { PACKAGE_METADATA } from "../../../src/generatedPackageMetadata.js"; import { PRODUCT_IDENTITY, SDK_IDENTITY } from "../../../src/identity.js"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; +import { + expandToolAvailability, + sessionProvider, +} from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { PROMPT_CONTRACTS } from "../../../src/contracts/promptContracts.js"; import { createServer } from "../../../src/server/createServer.js"; import { createServerIdentity } from "../../../src/serverIdentity.js"; @@ -70,20 +74,18 @@ const statusCapability = ( readonly reason: string; } = { available: true }, ) => ({ - operation, - effects: { - mutates_artifact: false, - launches_process: false, - may_show_ui: false, - may_access_network: false, - may_write_filesystem: false, - changes_permissions: false, - requires_root: false, - }, - limitations: [], - ...(availability.available - ? { available: true as const, reason: null, availability_code: null } - : availability), + provider: { id: "status-fixture", name: "Status fixture", version: null }, + shared_limitations: [], + groups: [ + { + operations: [operation], + effects: [], + limitations: [], + ...(availability.available + ? { available: true as const, reason: null, availability_code: null } + : availability), + }, + ], }); describe("server and catalog identity", () => { @@ -326,15 +328,10 @@ const activeTargetReasons = async ( name: "binary_session", arguments: {}, }); - const availability = z - .object({ - result: z.object({ - tool_availability: z.array( - z.object({ name: z.string(), reason: z.string() }).loose(), - ), - }), - }) - .parse(status.structuredContent).result.tool_availability; + const availability = expandToolAvailability( + z.object({ result: sessionProvider }).parse(status.structuredContent).result + .tool_availability, + ); return Object.fromEntries( ["inspect_macho", "inspect_artifact", "inspect_managed_artifact"].map( (name) => [name, availability.find((tool) => tool.name === name)?.reason], @@ -347,6 +344,32 @@ const assertLiveIdentity = async (client: Client): Promise => { name: "binary_session", arguments: {}, }); + const status = z + .object({ result: sessionProvider }) + .parse(identity.structuredContent).result; + expect(Object.keys(status.server_identity.catalog).sort()).toEqual([ + "counts", + "digests", + ]); + const availability = expandToolAvailability(status.tool_availability); + expect(availability.map(({ name }) => name).sort()).toEqual( + TOOL_CONTRACTS.map(({ name }) => name).sort(), + ); + for (const contract of TOOL_CONTRACTS) { + expect(availability).toContainEqual( + expect.objectContaining({ + name: contract.name, + surface: contract.kind, + effects: contract.effects, + annotations: { + read_only: contract.annotations.readOnlyHint ?? false, + destructive: contract.annotations.destructiveHint ?? false, + idempotent: contract.annotations.idempotentHint ?? false, + open_world: contract.annotations.openWorldHint ?? true, + }, + }), + ); + } expect(identity.structuredContent).toMatchObject({ result: { server_identity: { @@ -370,7 +393,17 @@ const assertSessionIdentity = async (client: Client): Promise => { name: "binary_session", arguments: { expected_package_version: "1.2.0" }, }); - expect(status.structuredContent).toMatchObject({ + const parsed = z + .object({ result: sessionProvider }) + .parse(status.structuredContent); + expect({ + result: { + ...parsed.result, + tool_availability: expandToolAvailability( + parsed.result.tool_availability, + ), + }, + }).toMatchObject({ result: { server_identity: { catalog: { diff --git a/tests/boundary/mcp/serverWorkflowMcp.test.ts b/tests/boundary/mcp/serverWorkflowMcp.test.ts index 91a50252a..c72fca5cb 100644 --- a/tests/boundary/mcp/serverWorkflowMcp.test.ts +++ b/tests/boundary/mcp/serverWorkflowMcp.test.ts @@ -3,7 +3,6 @@ import { ok as resultOk } from "../../../src/domain/result.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import type { CallToolResult } from "@modelcontextprotocol/server"; import { afterEach, expect, it } from "vitest"; -import { z } from "zod"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import type { @@ -12,8 +11,11 @@ import type { } from "../../../src/application/AnalysisProvider.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; import { createServer } from "../../../src/server/createServer.js"; -import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; -import { toolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; const resources: Array<{ close(): Promise }> = []; @@ -65,7 +67,7 @@ it("executes a realistic workflow: list methods, decompile selected, get xrefs", case "xrefs": return Promise.resolve(ok(["0x1000"])); default: - return Promise.resolve(ok(null)); + throw new Error(`Unexpected analysis operation: ${name}`); } }, }); @@ -126,11 +128,10 @@ it("advertises the complete currently available inventory with a session", async arguments: {}, }), ); - const inventory = z - .object({ - result: z.object({ tool_availability: z.array(toolAvailability) }), - }) - .parse(status).result.tool_availability; + const inventory = expandToolAvailability( + toolContract("binary_session").outputSchema.parse(status).result + .tool_availability, + ); for (const availability of inventory) { if (availability.available) continue; expect(availability.reason).not.toBe("available"); diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index 302f91ad7..42e92e501 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -26,6 +26,8 @@ import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { MAX_JSON_DEPTH } from "../../../src/domain/jsonValue.js"; import { parseAnalysisSnapshot } from "../../../src/domain/analysisSnapshot.js"; import { INVESTIGATION_EXAMPLES } from "../../../src/contracts/investigationExamples.js"; +import { expandProviderCapabilities } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; import { ok as resultOk } from "../../../src/domain/result.js"; import { createSessionMcpHarness, @@ -294,15 +296,12 @@ describe("target-free MCP workflow", () => { expect(openedStatus).toMatchObject({ result: { path: await realpath(first) }, }); + const sessionStatus = + toolContract("binary_session").outputSchema.parse(openedStatus); expect( - z - .object({ - result: z.object({ - capabilities: z.array(z.object({ operation: z.string() })), - }), - }) - .parse(openedStatus) - .result.capabilities.map(({ operation }) => operation), + expandProviderCapabilities(sessionStatus.result.capabilities).map( + ({ operation }) => operation, + ), ).not.toContain("inventory_artifact"); expect( text(await mcp.callTool({ name: "current_document", arguments: {} })), diff --git a/tests/boundary/mcp/wasmArtifactMcp.test.ts b/tests/boundary/mcp/wasmArtifactMcp.test.ts index 62b85b6e1..0a4a2610c 100644 --- a/tests/boundary/mcp/wasmArtifactMcp.test.ts +++ b/tests/boundary/mcp/wasmArtifactMcp.test.ts @@ -1,17 +1,93 @@ -import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { Client } from "@modelcontextprotocol/client"; import { Ajv2020 } from "ajv/dist/2020.js"; import { expect, it, onTestFinished, vi } from "vitest"; +import { readFile, realpath, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { WasmArtifactService } from "../../../src/application/wasm/WasmArtifactService.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; import { ok } from "../../../src/domain/result.js"; -import { createServer } from "../../../src/server/createServer.js"; +import { createApplicationMcpHarness } from "../../fixtures/applicationMcpHarness.js"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { wasmArtifactFixture, wasmArtifactExecution, } from "../../fixtures/wasm/artifact.js"; +const wasmToolSchemas = async (client: Client) => { + const advertised = (await client.listTools()).tools.find( + (tool) => tool.name === "inspect_wasm_artifact", + ); + if (advertised?.outputSchema === undefined) + throw new Error("WASM recovery must name an advertised static tool"); + const ajv = new Ajv2020({ strict: false, validateFormats: false }); + const inputSchema: Record = advertised.inputSchema; + const outputSchema: Record = advertised.outputSchema; + expect(ajv.validateSchema(inputSchema)).toBe(true); + expect(ajv.validateSchema(outputSchema)).toBe(true); + return { name: advertised.name, ajv, inputSchema, outputSchema }; +}; + +it("directs rejected native WASM opening to the advertised static tool while preserving missing-WABT diagnostics", async () => { + const directory = await createTestTempDirectory("rea-wasm-recovery-"); + const path = join( + await realpath(directory), + process.platform === "win32" ? "module space.bin" : 'module "quoted".bin', + ); + await writeFile(path, Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0])); + const createClient = vi.fn(() => { + throw new Error("native provider must not start for WASM"); + }); + const session = createTestBinarySession(createClient); + const harness = await createApplicationMcpHarness({ + session, + serverOptions: { environment: {} }, + }); + onTestFinished(harness.close); + const { client } = harness; + const { name, ajv, inputSchema } = await wasmToolSchemas(client); + const input = { path }; + expect( + toolContract("inspect_wasm_artifact").inputSchema.safeParse(input).success, + ).toBe(true); + const opening = await client.callTool({ + name: "open_binary", + arguments: input, + }); + expect(opening.isError).toBe(true); + const failure = parseMcpToolError(opening); + expect(failure).toMatchObject({ + error: { + code: "target_unavailable", + category: "unsupported_target", + details: { constraint: "wasm_requires_static_inspection" }, + }, + }); + expect(failure.error.remediation.action).toContain(JSON.stringify({ path })); + expect(ajv.validate(inputSchema, input)).toBe(true); + const inspection = await client.callTool({ + name, + arguments: input, + }); + expect(inspection.isError).toBe(true); + expect(parseMcpToolError(inspection)).toMatchObject({ + error: { + code: "capability_unavailable", + details: { + operation: "inspect_wasm_artifact", + reason: expect.stringContaining("REA_WABT_BIN_DIRECTORY"), + }, + }, + }); + await client.ping(); + expect(createClient).not.toHaveBeenCalled(); + expect(await readFile(path)).toEqual( + Buffer.from([0, 0x61, 0x73, 0x6d, 1, 0, 0, 0]), + ); +}); + it("advertises exact valid schemas and records inline WASM interface evidence without a binary target", async () => { const value = wasmArtifactFixture(); const close = vi.fn(() => Promise.resolve()); @@ -22,30 +98,13 @@ it("advertises exact valid schemas and records inline WASM interface evidence wi const session = createTestBinarySession(() => { throw new Error("deep provider must not start"); }); - const server = createServer( - { kind: "session", session }, - { wasmArtifact: service }, - ); - const client = new Client({ name: "wasm-artifact-contract", version: "1" }); - onTestFinished(async () => { - await client.close(); - await server.close(); - await session.close(); + const harness = await createApplicationMcpHarness({ + session, + serverOptions: { wasmArtifact: service }, }); - const [clientTransport, serverTransport] = - InMemoryTransport.createLinkedPair(); - await server.connect(serverTransport); - await client.connect(clientTransport); - const advertised = (await client.listTools()).tools.find( - (tool) => tool.name === "inspect_wasm_artifact", - ); - if (advertised?.outputSchema === undefined) - throw new Error("WASM interface inspection must publish both schemas"); - const ajv = new Ajv2020({ strict: false, validateFormats: false }); - const inputSchema: Record = advertised.inputSchema; - const outputSchema: Record = advertised.outputSchema; - expect(ajv.validateSchema(inputSchema)).toBe(true); - expect(ajv.validateSchema(outputSchema)).toBe(true); + onTestFinished(harness.close); + const { client } = harness; + const { ajv, inputSchema, outputSchema } = await wasmToolSchemas(client); expect(ajv.validate(inputSchema, { path: value.artifact.path })).toBe(true); expect( ajv.validate(inputSchema, { @@ -79,6 +138,6 @@ it("advertises exact valid schemas and records inline WASM interface evidence wi }) ).isError, ).toBe(true); - await server.close(); + await harness.close(); expect(close).toHaveBeenCalled(); }); diff --git a/tests/boundary/process/darwinProcessRunTokenReader.test.ts b/tests/boundary/process/darwinProcessRunTokenReader.test.ts index 6c0025d78..db492df83 100644 --- a/tests/boundary/process/darwinProcessRunTokenReader.test.ts +++ b/tests/boundary/process/darwinProcessRunTokenReader.test.ts @@ -22,9 +22,16 @@ import { createSystemProcessOwnershipHost, systemProcessOwnershipHost, } from "../../../src/process/ProcessOwnershipObservation.js"; -import { createDarwinProcessRunTokenReader } from "../../../src/process/DarwinProcessRunTokenReader.js"; +import { + createDarwinProcessRunTokenReader, + DarwinProcessOwnershipInspectionError, +} from "../../../src/process/DarwinProcessRunTokenReader.js"; import { execFileOutput } from "../../../src/process/ExecFileOutput.js"; -import { normalizeCaptureFailure } from "../../../src/process/capture/ProcessCaptureError.js"; +import { + normalizeCaptureFailure, + ProcessCaptureError, +} from "../../../src/process/capture/ProcessCaptureError.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { PlaywrightElectronActiveProvider } from "../../../src/browser/PlaywrightElectronActiveProvider.js"; import { electronActiveObservationInputSchema } from "../../../src/domain/javascript/electronActiveObservation.js"; @@ -168,8 +175,19 @@ it("reports an actionable missing Swift compiler without installing it", async ( } }); +const preparationFailure = async ( + reader: ReturnType, +): Promise => { + const cause = await reader.prepare().then( + () => undefined, + (failure: unknown) => failure, + ); + expect(cause).toBeInstanceOf(DarwinProcessOwnershipInspectionError); + return cause as DarwinProcessOwnershipInspectionError; +}; + it.skipIf(process.platform === "win32")( - "preserves compiler diagnostics when Swift preparation fails", + "reports the compiler exit code with its diagnostics when preparation fails", async () => { const directory = await createTestTempDirectory( "rea-process-token-compiler-failure-test-", @@ -177,13 +195,136 @@ it.skipIf(process.platform === "win32")( const executable = join(directory, "failing-xcrun"); await writeFile( executable, - '#!/usr/bin/env node\nprocess.stderr.write("module cache is not writable\\n"); process.exitCode = 1;\n', + '#!/usr/bin/env node\nprocess.stderr.write("module cache is not writable\\n" + "x".repeat(12000)); process.exitCode = 1;\n', ); await chmod(executable, 0o755); const reader = createDarwinProcessRunTokenReader({ xcrun: executable }); try { - await expect(reader.prepare()).rejects.toThrow( - /compilation failed.*module cache is not writable/su, + const failure = await preparationFailure(reader); + expect(failure.compilerOutcome).toMatchObject({ + outcome: "exited", + code: 1, + signal: null, + stderr: expect.stringMatching(/^module cache is not writable\n/u), + stderrTruncated: true, + killed: false, + }); + expect(failure.message).toContain( + "(compiler result: exit code 1): module cache is not writable", + ); + expect(failure.message).toContain("[diagnostics truncated]"); + const normalized = normalizeCaptureFailure(failure, undefined); + if (!(normalized instanceof ProcessCaptureError)) + throw new Error("Expected a normalized process capture error"); + expect(projectAnalysisError(normalized)).toMatchObject({ + code: "process_capture_failed", + details: { + execution_failure: expect.stringContaining( + "compiler result: exit code 1", + ), + captured_output: { + stderr: expect.stringContaining("module cache is not writable"), + exit_code: 1, + truncated: true, + }, + }, + }); + } finally { + await reader.close(); + } + }, +); + +it.skipIf(process.platform === "win32")( + "reports compiler diagnostics captured before the compilation deadline", + async () => { + const directory = await createTestTempDirectory( + "rea-process-token-compiler-timeout-test-", + ); + const executable = join(directory, "hanging-xcrun"); + await writeFile( + executable, + '#!/usr/bin/env node\nprocess.stderr.write("error: PCH was compiled with module cache path that differs\\n", () => { setInterval(() => {}, 1000); });\n', + ); + await chmod(executable, 0o755); + const reader = createDarwinProcessRunTokenReader({ + xcrun: executable, + // Headroom for the child interpreter's cold start under suite load. + compileTimeoutMs: 3_000, + }); + try { + await expect(preparationFailure(reader)).resolves.toMatchObject({ + message: + "macOS process ownership helper compilation failed via xcrun (compiler result: timed out after 3000 ms): error: PCH was compiled with module cache path that differs", + compilerOutcome: { + outcome: "timed_out", + code: "ETIMEDOUT", + timeoutMs: 3_000, + }, + }); + } finally { + await reader.close(); + } + }, +); + +it.skipIf(process.platform === "win32")( + "reports the stop signal when the compiler dies without an exit code", + async () => { + const directory = await createTestTempDirectory( + "rea-process-token-compiler-signal-test-", + ); + const executable = join(directory, "segv-xcrun"); + await writeFile( + executable, + '#!/usr/bin/env node\nprocess.stderr.write("fatal: segmentation fault during type checking\\n");\nprocess.kill(process.pid, "SIGSEGV");\n', + ); + await chmod(executable, 0o755); + const reader = createDarwinProcessRunTokenReader({ xcrun: executable }); + try { + await expect(preparationFailure(reader)).resolves.toMatchObject({ + message: + "macOS process ownership helper compilation failed via xcrun (compiler result: stopped by signal SIGSEGV): fatal: segmentation fault during type checking", + compilerOutcome: { + outcome: "signaled", + code: null, + signal: "SIGSEGV", + }, + }); + } finally { + await reader.close(); + } + }, +); + +it.skipIf(process.platform === "win32")( + "bounds embedded compiler diagnostics and marks the truncation", + async () => { + const directory = await createTestTempDirectory( + "rea-process-token-compiler-verbose-test-", + ); + const executable = join(directory, "verbose-xcrun"); + const diagnostic = "x".repeat(600); + await writeFile( + executable, + `#!/usr/bin/env node\nprocess.stderr.write(${JSON.stringify(`${diagnostic}\n`)}); process.exitCode = 1;\n`, + ); + await chmod(executable, 0o755); + const reader = createDarwinProcessRunTokenReader({ xcrun: executable }); + try { + const failure = await preparationFailure(reader); + const outcome = failure.compilerOutcome; + if (outcome === undefined) throw new Error("Expected compiler outcome"); + expect(outcome.stderrBytes).toBe(601); + expect(outcome.stderrTruncated).toBe(false); + expect(failure.message).toContain( + `: ${"x".repeat(512)}… [diagnostics truncated]`, + ); + const excerpt = failure.message.split("): ")[1]?.split("…")[0]; + expect(excerpt).toBe("x".repeat(512)); + expect(Buffer.byteLength(excerpt ?? "", "utf8")).toBeLessThanOrEqual(512); + expect(Buffer.byteLength(outcome.stderr, "utf8")).toBeLessThanOrEqual( + 8 * 1024, ); } finally { await reader.close(); diff --git a/tests/boundary/process/filesystemSnapshotFifoRace.test.ts b/tests/boundary/process/filesystemSnapshotFifoRace.test.ts index 5843158e3..871a1d8ca 100644 --- a/tests/boundary/process/filesystemSnapshotFifoRace.test.ts +++ b/tests/boundary/process/filesystemSnapshotFifoRace.test.ts @@ -1,6 +1,5 @@ import { randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; -import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; diff --git a/tests/boundary/process/jadxIntegration.test.ts b/tests/boundary/process/jadxIntegration.test.ts index c377a7cfb..86e2c5ab7 100644 --- a/tests/boundary/process/jadxIntegration.test.ts +++ b/tests/boundary/process/jadxIntegration.test.ts @@ -277,6 +277,40 @@ it("fetches all class pages rather than silently retaining the first upstream pa await verifyCleanup(launches); }); +it("preserves duplicate class-name rows and occurrence counts", async () => { + const { service, apk, launches, provider } = await setup( + "duplicate-class-name", + ); + const outcome = await service.execute("search_android_classes", { + path: apk, + query: "fixture.Target", + }); + if (!outcome.ok) throw outcome.error; + const result = androidResultSchemas.search_android_classes.parse( + outcome.value.normalized_result, + ); + expect(result.total_classes).toBe(402); + expect(result.matches).toEqual(["fixture.Target", "fixture.Target"]); + expect(result.duplicate_names).toEqual([ + { name: "fixture.Target", occurrences: 2 }, + ]); + const full = await service.execute("search_android_classes", { + path: apk, + query: "", + }); + if (!full.ok) throw full.error; + const fullResult = androidResultSchemas.search_android_classes.parse( + full.value.normalized_result, + ); + expect(fullResult.total_classes).toBe(402); + expect(fullResult.matches).toHaveLength(402); + expect(fullResult.duplicate_names).toEqual([ + { name: "fixture.Target", occurrences: 2 }, + ]); + await provider.close(); + await verifyCleanup(launches); +}); + it("exposes overload candidates and executes only an explicitly selected ambiguous method", async () => { const { service, apk, launches, provider } = await setup(); const input = { @@ -504,7 +538,7 @@ it("rejects bytes changed after target admission without launching an engine", a }); it("retains an uncertain workspace and blocks later launches after cleanup cannot be verified", async () => { - const { service, apk, launches, provider } = await setup("cleanup-failure"); + const { service, apk, launches } = await setup("cleanup-failure"); const outcome = await service.execute("inspect_android_package", { path: apk, }); diff --git a/tests/boundary/process/packagePackIsolation.test.ts b/tests/boundary/process/packagePackIsolation.test.ts index 48c88eaf2..0ad8a900c 100644 --- a/tests/boundary/process/packagePackIsolation.test.ts +++ b/tests/boundary/process/packagePackIsolation.test.ts @@ -1,12 +1,5 @@ import { execFile } from "node:child_process"; -import { - access, - mkdir, - readFile, - readdir, - rm, - writeFile, -} from "node:fs/promises"; +import { access, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { promisify } from "node:util"; import { pathToFileURL } from "node:url"; diff --git a/tests/boundary/process/mcpBundle.test.ts b/tests/boundary/process/runtimeBundle.test.ts similarity index 57% rename from tests/boundary/process/mcpBundle.test.ts rename to tests/boundary/process/runtimeBundle.test.ts index 3895b2ce8..ab09521f3 100644 --- a/tests/boundary/process/mcpBundle.test.ts +++ b/tests/boundary/process/runtimeBundle.test.ts @@ -4,15 +4,16 @@ import { pathToFileURL } from "node:url"; import { describe, expect } from "vitest"; -import { bundleMcpRuntime } from "../../../scripts/bundle-mcp-runtime.mjs"; +import { bundleRuntime } from "../../../scripts/bundle-runtime.mjs"; import { workspaceCliTest } from "../../support/cli/workspaceCliFixture.js"; -describe("MCP runtime bundle", () => { +describe("CLI and MCP runtime bundles", () => { workspaceCliTest( "preserves module locations, lazy side effects, package resolution and worker entrypoints", async ({ processes, workspace }) => { await workspace.mkdir("dist/provider"); await workspace.mkdir("dist/composition"); + await workspace.mkdir("dist/cli"); await workspace.mkdir("bridge"); await workspace.mkdir("node_modules/external-probe"); const files = { @@ -45,8 +46,17 @@ globalThis.effects.push('lazy'); export const lazyUrl = import.meta.url;`, "dist/composition/javascriptAnalysisWorkerProcess.js": 'process.stdout.write("worker-entry");', + "dist/generatedPackageMetadata.js": + 'export const generatedPackageMetadata = { version: "1.0.0" };', + "dist/cli.js": 'export { facts, run } from "./main.js";', + "dist/cliOutput.js": 'export { facts } from "./provider/probe λ.js";', + "dist/cli/streamedJsonOutput.js": + 'export { facts } from "../provider/probe λ.js";', + "dist/mcpDoctor.js": 'export { run } from "./main.js";', "dist/main.js": ` import { facts } from './provider/probe λ.js'; +import { generatedPackageMetadata } from './generatedPackageMetadata.js'; +export { facts }; import { execFileSync } from 'node:child_process'; import { fileURLToPath } from 'node:url'; globalThis.effects.push('main'); @@ -54,7 +64,7 @@ export const run = async () => { const beforeLazy = [...globalThis.effects]; const { lazyUrl } = await import('./lazy.js'); return { - ...facts, beforeLazy, afterLazy: globalThis.effects, lazyUrl, + ...facts, version: generatedPackageMetadata.version, beforeLazy, afterLazy: globalThis.effects, lazyUrl, workerOutput: execFileSync(process.execPath, [fileURLToPath(facts.worker)], { encoding: 'utf8' }) }; };`, @@ -76,9 +86,23 @@ export const run = async () => { return JSON.parse(result.stdout) as unknown; }; const original = await invoke("dist/main.js"); - await bundleMcpRuntime(workspace.root); + await bundleRuntime(workspace.root); const bundled = await invoke("dist/mcp/main.js"); expect(bundled).toEqual(original); + expect(await invoke("dist/cli-runtime/cli.js")).toEqual( + await invoke("dist/cli.js"), + ); + const cliDirectory = join(workspace.root, "dist/cli-runtime"); + const identity = await processes.run( + process.execPath, + [ + "--input-type=module", + "--eval", + `const { facts } = await import(${JSON.stringify(pathToFileURL(join(cliDirectory, "cli.js")).href)}); const output = await import(${JSON.stringify(pathToFileURL(join(cliDirectory, "cliOutput.js")).href)}); const streamed = await import(${JSON.stringify(pathToFileURL(join(cliDirectory, "streamedJsonOutput.js")).href)}); process.stdout.write(JSON.stringify(facts === output.facts && facts === streamed.facts));`, + ], + { cwd: workspace.root }, + ); + expect(identity).toMatchObject({ exitCode: 0, stdout: "true" }); expect(bundled).toMatchObject({ resource: "bridge resource\n", external: "external", @@ -93,6 +117,14 @@ export const run = async () => { expect( (await readdir(join(workspace.root, "dist/mcp"))).length, ).toBeGreaterThan(1); + await workspace.write( + "dist/generatedPackageMetadata.js", + 'export const generatedPackageMetadata = { version: "0.0.0" };', + ); + const previousRelease = await invoke("dist/main.js"); + expect(previousRelease).toMatchObject({ version: "0.0.0" }); + expect(await invoke("dist/mcp/main.js")).toEqual(previousRelease); + expect(await invoke("dist/cli-runtime/cli.js")).toEqual(previousRelease); }, ); @@ -100,18 +132,26 @@ export const run = async () => { "rejects module-relative resolution and unknown import metadata during the build", async ({ workspace }) => { await workspace.mkdir("dist"); + await workspace.mkdir("dist/cli"); await workspace.write("package.json", '{"type":"module"}'); - for (const expression of [ - "import.meta.resolve('./worker.js')", - "import.meta.env", - ]) { - await workspace.write( - "dist/main.js", - `export const value = ${expression};`, - ); - await expect(bundleMcpRuntime(workspace.root)).rejects.toThrow( - "Unsupported import.meta use in MCP bundle", - ); + for (const path of [ + "dist/cliOutput.js", + "dist/cli/streamedJsonOutput.js", + "dist/mcpDoctor.js", + ]) + await workspace.write(path, "export {};\n"); + for (const entry of ["dist/main.js", "dist/cli.js"]) { + for (const expression of [ + "import.meta.resolve('./worker.js')", + "import.meta.env", + ]) { + await workspace.write("dist/main.js", "export {};\n"); + await workspace.write("dist/cli.js", "export {};\n"); + await workspace.write(entry, `export const value = ${expression};`); + await expect(bundleRuntime(workspace.root)).rejects.toThrow( + "Unsupported import.meta use in runtime bundle", + ); + } } }, ); diff --git a/tests/boundary/providers/ida/calleeRelationships.test.ts b/tests/boundary/providers/ida/calleeRelationships.test.ts index ec46be03d..9d8ed638d 100644 --- a/tests/boundary/providers/ida/calleeRelationships.test.ts +++ b/tests/boundary/providers/ida/calleeRelationships.test.ts @@ -40,34 +40,41 @@ it("shows the callee edges of an ARM64 procedure in the call graph", async () => result.value, "the root lists both observed callees instead of an empty observed list", ).toMatchObject({ - "0": [ - { - address: "0x1000", - status: "ok", - calls: ["0x2000", "0x3000"], - limitations: expect.arrayContaining([ - "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", - ]), - }, - ], - "1": [ - { - address: "0x2000", - status: "ok", - calls: [], - limitations: expect.arrayContaining([ - "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", - ]), - }, - { - address: "0x3000", - status: "ok", - calls: [], - limitations: expect.arrayContaining([ - "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", - ]), - }, - ], + graph: { + "0": [ + { + address: "0x1000", + status: "ok", + calls: ["0x2000", "0x3000"], + limitations: expect.arrayContaining([ + "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", + ]), + }, + ], + "1": [ + { + address: "0x2000", + status: "ok", + calls: [], + limitations: expect.arrayContaining([ + "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", + ]), + }, + { + address: "0x3000", + status: "ok", + calls: [], + limitations: expect.arrayContaining([ + "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", + ]), + }, + ], + }, + limitations: expect.arrayContaining([ + expect.stringContaining( + "Provider version identifies the REA IDA adapter.", + ), + ]), }); expect( requestedRoots(producer), @@ -122,13 +129,16 @@ it("traverses every ARM64 callee in a call path without a goal and records no fa { source_address: "0x1000", target_address: "0x3000" }, ], failures: [], - truncated: true, + truncated: false, limitations: expect.arrayContaining([ "This relationship read does not establish calls to targets IDA does not recognize as functions; imports and other non-function call targets remain unknown.", ]), - residual_unknowns: expect.arrayContaining([ - expect.stringContaining("Call relationships for 0x1000 are limited:"), - ]), + residual_unknowns: [], + relationship_coverage: { + "0x1000": { status: "complete" }, + "0x2000": { status: "complete" }, + "0x3000": { status: "complete" }, + }, }); expect( requestedRoots(producer), @@ -180,15 +190,17 @@ it("preserves missing callgraph coverage through direct, graph, and trace workfl }); if (!graph.ok) throw graph.error; expect(graph.value).toMatchObject({ - "0": [ - { - status: "ok", - calls: [], - limitations: expect.arrayContaining([ - expect.stringContaining("callgraph was unavailable"), - ]), - }, - ], + graph: { + "0": [ + { + status: "ok", + calls: [], + limitations: expect.arrayContaining([ + expect.stringContaining("callgraph was unavailable"), + ]), + }, + ], + }, }); const trace = await application.execute("trace_call_path", { @@ -203,7 +215,9 @@ it("preserves missing callgraph coverage through direct, graph, and trace workfl expect.stringContaining("callgraph was unavailable"), ]), residual_unknowns: expect.arrayContaining([ - expect.stringContaining("Call relationships for 0x1000 are limited:"), + expect.stringContaining( + "Call relationship coverage is unknown for 0x1000:", + ), ]), }); expect( @@ -234,13 +248,18 @@ it("records a failed ARM64 call graph read as the node's error, not as a complet result.value, "the transport-truncated graph is a recorded failure and no callee list is invented", ).toEqual({ - "0": [ - { - address: "0x1000", - status: "error", - error: expect.objectContaining({ code: "unreadable_output" }), - }, - ], + graph: { + "0": [ + { + address: "0x1000", + status: "error", + error: expect.objectContaining({ code: "unreadable_output" }), + }, + ], + }, + limitations: expect.arrayContaining([ + expect.stringContaining("Live results are never replayed"), + ]), }); } finally { expect((await client.close()).ok).toBe(true); @@ -268,12 +287,15 @@ it("preserves an explicit root cap below the edge limit through every relationsh }); if (!graph.ok) throw graph.error; expect(graph.value).toMatchObject({ - "0": [ - { - calls: ["0x2000"], - limitations: expect.arrayContaining([limitation]), - }, - ], + graph: { + "0": [ + { + calls: ["0x2000"], + limitations: expect.arrayContaining([limitation]), + }, + ], + }, + limitations: expect.arrayContaining([limitation]), }); const trace = await application.execute("trace_call_path", { start: "main", diff --git a/tests/boundary/providers/ida/selectorProvenance.test.ts b/tests/boundary/providers/ida/selectorProvenance.test.ts index e18a1d3af..40e92ae1c 100644 --- a/tests/boundary/providers/ida/selectorProvenance.test.ts +++ b/tests/boundary/providers/ida/selectorProvenance.test.ts @@ -58,14 +58,17 @@ describe("provider-derived selectors in IDA composed workflows", () => { result.value, "the resolved root stays ok and only the derived node carries the producer fault", ).toMatchObject({ - "0": [{ address: "0x1000", status: "ok" }], - "1": [ - { - address: "0x2000", - status: "error", - error: { code: "unreadable_output" }, - }, - ], + graph: { + "0": [{ address: "0x1000", status: "ok" }], + "1": [ + { + address: "0x2000", + status: "error", + error: { code: "unreadable_output" }, + }, + ], + }, + limitations: expect.any(Array), }); } finally { expect((await client.close()).ok).toBe(true); diff --git a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts index 7849cfb82..7f4408811 100644 --- a/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts +++ b/tests/boundary/providers/managed/managedMemberComparisonFromMetadata.test.ts @@ -459,9 +459,83 @@ describe("managed member comparison of ambiguous undecoded signatures", () => { }); }); -const inspect = (bytes: Buffer, path: string) => { +describe("managed member comparison of selected types", () => { + it("keeps members outside the other side's selection unknown", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/left-all.dll"); + const right = inspect(buildManagedPeFixture(), "/tmp/right-selected.dll", [ + "Missing.Type", + ]); + const result = compareManagedMembers( + { evidenceId: left.evidenceId, result: left.result }, + { evidenceId: right.evidenceId, result: right.result }, + ); + + expect(result.summary).toMatchObject({ added: 0, removed: 0, unknown: 2 }); + expect(result.methods[0]).toMatchObject({ + status: "unknown", + left: { token: "0x06000001" }, + right: null, + }); + expect(result.coverage.status).toBe("partial"); + expect(result.limitations).toContain( + "The observations select different declaring types; a member outside the other side's selection is unknown, not added or removed.", + ); + }); + + it("does not remap members across types neither side observed", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/left-a.dll", [ + "Fixture.Program", + ]); + const right = inspect( + buildManagedPeFixture({ + mvid: Buffer.from("00112233445566778899aabbccddeefe", "hex"), + typeName: "A", + typeNamespace: "X", + }), + "/tmp/right-b.dll", + ["X.A"], + ); + const result = compareManagedMembers( + { evidenceId: left.evidenceId, result: left.result }, + { evidenceId: right.evidenceId, result: right.result }, + ); + + expect(result.matching.exact_il_signature).toBe(0); + expect(result.summary).toMatchObject({ + unchanged: 0, + added: 0, + removed: 0, + unknown: 4, + }); + }); + + it("compares two identical selections within those types", () => { + const left = inspect(buildManagedPeFixture(), "/tmp/left-sel.dll", [ + "Fixture.Program", + ]); + const right = inspect(buildManagedPeFixture(), "/tmp/right-sel.dll", [ + "Fixture.Program", + ]); + const result = compareManagedMembers( + { evidenceId: left.evidenceId, result: left.result }, + { evidenceId: right.evidenceId, result: right.result }, + ); + + expect(result.methods[0]).toMatchObject({ status: "unchanged" }); + expect(result.coverage.status).toBe("complete-within-inputs"); + expect(result.limitations).toContain( + "Both observations cover only the selected declaring types (Fixture.Program); member totals and statuses describe those types, not the whole assembly.", + ); + }); +}); + +const inspect = (bytes: Buffer, path: string, typeNames?: string[]) => { const target = managedPeFixtureTarget(bytes, path); - const result = inspectManagedMembersBytes(bytes, target); + const result = inspectManagedMembersBytes( + bytes, + target, + typeNames === undefined ? {} : { typeNames }, + ); return { result, evidenceId: `ev_${hash(Buffer.from(path))}`, diff --git a/tests/boundary/providers/managed/managedNativeOsMachine.test.ts b/tests/boundary/providers/managed/managedNativeOsMachine.test.ts new file mode 100644 index 000000000..9a31b99d4 --- /dev/null +++ b/tests/boundary/providers/managed/managedNativeOsMachine.test.ts @@ -0,0 +1,55 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; +import { parseBinaryTarget } from "../../../../src/application/BinaryTargetResolver.js"; +import { buildManagedPeFixture } from "../../../../src/dotnet/ManagedPe.fixture.js"; +import { ManagedStaticProvider } from "../../../../src/dotnet/ManagedStaticProvider.js"; + +// ReadyToRun images compiled for a non-Windows OS store the machine XORed with +// IMAGE_FILE_MACHINE_NATIVE_OS_OVERRIDE (dotnet/runtime src/coreclr/inc/pedecoder.h). +it.each([ + ["Linux x86", 0x014c ^ 0x7b79, 0, "x86"], + ["Linux ARM", 0x01c4 ^ 0x7b79, 0, "arm"], + ["Apple ARM", 0x01c4 ^ 0x4644, 0, "arm"], + ["FreeBSD x86", 0x014c ^ 0xadc4, 0, "x86"], +] as const)( + "opens and inspects the %s ReadyToRun machine through the public path", + async (_label, machine, cliFlags, architecture) => { + const bytes = buildManagedPeFixture({ cliFlags, readyToRun: true }); + bytes.writeUInt16LE(machine, bytes.readUInt32LE(0x3c) + 4); + const directory = await createTestTempDirectory("rea-managed-os-machine-"); + const path = join(directory, "fixture.dll"); + await writeFile(path, bytes); + const parsed = await parseBinaryTarget(path); + if (!parsed.ok) throw parsed.error; + expect(parsed.value).toMatchObject({ + kind: "executable", + format: "pe", + architecture, + managed: true, + }); + const client = new ManagedStaticProvider().createClient(parsed.value); + try { + for (const operation of [ + "inspect_managed_artifact", + "inspect_managed_members", + "inspect_managed_native_boundaries", + ] as const) { + const observed = await client.execute(operation, {}); + if (!observed.ok) throw observed.error; + expect(observed.value.result).toMatchObject({ + module: { name: "Fixture.dll" }, + }); + } + const artifact = await client.execute("inspect_managed_artifact", {}); + if (!artifact.ok) throw artifact.error; + expect(artifact.value.result).toMatchObject({ + pe: { machine, architecture }, + classification: { managed_architecture: architecture }, + }); + } finally { + await client.close(); + } + }, +); diff --git a/tests/boundary/providers/managed/managedStaticProvider.test.ts b/tests/boundary/providers/managed/managedStaticProvider.test.ts index 00e92cda4..1d1a750c3 100644 --- a/tests/boundary/providers/managed/managedStaticProvider.test.ts +++ b/tests/boundary/providers/managed/managedStaticProvider.test.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -142,6 +143,57 @@ const asManagedMemberResult = (execution: AnalysisExecution) => const asManagedNativeBoundaryResult = (execution: AnalysisExecution) => managedNativeBoundaryInspectionSchema.parse(execution.result); +it("preserves custom attribute value evidence through the filesystem provider when its constructor is invalid", async () => { + const directory = await createTestTempDirectory("rea-managed-attribute-"); + const path = join(directory, "fixture.exe"); + await writeFile(path, buildManagedPeFixture({ customAttributeTypeRaw: 19 })); + const parsed = await parseBinaryTarget(path); + if (!parsed.ok) throw parsed.error; + const frameworkText = Buffer.from(".NETCoreApp,Version=v8.0", "utf8"); + const attributeValue = Buffer.concat([ + Buffer.from([1, 0, frameworkText.length]), + frameworkText, + Buffer.alloc(2), + ]); + const client = new ManagedStaticProvider().createClient(parsed.value); + try { + const execution = await client.execute("inspect_managed_artifact", {}); + if (!execution.ok) throw execution.error; + + const result = asManagedResult(execution.value); + expect(result).toMatchObject({ + attributes: [ + { + parent_token: "0x20000001", + constructor_token: null, + type_name: null, + value_length: attributeValue.length, + value_sha256: createHash("sha256") + .update(attributeValue) + .digest("hex"), + token: "0x0c000001", + }, + ], + coverage: { + state: "partial", + }, + }); + const attribute = result.attributes[0]; + if (attribute === undefined) throw new Error("Fixture omitted attribute"); + expect(result.coverage.issues).toEqual([ + expect.objectContaining({ + code: "invalid-row", + scope: "metadata.CustomAttribute:0x0c000001", + offset: attribute.row_offset + 2, + detail: + "CustomAttribute constructor coded index 0x13 is invalid: coded index selects row 2 in table 10, which has 1 rows", + }), + ]); + } finally { + await client.close(); + } +}); + it("retains U+FEFF metadata through the filesystem target and provider", async () => { const directory = await createTestTempDirectory("rea-managed-bom-"); const path = join(directory, "fixture.exe"); diff --git a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts index 812cc2df7..eedf1eb80 100644 --- a/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts +++ b/tests/boundary/providers/native/artifactProvider.machOSlices.test.ts @@ -291,3 +291,61 @@ describe("artifact Mach-O arm64e variant slices", () => { }, ); }); + +// macOS 26 lipo leaves capabilities blank, even for a LIB64 x86_64 slice. +describe("artifact Mach-O LIB64 slices", () => { + const LIB64 = { type: 0x01000007, subtype: 0x80000003 } as const; + const lib64Slices = async (x86Subtype: string) => { + const root = await createTestTempDirectory("rea-slices-lib64-"); + const binary = join(root, "fat"); + const x86 = machoImage({ cpu: LIB64 }); + const arm = machoImage({ cpu: CPU.arm64 }); + await writeFile( + binary, + fatImage([ + { cpu: LIB64, bytes: x86 }, + { cpu: CPU.arm64, bytes: arm }, + ]), + ); + const reader = new MachOSliceArtifactReader( + binary, + {}, + lipoRunner( + [ + "architecture x86_64", + " cputype CPU_TYPE_X86_64", + ` cpusubtype ${x86Subtype}`, + " capabilities ", + " offset 4096", + ` size ${String(x86.length)}`, + " align 2^12 (4096)", + "architecture arm64", + " cputype CPU_TYPE_ARM64", + " cpusubtype CPU_SUBTYPE_ARM64_ALL", + " capabilities ", + " offset 8192", + ` size ${String(arm.length)}`, + " align 2^12 (4096)", + "", + ].join("\n"), + ), + ); + onTestFinished(() => reader.close()); + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + return entries.map(({ path, byteOffset }) => [path, byteOffset]); + }; + + it("matches an x86_64 LIB64 slice whose lipo capabilities are blank", async () => { + expect(await lib64Slices("CPU_SUBTYPE_X86_ALL")).toEqual([ + ["slices/x86_64", 4096], + ["slices/arm64", 8192], + ]); + }); + + it("still rejects a blank-capability subtype that names other bits", async () => { + await expect(lib64Slices("CPU_SUBTYPE_X86_64_H")).rejects.toMatchObject({ + reason: "integrity", + }); + }); +}); diff --git a/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts b/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts new file mode 100644 index 000000000..f451c3863 --- /dev/null +++ b/tests/boundary/providers/native/artifactProvider.universalExtraction.test.ts @@ -0,0 +1,73 @@ +import { access, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, vi } from "vitest"; + +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; + +import { ArtifactProvider } from "../../../../src/artifacts/ArtifactProvider.js"; +import { ArtifactReaderFailure } from "../../../../src/artifacts/ArtifactReader.js"; +import { artifactExtractionExecutionSchema } from "../../../../src/contracts/artifactToolContracts.js"; +import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; + +const sliceReader = vi.hoisted(() => ({ + reached: vi.fn(), + failure: null as unknown, +})); + +vi.mock("../../../../src/artifacts/MachOSliceArtifactReader.js", () => ({ + MachOSliceArtifactReader: class { + async *entries() { + sliceReader.reached(); + yield* []; + throw sliceReader.failure; + } + + close() { + return Promise.resolve(); + } + + provenance() { + return []; + } + }, +})); + +it("routes universal Mach-O extraction through slice inventory", async () => { + sliceReader.reached.mockClear(); + sliceReader.failure = new ArtifactReaderFailure( + "unavailable", + "fixture reached universal slice inventory", + ); + const root = await createTestTempDirectory("rea-extract-universal-"); + const image = join(root, "Universal"); + await writeFile(image, Buffer.from([0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 0])); + const output = join(root, "output"); + const result = await new ArtifactProvider(process.env) + .createClient(universalTarget(image)) + .execute( + "extract_artifact", + artifactExtractionExecutionSchema.parse({ output_root: output }), + ); + + expect(result).toMatchObject({ + ok: false, + error: { + _tag: "ArtifactOperationError", + reason: "unavailable", + detail: "fixture reached universal slice inventory", + }, + }); + expect(sliceReader.reached).toHaveBeenCalledOnce(); + await expect(access(output)).rejects.toThrow(); +}); + +const universalTarget = (path: string): BinaryTarget => ({ + path, + sourcePath: path, + sha256: "0".repeat(64), + kind: "executable", + format: "mach-o-universal", + architecture: "arm64", + availableArchitectures: ["arm64"], +}); diff --git a/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts b/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts index 99adc61f1..87912a31d 100644 --- a/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts +++ b/tests/boundary/providers/native/nativeLoadKinds.golden.test.ts @@ -95,6 +95,74 @@ it("retains empty dylib-use options and rejects missing or ambiguous encoding fi } }); +it.each(["vmaddr", "vmsize", "uuid", "entryoff"])( + "rejects a required %s label truncated before its value", + async (key) => { + const raw = await nativeFixture("otool-load.txt"); + const truncated = raw.replace( + new RegExp(`^(\\s*${key})\\s+\\S.*$`, "mu"), + "$1", + ); + expect(truncated).not.toBe(raw); + expect(() => parseOtoolLoadCommands(raw)).not.toThrow(); + // Only the empty dylib-use options row may omit its value. + expect(() => parseOtoolLoadCommands(truncated)).toThrow( + `otool field ${key} has no value`, + ); + }, +); + +it("retains the unnamed segment of a Mach-O object file", () => { + // Captured from `otool -l` on a clang `-c` object: MH_OBJECT sections live + // in one segment whose segname is empty, printed as a bare label. + const objectSegment = [ + "Load command 0", + " cmd LC_SEGMENT_64", + " cmdsize 152", + " segname ", + " vmaddr 0x0000000000000000", + " vmsize 0x0000000000000018", + " fileoff 392", + " filesize 24", + " maxprot 0x00000007", + " initprot 0x00000007", + " nsects 1", + " flags 0x0", + "Section", + " sectname __text", + " segname __TEXT", + " addr 0x0000000000000000", + " size 0x0000000000000018", + " offset 392", + " align 2^2 (4)", + " reloff 0", + " nreloc 0", + " flags 0x80000400", + " reserved1 0", + " reserved2 0", + "", + ].join("\n"); + const load = parseOtoolLoadCommands(withOtoolMachHeader(objectSegment)); + expect(load.segments).toMatchObject([ + { name: "unknown", sections: [{ segment: "__TEXT", name: "__text" }] }, + ]); + // An all-NUL section name is printed the same way and stays "unknown". + expect( + parseOtoolLoadCommands( + withOtoolMachHeader( + objectSegment.replace(" sectname __text", " sectname "), + ), + ).segments, + ).toMatchObject([{ sections: [{ segment: "__TEXT", name: "unknown" }] }]); + expect(() => + parseOtoolLoadCommands( + withOtoolMachHeader( + objectSegment.replace(" vmaddr 0x0000000000000000", " vmaddr"), + ), + ), + ).toThrow("otool field vmaddr has no value"); +}); + it("preserves dependency install names that begin with the word stamp", () => { const load = parseOtoolLoadCommands( withOtoolMachHeader( diff --git a/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts b/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts index 49d80a601..ea81dacaa 100644 --- a/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts +++ b/tests/boundary/providers/native/nativeMachoHeader.golden.test.ts @@ -9,6 +9,78 @@ import { withOtoolMachHeader, } from "../../../fixtures/nativeCommands.js"; +it("retains the delayed dependency in the complete real framed otool output", async () => { + const output = await nativeFixture("otool-dylib-use.txt"); + const runner = new NativeFixtureRunner({ "otool:-h": output }); + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + nativeMachoTarget("/owned/fixture"), + ); + const result = await client.execute("inspect_macho", {}); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + load_commands: { + items: expect.arrayContaining([ + expect.objectContaining({ + index: 16, + kind: "LC_LOAD_DYLIB", + fields: expect.objectContaining({ options: "delay-init" }), + }), + ]), + }, + dependencies: { + items: expect.arrayContaining([ + expect.objectContaining({ + path: "@rpath/libdelay.dylib", + current_version: "0.0.0", + compatibility_version: "1.0.0", + }), + ]), + }, + }); + expect(parseOtoolLoadCommands(output).commands).toHaveLength(23); +}); + +it("rejects a repeated options field even when the duplicate has no value", async () => { + const output = (await nativeFixture("otool-dylib-use.txt")).replace( + " options delay-init ", + " options delay-init\n options", + ); + expect(() => parseOtoolLoadCommands(output)).toThrow(/otool/u); +}); + +it.each([ + "vmaddr", + "vmsize", + "uuid", + "entryoff", + "addr", + "current version", + "time stamp", +])( + "rejects a required %s label with no value and retains its producer output", + async (field) => { + const original = await nativeFixture("otool-load.txt"); + const output = original.replace( + new RegExp(`^([ \\t]*${field})[ \\t]+\\S[^\\n]*$`, "mu"), + "$1", + ); + if (output === original) throw new Error(`Fixture has no ${field} value`); + const runner = new NativeFixtureRunner({ "otool:-h": output }); + const client = new NativeMacOSProvider({}, runner, "darwin").createClient( + nativeMachoTarget("/owned/fixture"), + ); + const result = await client.execute("inspect_macho", {}); + if (result.ok) throw new Error(`Expected missing ${field} value rejection`); + expect(result.error).toMatchObject({ + _tag: "AnalysisOutputError", + capturedOutput: { stdout: output, stderr: "", truncated: false }, + }); + expect(result.error.cause).toMatchObject({ + message: expect.stringContaining(field), + }); + }, +); + it("collects the actual Mach header alongside the native load commands", async () => { // The provider always invokes `otool -h -l`, which the runner resolves // via the "otool:-h" key; no bare `otool` override is needed. diff --git a/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts b/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts index 04f88284b..0fd9e6f17 100644 --- a/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts +++ b/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts @@ -32,6 +32,10 @@ it("does not attribute the first universal slice's UUID and segments to the sele if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ uuid: "01234567-89AB-CDEF-0123-456789ABCDEF", + selected_architecture: { + name: "arm64", + container_file_offset: 32768, + }, architectures: { total: 2 }, }); const normalized = inspectMachoSchema.parse(result.value.result); @@ -125,16 +129,19 @@ it("projects universal segment evidence into the selected slice's container offs start: 32768, end: 40960, }); + expect(result.value.locations).toContainEqual({ + kind: "file-offset", + offset: 36864, + }); expect(result.value.locations).not.toContainEqual({ kind: "file-offset-range", start: 0, end: 8192, }); - // The normalized tool field remains slice-relative; only evidence is projected. - expect( - inspectMachoSchema.parse(result.value.result).segments.items[0] - ?.file_offset, - ).toBe(0); + // Normalized tool fields keep their slice-relative meaning. + const normalized = inspectMachoSchema.parse(result.value.result); + expect(normalized.segments.items[0]?.file_offset).toBe(0); + expect(normalized.entrypoints.items[0]?.file_offset).toBe(4096); }); it.each([ @@ -168,6 +175,12 @@ it.each([ start: offset, end: offset + 8192, }); + expect(result.value.result).toMatchObject({ + selected_architecture: { + name: "arm64", + container_file_offset: offset, + }, + }); if (offset > 0) expect(result.value.locations).not.toContainEqual({ kind: "file-offset-range", @@ -206,7 +219,10 @@ it.each([ end: 8192, }); expect(result.value.limitations).toContain( - "Segment evidence file offsets are unavailable because the selected Mach-O slice's container offset was not observed.", + "The selected Mach-O slice's container file offset was not observed; slice-based evidence locations are unavailable.", ); + expect(result.value.result).toMatchObject({ + selected_architecture: { container_file_offset: null }, + }); }, ); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index aa809b6fb..302d51d50 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -597,10 +597,19 @@ describe("provider health and availability", () => { expect(session.status()).toMatchObject({ capabilities: expect.arrayContaining([ expect.objectContaining({ - operation: "address_name", - available: false, - reason: - "ProviderAdapterError: Provider fixture adapter failed during address_name", + provider: { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }, + groups: expect.arrayContaining([ + expect.objectContaining({ + operations: expect.arrayContaining(["address_name"]), + available: false, + reason: + "ProviderAdapterError: Provider fixture adapter failed during address_name", + }), + ]), }), ]), }); @@ -609,9 +618,18 @@ describe("provider health and availability", () => { expect(session.status()).toMatchObject({ capabilities: expect.arrayContaining([ expect.objectContaining({ - operation: "address_name", - available: true, - reason: null, + provider: { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }, + groups: expect.arrayContaining([ + expect.objectContaining({ + operations: expect.arrayContaining(["address_name"]), + available: true, + reason: null, + }), + ]), }), ]), }); @@ -657,9 +675,18 @@ describe("provider health and availability", () => { expect(session.status()).toMatchObject({ capabilities: expect.arrayContaining([ expect.objectContaining({ - operation: "address_name", - available: true, - reason: null, + provider: { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }, + groups: expect.arrayContaining([ + expect.objectContaining({ + operations: expect.arrayContaining(["address_name"]), + available: true, + reason: null, + }), + ]), }), ]), }); diff --git a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts index 915545a69..cd60028d4 100644 --- a/tests/composition/analysis-sessions/managedApplicationGraph.test.ts +++ b/tests/composition/analysis-sessions/managedApplicationGraph.test.ts @@ -629,3 +629,55 @@ describe("managed application graph coverage", () => { }); }); }); + +describe("managed application graph type selection", () => { + const project = (result: Parameters[2]["result"]) => { + const bytes = buildManagedPeFixture(); + const binary = managedPeFixtureTarget(bytes, "/fixture/ManagedInterop.exe"); + const projection = projectManagedApplicationGraphEvidence({ + managed_members: createEvidence(binary, MANAGED_STATIC_PROVIDER, { + operation: "inspect_managed_members", + parameters: {}, + result, + rawResult: null, + limitations: [], + }), + }); + if (!projection.ok) throw new Error("projection failed"); + return managedApplicationGraphResultSchema.parse( + parseEvidence(projection.value).normalized_result, + ); + }; + + it("marks a graph from selected member inspection partial", () => { + const bytes = buildManagedPeFixture(); + const binary = managedPeFixtureTarget(bytes, "/fixture/ManagedInterop.exe"); + const selected = inspectManagedMembersBytes(bytes, binary, { + typeNames: ["Fixture.Program"], + }); + + expect(selected.coverage.state).toBe("complete"); + const result = project(selected); + expect(result.coverage).toMatchObject({ status: "partial" }); + expect(result.limitations).toContain( + "Managed member Evidence selected only these declaring types: Fixture.Program; type, method, and field nodes of other types are omitted, not absent.", + ); + expect(result.limitations).not.toContain( + "At least one supplied managed Evidence record has partial coverage.", + ); + }); + + it("accepts member Evidence recorded before type selection existed", () => { + const bytes = buildManagedPeFixture(); + const binary = managedPeFixtureTarget(bytes, "/fixture/ManagedInterop.exe"); + const { selection: _selection, ...legacy } = inspectManagedMembersBytes( + bytes, + binary, + ); + + expect(managedMemberInspectionSchema.parse(legacy).selection).toBeNull(); + expect(project(legacy).coverage).toMatchObject({ + status: "complete-within-inputs", + }); + }); +}); diff --git a/tests/composition/analysis-sessions/providerBinding.test.ts b/tests/composition/analysis-sessions/providerBinding.test.ts index b6d726fdb..9ed21abfa 100644 --- a/tests/composition/analysis-sessions/providerBinding.test.ts +++ b/tests/composition/analysis-sessions/providerBinding.test.ts @@ -191,6 +191,38 @@ describe("target-bound provider routing", () => { }); }); +describe("selected provider capability provenance", () => { + it("labels selected-provider capabilities with the resolved identity", async () => { + const target = await databaseTarget(); + const alpha = deepProvider("alpha"); + const session = selectableSession([alpha.provider], auxiliaryProvider()); + + expect((await session.open(target, { providerId: "alpha" })).ok).toBe(true); + const resolved = { id: "alpha", name: "alpha provider", version: "1" }; + const status = bindingStatus(session); + expect(status).toMatchObject({ + analysis_provider_binding: { provider: resolved }, + analysis_provider_candidates: [ + { + provider: resolved, + selected: true, + capabilities: [{ provider: resolved }], + }, + ], + }); + expect( + z + .object({ capabilities: z.array(z.object({ provider: z.unknown() })) }) + .parse(status) + .capabilities.map(({ provider }) => provider), + ).toEqual([ + { id: "alpha", name: "alpha provider", version: "1" }, + { id: "auxiliary", name: "auxiliary provider", version: "1" }, + ]); + await session.close(); + }); +}); + interface ClientState { readonly calls: string[]; closed: boolean; diff --git a/tests/conformance/c/mips-relocatable.c b/tests/conformance/c/mips-relocatable.c index 7552a918c..12eb4da5c 100644 --- a/tests/conformance/c/mips-relocatable.c +++ b/tests/conformance/c/mips-relocatable.c @@ -1,7 +1,8 @@ /* Relocatable (ET_REL) static-analysis fixture, built with -c -G0 and never * executed or linked by REA. It exercises the relocations Linux MIPS kernel * modules use: R_MIPS_26 calls, HI16/LO16 address pairs (one HI16 shared by - * several LO16s, and a LO16 >= 0x8000 that forces a HI16 carry), R_MIPS_32 + * several LO16s, two HI16s resolved by one LO16, and a LO16 >= 0x8000 that + * forces a HI16 carry), R_MIPS_32 * data words, and undefined external code and data symbols. powf is an * external whose float prototype Ghidra's C library archive supplies by name, * so soft-float builds must give it the soft-float prototype too. @@ -19,6 +20,8 @@ struct rea_mips_rel_pair rea_mips_rel_pair = { 0x1111, 0x2222 }; /* Places rea_mips_rel_carry past a signed 16-bit low-half boundary. */ char rea_mips_rel_padding[0x8000] = { 1 }; int rea_mips_rel_carry = 0x3333; +/* Referenced only by rea_mips_rel_highs, so its single LO16 stays unique. */ +int rea_mips_rel_highs_target = 0x4444; static const char rea_mips_rel_marker[] = "rea-mips-relocatable-fixture"; __attribute__((noinline)) int rea_mips_rel_leaf(int value) @@ -45,6 +48,26 @@ __attribute__((noinline)) int rea_mips_rel_scaled(int value) return value ^ *(int *)&scaled; } +/* Most GCC-built kernel modules checked contain LO16s that resolve several + * pending HI16s; Clang does not produce that from C. Two %hi and exactly one + * %lo of a symbol nothing else references: any assembler that orders each + * HI16 before a matching LO16 (GNU as, every LLVM version) must emit both + * HI16s ahead of that single LO16, which therefore resolves both. + */ +__asm__(".text\n" + ".globl rea_mips_rel_highs\n" + ".type rea_mips_rel_highs, @function\n" + ".set push\n" + ".set noreorder\n" + "rea_mips_rel_highs:\n" + " lui $2, %hi(rea_mips_rel_highs_target)\n" + " lui $3, %hi(rea_mips_rel_highs_target)\n" + " addiu $2, $2, %lo(rea_mips_rel_highs_target)\n" + " jr $31\n" + " move $3, $2\n" + ".set pop\n" + ".size rea_mips_rel_highs, .-rea_mips_rel_highs\n"); + int rea_mips_rel_entry(int value) { return rea_mips_rel_leaf(value) + rea_mips_rel_far(value) + diff --git a/tests/conformance/ghidra/mips16-recordless.ld b/tests/conformance/ghidra/mips16-recordless.ld new file mode 100644 index 000000000..8d90dc432 --- /dev/null +++ b/tests/conformance/ghidra/mips16-recordless.ld @@ -0,0 +1,22 @@ +/* Build an older-style shared image from source without ABI declarations. */ +PHDRS { + headers PT_PHDR PHDRS; + text PT_LOAD FILEHDR PHDRS FLAGS(5); + data PT_LOAD FLAGS(6); + dynamic PT_DYNAMIC FLAGS(6); +} +SECTIONS { + . = SIZEOF_HEADERS; + .hash : { *(.hash) } :text + .gnu.hash : { *(.gnu.hash) } :text + .dynsym : { *(.dynsym) } :text + .dynstr : { *(.dynstr) } :text + .rel.dyn : { *(.rel.*) } :text + .text : { *(.text*) } :text + . = ALIGN(0x10000); + .dynamic : { *(.dynamic) } :data :dynamic + .got : { *(.got*) } :data + .data : { *(.data*) } :data + .bss : { *(.bss*) *(COMMON) } :data + /DISCARD/ : { *(.MIPS.abiflags) *(.reginfo) *(.pdr) *(.mdebug.*) *(.comment) *(.note.*) } +} diff --git a/tests/conformance/ghidra/mips16-shared-fixture.mjs b/tests/conformance/ghidra/mips16-shared-fixture.mjs new file mode 100644 index 000000000..37ebddeab --- /dev/null +++ b/tests/conformance/ghidra/mips16-shared-fixture.mjs @@ -0,0 +1,156 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const execute = promisify(execFile); +const source = fileURLToPath(new URL("./mips16-shared.S", import.meta.url)); +const linkerScript = fileURLToPath( + new URL("./mips16-recordless.ld", import.meta.url), +); +const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); + +/** BYO GNU cross-binutils; generated controls are never executed or byte-patched. */ +export async function buildMips16SharedFixture(directory, byteOrder, variant) { + assert(["little", "big"].includes(byteOrder)); + assert( + [ + "declared", + "recordless", + "symbol-relative", + "section-relocations", + ].includes(variant), + ); + const assembler = process.env.REA_MIPS_AS ?? "mips-linux-gnu-as"; + const linker = process.env.REA_MIPS_LD ?? "mips-linux-gnu-ld"; + const endian = byteOrder === "little" ? "-EL" : "-EB"; + const prefix = join(directory, `${byteOrder}-${variant}`); + const object = `${prefix}.o`; + const path = `${prefix}.so`; + const assemblerVersion = ( + await execute(assembler, ["--version"], { timeout: 10000 }) + ).stdout; + const linkerVersion = ( + await execute(linker, ["--version"], { timeout: 10000 }) + ).stdout; + assert( + assemblerVersion.includes("GNU assembler"), + "Use GNU's MIPS assembler: other assemblers have different compressed-instruction behavior.", + ); + assert(linkerVersion.includes("GNU ld")); + await execute( + assembler, + [ + "-32", + "-mips32r2", + endian, + ...(variant === "symbol-relative" + ? ["--defsym", "REA_MIPS_SYMBOL_RELATIVE=1"] + : []), + source, + "-o", + object, + ], + { timeout: 60000 }, + ); + await execute( + linker, + [ + endian, + "-shared", + ...(variant === "section-relocations" ? ["--emit-relocs"] : []), + ...(variant === "recordless" ? ["-T", linkerScript] : []), + object, + "-o", + path, + ], + { timeout: 60000 }, + ); + const bytes = await readFile(path); + const little = byteOrder === "little"; + const u16 = (offset) => + little ? bytes.readUInt16LE(offset) : bytes.readUInt16BE(offset); + const u32 = (offset) => + little ? bytes.readUInt32LE(offset) : bytes.readUInt32BE(offset); + assert.equal( + bytes.subarray(0, 6).toString("hex"), + little ? "7f454c460101" : "7f454c460102", + ); + assert.equal(u16(16), 3); + assert.equal(u16(18), 8); + assert.equal(u32(36), 0x74001007); // noreorder/PIC/CPIC plus o32/MIPS16/MIPS32r2. + const headers = Array.from( + { length: u16(44) }, + (_, i) => u32(28) + i * u16(42), + ); + const fileOffset = (address, size) => { + const mappings = headers.filter( + (offset) => + u32(offset) === 1 && + address >= u32(offset + 8) && + address - u32(offset + 8) <= u32(offset + 16) - size, + ); + assert.equal(mappings.length, 1); + return u32(mappings[0] + 4) + address - u32(mappings[0] + 8); + }; + const readelf = await execute( + process.env.REA_MIPS_READELF ?? "readelf", + ["-W", "-h", "-l", "-s", "-r", path], + { timeout: 10000 }, + ); + const symbols = {}; + for (const name of [ + "rea_mips16_probe", + "rea_mips32_probe", + "rea_mips16_pointer", + ]) { + const addresses = new Set( + readelf.stdout + .split("\n") + .map((line) => line.trim().split(/\s+/u)) + .filter( + (fields) => + fields[0]?.endsWith(":") === true && + ["FUNC", "OBJECT"].includes(fields[3]) && + fields.at(-1) === name, + ) + .map((fields) => Number.parseInt(fields[1], 16)), + ); + assert.equal(addresses.size, 1, `Missing or conflicting ${name}`); + symbols[name] = [...addresses][0]; + } + assert.equal(symbols.rea_mips16_probe & 1, 1); + assert.equal(symbols.rea_mips32_probe & 1, 0); + const compressedOffset = fileOffset(symbols.rea_mips16_probe - 1, 8); + assert.equal( + bytes.subarray(compressedOffset, compressedOffset + 8).toString("hex"), + little ? "5a6a8eea074aa0e8" : "6a5aea8e4a07e8a0", + ); + const standardOffset = fileOffset(symbols.rea_mips32_probe, 12); + assert.equal( + bytes.subarray(standardOffset, standardOffset + 12).toString("hex"), + little ? "030082240800e00300000000" : "2482000303e0000800000000", + ); + const pointerOffset = fileOffset(symbols.rea_mips16_pointer, 4); + const rawPointer = u32(pointerOffset); + if (variant !== "symbol-relative") + assert.equal(rawPointer, symbols.rea_mips16_probe); + return { + path, + sha256: hash(bytes), + byte_order: byteOrder, + variant, + elf_flags: u32(36), + symbols, + raw_pointer: rawPointer, + source_sha256: hash(await readFile(source)), + linker_script_sha256: + variant === "recordless" ? hash(await readFile(linkerScript)) : null, + assembler_version: assemblerVersion.split("\n")[0], + linker_version: linkerVersion.split("\n")[0], + readelf_output: readelf.stdout, + }; +} diff --git a/tests/conformance/ghidra/mips16-shared.S b/tests/conformance/ghidra/mips16-shared.S new file mode 100644 index 000000000..d3d41802e --- /dev/null +++ b/tests/conformance/ghidra/mips16-shared.S @@ -0,0 +1,41 @@ +/* Source-owned mixed-instruction shared-image control. */ +.abicalls +.module fp=xx +.module nooddspreg +.text +.set noreorder +.set nomicromips +.set mips16 +.globl rea_mips16_probe +.type rea_mips16_probe, @function +.ent rea_mips16_probe +rea_mips16_probe: + li $2, 90 + xor $2, $4 + addiu $2, 7 + jrc $31 +.end rea_mips16_probe +.size rea_mips16_probe, . - rea_mips16_probe +.ifndef REA_MIPS_SYMBOL_RELATIVE +.hidden rea_mips16_probe +.endif + +.set nomips16 +.globl rea_mips32_probe +.type rea_mips32_probe, @function +.ent rea_mips32_probe +rea_mips32_probe: + addiu $2, $4, 3 + jr $31 + nop +.end rea_mips32_probe +.size rea_mips32_probe, . - rea_mips32_probe + +.data +.balign 4 +.globl rea_mips16_pointer +.type rea_mips16_pointer, @object +.size rea_mips16_pointer, 4 +rea_mips16_pointer: + .word rea_mips16_probe +.section .note.GNU-stack, "", @progbits diff --git a/tests/fixtures/analysisView.ts b/tests/fixtures/analysisView.ts index d3be2b1e3..fb140c08e 100644 --- a/tests/fixtures/analysisView.ts +++ b/tests/fixtures/analysisView.ts @@ -1,3 +1,5 @@ +import { apiSurfaceHar } from "./historicalHar.js"; +import { webNetworkCaptureSchema } from "../../src/domain/webNetworkCapture.js"; import { BINARY_LAYOUT_TEST_PROVIDER } from "./binaryDiagnostics/layout.js"; import { binaryLayoutFixture } from "./binaryDiagnostics/layout.js"; import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../src/contracts/javascript/javascriptRuntimeReconciliationExample.js"; @@ -193,3 +195,75 @@ export const analysisViewJavaScriptEvidence = ( limitations: analysis.limitations, }, ); + +/** Capture parent with explicit producer ordinals, including excluded URL values. */ +export const analysisViewCaptureFixture = () => + webNetworkCaptureSchema.parse({ + artifact: { + path: "/fixtures/surface.har", + sha256: "c".repeat(64), + bytes: 1024, + }, + format: "har", + decoder: { id: "fixture", name: "fixture", version: "1" }, + container: { + reported: {}, + numeric_literals: [], + redactions: [], + records_pointer: "/log/entries", + }, + total_records: 15, + records: [ + ...apiSurfaceHar().log.entries, + { request: { method: "GET", url: null } }, + { request: { method: "GET", url: "invalid URL" } }, + ].map((reported, ordinal) => ({ + ordinal, + location: { kind: "json-pointer", pointer: `/log/entries/${ordinal}` }, + reported, + binary_fields: [], + numeric_literals: [], + redactions: [], + limitations: [], + })), + runtime_attribution: "unknown", + limitations: [], + }); + +export const analysisViewCaptureEvidence = ( + capture = analysisViewCaptureFixture(), +): Evidence => + createEvidence( + capture.artifact.sha256 === null + ? undefined + : { + path: capture.artifact.path, + sha256: capture.artifact.sha256, + format: "file", + }, + { id: "fixture", name: "fixture", version: "1" }, + { + operation: "inspect_web_network_capture", + parameters: { sensitive_value_count: 0 }, + result: capture, + confidence: "observed", + }, + ); + +/** A small encoded capture whose many short objects expand during validation. */ +export const analysisViewObjectDenseCapture = (headerCount = 100_000) => { + const capture = analysisViewCaptureFixture(); + capture.records = capture.records.slice(0, 1); + capture.total_records = 1; + capture.records[0]!.reported = { + request: { + method: "GET", + url: "https://api.test/items/1", + headers: Array.from({ length: headerCount }, () => ({ + name: "x", + value: "y", + })), + }, + }; + return capture; +}; diff --git a/tests/fixtures/android/jadx-mcp.mjs b/tests/fixtures/android/jadx-mcp.mjs index 4c008a7f4..212cf4f5b 100644 --- a/tests/fixtures/android/jadx-mcp.mjs +++ b/tests/fixtures/android/jadx-mcp.mjs @@ -26,6 +26,7 @@ const classes = [ { length: generatedCount }, (_, index) => `fixture.Class${index}`, ), + ...(mode === "duplicate-class-name" ? ["fixture.Target"] : []), ]; const methods = [ { diff --git a/tests/fixtures/applicationMcpHarness.ts b/tests/fixtures/applicationMcpHarness.ts index f9e8f38b1..95196e621 100644 --- a/tests/fixtures/applicationMcpHarness.ts +++ b/tests/fixtures/applicationMcpHarness.ts @@ -14,12 +14,22 @@ export interface ApplicationMcpHarness { } /** Connect application workflows through the production MCP registration. */ -export async function createApplicationMcpHarness(): Promise { - const session = createTestBinarySession(() => ({ - execute: () => Promise.resolve(observed(null)), - close: () => Promise.resolve(resultOk(null)), - })); - const server = createServer({ kind: "session", session }); +export async function createApplicationMcpHarness( + options: { + readonly session?: BinarySession; + readonly serverOptions?: Parameters[1]; + } = {}, +): Promise { + const session = + options.session ?? + createTestBinarySession(() => ({ + execute: () => Promise.resolve(observed(null)), + close: () => Promise.resolve(resultOk(null)), + })); + const server = createServer( + { kind: "session", session }, + options.serverOptions, + ); const client = new Client({ name: "application-workflow-test", version: "1", diff --git a/tests/fixtures/browserScenarioCancellation.mjs b/tests/fixtures/browserScenarioCancellation.mjs index 2e0918f08..75c278f05 100644 --- a/tests/fixtures/browserScenarioCancellation.mjs +++ b/tests/fixtures/browserScenarioCancellation.mjs @@ -103,9 +103,12 @@ export const createBrowserScenarioProvider = () => const composition = pathToFileURL( root + "/dist/composition/browserScenario.js", ).href; +const bundledComposition = pathToFileURL( + root + "/dist/cli-runtime/composition__browserScenario-", +).href; registerHooks({ load(url, context, nextLoad) { - if (url === composition) + if (url === composition || url.startsWith(bundledComposition)) return { format: "module", shortCircuit: true, diff --git a/tests/fixtures/clientEnvironment.ts b/tests/fixtures/clientEnvironment.ts index 91ccb92cf..35fc6843c 100644 --- a/tests/fixtures/clientEnvironment.ts +++ b/tests/fixtures/clientEnvironment.ts @@ -15,6 +15,8 @@ export const CLIENT_LOCATION_ENVIRONMENT = [ "APPDATA", "CLAUDE_CONFIG_DIR", "CODEX_HOME", + "DSH_HOME", + "DSH_AGENTS_HOME", "COPILOT_HOME", "GROK_HOME", "GEMINI_CLI_HOME", diff --git a/tests/fixtures/historicalHar.ts b/tests/fixtures/historicalHar.ts index 1c2542105..217b9baff 100644 --- a/tests/fixtures/historicalHar.ts +++ b/tests/fixtures/historicalHar.ts @@ -34,3 +34,41 @@ export const historicalHar = () => ({ ], }, }); + +/** Request paths exercising only the public API surface format rules. */ +export const apiSurfaceHar = () => { + const fixture = historicalHar(); + const entry = fixture.log.entries[0]!; + const paths = [ + "/users/8812", + "/users/9031", + "/ids/550e8400-e29b-41d4-a716-446655440000", + "/ids/abcdef0123456789abcdef0123456789", + "/dates/2024-02-29", + "/api/v1/users", + "/api/v2/users", + "/api/Report2", + "/api/Report3", + "/api/CreateDocument", + "/api/ListDocuments", + "/ids/abc123", + "/dates/2023-02-29", + ]; + fixture.log.entries = paths.map((path, index) => ({ + ...entry, + request: { + ...entry.request, + url: `https://api.example.test${path}?not-in-view=value`, + }, + response: { + ...entry.response, + status: index === 1 ? 404 : 200, + content: { + ...entry.response.content, + mimeType: + index === 1 ? "text/plain" : "application/json; charset=utf-8", + }, + }, + })); + return fixture; +}; diff --git a/tests/fixtures/keyedArchivePrototypeKeys.ts b/tests/fixtures/keyedArchivePrototypeKeys.ts new file mode 100644 index 000000000..421935d4d --- /dev/null +++ b/tests/fixtures/keyedArchivePrototypeKeys.ts @@ -0,0 +1,58 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; + +import { keyedArchiveResultSchema } from "../../src/domain/apple/keyedArchive.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +/** Inert XML with producer-equivalent key spellings and unchanged ordinary data. */ +export const keyedArchivePrototypeKeyFixture = async () => { + const directory = await createTestTempDirectory("rea-keyed-xml-keys-"); + const path = join(directory, "archive.plist"); + const bytes = Buffer.from(` + + $archiverNSKeyedArchiver + $version100000 + $objects$null + __proto__omitted outer value + kepta b…c
d + __rea_prototype_key__ordinary key + nested + + omitted nested value + value7 + + + $toproot + CF$UID1 + +`); + await writeFile(path, bytes); + return { path, digest: createHash("sha256").update(bytes).digest("hex") }; +}; + +/** Require original source identity, ordinary values and the exact omission count. */ +export const expectKeyedArchivePrototypeKeys = ( + value: unknown, + digest: string, +) => { + const result = keyedArchiveResultSchema.parse(value); + expect(result.archive_sha256).toBe(digest); + expect(result.roots).toEqual({ root: { CF$UID: 1 } }); + expect(result.objects).toHaveLength(2); + expect(result.objects[1]).toMatchObject({ + id: 1, + kind: "dictionary", + value: { + kept: "a\rb\u0085c\u2028d", + __rea_prototype_key__: "ordinary key", + nested: { value: 7 }, + }, + }); + expect(result.limitations).toContain( + "2 dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.", + ); + expect(JSON.stringify(result)).not.toContain("omitted outer value"); + expect(JSON.stringify(result)).not.toContain("omitted nested value"); +}; diff --git a/tests/fixtures/native-macos/otool-dylib-use.md b/tests/fixtures/native-macos/otool-dylib-use.md new file mode 100644 index 000000000..750f04426 --- /dev/null +++ b/tests/fixtures/native-macos/otool-dylib-use.md @@ -0,0 +1,14 @@ +# Dylib-use otool capture + +`otool-dylib-use.txt` is the actual `otool -h -l` stdout retained by the +[Apple CLI/MCP verification job](https://github.com/morluto/rea/actions/runs/38176410166/job/114579755550). +Only its first filename line was replaced with `/owned/fixture:`. + +The source-built application comes from `scripts/verify/apple/macos-bundle.mjs`. +Its 23 load commands include a delayed dependency on `@rpath/libdelay.dylib`. +That dylib-use command has an `options delay-init` row instead of the legacy +`time stamp` row. The other three dylib commands retain the legacy form. + +The capture establishes actual producer text, command framing and dependency +fields. Linux replay tests do not establish real macOS provider execution; +the Apple CLI/MCP verification lane supplies that coverage. diff --git a/tests/fixtures/native-macos/otool-dylib-use.txt b/tests/fixtures/native-macos/otool-dylib-use.txt new file mode 100644 index 000000000..28d40b4df --- /dev/null +++ b/tests/fixtures/native-macos/otool-dylib-use.txt @@ -0,0 +1,275 @@ +/owned/fixture: +Mach header + magic cputype cpusubtype caps filetype ncmds sizeofcmds flags + 0xfeedfacf 16777228 0 0x00 2 23 1632 0x00200085 +Load command 0 + cmd LC_SEGMENT_64 + cmdsize 72 + segname __PAGEZERO + vmaddr 0x0000000000000000 + vmsize 0x0000000100000000 + fileoff 0 + filesize 0 + maxprot 0x00000000 + initprot 0x00000000 + nsects 0 + flags 0x0 +Load command 1 + cmd LC_SEGMENT_64 + cmdsize 552 + segname __TEXT + vmaddr 0x0000000100000000 + vmsize 0x0000000000004000 + fileoff 0 + filesize 16384 + maxprot 0x00000005 + initprot 0x00000005 + nsects 6 + flags 0x0 +Section + sectname __text + segname __TEXT + addr 0x00000001000006b0 + size 0x0000000000000080 + offset 1712 + align 2^2 (4) + reloff 0 + nreloc 0 + flags 0x80000400 + reserved1 0 + reserved2 0 +Section + sectname __stubs + segname __TEXT + addr 0x0000000100000730 + size 0x0000000000000024 + offset 1840 + align 2^2 (4) + reloff 0 + nreloc 0 + flags 0x80000408 + reserved1 0 (index into indirect symbol table) + reserved2 12 (size of stubs) +Section + sectname __delay_stubs + segname __TEXT + addr 0x0000000100000754 + size 0x0000000000000028 + offset 1876 + align 2^2 (4) + reloff 0 + nreloc 0 + flags 0x80000400 + reserved1 0 + reserved2 0 +Section + sectname __delay_helper + segname __TEXT + addr 0x000000010000077c + size 0x00000000000000a0 + offset 1916 + align 2^2 (4) + reloff 0 + nreloc 0 + flags 0x80000400 + reserved1 0 + reserved2 0 +Section + sectname __cstring + segname __TEXT + addr 0x000000010000081c + size 0x0000000000000037 + offset 2076 + align 2^0 (1) + reloff 0 + nreloc 0 + flags 0x00000002 + reserved1 0 + reserved2 0 +Section + sectname __unwind_info + segname __TEXT + addr 0x0000000100000854 + size 0x0000000000000058 + offset 2132 + align 2^2 (4) + reloff 0 + nreloc 0 + flags 0x00000000 + reserved1 0 + reserved2 0 +Load command 2 + cmd LC_SEGMENT_64 + cmdsize 152 + segname __DATA_CONST + vmaddr 0x0000000100004000 + vmsize 0x0000000000004000 + fileoff 16384 + filesize 16384 + maxprot 0x00000003 + initprot 0x00000003 + nsects 1 + flags 0x10 +Section + sectname __got + segname __DATA_CONST + addr 0x0000000100004000 + size 0x0000000000000028 + offset 16384 + align 2^3 (8) + reloff 0 + nreloc 0 + flags 0x00000006 + reserved1 3 (index into indirect symbol table) + reserved2 0 +Load command 3 + cmd LC_SEGMENT_64 + cmdsize 152 + segname __DATA + vmaddr 0x0000000100008000 + vmsize 0x0000000000004000 + fileoff 32768 + filesize 16384 + maxprot 0x00000003 + initprot 0x00000003 + nsects 1 + flags 0x0 +Section + sectname __data + segname __DATA + addr 0x0000000100008000 + size 0x0000000000000004 + offset 32768 + align 2^2 (4) + reloff 0 + nreloc 0 + flags 0x00000000 + reserved1 0 + reserved2 0 +Load command 4 + cmd LC_SEGMENT_64 + cmdsize 72 + segname __LINKEDIT + vmaddr 0x000000010000c000 + vmsize 0x0000000000008000 + fileoff 49152 + filesize 19280 + maxprot 0x00000001 + initprot 0x00000001 + nsects 0 + flags 0x0 +Load command 5 + cmd LC_DYLD_CHAINED_FIXUPS + cmdsize 16 + dataoff 49152 + datasize 160 +Load command 6 + cmd LC_DYLD_EXPORTS_TRIE + cmdsize 16 + dataoff 49312 + datasize 48 +Load command 7 + cmd LC_SYMTAB + cmdsize 24 + symoff 49368 + nsyms 10 + stroff 49560 + strsize 176 +Load command 8 + cmd LC_DYSYMTAB + cmdsize 80 + ilocalsym 0 + nlocalsym 3 + iextdefsym 3 + nextdefsym 2 + iundefsym 5 + nundefsym 5 + tocoff 0 + ntoc 0 + modtaboff 0 + nmodtab 0 + extrefsymoff 0 + nextrefsyms 0 + indirectsymoff 49528 + nindirectsyms 8 + extreloff 0 + nextrel 0 + locreloff 0 + nlocrel 0 +Load command 9 + cmd LC_LOAD_DYLINKER + cmdsize 32 + name /usr/lib/dyld (offset 12) +Load command 10 + cmd LC_UUID + cmdsize 24 + uuid 2F026098-89DD-47C5-A6E4-7C567C2ACD30 +Load command 11 + cmd LC_BUILD_VERSION + cmdsize 32 + platform 1 + minos 15.0 + sdk 15.5 + ntools 1 + tool 3 + version 1167.5 +Load command 12 + cmd LC_SOURCE_VERSION + cmdsize 16 + version 0.0 +Load command 13 + cmd LC_MAIN + cmdsize 24 + entryoff 1712 + stacksize 0 +Load command 14 + cmd LC_LOAD_DYLIB + cmdsize 64 + name @rpath/Core.framework/Versions/A/Core (offset 24) + time stamp 2 Thu Jan 1 00:00:02 1970 + current version 0.0.0 +compatibility version 0.0.0 +Load command 15 + cmd LC_LOAD_WEAK_DYLIB + cmdsize 48 + name @rpath/libgone.dylib (offset 24) + time stamp 2 Thu Jan 1 00:00:02 1970 + current version 0.0.0 +compatibility version 0.0.0 +Load command 16 + cmd LC_LOAD_DYLIB + cmdsize 56 + name @rpath/libdelay.dylib (offset 28) + options delay-init + current version 0.0.0 +compatibility version 1.0.0 +Load command 17 + cmd LC_LOAD_DYLIB + cmdsize 56 + name /usr/lib/libSystem.B.dylib (offset 24) + time stamp 2 Thu Jan 1 00:00:02 1970 + current version 1351.0.0 +compatibility version 1.0.0 +Load command 18 + cmd LC_RPATH + cmdsize 48 + path @executable_path/../Overrides (offset 12) +Load command 19 + cmd LC_RPATH + cmdsize 48 + path @executable_path/../Frameworks (offset 12) +Load command 20 + cmd LC_FUNCTION_STARTS + cmdsize 16 + dataoff 49360 + datasize 8 +Load command 21 + cmd LC_DATA_IN_CODE + cmdsize 16 + dataoff 49368 + datasize 0 +Load command 22 + cmd LC_CODE_SIGNATURE + cmdsize 16 + dataoff 49744 + datasize 18688 diff --git a/tests/fixtures/wasm/README.md b/tests/fixtures/wasm/README.md index 9743996dc..1ab586d9c 100644 --- a/tests/fixtures/wasm/README.md +++ b/tests/fixtures/wasm/README.md @@ -9,5 +9,6 @@ custom section. The custom payload is retained as an `@custom` annotation in dec Upstream tag commit: `ff0ef7e0009402740c805a9744c09b05be063e48`. The separate real-tool verifier generates its own fixtures with caller-supplied WABT and compares exact producer output against the public CLI/MCP surfaces. -`custom-names.wat` covers newlines, quotes, Unicode, backslashes and embedded -section markers in custom names; objdump displays those names without escaping. +`custom-names.wat` covers newlines, quotes, Unicode, backslashes, embedded +section markers and NUL in custom names; objdump displays those names without +escaping and ends each at its first NUL. diff --git a/tests/fixtures/wasm/custom-names.wat b/tests/fixtures/wasm/custom-names.wat index c3c2dc988..2231c040d 100644 --- a/tests/fixtures/wasm/custom-names.wat +++ b/tests/fixtures/wasm/custom-names.wat @@ -1,4 +1,5 @@ (module (@custom "test\0aline" "payload") (@custom "fake\22\0aSection Details:\0a Type start=0x0000000a end=0x0000000b (size=0x00000001) count: 0\0a\22end" "payload") - (@custom "unicode-\c3\a9\\\t" "payload")) + (@custom "unicode-\c3\a9\\\t" "payload") + (@custom "visible\00hidden" "payload")) diff --git a/tests/support/javascriptApplicationScope.ts b/tests/support/javascriptApplicationScope.ts index c9c920f02..7f5d51b80 100644 --- a/tests/support/javascriptApplicationScope.ts +++ b/tests/support/javascriptApplicationScope.ts @@ -12,9 +12,10 @@ export const readJavaScriptArtifactFiles = ( reader: ArtifactReader, snapshot: ArtifactInventorySnapshot, signal?: AbortSignal, + sourcePaths?: readonly string[], ) => withArtifactResourceScope((scope) => - readOwned(reader, snapshot, scope, signal), + readOwned(reader, snapshot, scope, signal, sourcePaths), ); /** Run a JavaScript analysis with an awaited, test-owned artifact scope. */ diff --git a/turbo.json b/turbo.json index 03ce42b73..a177552d3 100644 --- a/turbo.json +++ b/turbo.json @@ -10,7 +10,7 @@ "package-lock.json", "scripts/check-dependency-install.mjs", "scripts/clean-build-output.mjs", - "scripts/bundle-mcp-runtime.mjs", + "scripts/bundle-runtime.mjs", "scripts/generate-package-metadata.mjs", "src/**/*.ts", "!src/**/*.test.ts",