Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8005cad374 | ||
|
|
75881af7aa | ||
|
|
8d3cfef753 | ||
|
|
975dff2e25 | ||
|
|
1b527f4898 | ||
|
|
b09dd333ab | ||
|
|
60fbb2f06e | ||
|
|
8b4c96e6c6 | ||
|
|
95a77b6e25 | ||
|
|
67966c69a5 | ||
|
|
279c942fe1 | ||
|
|
54e2890b30 | ||
|
|
0f605bca09 | ||
|
|
423b11c8ef | ||
|
|
52331ab637 | ||
|
|
3e931a7e35 | ||
|
|
6edd5cf045 | ||
|
|
1a6ccec636 | ||
|
|
afc9d819aa | ||
|
|
7b3011b41e | ||
|
|
6849ecf70a | ||
|
|
926c2d6ba7 | ||
|
|
c7345512ea | ||
|
|
53e7fadbd0 | ||
|
|
215001a1ce | ||
|
|
9515d2fcd3 | ||
|
|
ac9677bd6c | ||
|
|
69fe5b6010 | ||
|
|
684199ba0b | ||
|
|
a17e257f43 | ||
|
|
5502ca9c97 | ||
|
|
be75ed5e4b | ||
|
|
3f8c51e9b5 | ||
|
|
e0e49e9dd5 | ||
|
|
be6272f261 | ||
|
|
32d13c431e | ||
|
|
0fa0b04628 | ||
|
|
ffe0574e17 | ||
|
|
87d5ec93d5 | ||
|
|
f1c58cc53f | ||
|
|
6636045c9c | ||
|
|
b74d9a1bb2 | ||
|
|
6e0d5d64d3 | ||
|
|
d6f698d8af | ||
|
|
bbd01923e9 | ||
|
|
5a77a962ec | ||
|
|
6585293c56 | ||
|
|
be44b5098e | ||
|
|
6db97d5162 | ||
|
|
39da0b150b | ||
|
|
3967cfa43e | ||
|
|
2014ee7d68 | ||
|
|
5100a1a1a5 | ||
|
|
de0d341524 | ||
|
|
d1f04512af | ||
|
|
514df0918c | ||
|
|
8fb79c67ee | ||
|
|
6a41a29963 | ||
|
|
c12a532b9c | ||
|
|
b28c0337fc | ||
|
|
f4e437e544 | ||
|
|
8ba0a887ff | ||
|
|
f9ea9a9268 | ||
|
|
78778d91a4 | ||
|
|
d510f3e928 | ||
|
|
043fd2579d | ||
|
|
a761981897 | ||
|
|
2653b194b6 | ||
|
|
16d9c5adef | ||
|
|
b582b6ecb4 | ||
|
|
ba056e08a4 | ||
|
|
cb13a27bfa | ||
|
|
3d79903fb1 |
@@ -2,7 +2,7 @@
|
||||
name: reverse-engineer-anything
|
||||
description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.
|
||||
metadata:
|
||||
version: "36"
|
||||
version: "37"
|
||||
---
|
||||
|
||||
# REA
|
||||
@@ -25,57 +25,18 @@ a skill installed from repository main may describe capabilities absent from
|
||||
an older npm release. Keep complete inline Evidence when that server does not
|
||||
advertise retained references.
|
||||
|
||||
When tools are absent or registration is stale:
|
||||
If tools are absent, a call is blocked by the client, or a required provider is
|
||||
unavailable, read [connection and recovery](references/connection-and-recovery.md).
|
||||
Diagnose the observed blocker, continue useful supported work, and resume the
|
||||
original task after repair. Client approval, shell PATH, registration and engine
|
||||
availability have different remedies; avoid repeating setup for all of them.
|
||||
|
||||
1. Diagnose without changing files. For Codex, run
|
||||
`npx -y rea-agents@latest doctor --client codex --json`. Substitute the current
|
||||
supported client: `claude_code`, `claude_desktop`, `codex`, `cursor`,
|
||||
`gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`,
|
||||
`commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`, `hermes`, or `grok_bot`. `grok_bot` has no local
|
||||
file registration: doctor reports the Grok Bot chat step, and setup does not
|
||||
write `mcp.json`. If the client is unknown, use `doctor --json` and
|
||||
inspect its registration results before choosing a setup scope.
|
||||
2. Distinguish the reason. Missing, malformed, or stale registration needs a
|
||||
scoped configuration repair. An aligned registration with no tools in the
|
||||
active session needs a restart/reconnection; doctor cannot prove that the
|
||||
current agent has connected. A missing provider affects only tasks requiring
|
||||
that provider: static JavaScript inspection needs neither Hopper nor Ghidra,
|
||||
and Android inspection has separate bring-your-own JADX/Java prerequisites.
|
||||
3. For a configuration repair, prepare the read-only plan:
|
||||
`npx -y rea-agents@latest setup --client codex --dry-run --json`.
|
||||
Use the current client's ID, show its exact proposed paths, backups, and
|
||||
changes, and obtain approval before setup writes configuration or installs
|
||||
Hopper. Setup normally installs the matching bundled skill too; include that
|
||||
replacement in the reviewed plan. After approval, apply the same scope with
|
||||
`npx -y rea-agents@latest setup --client codex --yes`. Add `--install-hopper`
|
||||
only if that separate installation was needed and explicitly approved.
|
||||
4. Restart/reconnect the affected agent. Verify that REA tools actually appear
|
||||
in the session, then resume the original investigation. If they remain
|
||||
absent, inspect the client's MCP launch error rather than repeating setup.
|
||||
|
||||
REA setup never installs or upgrades Node.js, npm, Homebrew, Java, Ghidra, IDA,
|
||||
JADX, Binwalk, or Unblob. Use existing prerequisites; do not install unrelated
|
||||
software to repair
|
||||
MCP registration. For an unsupported client, use manual stdio registration
|
||||
with a version-pinned `rea-agents` package or continue through the CLI.
|
||||
|
||||
A concrete CLI fallback for an operator-supplied JavaScript tree or ASAR is:
|
||||
|
||||
```bash
|
||||
npx -y rea-agents@latest analyze-javascript-application /absolute/path/to/app --json > app-evidence.json
|
||||
jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json
|
||||
npx -y rea-agents@latest inspect-analysis-view app-view.json
|
||||
```
|
||||
|
||||
No MCP registration or native engine is required. The CLI returns the complete
|
||||
Evidence record directly, and a real application can produce hundreds of
|
||||
megabytes, so save it to a file rather than reading it into context. Start from
|
||||
the summary view, then request a module page
|
||||
(`{"kind": "page", "collection": "modules", "offset": 0, "limit": 32}`) or one
|
||||
module the same way, or pass the saved Evidence inline to
|
||||
`trace-application-feature`. Read graph, limitations, and unknowns with the
|
||||
same care as an MCP result. This fallback does not establish that MCP is
|
||||
configured. Native CLI tasks still require their selected engine.
|
||||
Prefer an existing `rea` CLI. When none is installed and REA package acquisition
|
||||
is within the authorized scope, use `npx -y rea-agents@latest` in its place.
|
||||
Configuration repair starts with a read-only setup plan; show paths, backups and
|
||||
skill replacement before applying authorized changes. Preserve existing approval
|
||||
and target-execution boundaries. REA setup never installs or upgrades unrelated
|
||||
software, including Node.js, npm, Homebrew, Java, Ghidra, IDA, JADX, Binwalk or Unblob.
|
||||
|
||||
## Route the target first
|
||||
|
||||
@@ -140,9 +101,11 @@ after opening; IDA does not supply `binary_overview`. Consult session availabili
|
||||
when composing broader workflows. Modern direct callers and unsupported dossier
|
||||
facets remain unknown; live IDA results are not replayed from snapshots.
|
||||
|
||||
If the app is missing, ask which app to inspect. Resolve a human-readable app
|
||||
name to one clear installed artifact when possible; ask only when matches are
|
||||
ambiguous. Never choose an example app on the user's behalf.
|
||||
If the user says only "my app" without a name, path, or target already selected
|
||||
for this task, ask which app before searching for artifacts. Paths in unrelated
|
||||
saved reports and nearby test fixtures do not select the user's target. Resolve
|
||||
an actual human-readable app name to one clear installed artifact when possible;
|
||||
ask when matches are ambiguous. Never choose an example app on the user's behalf.
|
||||
|
||||
In a target-free session, use `open_binary` to bind any archive/package or
|
||||
native target whose analysis tool operates on the active target. Do not call an
|
||||
@@ -157,7 +120,11 @@ For a JavaScript application or ELF layout whose size is unknown or large,
|
||||
call `analyze_javascript_application` or `inspect_binary_layout` with
|
||||
`"detail": "summary"`, then inspect the retained `parent_evidence_id` with
|
||||
`inspect_analysis_view`.
|
||||
Do not repeat an identical tool call. Make a focused follow-up only when the
|
||||
Page module identities and select the returned `node_id` before requesting an
|
||||
item; a single file path can represent several graph roles. Do not repeat an
|
||||
identical analysis or rejected call without a changed input or resolved blocker.
|
||||
Recheck session state when a lifecycle change makes that status useful.
|
||||
Make a focused follow-up only when the
|
||||
returned result leaves a specific question unanswered.
|
||||
|
||||
If MCP reports `resource_constraint` with `details.resource: "transport"`, call
|
||||
@@ -168,6 +135,14 @@ session to a selected path. The analysis remains complete in the ledger; avoid
|
||||
repeating it merely to request the same oversized response. Complete CLI JSON
|
||||
output also streams without a single MCP frame.
|
||||
|
||||
A client can also truncate a successful tool result without a REA transport
|
||||
error. Treat that preview as incomplete; use the retained parent ID for smaller
|
||||
views or export when the full record is needed. Do not parse the preview as
|
||||
canonical JSON or repeat the producer. For JavaScript, read the
|
||||
[module-view and trace recipes](references/javascript-applications.md) before
|
||||
drilling into a large graph; a broad bidirectional trace can expand unrelated
|
||||
containment relationships even after summary analysis.
|
||||
|
||||
Every conclusion must distinguish observations, inferences, and unknowns. Cite
|
||||
Evidence IDs, preserve limitations and incomplete coverage, and never imply
|
||||
that static analysis observed execution. Runtime requests execute the declared
|
||||
@@ -175,7 +150,9 @@ target and lifecycle; do not broaden the target or action beyond those fields.
|
||||
|
||||
Within the user's requested investigation, call available analysis tools
|
||||
directly. REA does not require permission grants or per-call approval flags.
|
||||
Follow the declared request scope and the host's actual access requirements.
|
||||
The agent client can independently require approval. Follow the declared request
|
||||
scope and the host's actual access requirements; use the recovery guide for an
|
||||
observed client-policy rejection.
|
||||
|
||||
## Plan broader investigations
|
||||
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
# Connection and task recovery
|
||||
|
||||
Read this guide only for an observed connection, command, approval or provider
|
||||
blocker. Keep the user's target and question in view. Explain what prevented the
|
||||
operation and the next useful step; after repair, continue the investigation.
|
||||
|
||||
Commands below use an installed `rea`. If none is available, use the package
|
||||
invocation in `SKILL.md` when acquiring REA is authorized. Follow the connected
|
||||
server's schemas and the installed CLI's help rather than assuming repository
|
||||
main and an older release expose the same arguments.
|
||||
|
||||
## Choose the remedy from the observed failure
|
||||
|
||||
| Observation | Meaning and next step |
|
||||
| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. |
|
||||
| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. |
|
||||
| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. |
|
||||
| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. |
|
||||
| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Resolve only the prerequisite needed for the task; a complete tool catalog does not imply every operation is available. |
|
||||
| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. |
|
||||
|
||||
If the client cannot launch its shell/sandbox helper or read the installed skill,
|
||||
inspect its launch diagnostic. For example, Codex can refuse helper aliases when
|
||||
CODEX_HOME is under `/tmp`; use an authorized profile outside the OS temporary
|
||||
directory. REA setup cannot repair the client's executor. Preserve the sandbox
|
||||
instead of silently disabling it to make an analysis command work.
|
||||
|
||||
Static JavaScript and ordinary ZIP inventory need neither Hopper nor Ghidra.
|
||||
HAR inspection is offline and does not require a live browser. Native
|
||||
decompilation, Android analysis and firmware extraction have their own
|
||||
bring-your-own prerequisites. Use supported inventory/static evidence when it
|
||||
answers part of the question, and name the unavailable evidence separately.
|
||||
Do not substitute another target. Label observations from ordinary shell tools
|
||||
as such; only actual REA output supplies REA Evidence.
|
||||
|
||||
## Repair registration when needed
|
||||
|
||||
For Codex:
|
||||
|
||||
```bash
|
||||
rea doctor --client codex --json
|
||||
rea setup --client codex --dry-run --json
|
||||
```
|
||||
|
||||
Use the current supported client's ID: `claude_code`, `claude_desktop`, `codex`,
|
||||
`cursor`, `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`,
|
||||
`copilot_cli`, `commandcode`, `qwen_code`, `vscode`, `grok_build`, `omp`, `pi`,
|
||||
`hermes`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a
|
||||
scope. `grok_bot` uses a chat registration step rather than local `mcp.json`.
|
||||
For unsupported clients, use version-pinned manual stdio registration or CLI.
|
||||
|
||||
Show the plan's exact paths, backups, changes and bundled-skill replacement.
|
||||
Obtain approval for writes when that concrete scope is not already authorized,
|
||||
then apply the same scope with `rea setup --client codex --yes`. Add
|
||||
`--install-hopper` only when separately needed and explicitly approved. Do not
|
||||
install or upgrade unrelated prerequisites. Restart/reconnect, verify tools
|
||||
actually appear, and resume the original task. If they remain absent, inspect
|
||||
the launch failure rather than looping through setup.
|
||||
|
||||
## Codex tool approval is separate from REA readiness
|
||||
|
||||
REA tools have no permission-grant handshake. Codex can still require approval
|
||||
because an operation creates owned resources, launches a helper or has cleanup
|
||||
effects. An advertised tool or successful doctor result does not override that
|
||||
client policy. `workspace-write` alone does not resolve an MCP approval
|
||||
rejection, and `approval_policy = "never"` does not mean every MCP tool is
|
||||
preapproved.
|
||||
|
||||
When interactive approval is available, use the client's approval flow for the
|
||||
requested operation. In an unattended run, continue through an authorized CLI
|
||||
workflow if the execution environment permits it, or report the blocked
|
||||
operation and the approval needed to proceed. Do not retry the same rejected
|
||||
call, silently broaden approvals, or rewrite MCP effects as read-only.
|
||||
|
||||
For an operator who explicitly wants to preapprove named operations in a
|
||||
disposable Codex profile, current Codex supports per-tool configuration. For
|
||||
example, opening and closing an analysis session can be scoped independently:
|
||||
|
||||
```toml
|
||||
[mcp_servers.rea.tools.open_binary]
|
||||
approval_mode = "approve"
|
||||
|
||||
[mcp_servers.rea.tools.close_binary]
|
||||
approval_mode = "approve"
|
||||
```
|
||||
|
||||
Use the actual server ID and only the operations whose effects were approved;
|
||||
this example does not approve inspection, extraction or runtime execution.
|
||||
Show the proposed config edit and preserve unrelated entries and backups before
|
||||
writing. Managed policy can constrain client settings. Consult the
|
||||
[Codex configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference)
|
||||
for the installed client's supported settings. This is an optional recovery
|
||||
choice, not a prerequisite for every REA task.
|
||||
|
||||
## Continue through an installed CLI
|
||||
|
||||
Check the command environment only if the launcher fails:
|
||||
|
||||
```bash
|
||||
command -v rea
|
||||
command -v node
|
||||
node --version
|
||||
rea --version
|
||||
```
|
||||
|
||||
Use the public launcher at its known installation path with a supported existing
|
||||
Node on PATH when the child shell loses a prefix installation. Avoid internal
|
||||
`dist/cli.js` entrypoints or global toolchain changes. MCP may have its own valid
|
||||
launcher even when a shell cannot resolve `rea`.
|
||||
|
||||
For a user-supplied JavaScript tree or ASAR, select output files in the authorized
|
||||
workspace and save complete Evidence before reading a view:
|
||||
|
||||
```bash
|
||||
rea analyze-javascript-application /absolute/path/to/app --json > app-evidence.json
|
||||
jq -c '{source: {kind: "inline", evidence: .}, view: {kind: "summary"}}' app-evidence.json > app-view.json
|
||||
rea inspect-analysis-view app-view.json
|
||||
```
|
||||
|
||||
These commands do not register MCP or install an analysis engine. CLI JavaScript
|
||||
JSON is complete and can be very large; redirect it rather than loading it into
|
||||
context. Use an existing JSON processor if `jq` is absent, without installing
|
||||
one just for this recipe. Inspect a module page or item using the same inline
|
||||
source and the [JavaScript recipes](javascript-applications.md). Preserve graph
|
||||
coverage, limitations and unknowns. Native CLI analysis still requires its
|
||||
selected engine, and all CLI operations retain the host's execution permissions.
|
||||
@@ -1,20 +1,90 @@
|
||||
# JavaScript and Electron application artifacts
|
||||
|
||||
Use `analyze_javascript_application` directly on the operator-supplied ASAR or
|
||||
extracted tree. The complete result, graph, and Evidence context are returned
|
||||
inline when they fit the MCP frame. Source-map contents are part of the static analysis. Start from its
|
||||
findings, coverage, unknowns, and graph context, then make a focused follow-up
|
||||
only when a specific question remains unanswered.
|
||||
extracted tree. When size is unknown or large and the connected schema advertises
|
||||
`detail`, request `detail: "summary"`. This returns an `inspect_analysis_view`
|
||||
Evidence record with a distinct ID and a `normalized_result.parent_evidence_id`
|
||||
for the complete retained application analysis. Source-map contents are part of
|
||||
that static analysis. Read findings, coverage, limitations and unknowns, then
|
||||
follow a specific unanswered question. Older servers without `detail` return
|
||||
the complete application Evidence inline when it fits.
|
||||
|
||||
## Read a module without repeating analysis
|
||||
|
||||
Use the actual returned parent ID, on the same connection, as `source.evidence_id`.
|
||||
The following IDs are placeholders to replace with returned values. When the
|
||||
module's node ID is not already known, page identities before selecting an item.
|
||||
Pages return only node IDs, paths and roles; choose a useful page size and follow
|
||||
`coverage.next_offset` when another page matters:
|
||||
|
||||
```json
|
||||
{
|
||||
"source": {
|
||||
"kind": "retained-evidence",
|
||||
"evidence_id": "ev_ACTUAL_PARENT_ID"
|
||||
},
|
||||
"view": { "kind": "page", "collection": "modules", "offset": 0, "limit": 16 }
|
||||
}
|
||||
```
|
||||
|
||||
Select the relevant page entry by its exact `node_id` and recorded role:
|
||||
|
||||
```json
|
||||
{
|
||||
"source": {
|
||||
"kind": "retained-evidence",
|
||||
"evidence_id": "ev_ACTUAL_PARENT_ID"
|
||||
},
|
||||
"view": {
|
||||
"kind": "item",
|
||||
"collection": "modules",
|
||||
"selector": { "node_id": "jag_node_ACTUAL_NODE_ID" }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Module pages include asset, bundled and source module roles, so one path can
|
||||
identify several entries. Use `selector.path` only when uniqueness is established;
|
||||
the returned node ID avoids needing a complete inventory to prove uniqueness. On ambiguity,
|
||||
select the matching node ID from the page or the error's candidates. Follow
|
||||
`next_offset` when another page matters. An item exposes that module node's
|
||||
recorded observations, identity, hashes, exports and source ranges; it does not
|
||||
include every child syntax observation, related edge or original source file.
|
||||
An absent item fact is not evidence of absence from the whole application.
|
||||
|
||||
## Trace only the remaining question
|
||||
|
||||
Use `trace_application_feature` on the parent application Evidence for one
|
||||
literal node ID, route, string, API, IPC channel, module, or native export.
|
||||
Select `incoming` for relationships leading to the seed, `outgoing` for those
|
||||
leaving it, and `both` when the question needs both. Trace traversal includes
|
||||
containment relationships, so even a narrow seed can expand many nodes; a
|
||||
bidirectional trace can reach unrelated sibling modules. A module item or page
|
||||
is preferable when its observations already answer the question. Do not treat
|
||||
all reached nodes as participants in the feature.
|
||||
|
||||
Reuse retained parent Evidence when the server advertises it, or supply complete
|
||||
Evidence inline. Include Hopper or Ghidra Evidence only when its artifact digest
|
||||
matches exactly.
|
||||
|
||||
## Recover an oversized display
|
||||
|
||||
An oversized response reports `resource_constraint` and
|
||||
`details.resource: "transport"`. Reuse
|
||||
`details.reported_limits.evidence_reference` with `inspect_analysis_view` for a
|
||||
summary, one module, or a stable page of module identities. Use
|
||||
`trace_application_feature` once a module, route, or string seed is known.
|
||||
summary, one module, or a stable page of module identities.
|
||||
`export_evidence_bundle` writes the complete canonical session to a
|
||||
caller-selected file. These workflows keep the original Evidence and coverage;
|
||||
a broad follow-up can also exceed framing.
|
||||
|
||||
A client can truncate a successful result inside its text envelope without a
|
||||
REA `resource_constraint`. Treat a truncation notice or incomplete JSON as a
|
||||
display limitation; do not claim the full record was examined. Reuse the known
|
||||
parent ID for smaller views, or export complete Evidence to a selected file
|
||||
while the session remains open. If the parent ID was lost from the preview,
|
||||
use the prior analysis summary or explicit ledger entries to recover it; do
|
||||
not invent an ID or rerun analysis just to reproduce the same display.
|
||||
|
||||
BrowserWindow preferences, preload and contextBridge surfaces, IPC
|
||||
registrations, utility processes, and native binding requests are static syntax
|
||||
observations. Only a unique exact literal IPC channel match supports an inferred
|
||||
@@ -22,13 +92,6 @@ pairing. Dynamic or ambiguous channels remain unresolved. A requested `.node`
|
||||
member is not a verified native export. Never claim runtime reachability,
|
||||
registration, defaults, or policy enforcement from static analysis.
|
||||
|
||||
Use `trace_application_feature` on existing application Evidence for one
|
||||
literal node ID, route, string, API, IPC channel, module, or native export.
|
||||
Choose a direction and reuse the returned application Evidence ID when the
|
||||
connected server advertises retained references, or supply complete Evidence
|
||||
inline. Include
|
||||
Hopper or Ghidra Evidence only when its artifact digest matches exactly.
|
||||
|
||||
For version comparison, analyze each version once, then call
|
||||
`compare_application_versions`. Accept only its digest, source-map, structural
|
||||
fingerprint, or non-module semantic matches. Module ordinals and minified names
|
||||
|
||||
@@ -152,7 +152,7 @@ context. PSP/stack/device state remains unmodeled. See [DOS guide](https://githu
|
||||
|
||||
### Function annotations in Ghidra
|
||||
|
||||
Use `annotate_native_function` for one function name and/or entry comments, with at least one explicit change. Review its annotation readback and refreshed dossier inline. Changes are atomic and session-scoped; empty comments clear them, omitted fields preserve them. Later MCP calls observe edits until close. CLI `annotate-native-function` returns the updated analysis before discarding the session. Original executable bytes are unchanged; edits invalidate immutable snapshots. Windows P0 does not admit database mutations.
|
||||
Use `annotate_native_function` for one function name and/or entry comments, with at least one explicit change. Review its annotation readback and refreshed dossier inline. Changes are atomic and session-scoped; empty comments clear them, omitted fields preserve them. Later MCP calls observe edits until close. CLI `annotate-native-function` returns the updated analysis before discarding the session. Original executable bytes are unchanged; edits invalidate immutable snapshots. Windows P0 does not admit database mutations. For naming only, `set_address_name` / `set_addresses_names` also work with Ghidra: they name function entries or data addresses (primary labels) and return compact per-address success booleans instead of a dossier.
|
||||
|
||||
## Offline PE resource inventory
|
||||
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"_comment": "Test files are intentionally NOT excluded: the gate used to report zero clones because it skipped **/*.test.ts, hiding all test-tree duplication. Current actual is ~2.4% lines across src+ and tests/, down from 3.4% when tests were included. Lower this as duplication is removed.",
|
||||
"threshold": 2.5,
|
||||
"_comment": "Test files are intentionally NOT excluded: the gate used to report zero clones because it skipped **/*.test.ts, hiding all test-tree duplication. Current actual is ~2.47% lines across src+ and tests/, down from 3.4% when tests were included. The 2.6% limit leaves modest headroom for ordinary changes; lower it as duplication is removed.",
|
||||
"threshold": 2.6,
|
||||
"reporters": ["consoleFull"],
|
||||
"ignore": [
|
||||
"node_modules/**",
|
||||
|
||||
@@ -120,6 +120,8 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
);
|
||||
private static final String[] CAPABILITIES = {
|
||||
"annotate_native_function",
|
||||
"set_address_name",
|
||||
"set_addresses_names",
|
||||
"inspect_native_load_image",
|
||||
"read_bytes",
|
||||
"address_to_file_offset",
|
||||
@@ -374,6 +376,12 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
case "resolve_native_call_targets" -> resolveNativeCallTargets(request.params);
|
||||
case "xrefs" -> xrefs(request.params);
|
||||
case "analyze_function" -> analyzeFunction(request.params);
|
||||
case "set_address_name", "set_addresses_names" -> {
|
||||
if (!descriptor.transport.equals("unix-socket"))
|
||||
throw new RequestFailure("method_unavailable", "Windows P0 does not admit database mutation");
|
||||
yield request.method.equals("set_address_name")
|
||||
? setAddressName(request.params) : setAddressesNames(request.params);
|
||||
}
|
||||
case "annotate_native_function" -> {
|
||||
if (!descriptor.transport.equals("unix-socket"))
|
||||
throw new RequestFailure("method_unavailable", "Windows P0 does not admit database mutation");
|
||||
@@ -417,7 +425,8 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
result.addProperty("analysis_timed_out", timedOut);
|
||||
result.add("capabilities", GSON.toJsonTree(
|
||||
java.util.Arrays.stream(CAPABILITIES)
|
||||
.filter(value -> !readOnly || !value.equals("annotate_native_function"))
|
||||
.filter(value -> !readOnly || !(value.equals("annotate_native_function")
|
||||
|| value.equals("set_address_name") || value.equals("set_addresses_names")))
|
||||
.toArray(String[]::new)
|
||||
));
|
||||
result.add("target", target);
|
||||
@@ -1027,11 +1036,11 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
for (String field : List.of("name", "comment", "inline_comment")) {
|
||||
if (params.has(field)) validateAnnotationText(requireText(params, field), field);
|
||||
}
|
||||
String leafName = params.has("name")
|
||||
? annotationLeafName(function, requireString(params, "name")) : null;
|
||||
int transaction = currentProgram.startTransaction("REA function annotations");
|
||||
boolean commit = false;
|
||||
try {
|
||||
String leafName = params.has("name")
|
||||
? symbolLeafName(function.getSymbol(), requireString(params, "name")) : null;
|
||||
// Validate edits through Ghidra's own setters, including name rules.
|
||||
// A later invalid name must roll back earlier comment writes.
|
||||
for (String field : List.of("comment", "inline_comment")) {
|
||||
@@ -1080,22 +1089,33 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
}
|
||||
}
|
||||
|
||||
private String annotationLeafName(Function function, String requested) {
|
||||
private String symbolLeafName(Symbol symbol, String requested) throws ghidra.util.exception.InvalidInputException {
|
||||
// Fully qualified readback must be usable as an idempotent rename input.
|
||||
if (requested.equals(procedureName(function))) return function.getName();
|
||||
String namespace = function.getParentNamespace().isGlobal()
|
||||
? "" : function.getParentNamespace().getName(true);
|
||||
if (requested.equals(symbol.getName(true))) return symbol.getName();
|
||||
String namespace = symbol.getParentNamespace().isGlobal()
|
||||
? "" : symbol.getParentNamespace().getName(true);
|
||||
String prefix = namespace + "::";
|
||||
if (!namespace.isEmpty() && requested.startsWith(prefix)) {
|
||||
String leaf = requested.substring(prefix.length());
|
||||
if (!leaf.isEmpty()) return leaf;
|
||||
throw new RequestFailure("invalid_function_name",
|
||||
"Function name has an empty leaf within namespace " + namespace + " at " +
|
||||
canonicalAddress(function.getEntryPoint()) + ": " + requested);
|
||||
throw new ghidra.util.exception.InvalidInputException(
|
||||
"Symbol name has an empty leaf within namespace " + namespace + " at " +
|
||||
canonicalAddress(symbol.getAddress()) + ": " + requested);
|
||||
}
|
||||
return requested;
|
||||
}
|
||||
|
||||
private void requireAvailableAddressName(Address address,
|
||||
ghidra.program.model.symbol.Namespace namespace, String name)
|
||||
throws ghidra.util.exception.DuplicateNameException {
|
||||
for (Symbol owner : currentProgram.getSymbolTable().getSymbols(name, namespace)) {
|
||||
if (!owner.getAddress().equals(Address.NO_ADDRESS) && !owner.getAddress().equals(address))
|
||||
throw new ghidra.util.exception.DuplicateNameException(
|
||||
"Symbol name " + name + " is already owned at " +
|
||||
canonicalAddress(owner.getAddress()) + " in namespace " + namespace.getName(true));
|
||||
}
|
||||
}
|
||||
|
||||
private static String requireText(JsonObject params, String field) {
|
||||
JsonElement value = params.get(field);
|
||||
if (value == null || !value.isJsonPrimitive() || !value.getAsJsonPrimitive().isString())
|
||||
@@ -1416,6 +1436,67 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
return symbol == null ? JsonNull.INSTANCE : GSON.toJsonTree(symbol.getName(true));
|
||||
}
|
||||
|
||||
// Analyst naming: a function entry renames the function, other addresses get a primary label.
|
||||
private boolean applyAddressName(Address address, String name) throws Exception {
|
||||
if (!currentProgram.getMemory().contains(address))
|
||||
throw new RequestFailure("invalid_request", "Address is not mapped: " + canonicalAddress(address));
|
||||
validateAnnotationText(name, "name");
|
||||
int transaction = currentProgram.startTransaction("REA address name");
|
||||
boolean commit = false;
|
||||
try {
|
||||
Function function = currentProgram.getFunctionManager().getFunctionAt(address);
|
||||
var table = currentProgram.getSymbolTable();
|
||||
Symbol primary = function == null ? table.getPrimarySymbol(address) : function.getSymbol();
|
||||
boolean existing = primary != null && !primary.isDynamic();
|
||||
String expected = existing ? symbolLeafName(primary, name) : name;
|
||||
requireAvailableAddressName(address,
|
||||
existing ? primary.getParentNamespace() : currentProgram.getGlobalNamespace(), expected);
|
||||
if (function != null) {
|
||||
function.setName(expected, SourceType.USER_DEFINED);
|
||||
} else if (existing) {
|
||||
primary.setName(expected, SourceType.USER_DEFINED);
|
||||
} else {
|
||||
Symbol created = table.createLabel(address, expected, SourceType.USER_DEFINED);
|
||||
created.setPrimary();
|
||||
}
|
||||
invalidateAnalysisCaches();
|
||||
Symbol readback = currentProgram.getSymbolTable().getPrimarySymbol(address);
|
||||
commit = readback != null && expected.equals(readback.getName());
|
||||
return commit;
|
||||
}
|
||||
catch (ghidra.util.exception.InvalidInputException | ghidra.util.exception.DuplicateNameException exception) {
|
||||
return false;
|
||||
}
|
||||
finally {
|
||||
currentProgram.endTransaction(transaction, commit);
|
||||
}
|
||||
}
|
||||
|
||||
private JsonElement setAddressName(JsonObject params) throws Exception {
|
||||
requireKeys(params, Set.of("document", "address", "name"));
|
||||
requireDocument(params);
|
||||
return GSON.toJsonTree(applyAddressName(requireAddress(params, "address"), requireString(params, "name")));
|
||||
}
|
||||
|
||||
private JsonElement setAddressesNames(JsonObject params) throws Exception {
|
||||
requireKeys(params, Set.of("document", "names"));
|
||||
requireDocument(params);
|
||||
if (!params.has("names") || !params.get("names").isJsonObject())
|
||||
throw new RequestFailure("invalid_request", "names must be an object of address -> name");
|
||||
JsonObject result = new JsonObject();
|
||||
for (Map.Entry<String, JsonElement> entry : params.getAsJsonObject("names").entrySet()) {
|
||||
boolean ok;
|
||||
try {
|
||||
ok = applyAddressName(parseReaAddress(entry.getKey()), entry.getValue().getAsString());
|
||||
}
|
||||
catch (RequestFailure failure) {
|
||||
ok = false;
|
||||
}
|
||||
result.addProperty(entry.getKey(), ok);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private JsonArray listDocuments(JsonObject params) {
|
||||
requireKeys(params, Set.of());
|
||||
JsonArray result = new JsonArray();
|
||||
|
||||
+10
-3
@@ -166,9 +166,16 @@ rea import-reference-source /absolute/path/to/source
|
||||
```
|
||||
|
||||
The import records hashes and metadata for the supplied files, separately
|
||||
from observations of the current app. File names do not automatically exclude
|
||||
files. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore
|
||||
patterns when you want to exclude selected paths.
|
||||
from observations of the current app. It honors the source tree's `.gitignore`
|
||||
and applies default exclusions for common generated and dependency paths such
|
||||
as `node_modules/`, `dist/`, and `*.log`. Pattern-based exclusions record the
|
||||
matched pattern and whether it came from project, default, caller, or sensitive
|
||||
path policy. Set `REA_REFERENCE_SECRET_PATTERNS_JSON` to a JSON array of ignore
|
||||
patterns when you want to mark selected paths as sensitive; those patterns are
|
||||
reported separately as configured-secret exclusions.
|
||||
|
||||
Re-import saved graphs whose pattern-based exclusions lack `pattern` before
|
||||
using them in source-to-bundle comparisons.
|
||||
|
||||
JavaScript and TypeScript import parsing requires valid UTF-8. Malformed source
|
||||
bytes retain their original hashes and sizes with a decoding diagnostic; REA
|
||||
|
||||
@@ -29,6 +29,13 @@ section names or the record being inside the initial 4 KiB header probe.
|
||||
Program and section declarations must agree; duplicate, conflicting, truncated
|
||||
or out-of-file records are rejected. Stripped section tables and section-only
|
||||
records are supported. Extended table numbering is explicitly unsupported.
|
||||
For carved ELFs whose section table extends beyond the captured file, REA
|
||||
examines every complete captured section header and records the unavailable
|
||||
headers as a limitation. A valid program-header ABI record can still establish
|
||||
the supported declaration. Available section declarations must agree with it;
|
||||
their duplicate, conflicting or unreadable records remain errors. A missing
|
||||
program-header record does not make ABI declarations in uncaptured sections
|
||||
absent or supply a default ABI.
|
||||
The field layout follows
|
||||
[`Elf_MIPS_ABIFlags_v0`](https://github.com/bminor/glibc/blob/master/elf/elf.h).
|
||||
|
||||
@@ -92,6 +99,11 @@ observations and does not relabel the upstream result as verified.
|
||||
|
||||
CLI/MCP discovery and decompilation, direct calls, target/profile/Evidence
|
||||
identity, snapshot round-trip and final process cleanup remain covered.
|
||||
For both byte orders, the lane also carves away the section table while retaining
|
||||
all program-header payloads. It checks the actual Ghidra language, entry-address
|
||||
decompilation, distinct artifact digest and missing-section limitations through
|
||||
CLI Evidence and an MCP-produced snapshot. Caller-selected Ghidra heap limits
|
||||
are passed to these child workflows.
|
||||
Pseudocode presence is a liveness check, not proof of semantic equivalence.
|
||||
Instruction decoding and reader agreement likewise do not establish runtime,
|
||||
delay-slot execution or complete floating-point semantics. Assertion regressions
|
||||
|
||||
+45
-4
@@ -68,7 +68,15 @@ selects the npm release, and persistent MCP registrations are pinned to the
|
||||
version that performed setup. Installing newer instructions does not update a
|
||||
running server or its registration.
|
||||
|
||||
The release checked on **2026-10-07** was **5.0.0** (133 MCP tools), published
|
||||
The npm release checked on **2026-10-10** was **6.3.0**, published from
|
||||
[`74cf9e14`](https://github.com/morluto/rea/commit/74cf9e1401fcd89482fe39c27cc898f5858e9b72),
|
||||
with bundled skill version 34. It does not include automatic Pi or Hermes
|
||||
registration: `setup --client pi` and `setup --client hermes` reject those IDs
|
||||
before writing configuration. Their setup workflows below require a source build
|
||||
or a later release containing those integrations; `@latest` alone cannot select
|
||||
unpublished changes.
|
||||
|
||||
The earlier release checked on **2026-10-07** was **5.0.0** (133 MCP tools), published
|
||||
from the fixed checkpoint
|
||||
[`b33236ec`](https://github.com/morluto/rea/releases/tag/rea-agents-5.0.0).
|
||||
The public CLI, MCP catalog and target-free session, and isolated update from
|
||||
@@ -142,7 +150,29 @@ listed after the table because its connector is not one of these files:
|
||||
| Pi | `pi` |
|
||||
| Hermes | `hermes` |
|
||||
|
||||
For GitHub Copilot CLI 1.0.95, `gpt-4.1` model metadata can block ordinary
|
||||
chat with REA's full input-schema profile before sending a model request
|
||||
(`compaction_static_context_blocked`; [#1554](https://github.com/morluto/rea/issues/1554)).
|
||||
For the native offline OpenAI-compatible BYOK adapter, full-profile chat and
|
||||
analysis also passed with a distinct `COPILOT_PROVIDER_MODEL_ID` such as
|
||||
`rea-gpt-4.1`, `COPILOT_PROVIDER_WIRE_MODEL=gpt-4.1`, and that distinct ID selected
|
||||
through `copilot --model rea-gpt-4.1`. The local fixture requested
|
||||
`COPILOT_PROVIDER_MAX_PROMPT_TOKENS=1000000`; configure the limit supported by
|
||||
your backend. This keeps every full schema and the actual wire model intact.
|
||||
It does not establish the effective client budget or fix the built-in-ID path.
|
||||
The existing compact profile passed native chat and analysis with the complete
|
||||
tool inventory on Linux using a loopback model fixture. After setup, add
|
||||
`"REA_MCP_INPUT_SCHEMA_PROFILE": "compact"` to the `env` object of the
|
||||
`mcpServers.rea` entry in Copilot's `mcp-config.json`, preserving its other
|
||||
settings, then restart the client. Compact advertisements can reduce nested
|
||||
schema constraints; REA still validates complete canonical inputs. Full-profile
|
||||
`gpt-5.4` metadata is also verified locally. Live model API acceptance and
|
||||
Windows remain unverified; see [native client verification](testing.md).
|
||||
|
||||
For [Pi](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/mcp.md),
|
||||
automatic setup requires a build containing the integration; npm 6.3.0 does not
|
||||
include it. See [released package and main](#released-package-and-main).
|
||||
In a supported build,
|
||||
setup writes a `type: "stdio"` entry under `mcpServers.rea` in the user-level
|
||||
`~/.pi/agent/mcp.json` and installs the bundled skill under `~/.agents/skills`.
|
||||
Use `rea setup --client pi --dry-run --json` to inspect the plan, then
|
||||
@@ -201,7 +231,10 @@ registration. Doctor treats an `enabled: false` entry as active when
|
||||
`enabledServers` lists `rea`, as OMP does, unless `disabledServers` also lists
|
||||
it. Run setup under each profile that should load REA.
|
||||
|
||||
For Hermes, setup writes a `mcp_servers.rea` entry to `config.yaml`, preserving
|
||||
For Hermes, automatic setup also requires a build containing the integration;
|
||||
npm 6.3.0 does not include it. See
|
||||
[released package and main](#released-package-and-main).
|
||||
In a supported build, setup writes a `mcp_servers.rea` entry to `config.yaml`, preserving
|
||||
comments and unrelated settings. It resolves that file from `HERMES_HOME`, else
|
||||
from the platform default Hermes itself uses — `%LOCALAPPDATA%\hermes` on
|
||||
Windows, `~/.hermes` elsewhere — including any `HERMES_DATA_DIR_SUFFIX`. Hermes
|
||||
@@ -334,6 +367,12 @@ Hopper installation or any file write. Declining or pressing Ctrl-C makes no
|
||||
changes. Agent configuration writes preserve unrelated entries and comments,
|
||||
create backups, use atomic replacement, and verify their result. Setup and
|
||||
uninstall retain an existing `.rea.backup` rather than replacing the first snapshot.
|
||||
Setup also retains valid environment overrides in REA's existing server entry,
|
||||
such as `REA_MCP_INPUT_SCHEMA_PROFILE` and `REA_MCP_MAX_RESPONSE_BYTES`.
|
||||
Detected provider settings replace only matching environment keys.
|
||||
Setup and doctor report a malformed REA environment map or invalid runtime
|
||||
setting before setup writes files. Repair the named setting in the client
|
||||
configuration; setup does not silently discard the rest of the map.
|
||||
|
||||
After setup, REA reports which agents, analysis tools, and workflow files passed
|
||||
its final checks. Restart any agent named in the completion message, then begin
|
||||
@@ -598,8 +637,10 @@ dossiers, instructions, recovered data types, measured load mappings, loaded
|
||||
memory bytes, and observed file offsets. Independent load-image attestation
|
||||
supports DOS MZ and explicitly selected COM; PE returns its measurements with that limitation.
|
||||
On Linux and macOS, `annotate_native_function` also edits a function name and/or
|
||||
entry comments atomically and returns refreshed analysis. These session metadata
|
||||
edits leave executable bytes unchanged and are discarded on close. GUI controls
|
||||
entry comments atomically and returns refreshed analysis, and `set_address_name`
|
||||
and `set_addresses_names` name a function entry or any other mapped address
|
||||
(creating or renaming its primary label) with per-address success results. These
|
||||
session metadata edits leave executable bytes unchanged and are discarded on close. GUI controls
|
||||
require Hopper; Windows P0 remains read-only.
|
||||
|
||||
Windows P0 admits native x86 and x86-64 PE applications and DLLs on fixed local NTFS volumes.
|
||||
|
||||
@@ -397,6 +397,15 @@ unpaired Unicode surrogate; a rejection identifies the field and UTF-16 index
|
||||
and leaves every annotation unchanged. CRLF, supplementary Unicode characters,
|
||||
and combining characters are preserved. Windows P0 remains read-only. Ghidra has no
|
||||
GUI authority, and REA never falls back automatically to Hopper.
|
||||
`set_address_name` and `set_addresses_names` give Ghidra sessions the same
|
||||
compact naming results as Hopper: a local function entry renames that function,
|
||||
and any other mapped address, such as a global, receives a new or renamed primary
|
||||
label. Names owned by another address in the same namespace are rejected;
|
||||
the same leaf name remains valid in a different namespace. A batch applies each
|
||||
address separately and reports one success boolean per address, preserving the
|
||||
other entries when one fails. Ghidra batches do not move or swap names. Use them
|
||||
to name many globals and helpers cheaply; use
|
||||
`annotate_native_function` when you also want comments and a refreshed dossier.
|
||||
Ghidra name edits preserve the existing namespace. Supply either a leaf name
|
||||
such as `renamed` or a fully qualified name in that namespace, such as
|
||||
`alpha::renamed`. The returned qualified name can be reused as an idempotent
|
||||
|
||||
@@ -114,6 +114,12 @@ from launch, must be ordered, and must fall within `timeout_ms`. Inputs marked
|
||||
Environment overrides are recorded in the scenario commitment; inherited
|
||||
values are not copied into Evidence.
|
||||
|
||||
Terminal dimensions must satisfy `columns × (rows + scrollback) ≤ 1,000,000`.
|
||||
Every scheduled resize uses the same selected scrollback and must fit this
|
||||
working-buffer budget. REA rejects oversized scenarios before launching the
|
||||
command; it never clamps the requested dimensions. This limit is independent
|
||||
of `limits.output_bytes`, which bounds retained terminal observations.
|
||||
|
||||
The capture contains raw PTY output chunks and rendered terminal states,
|
||||
interaction dispatch outcomes, exit reason, sampled process-tree observations,
|
||||
and settlement status. When filesystem paths are selected, REA records
|
||||
|
||||
+89
-10
@@ -99,9 +99,10 @@ result that its client offloads, and Pi exercises default codemode execution.
|
||||
Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package.
|
||||
Receipts retain client versions, result digests, host coverage and owned-process
|
||||
cleanup. These POSIX lanes use a deterministic loopback model, so they do not
|
||||
prove live model-provider or native Windows compatibility. Qwen and Pi use
|
||||
caller-configured skill directories to isolate the fixture account from the OS
|
||||
home; this does not prove their default home-directory discovery.
|
||||
prove live model-provider or native Windows compatibility. Qwen, Pi and Hermes
|
||||
bind both `HOME` and `USERPROFILE` to the disposable account. Qwen and Pi discover
|
||||
the shared personal skill through their default search paths without an explicit
|
||||
skill setting; Hermes discovers it through the selected `HERMES_HOME`.
|
||||
Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of
|
||||
a named sticky profile with `HERMES_HOME` still pointing to its root.
|
||||
|
||||
@@ -159,11 +160,39 @@ Use `-- chat` for ordinary
|
||||
chat or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed package.
|
||||
The native offline BYOK adapter uses a loopback OpenAI completions/SSE fixture
|
||||
and `gpt-5.4` model metadata; model inference and token usage are synthetic.
|
||||
Set `REA_VERIFY_COPILOT_MODEL` to examine another model configuration. In the
|
||||
verified client, `gpt-4.1` blocks the complete catalog before HTTP with
|
||||
Set `REA_VERIFY_COPILOT_MODEL` to examine another client model ID and
|
||||
`REA_VERIFY_COPILOT_WIRE_MODEL` for its backend model name (defaults to the client
|
||||
ID). Every actual HTTP request's model is checked and recorded separately.
|
||||
Both `HOME` and `USERPROFILE` point to the disposable account; native `skill add`
|
||||
still selects the installed skill directory explicitly.
|
||||
|
||||
The same native client passed full-profile chat and analysis with the actual
|
||||
wire model `gpt-4.1` under a distinct BYOK model ID:
|
||||
|
||||
```bash
|
||||
REA_VERIFY_COPILOT_MODEL=rea-gpt-4.1 REA_VERIFY_COPILOT_WIRE_MODEL=gpt-4.1 npm run verify:copilot-client -- chat
|
||||
REA_VERIFY_COPILOT_MODEL=rea-gpt-4.1 REA_VERIFY_COPILOT_WIRE_MODEL=gpt-4.1 npm run verify:copilot-client -- call
|
||||
```
|
||||
|
||||
These runs retain full canonical input/output schemas, all tools, installed-skill
|
||||
loading and complete Evidence readback. They use the fixture's requested
|
||||
one-million-token BYOK prompt setting; effective capacity and live API acceptance
|
||||
remain unknown. With the built-in `gpt-4.1` model ID, the verified client still
|
||||
blocks the full input-schema profile before HTTP with
|
||||
`compaction_static_context_blocked`, even when requesting a larger BYOK prompt
|
||||
capacity. The effective capacity is unknown; this lane does not establish a
|
||||
fix for that client/model limit. It preserves the complete catalog and schemas.
|
||||
capacity. The effective capacity is unknown. The default verifier preserves
|
||||
the complete catalog and full schemas. To verify the existing compact profile:
|
||||
|
||||
```bash
|
||||
REA_VERIFY_COPILOT_MODEL=gpt-4.1 REA_VERIFY_COPILOT_SCHEMA_PROFILE=compact npm run verify:copilot-client -- chat
|
||||
REA_VERIFY_COPILOT_MODEL=gpt-4.1 REA_VERIFY_COPILOT_SCHEMA_PROFILE=compact npm run verify:copilot-client -- call
|
||||
```
|
||||
|
||||
Both native workflows passed with the complete tool inventory. The verifier selects the
|
||||
profile only in its disposable Copilot registration and records it in the
|
||||
receipt. Compact advertisements can reduce nested validation structure; the
|
||||
server still validates complete canonical inputs. This is a tested alternate
|
||||
workflow, not a fix for the full-profile client/model limit tracked in #1554.
|
||||
Live model APIs, native Windows and default OS-home skill discovery remain
|
||||
unverified.
|
||||
|
||||
@@ -232,8 +261,9 @@ unverified; skill discovery uses an explicit isolated custom directory.
|
||||
verified with `@deepseek-ai/dsh@0.2.0-rc.2`) and Git. Set
|
||||
`REA_VERIFY_DEEPSEEK_COMMAND` to select its executable. This optional POSIX lane
|
||||
installs the skill through `rea setup --skill`, writes the native Cordis MCP
|
||||
patch into an isolated `DSH_HOME`, and uses `DSH_AGENTS_HOME` for native personal
|
||||
skill discovery. A separate Git root prevents inherited project skills from
|
||||
patch into an isolated `DSH_HOME`, and binds `HOME` and `USERPROFILE` to the
|
||||
disposable account. The native default `~/.agents/skills` search path discovers
|
||||
the installed skill without a `DSH_AGENTS_HOME` override. A separate Git root prevents inherited project skills from
|
||||
masking a missing installation. The default `call` mode checks skill loading,
|
||||
complete catalog discovery, the actual forwarded input schemas and regexes,
|
||||
and full JavaScript Evidence for a Unicode path. Use `-- chat` for ordinary
|
||||
@@ -1053,6 +1083,39 @@ real local Codex CLI with:
|
||||
npm run verify:agent
|
||||
```
|
||||
|
||||
The lane runs scenarios sequentially in standalone Codex (`--no-daemon`) with a
|
||||
disposable HOME and CODEX_HOME under `~/.cache/rea-agent-evaluations/`. Set
|
||||
`REA_AGENT_EVAL_ROOT` to choose another parent directory outside the OS temporary
|
||||
directory: current Codex refuses helper aliases under `/tmp`, preventing shell
|
||||
commands and skill reads even if MCP works. Each owned run directory is removed
|
||||
after verification unless fixture retention is requested. The lane
|
||||
installs the packaged skill and MCP registration through public `rea setup`,
|
||||
rather than copying a skill into a project that also inherits the caller's
|
||||
skills and configuration. Only authentication is copied, when present, and it
|
||||
is removed even with `REA_AGENT_EVAL_KEEP_FIXTURES=true`. No user configuration,
|
||||
rules, plugins or history is copied; shell snapshots and additional agents are
|
||||
disabled. Existing API-key environment authentication remains available.
|
||||
The native missing-provider scenario preapproves only `open_binary` and
|
||||
`close_binary` in the disposable client so it reaches REA's actual provider
|
||||
boundary instead of stopping at Codex approval. This does not validate a real
|
||||
native engine or authorize arbitrary runtime execution.
|
||||
|
||||
Select a smaller run with `REA_AGENT_EVAL_SCENARIOS`, a comma-separated list of
|
||||
scenario IDs. `asar-zh` exercises the packaged desktop rubric from a Chinese
|
||||
request. `javascript-module-view` exercises summary analysis, module paging and
|
||||
item inspection in a tree with unrelated modules; its routing and text checks
|
||||
remain heuristic, so review the retained transcript for selector choice and
|
||||
coverage claims. Set `REA_CODEX_CLI` to an installed Codex executable and
|
||||
`REA_AGENT_EVAL_MODEL` to an account-supported model. Use an external resource
|
||||
limit for the whole process tree when testing on a constrained host; a Node
|
||||
heap limit alone does not bound client and child-process memory.
|
||||
|
||||
`missing-target` leaves desktop fixtures visible without selecting one in the
|
||||
request. It requires zero REA calls and a structured request for an app name or
|
||||
artifact path. Its `expectedFirstTool` is `null`, and
|
||||
`targetClarificationPassed` grades that limited routing outcome independently
|
||||
of analysis prose heuristics. Analyzing a nearby example cannot pass this case.
|
||||
|
||||
The packaged desktop and parser-comparison scenarios now assess a closed set of
|
||||
known fixture claims. Each final answer must be a strict JSON manifest containing
|
||||
exactly the requested claim IDs, values, Evidence IDs, and Evidence authority and
|
||||
@@ -1077,9 +1140,21 @@ navigation-context, and address-context scenarios have no factual rubric and
|
||||
remain `not_assessed`; the managed workflow requires both artifact and member
|
||||
inspection to answer its type and entry-point question.
|
||||
|
||||
The closed factual scenarios request complete producer Evidence on the initial
|
||||
analysis so their source selectors can authenticate all configured facts.
|
||||
Summary/page/item workflows are exercised separately by `javascript-module-view`;
|
||||
its heuristic gate does not establish full-producer factual correctness.
|
||||
|
||||
All scenarios retain routing, workflow, validation, repetition, process-exit,
|
||||
and token-use gates. Configured factual scenarios additionally require a factual
|
||||
pass. Their answer-text heuristics are diagnostic and do not affect the gate.
|
||||
Validation counts include complete text-only REA `invalid_request` envelopes
|
||||
and failed SDK calls whose arguments violate the named current REA contract.
|
||||
Client approval rejections and provider unavailability remain distinct failures;
|
||||
truncated JSON previews are not parsed into invented error codes.
|
||||
Repeated `binary_session` observations separated by a successful `open_binary`
|
||||
or `close_binary` are treated as lifecycle verification. An unchanged retry,
|
||||
including one after a failed lifecycle call, still counts as repetition.
|
||||
Scenarios without a rubric retain the legacy text-heuristic gate.
|
||||
`answerTermCoverageMet` checks case-insensitive substrings,
|
||||
`epistemicCuePresent` checks keywords, and `finalCitesEvidence` checks only an ID's
|
||||
@@ -1087,7 +1162,11 @@ presence. These metrics can still accept fabricated prose and must not be read
|
||||
as factual assessments. Set `REA_AGENT_EVAL_TRANSCRIPT_DIR` to retain complete
|
||||
tool results and final answers for review.
|
||||
|
||||
Report schema version 3 changes the top-level `factualCorrectness` from a constant
|
||||
Report schema version 4 adds negative routing scenarios with nullable
|
||||
`expectedFirstTool` and a `targetClarificationPassed` outcome. Consumers must
|
||||
allow an expected absence of REA calls; this does not make an analysis scenario
|
||||
pass without its required tools. Version 3 changed the top-level
|
||||
`factualCorrectness` from a constant
|
||||
string to an assessment summary with status, scope, and assessed/passed/failed/
|
||||
not-assessed scenario counts. Scenario records include the factual assessment and
|
||||
configured claim IDs. The evaluation scope is
|
||||
|
||||
+7
-4
@@ -85,19 +85,22 @@ if [[ "$version" =~ \+ ]]; then
|
||||
fail "npm cannot install an exact version with build metadata. Pass a version without build metadata, then retry."
|
||||
fi
|
||||
|
||||
prefix_args=()
|
||||
install_args=(install --global)
|
||||
if [[ "$platform" == "Linux" ]]; then
|
||||
prefix_args=(--prefix "$HOME/.local")
|
||||
install_args+=(--prefix "$HOME/.local")
|
||||
install_bin="$HOME/.local/bin/rea"
|
||||
else
|
||||
npm_prefix="$(npm prefix --global)" || fail "the npm global prefix could not be read. Repair the npm configuration, then retry."
|
||||
install_bin="$npm_prefix/bin/rea"
|
||||
fi
|
||||
install_args+=("$PACKAGE@$version")
|
||||
|
||||
printf 'REA install plan\n'
|
||||
printf ' Version: %s\n' "$version"
|
||||
printf ' Runtime: Node.js %s\n' "$node_version"
|
||||
printf ' Command: npm install --global %s@%s\n' "$PACKAGE" "$version"
|
||||
printf ' Command:'
|
||||
printf ' %q' npm "${install_args[@]}"
|
||||
printf '\n'
|
||||
printf ' Binary: %s\n' "$install_bin"
|
||||
printf ' Setup: %s\n' "$([[ "$start_setup" == true ]] && printf 'start when a terminal is available' || printf 'skipped')"
|
||||
|
||||
@@ -107,7 +110,7 @@ if [[ "$dry_run" == true ]]; then
|
||||
fi
|
||||
|
||||
printf 'Installing %s@%s...\n' "$PACKAGE" "$version"
|
||||
npm install --global ${prefix_args[@]+"${prefix_args[@]}"} "$PACKAGE@$version" || fail "npm could not install REA. Check registry access and npm permissions, then retry."
|
||||
npm "${install_args[@]}" || fail "npm could not install REA. Check registry access and npm permissions, then retry."
|
||||
[[ -x "$install_bin" ]] || fail "npm completed without installing the rea command. Check the npm global bin directory and PATH, then retry."
|
||||
installed_version="$("$install_bin" --version 2>/dev/null | tr -d '[:space:]')" || fail "the installed REA version could not be read. Reinstall the requested version, then retry."
|
||||
[[ "$installed_version" == "$version" ]] || fail "installed version $installed_version does not match $version. Reinstall the requested version, then retry."
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
import { analyzeJavaScriptApplication } from "../../dist/application/javascript/JavaScriptApplicationService.js";
|
||||
import { ArtifactResourceScope } from "../../dist/artifacts/ArtifactResourceScope.js";
|
||||
|
||||
/** Analyze one JavaScript application and close its caller-owned resources. */
|
||||
export const analyzeJavaScriptApplicationWithOwnedScope = async (input) => {
|
||||
const scope = new ArtifactResourceScope();
|
||||
let outcome;
|
||||
try {
|
||||
const result = await analyzeJavaScriptApplication(input, scope);
|
||||
outcome = result.ok
|
||||
? { kind: "completed", value: result.value }
|
||||
: { kind: "failed", cause: result.error };
|
||||
} catch (cause) {
|
||||
outcome = { kind: "failed", cause };
|
||||
}
|
||||
try {
|
||||
await scope.close();
|
||||
} catch (cleanupCause) {
|
||||
if (outcome.kind === "failed")
|
||||
throw new AggregateError(
|
||||
[outcome.cause, cleanupCause],
|
||||
"JavaScript analysis and artifact cleanup both failed",
|
||||
);
|
||||
throw cleanupCause;
|
||||
}
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
return outcome.value;
|
||||
};
|
||||
@@ -21,7 +21,6 @@ export async function verifyGhidraEntryAliases({
|
||||
new URL("../../tests/conformance/ghidra/entry-aliases.c", import.meta.url),
|
||||
);
|
||||
const path = join(workspace, "entry-aliases");
|
||||
const prefix = process.platform === "darwin" ? "_" : "";
|
||||
try {
|
||||
await exec(compiler, ["-O0", "-g", "-fno-inline", source, "-o", path], {
|
||||
env,
|
||||
@@ -29,7 +28,8 @@ export async function verifyGhidraEntryAliases({
|
||||
});
|
||||
const bytes = await readFile(path);
|
||||
await call("close_binary");
|
||||
await call("open_binary", { path, provider_id: "ghidra" });
|
||||
const imported = await call("open_binary", { path, provider_id: "ghidra" });
|
||||
const prefix = imported.format === "mach-o" ? "_" : "";
|
||||
const names = await call("list_names");
|
||||
const alias = names.find(
|
||||
({ value }) => value === `${prefix}rea_entry_alias`,
|
||||
@@ -46,8 +46,10 @@ export async function verifyGhidraEntryAliases({
|
||||
);
|
||||
assert.ok(canonical, "Fixture alias is not at a function entry");
|
||||
const xrefsSelectors = [];
|
||||
// CLI xrefs keeps bare hexadecimal selectors as addresses. A Mach-O
|
||||
// primary label such as "dead" is tested below as a procedure name instead.
|
||||
for (const value of [
|
||||
canonical.value,
|
||||
`${prefix}rea_alias_target`,
|
||||
alias.value,
|
||||
`${prefix}rea_xrefs_data`,
|
||||
`${prefix}rea_xrefs_unreferenced`,
|
||||
@@ -89,6 +91,18 @@ export async function verifyGhidraEntryAliases({
|
||||
);
|
||||
assert.ok(interior);
|
||||
assert.notEqual(interior.address, alias.address);
|
||||
assert.equal(
|
||||
interior.symbol.type,
|
||||
"label",
|
||||
"Fixture interior control must remain an imported label",
|
||||
);
|
||||
assert.equal(
|
||||
(await call("list_procedures")).some(
|
||||
({ address }) => address === interior.address,
|
||||
),
|
||||
false,
|
||||
"Fixture interior control must not be a Ghidra procedure entry",
|
||||
);
|
||||
const failure = await reject("procedure_address", {
|
||||
procedure: interior.value,
|
||||
});
|
||||
@@ -153,7 +167,11 @@ export async function verifyGhidraEntryAliases({
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const evidence = JSON.parse(stdout);
|
||||
assert.deepEqual(evidence.normalized_result, expected);
|
||||
assert.deepEqual(
|
||||
evidence.normalized_result,
|
||||
expected,
|
||||
`CLI xrefs mismatch for ${JSON.stringify(selector)}`,
|
||||
);
|
||||
assert.equal(evidence.parameters.address, selector.at(-1));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -54,6 +54,15 @@ export async function verifyGhidraNamespaceAnnotations({
|
||||
);
|
||||
return { namespace, address: procedure.address };
|
||||
});
|
||||
const names = await call("list_names");
|
||||
const globals = ["alpha", "outer::inner"].map((namespace) => {
|
||||
const symbol = names.find(
|
||||
(item) => item.value === `${namespace}::value` && item.symbol.primary,
|
||||
);
|
||||
assert.ok(symbol, `Missing demangled fixture global ${namespace}::value`);
|
||||
return { namespace, address: symbol.address };
|
||||
});
|
||||
await verifyAddressNameCollisions(call, procedures, names);
|
||||
const ambiguous = await reject("procedure_address", { procedure: "same" });
|
||||
assert.equal(ambiguous.code, "invalid_request");
|
||||
for (const { address } of entries)
|
||||
@@ -125,6 +134,20 @@ export async function verifyGhidraNamespaceAnnotations({
|
||||
});
|
||||
assert.deepEqual(unchanged.annotations, qualified.annotations);
|
||||
}
|
||||
await verifyAddressNames(call, [
|
||||
{ ...entries[0], kind: "function" },
|
||||
{ ...entries[2], kind: "function" },
|
||||
...globals.map((symbol) => ({ ...symbol, kind: "label" })),
|
||||
]);
|
||||
for (const { namespace, address } of entries) {
|
||||
assert.equal(
|
||||
await call("set_address_name", {
|
||||
address,
|
||||
name: `${namespace}::qualified`,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
}
|
||||
const { stdout } = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
@@ -159,6 +182,224 @@ export async function verifyGhidraNamespaceAnnotations({
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyAddressNameCollisions(call, procedures, names) {
|
||||
const find = (items, name) => {
|
||||
const symbol = items.find(
|
||||
(item) => item.value === name || item.value === `_${name}`,
|
||||
);
|
||||
assert.ok(symbol, `Missing naming collision fixture ${name}`);
|
||||
return symbol;
|
||||
};
|
||||
const first = find(procedures, "rea_name_first");
|
||||
const second = find(procedures, "rea_name_second");
|
||||
const firstLabel = find(names, "rea_name_data_first");
|
||||
const secondLabel = find(names, "rea_name_data_second");
|
||||
for (const [owner, other, name] of [
|
||||
[first, second, "owned_function"],
|
||||
[firstLabel, secondLabel, "owned_label"],
|
||||
]) {
|
||||
assert.equal(
|
||||
await call("set_address_name", { address: owner.address, name }),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await call("set_address_name", { address: other.address, name }),
|
||||
false,
|
||||
"A different address must not acquire the existing name",
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: other.address }),
|
||||
other.value,
|
||||
);
|
||||
assert.equal(
|
||||
await call("set_address_name", { address: owner.address, name }),
|
||||
true,
|
||||
"The owning address may repeat the same name",
|
||||
);
|
||||
}
|
||||
for (const [address, name, original] of [
|
||||
[first.address, "owned_label", "owned_function"],
|
||||
[firstLabel.address, "owned_function", "owned_label"],
|
||||
]) {
|
||||
assert.equal(await call("set_address_name", { address, name }), false);
|
||||
assert.equal(await call("address_name", { address }), original);
|
||||
}
|
||||
const alphaFunction = find(procedures, "alpha::same");
|
||||
const alphaLabel = find(names, "alpha::value");
|
||||
for (const name of ["value", "alpha::value"]) {
|
||||
assert.equal(
|
||||
await call("set_address_name", { address: alphaFunction.address, name }),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: alphaFunction.address }),
|
||||
alphaFunction.value,
|
||||
);
|
||||
}
|
||||
assert.equal(
|
||||
await call("set_address_name", {
|
||||
address: alphaLabel.address,
|
||||
name: "same",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: alphaLabel.address }),
|
||||
alphaLabel.value,
|
||||
);
|
||||
assert.deepEqual(
|
||||
await call("set_addresses_names", {
|
||||
names: {
|
||||
[secondLabel.address]: "batch_available",
|
||||
[second.address]: "owned_function",
|
||||
[firstLabel.address]: "owned_label",
|
||||
},
|
||||
}),
|
||||
{
|
||||
[secondLabel.address]: true,
|
||||
[second.address]: false,
|
||||
[firstLabel.address]: true,
|
||||
},
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: second.address }),
|
||||
second.value,
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: first.address }),
|
||||
"owned_function",
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: secondLabel.address }),
|
||||
"batch_available",
|
||||
);
|
||||
const before = await call("annotate_native_function", {
|
||||
procedure: second.address,
|
||||
comment: "Keep this comment",
|
||||
inline_comment: "Keep this inline comment",
|
||||
});
|
||||
assert.equal(
|
||||
await call("set_address_name", {
|
||||
address: second.address,
|
||||
name: "owned_function",
|
||||
}),
|
||||
false,
|
||||
);
|
||||
const unchanged = await call("annotate_native_function", {
|
||||
procedure: second.address,
|
||||
name: second.value,
|
||||
});
|
||||
assert.deepEqual(unchanged.annotations, before.annotations);
|
||||
assert.equal(
|
||||
await call("set_address_name", {
|
||||
address: first.address,
|
||||
name: "released_function",
|
||||
}),
|
||||
true,
|
||||
);
|
||||
assert.equal(
|
||||
await call("set_address_name", {
|
||||
address: second.address,
|
||||
name: "owned_function",
|
||||
}),
|
||||
true,
|
||||
"A name becomes available after its old owner releases it",
|
||||
);
|
||||
}
|
||||
|
||||
async function verifyAddressNames(call, symbols) {
|
||||
for (const { namespace, address, kind } of symbols) {
|
||||
assert.equal(
|
||||
await call("set_address_name", {
|
||||
address,
|
||||
name: `address_renamed_${kind}`,
|
||||
}),
|
||||
true,
|
||||
);
|
||||
const qualified = `${namespace}::address_qualified_${kind}`;
|
||||
assert.equal(
|
||||
await call("set_address_name", { address, name: qualified }),
|
||||
true,
|
||||
);
|
||||
assert.equal(await call("address_name", { address }), qualified);
|
||||
assert.equal(
|
||||
await call("set_address_name", { address, name: "other::literal" }),
|
||||
true,
|
||||
);
|
||||
const literal = await call("address_name", { address });
|
||||
assert.equal(literal, `${namespace}::other::literal`);
|
||||
assert.equal(
|
||||
await call("set_address_name", { address, name: literal }),
|
||||
true,
|
||||
);
|
||||
assert.equal(await call("address_name", { address }), literal);
|
||||
assert.equal(
|
||||
await call("set_address_name", { address, name: qualified }),
|
||||
true,
|
||||
);
|
||||
for (let repeat = 0; repeat < 2; repeat++) {
|
||||
const readback = await call("address_name", { address });
|
||||
assert.deepEqual(
|
||||
await call("set_addresses_names", { names: { [address]: readback } }),
|
||||
{ [address]: true },
|
||||
);
|
||||
assert.equal(await call("address_name", { address }), qualified);
|
||||
}
|
||||
assert.equal(
|
||||
await call("set_address_name", { address, name: `${namespace}::` }),
|
||||
false,
|
||||
);
|
||||
const session = await call("binary_session");
|
||||
for (const operation of [
|
||||
"set_address_name",
|
||||
"set_addresses_names",
|
||||
"address_name",
|
||||
"list_names",
|
||||
])
|
||||
assert.equal(
|
||||
session.capabilities.find((item) => item.operation === operation)
|
||||
?.available,
|
||||
true,
|
||||
`An invalid name must not mark ${operation} unavailable`,
|
||||
);
|
||||
assert.equal(await call("address_name", { address }), qualified);
|
||||
}
|
||||
const [valid, invalid] = symbols;
|
||||
assert.deepEqual(
|
||||
await call("set_addresses_names", {
|
||||
names: {
|
||||
[valid.address]: `${valid.namespace}::batch_renamed`,
|
||||
[invalid.address]: `${invalid.namespace}::`,
|
||||
},
|
||||
}),
|
||||
{ [valid.address]: true, [invalid.address]: false },
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: valid.address }),
|
||||
`${valid.namespace}::batch_renamed`,
|
||||
);
|
||||
assert.equal(
|
||||
await call("address_name", { address: invalid.address }),
|
||||
`${invalid.namespace}::address_qualified_${invalid.kind}`,
|
||||
);
|
||||
const names = await call("list_names");
|
||||
for (const { namespace, address, kind } of symbols) {
|
||||
const expected =
|
||||
address === valid.address
|
||||
? `${namespace}::batch_renamed`
|
||||
: `${namespace}::address_qualified_${kind}`;
|
||||
assert.ok(
|
||||
names.some(
|
||||
(item) =>
|
||||
item.address === address &&
|
||||
item.value === expected &&
|
||||
item.symbol.primary,
|
||||
),
|
||||
`Missing refreshed primary symbol ${expected}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function requireNamespaceDemangler(env) {
|
||||
const inspection = inspectGhidraInstallation({
|
||||
environment: process.env,
|
||||
|
||||
@@ -1,9 +1,18 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { cp, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import {
|
||||
chmod,
|
||||
copyFile,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import { delimiter, dirname, join, resolve } from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { z } from "zod";
|
||||
|
||||
import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js";
|
||||
import {
|
||||
@@ -15,32 +24,95 @@ import {
|
||||
agentEvaluationPassed,
|
||||
summarizeFactualCorrectness,
|
||||
} from "../dist/evaluation/AgentEvaluationReport.js";
|
||||
import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js";
|
||||
import { PRODUCT_IDENTITY } from "../dist/identity.js";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const verifierRun = createVerifierRun();
|
||||
const evaluationRoot = await mkdtemp(join(tmpdir(), "rea-agent-eval-"));
|
||||
const fixtureRoot = join(evaluationRoot, "targets");
|
||||
const skillDestination = join(
|
||||
evaluationRoot,
|
||||
".agents/skills/reverse-engineer-anything",
|
||||
);
|
||||
const timeoutMs = Number(process.env.REA_AGENT_EVAL_TIMEOUT_MS ?? 480_000);
|
||||
const codex = process.env.REA_CODEX_CLI ?? "codex";
|
||||
const optionalModel = process.env.REA_AGENT_EVAL_MODEL;
|
||||
|
||||
if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 10_000)
|
||||
throw new Error("REA_AGENT_EVAL_TIMEOUT_MS must be an integer >= 10000");
|
||||
const codexVersion = await readCommandVersion(codex);
|
||||
// Codex refuses helper aliases when CODEX_HOME is under the OS temp directory.
|
||||
// Keep the disposable account in a private cache, then remove the owned run.
|
||||
const evaluationParent = resolve(
|
||||
process.env.REA_AGENT_EVAL_ROOT ??
|
||||
join(homedir(), ".cache", "rea-agent-evaluations"),
|
||||
);
|
||||
await mkdir(evaluationParent, { recursive: true, mode: 0o700 });
|
||||
const evaluationRoot = await mkdtemp(join(evaluationParent, "run-"));
|
||||
const fixtureRoot = join(evaluationRoot, "targets");
|
||||
const account = join(evaluationRoot, "account");
|
||||
const codexHome = join(evaluationRoot, "codex");
|
||||
const authPath = join(codexHome, "auth.json");
|
||||
const skillDestination = join(
|
||||
account,
|
||||
".agents/skills/reverse-engineer-anything",
|
||||
);
|
||||
const evaluationEnvironment = {
|
||||
...Object.fromEntries(
|
||||
Object.entries(process.env).filter(
|
||||
([name]) => !/^(?:REA_|CODEX_|GHIDRA_|HOPPER_|IDA_)/u.test(name),
|
||||
),
|
||||
),
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
CODEX_HOME: codexHome,
|
||||
XDG_CONFIG_HOME: join(account, ".config"),
|
||||
XDG_DATA_HOME: join(account, ".local/share"),
|
||||
XDG_CACHE_HOME: join(account, ".cache"),
|
||||
REA_PROCESS_RUN_ID: verifierRun.run_id,
|
||||
PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH ?? ""}`,
|
||||
};
|
||||
|
||||
try {
|
||||
await mkdir(fixtureRoot, { recursive: true });
|
||||
await cp(
|
||||
join(repositoryRoot, "skills/reverse-engineer-anything"),
|
||||
skillDestination,
|
||||
{ recursive: true },
|
||||
const codexVersion = await readCommandVersion(codex);
|
||||
await Promise.all([
|
||||
mkdir(fixtureRoot, { recursive: true }),
|
||||
mkdir(account),
|
||||
mkdir(codexHome, { mode: 0o700 }),
|
||||
]);
|
||||
// Reuse authentication only. User config, plugins, rules, history and skills
|
||||
// must not make a fresh installed-skill trial pass accidentally.
|
||||
try {
|
||||
await copyFile(
|
||||
join(process.env.CODEX_HOME ?? join(homedir(), ".codex"), "auth.json"),
|
||||
authPath,
|
||||
);
|
||||
await chmod(authPath, 0o600);
|
||||
} catch (cause) {
|
||||
if (cause?.code !== "ENOENT") throw cause;
|
||||
// Environment/API-key authentication can work without a local auth file.
|
||||
}
|
||||
await writeFile(
|
||||
join(codexHome, "config.toml"),
|
||||
'approval_policy = "never"\nsandbox_mode = "read-only"\nweb_search = "disabled"\n[features]\nmulti_agent = false\nshell_snapshot = false\n[history]\npersistence = "none"\n',
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
for (const approval of ["--dry-run", "--yes"]) {
|
||||
await runProcess(
|
||||
process.execPath,
|
||||
[
|
||||
join(repositoryRoot, "scripts/rea.mjs"),
|
||||
"setup",
|
||||
"--client",
|
||||
"codex",
|
||||
approval,
|
||||
"--json",
|
||||
],
|
||||
evaluationRoot,
|
||||
120_000,
|
||||
evaluationEnvironment,
|
||||
);
|
||||
}
|
||||
const installedSkill = await readFile(
|
||||
join(skillDestination, "SKILL.md"),
|
||||
"utf8",
|
||||
);
|
||||
if (!installedSkill.includes(` version: "${PRODUCT_IDENTITY.skillVersion}"`))
|
||||
throw new Error("Setup did not install the current packaged REA skill");
|
||||
const requestedScenarioIds = new Set(
|
||||
(process.env.REA_AGENT_EVAL_SCENARIOS ?? "")
|
||||
.split(",")
|
||||
@@ -57,7 +129,11 @@ try {
|
||||
expectedFirstTool: "open_binary",
|
||||
requiresEvidence: false,
|
||||
requiredAnswerTermGroups: [["unavailable", "could not", "provider"]],
|
||||
prompt: `Explain what the shipped native program at ${targets.native} does. Base the answer on artifact analysis, and clearly state any facts that remain unavailable.`,
|
||||
// The verifier authorizes these named session operations, not arbitrary
|
||||
// writes or runtime execution. Missing-engine behavior stays observable.
|
||||
approvedTools: ["open_binary", "close_binary"],
|
||||
requiredToolSubsequence: ["open_binary", "close_binary"],
|
||||
prompt: `Explain what static artifact evidence establishes about the shipped native program at ${targets.native}. Do not execute the target or install tools. Clearly state unavailable provider evidence, and close any analysis session you opened before finishing.`,
|
||||
},
|
||||
{
|
||||
id: "asar",
|
||||
@@ -70,6 +146,38 @@ try {
|
||||
],
|
||||
prompt: `Explain how the desktop application at ${targets.javascript} exposes APIs to its renderer. Base the answer on the shipped application artifact, and state what remains unknown.`,
|
||||
},
|
||||
{
|
||||
id: "asar-zh",
|
||||
fixtureScenarioId: "asar",
|
||||
expectedFirstTool: "analyze_javascript_application",
|
||||
requiresEvidence: true,
|
||||
requiredAnswerTermGroups: [["profileapi"], ["profile:read"]],
|
||||
prompt: `請靜態分析已交付的 Electron 程式 ${targets.javascript},說明 renderer 如何透過 preload 讀取 profile,以及 main handler 的關係。不要執行目標或安裝分析引擎;引用完整的產出 Evidence,保留 runtime 未知事項。`,
|
||||
},
|
||||
{
|
||||
id: "javascript-module-view",
|
||||
expectedFirstTool: "analyze_javascript_application",
|
||||
requiresEvidence: true,
|
||||
requiredToolSubsequence: [
|
||||
"analyze_javascript_application",
|
||||
"inspect_analysis_view",
|
||||
"inspect_analysis_view",
|
||||
],
|
||||
requiredAnswerTermGroups: [
|
||||
["main.js"],
|
||||
["node", "identity"],
|
||||
["observations", "recorded"],
|
||||
],
|
||||
prompt: `Inspect the recorded module identity, exports and observations for main.js in the shipped JavaScript tree ${targets.largeJavascript}. Use REA without directly reading target files or executing the target. Keep results focused on that module, cite Evidence, and state the limits of what its module view establishes.`,
|
||||
},
|
||||
{
|
||||
id: "missing-target",
|
||||
expectedFirstTool: null,
|
||||
requiresEvidence: false,
|
||||
requiredAnswerTermGroups: [],
|
||||
prompt:
|
||||
'Reverse engineer my desktop app and explain how it saves user settings. Use static analysis; do not install software. End with one JSON object containing status ("needs_target" or "analyzed"), requested_input ("app_name_or_artifact_path" or null), and question (a string or null).',
|
||||
},
|
||||
{
|
||||
id: "javascript-export-shape",
|
||||
expectedFirstTool: "analyze_javascript_application",
|
||||
@@ -145,12 +253,14 @@ try {
|
||||
const results = [];
|
||||
for (const scenario of scenarios) {
|
||||
process.stderr.write(`Running Codex agent evaluation: ${scenario.id}\n`);
|
||||
const fixtureClaims = agentFixtureClaims(scenario.id, targets);
|
||||
const answerInstructions = agentFixtureAnswerInstructions(scenario.id);
|
||||
const fixtureId = scenario.fixtureScenarioId ?? scenario.id;
|
||||
const fixtureClaims = agentFixtureClaims(fixtureId, targets);
|
||||
const answerInstructions = agentFixtureAnswerInstructions(fixtureId);
|
||||
const execution = await runCodex(
|
||||
answerInstructions.length === 0
|
||||
? scenario.prompt
|
||||
: `${scenario.prompt}\n\n${answerInstructions}`,
|
||||
: `${scenario.prompt}\n\nThis closed factual rubric needs complete producer Evidence in the transcript. Request complete detail on each initial application analysis when supported; summary or view records alone are not accepted by this rubric.\n\n${answerInstructions}`,
|
||||
scenario.approvedTools ?? [],
|
||||
);
|
||||
const transcriptDirectory = process.env.REA_AGENT_EVAL_TRANSCRIPT_DIR;
|
||||
if (transcriptDirectory !== undefined) {
|
||||
@@ -175,6 +285,12 @@ try {
|
||||
...(fixtureClaims === undefined ? {} : { fixtureClaims }),
|
||||
},
|
||||
);
|
||||
// Fixtures are deliberately visible beside prior investigation artifacts.
|
||||
// Their existence does not identify the unnamed app in this request.
|
||||
const targetClarificationPassed =
|
||||
scenario.expectedFirstTool === null
|
||||
? validTargetClarification(metrics.finalMessage)
|
||||
: undefined;
|
||||
const result = {
|
||||
id: scenario.id,
|
||||
expectedFirstTool: scenario.expectedFirstTool,
|
||||
@@ -185,6 +301,9 @@ try {
|
||||
exitCode: execution.exitCode,
|
||||
stderr: execution.stderr,
|
||||
...metrics,
|
||||
...(targetClarificationPassed === undefined
|
||||
? {}
|
||||
: { targetClarificationPassed }),
|
||||
};
|
||||
results.push(result);
|
||||
process.stderr.write(
|
||||
@@ -194,13 +313,20 @@ try {
|
||||
|
||||
const factualCorrectness = summarizeFactualCorrectness(results);
|
||||
const summary = {
|
||||
schemaVersion: 3,
|
||||
schemaVersion: 4,
|
||||
evaluationScope: "routing_workflow_and_configured_fixture_claims",
|
||||
factualCorrectness,
|
||||
verifier_run: await completeVerifierRun(verifierRun),
|
||||
codex,
|
||||
codexVersion,
|
||||
model: optionalModel ?? null,
|
||||
installation: {
|
||||
method: "rea_setup_codex",
|
||||
isolatedHome: true,
|
||||
isolatedCodexHome: true,
|
||||
skillVersion: PRODUCT_IDENTITY.skillVersion,
|
||||
authentication: "caller_auth_only_removed_after_run",
|
||||
},
|
||||
scenarios: results,
|
||||
totals: {
|
||||
scenarios: results.length,
|
||||
@@ -253,12 +379,32 @@ try {
|
||||
`Agent routing/workflow/answer checks failed: ${failed.map(({ id }) => id).join(", ")}`,
|
||||
);
|
||||
} finally {
|
||||
// Even a deliberately retained fixture directory must never retain auth.
|
||||
await rm(authPath, { force: true });
|
||||
if (process.env.REA_AGENT_EVAL_KEEP_FIXTURES !== "true")
|
||||
await rm(evaluationRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function createTargets(root, includeManaged) {
|
||||
const fixtures = await createAgentEvaluationFixtures(root, repositoryRoot);
|
||||
const largeJavascript = join(root, "large-app");
|
||||
await mkdir(join(largeJavascript, "lib"), { recursive: true });
|
||||
await Promise.all([
|
||||
writeFile(
|
||||
join(largeJavascript, "package.json"),
|
||||
'{"name":"large-app","main":"main.js"}\n',
|
||||
),
|
||||
writeFile(
|
||||
join(largeJavascript, "main.js"),
|
||||
'const { ipcMain } = require("electron");\nipcMain.handle("profile:read", (_event, id) => ({ id }));\n',
|
||||
),
|
||||
...Array.from({ length: 180 }, (_, index) =>
|
||||
writeFile(
|
||||
join(largeJavascript, "lib", `formatter-${String(index)}.js`),
|
||||
`export const format = (value) => String(value).trim();\n`,
|
||||
),
|
||||
),
|
||||
]);
|
||||
|
||||
const managedOutput = join(root, "managed-output");
|
||||
if (includeManaged) {
|
||||
@@ -292,15 +438,29 @@ async function createTargets(root, includeManaged) {
|
||||
return {
|
||||
native: "/bin/true",
|
||||
...fixtures,
|
||||
largeJavascript,
|
||||
managed: join(managedOutput, "AgentEval.dll"),
|
||||
};
|
||||
}
|
||||
|
||||
async function runCodex(prompt) {
|
||||
function validTargetClarification(message) {
|
||||
const schema = z.strictObject({
|
||||
status: z.literal("needs_target"),
|
||||
requested_input: z.literal("app_name_or_artifact_path"),
|
||||
question: z.string().refine((value) => value.trim().length > 0),
|
||||
});
|
||||
try {
|
||||
return schema.safeParse(JSON.parse(message)).success;
|
||||
} catch (cause) {
|
||||
if (cause instanceof SyntaxError) return false;
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
async function runCodex(prompt, approvedTools) {
|
||||
const arguments_ = [
|
||||
"--no-daemon",
|
||||
"exec",
|
||||
"--ignore-user-config",
|
||||
"--ignore-rules",
|
||||
"--ephemeral",
|
||||
"--skip-git-repo-check",
|
||||
"--sandbox",
|
||||
@@ -312,18 +472,16 @@ async function runCodex(prompt) {
|
||||
evaluationRoot,
|
||||
"-c",
|
||||
'approval_policy="never"',
|
||||
"-c",
|
||||
`mcp_servers.rea.command=${JSON.stringify(process.execPath)}`,
|
||||
"-c",
|
||||
`mcp_servers.rea.args=${JSON.stringify([join(repositoryRoot, "scripts/rea.mjs"), "mcp"])}`,
|
||||
"-c",
|
||||
`mcp_servers.rea.startup_timeout_sec=${String(MCP_STARTUP_POLICY.codexStartupTimeoutSeconds)}`,
|
||||
...approvedTools.flatMap((name) => [
|
||||
"-c",
|
||||
`mcp_servers.rea.tools.${name}.approval_mode="approve"`,
|
||||
]),
|
||||
...(optionalModel === undefined ? [] : ["--model", optionalModel]),
|
||||
prompt,
|
||||
];
|
||||
const child = spawn(codex, arguments_, {
|
||||
cwd: evaluationRoot,
|
||||
env: process.env,
|
||||
env: evaluationEnvironment,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const events = [];
|
||||
@@ -362,9 +520,16 @@ async function runCodex(prompt) {
|
||||
return { events, exitCode, stderr: stderr.join("").slice(0, 32_768) };
|
||||
}
|
||||
|
||||
async function runProcess(command, arguments_, cwd, timeout) {
|
||||
async function runProcess(
|
||||
command,
|
||||
arguments_,
|
||||
cwd,
|
||||
timeout,
|
||||
env = process.env,
|
||||
) {
|
||||
const child = spawn(command, arguments_, {
|
||||
cwd,
|
||||
env,
|
||||
stdio: ["ignore", "ignore", "pipe"],
|
||||
});
|
||||
let stderr = "";
|
||||
|
||||
@@ -22,6 +22,13 @@ const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const command = process.env.REA_VERIFY_COPILOT_COMMAND ?? "copilot";
|
||||
const mode = process.argv[2] ?? "call";
|
||||
const model = process.env.REA_VERIFY_COPILOT_MODEL ?? "gpt-5.4";
|
||||
const wireModel = process.env.REA_VERIFY_COPILOT_WIRE_MODEL ?? model;
|
||||
const inputSchemaProfile =
|
||||
process.env.REA_VERIFY_COPILOT_SCHEMA_PROFILE ?? "full";
|
||||
assert(
|
||||
["full", "compact"].includes(inputSchemaProfile),
|
||||
"REA_VERIFY_COPILOT_SCHEMA_PROFILE must be full or compact",
|
||||
);
|
||||
assert(
|
||||
["chat", "call"].includes(mode),
|
||||
"Usage: verify-copilot-client.mjs [chat|call]",
|
||||
@@ -44,6 +51,7 @@ await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
NODE_OPTIONS: process.env.NODE_OPTIONS,
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
COPILOT_HOME: profile,
|
||||
COPILOT_OFFLINE: "true",
|
||||
@@ -54,7 +62,7 @@ const environment = {
|
||||
COPILOT_PROVIDER_BEARER_TOKEN: "",
|
||||
COPILOT_PROVIDER_HEADERS: "",
|
||||
COPILOT_PROVIDER_MODEL_ID: model,
|
||||
COPILOT_PROVIDER_WIRE_MODEL: model,
|
||||
COPILOT_PROVIDER_WIRE_MODEL: wireModel,
|
||||
COPILOT_PROVIDER_MAX_PROMPT_TOKENS: "1000000",
|
||||
COPILOT_PROVIDER_MAX_OUTPUT_TOKENS: "4096",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
@@ -112,10 +120,17 @@ const extractEvidence = (value) => {
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
const wireModels = [];
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: "copilot",
|
||||
onRequest: async ({ body, tools, requestIndex }) => {
|
||||
assert.equal(
|
||||
body.model,
|
||||
wireModel,
|
||||
"Copilot must send the selected wire model",
|
||||
);
|
||||
wireModels.push(body.model);
|
||||
// Auxiliary requests without tools cannot execute the analyst's workflow.
|
||||
if (tools.length === 0)
|
||||
return { role: "assistant", content: "REA fixture title" };
|
||||
@@ -311,6 +326,16 @@ try {
|
||||
assert.deepEqual(JSON.parse(repeated.stdout).appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), configured);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
if (inputSchemaProfile === "compact") {
|
||||
// Select the existing presentation in this disposable native-client profile.
|
||||
const customized = JSON.parse(configured);
|
||||
assert(customized.mcpServers.rea);
|
||||
customized.mcpServers.rea.env = {
|
||||
...customized.mcpServers.rea.env,
|
||||
REA_MCP_INPUT_SCHEMA_PROFILE: inputSchemaProfile,
|
||||
};
|
||||
await writeFile(configPath, JSON.stringify(customized, null, 2));
|
||||
}
|
||||
const installedSkill = await readFile(
|
||||
join(skillDirectory, "SKILL.md"),
|
||||
"utf8",
|
||||
@@ -408,6 +433,8 @@ try {
|
||||
clientVersion,
|
||||
mode,
|
||||
model,
|
||||
wireModel,
|
||||
inputSchemaProfile,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
@@ -421,7 +448,9 @@ try {
|
||||
skillDiscovery:
|
||||
"native COPILOT_HOME MCP configuration and caller-added isolated skill directory; default OS-home discovery unverified",
|
||||
capacityBoundary:
|
||||
"gpt-4.1 model configuration blocks the full catalog before HTTP even with a ten-million-token requested prompt override; gpt-5.4 is the verified local fixture configuration",
|
||||
inputSchemaProfile === "compact"
|
||||
? "Explicit compact profile: complete inventory and canonical server validation; advertised schemas may be reduced. This does not fix the full-profile gpt-4.1 context boundary."
|
||||
: "Complete canonical input and output schemas. Client model ID and wire model are recorded separately; effective token capacity and live provider compatibility remain unknown.",
|
||||
catalogSize,
|
||||
schemasChecked,
|
||||
spillReadRequested,
|
||||
@@ -429,6 +458,7 @@ try {
|
||||
skillLoaded,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
wireModels,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
|
||||
@@ -40,9 +40,9 @@ await Promise.all([
|
||||
]);
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
DSH_HOME: profile,
|
||||
DSH_AGENTS_HOME: join(account, ".agents"),
|
||||
OPENAI_API_KEY: "local-fixture-key",
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
@@ -328,7 +328,7 @@ try {
|
||||
"loopback custom OpenAI-compatible adapter; no live DeepSeek provider claim",
|
||||
modelContextWindow: 1000000,
|
||||
skillDiscovery:
|
||||
"native DSH_AGENTS_HOME personal directory; isolated project root",
|
||||
"native HOME shared personal directory; isolated project root",
|
||||
catalogSize: catalog?.length,
|
||||
patternCount,
|
||||
uniquePatterns: patterns.size,
|
||||
|
||||
@@ -54,6 +54,7 @@ await writeFile(
|
||||
);
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
[client === "pi" ? "PI_CODING_AGENT_DIR" : "HERMES_HOME"]: profile,
|
||||
PI_OFFLINE: "1",
|
||||
@@ -71,13 +72,6 @@ const original =
|
||||
})
|
||||
: "# Keep caller settings.\nmcp_servers:\n other:\n command: unrelated-server\n enabled: false\n";
|
||||
await writeFile(configPath, original);
|
||||
if (client === "pi") {
|
||||
// The CLI fixture account and native os.homedir() differ; use Pi's caller-owned skill setting.
|
||||
await writeFile(
|
||||
join(profile, "settings.json"),
|
||||
JSON.stringify({ skills: [join(account, ".agents", "skills")] }),
|
||||
);
|
||||
}
|
||||
const clientVersion = (
|
||||
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
|
||||
).stdout.trim();
|
||||
@@ -448,7 +442,7 @@ try {
|
||||
"loopback deterministic OpenAI-compatible endpoint; no live provider claim",
|
||||
skillDiscovery:
|
||||
client === "pi"
|
||||
? "caller-configured isolated shared directory; default OS home discovery unverified"
|
||||
? "native HOME shared personal directory; no explicit skill setting"
|
||||
: "native HERMES_HOME personal directory",
|
||||
stickyHermes,
|
||||
catalogSize: catalog?.length,
|
||||
|
||||
@@ -45,6 +45,7 @@ await Promise.all([
|
||||
]);
|
||||
const baseEnvironment = {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: account,
|
||||
USERPROFILE: account,
|
||||
QWEN_HOME: profile,
|
||||
QWEN_RUNTIME_DIR: runtime,
|
||||
@@ -71,9 +72,6 @@ const original = JSON.stringify(
|
||||
},
|
||||
telemetry: { enabled: false },
|
||||
model: { name: "rea-client-fixture" },
|
||||
// Qwen uses os.homedir(), while REA's isolated CLI account uses USERPROFILE.
|
||||
// This caller-owned setting lets the real client load the REA-installed bundle.
|
||||
skills: { directories: [sharedSkills] },
|
||||
mcpServers: { other: { command: "unrelated-server", disabled: true } },
|
||||
},
|
||||
null,
|
||||
@@ -112,7 +110,7 @@ const configured = JSON.parse(
|
||||
await readFile(join(profile, "settings.json"), "utf8"),
|
||||
);
|
||||
assert.deepEqual(configured.model, { name: "rea-client-fixture" });
|
||||
assert.deepEqual(configured.skills, { directories: [sharedSkills] });
|
||||
assert.equal(configured.skills, undefined);
|
||||
assert.deepEqual(configured.mcpServers.other, {
|
||||
command: "unrelated-server",
|
||||
disabled: true,
|
||||
@@ -384,7 +382,7 @@ try {
|
||||
modelFixture:
|
||||
"loopback deterministic OpenAI-compatible endpoint; no live model-provider claim",
|
||||
skillDiscovery:
|
||||
"caller-configured isolated shared directory; default OS home discovery unverified",
|
||||
"native HOME shared personal directory; no explicit skill setting",
|
||||
catalogSize: TOOL_CONTRACTS.length,
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, mkdir, readFile, rm } from "node:fs/promises";
|
||||
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -60,6 +60,9 @@ const env = {
|
||||
JAVA_HOME: installation.javaHome,
|
||||
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
|
||||
};
|
||||
for (const setting of ["GHIDRA_HEADLESS_MAXMEM", "GHIDRA_MAXMEM"]) {
|
||||
if (process.env[setting] !== undefined) env[setting] = process.env[setting];
|
||||
}
|
||||
const reports = [];
|
||||
let primaryFailure;
|
||||
try {
|
||||
@@ -227,6 +230,103 @@ try {
|
||||
fixture.sha256,
|
||||
"Analysis modified the source executable",
|
||||
);
|
||||
// Carved firmware keeps all program headers/load bytes but loses sections.
|
||||
const read16 = (offset) =>
|
||||
byteOrder === "little"
|
||||
? sourceBytes.readUInt16LE(offset)
|
||||
: sourceBytes.readUInt16BE(offset);
|
||||
const read32 = (offset) =>
|
||||
byteOrder === "little"
|
||||
? sourceBytes.readUInt32LE(offset)
|
||||
: sourceBytes.readUInt32BE(offset);
|
||||
const sectionOffset = read32(32);
|
||||
const cut = sectionOffset - 1;
|
||||
const programOffset = read32(28);
|
||||
const programCount = read16(44);
|
||||
assert.equal(read16(42), 32);
|
||||
assert.ok(cut >= programOffset + programCount * 32);
|
||||
for (let i = 0; i < programCount; i += 1) {
|
||||
const offset = programOffset + i * 32;
|
||||
assert.ok(read32(offset + 4) + read32(offset + 16) <= cut);
|
||||
}
|
||||
const carvedBytes = sourceBytes.subarray(0, cut);
|
||||
const carvedPath = join(workspace, `${byteOrder}-carved.elf`);
|
||||
const carvedSha256 = createHash("sha256")
|
||||
.update(carvedBytes)
|
||||
.digest("hex");
|
||||
await writeFile(carvedPath, carvedBytes);
|
||||
const carvedResolved = await parseBinaryTarget(carvedPath);
|
||||
if (!carvedResolved.ok) throw carvedResolved.error;
|
||||
assert.deepEqual(
|
||||
carvedResolved.value.mips.abiFlags,
|
||||
independent.mips.abiFlags,
|
||||
);
|
||||
assert.match(
|
||||
carvedResolved.value.mips.limitations.join(" "),
|
||||
/section header table/u,
|
||||
);
|
||||
const carvedTarget = await call("open_binary", {
|
||||
path: carvedPath,
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
assert.equal(carvedTarget.sha256, carvedSha256);
|
||||
const carvedImage = await call("inspect_native_load_image");
|
||||
assertMipsLoadedImage(carvedImage.observations, byteOrder, carvedSha256);
|
||||
const carvedProcedures = await call("list_procedures");
|
||||
assert.ok(
|
||||
carvedProcedures.some(
|
||||
(p) =>
|
||||
BigInt(p.address) === BigInt(independent.symbols.rea_mips_entry),
|
||||
),
|
||||
);
|
||||
const carvedDossier = functionDossierSchema.parse(
|
||||
await call("analyze_function", {
|
||||
procedure: independent.symbols.rea_mips_entry,
|
||||
}),
|
||||
);
|
||||
assert.ok(carvedDossier.pseudocode.trim().length > 0);
|
||||
const carvedSnapshotPath = join(
|
||||
workspace,
|
||||
`${byteOrder}-carved.snapshot.json`,
|
||||
);
|
||||
await call("close_binary", { snapshot_path: carvedSnapshotPath });
|
||||
const carvedSnapshot = parseAnalysisSnapshot(
|
||||
JSON.parse(await readFile(carvedSnapshotPath, "utf8")),
|
||||
);
|
||||
assert.equal(carvedSnapshot.target.sha256, carvedSha256);
|
||||
assert.ok(
|
||||
carvedSnapshot.evidence_bundle.records.some((record) =>
|
||||
parseEvidence(record).limitations.some((item) =>
|
||||
item.includes("section header table"),
|
||||
),
|
||||
),
|
||||
);
|
||||
const carvedCli = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"function",
|
||||
carvedPath,
|
||||
independent.symbols.rea_mips_entry,
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const carvedEvidence = parseEvidence(JSON.parse(carvedCli.stdout));
|
||||
assert.equal(carvedEvidence.subject?.digest.sha256, carvedSha256);
|
||||
assert.ok(
|
||||
carvedEvidence.limitations.some((item) =>
|
||||
item.includes("section header table"),
|
||||
),
|
||||
);
|
||||
assert.equal(
|
||||
functionDossierSchema.parse(carvedEvidence.normalized_result).procedure
|
||||
.address,
|
||||
carvedDossier.procedure.address,
|
||||
);
|
||||
assert.deepEqual(await readFile(carvedPath), carvedBytes);
|
||||
reports.push({
|
||||
...fixture,
|
||||
independent_reader: independent,
|
||||
@@ -241,6 +341,14 @@ try {
|
||||
"CLI/MCP and snapshot identity",
|
||||
],
|
||||
procedures: procedures.length,
|
||||
carved: {
|
||||
sha256: carvedSha256,
|
||||
source_bytes: carvedBytes.length,
|
||||
section_header_offset: sectionOffset,
|
||||
effective_language: carvedImage.observations.language_id,
|
||||
limitations: carvedResolved.value.mips.limitations,
|
||||
procedures: carvedProcedures.length,
|
||||
},
|
||||
status: "passed",
|
||||
});
|
||||
} catch (cause) {
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import { realpath } from "node:fs/promises";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import { analyzeJavaScriptApplication } from "../../../dist/application/javascript/JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplicationWithOwnedScope } from "../../lib/javascript-application-analysis.mjs";
|
||||
import {
|
||||
compareApplicationVersionsEvidence,
|
||||
compareJavaScriptExportShapesEvidence,
|
||||
@@ -199,9 +199,5 @@ function selectorKey(selector) {
|
||||
}
|
||||
|
||||
async function analyze(path) {
|
||||
const result = await analyzeJavaScriptApplication({
|
||||
input_path: path,
|
||||
});
|
||||
if (!result.ok) throw result.error;
|
||||
return result.value;
|
||||
return analyzeJavaScriptApplicationWithOwnedScope({ input_path: path });
|
||||
}
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
import { readFile, realpath } from "node:fs/promises";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import { analyzeJavaScriptApplication } from "../../../dist/application/javascript/JavaScriptApplicationService.js";
|
||||
import { reconcileJavaScriptRuntimeEvidence } from "../../../dist/application/javascript/JavaScriptRuntimeReconciliationService.js";
|
||||
import { analyzeJavaScriptApplicationWithOwnedScope } from "../../lib/javascript-application-analysis.mjs";
|
||||
import {
|
||||
completeVerifierRun,
|
||||
createVerifierRun,
|
||||
@@ -44,14 +44,13 @@ const canonicalLayers = await Promise.all(
|
||||
);
|
||||
const staticLayers = [];
|
||||
for (const layer of canonicalLayers) {
|
||||
const analyzed = await analyzeJavaScriptApplication({
|
||||
const analyzed = await analyzeJavaScriptApplicationWithOwnedScope({
|
||||
input_path: layer.path,
|
||||
format: "auto",
|
||||
});
|
||||
if (!analyzed.ok) throw analyzed.error;
|
||||
staticLayers.push({
|
||||
role: layer.role,
|
||||
analysis: analyzed.value,
|
||||
analysis: analyzed,
|
||||
runtime_mappings: [],
|
||||
});
|
||||
}
|
||||
|
||||
@@ -373,7 +373,13 @@ const readExecutableMetadata = async (
|
||||
if (!abiFlags.ok) return abiFlags;
|
||||
return ok({
|
||||
...parsed.value,
|
||||
mips: { ...parsed.value.mips, abiFlags: abiFlags.value },
|
||||
mips: {
|
||||
...parsed.value.mips,
|
||||
abiFlags: abiFlags.value.abiFlags,
|
||||
...(abiFlags.value.limitations.length === 0
|
||||
? {}
|
||||
: { limitations: abiFlags.value.limitations }),
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { readRegularFileText } from "./RegularFileRead.js";
|
||||
import {
|
||||
RegularFileCleanupFailure,
|
||||
readRegularFileText,
|
||||
} from "./RegularFileRead.js";
|
||||
import {
|
||||
effectiveClientServer,
|
||||
parseClientConfiguration,
|
||||
type ClientConfigurationDocument,
|
||||
} from "./ClientConfigurationDocument.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
import { clientRegistrationEnvironment } from "./ClientRegistrationEnvironment.js";
|
||||
|
||||
/** A named source failure that setup can explain without losing the bad path. */
|
||||
export interface ClientConfigurationFileError {
|
||||
readonly kind: "unreadable" | "malformed";
|
||||
readonly path: string;
|
||||
readonly detail: string;
|
||||
readonly cleanupFailure?: RegularFileCleanupFailure;
|
||||
}
|
||||
|
||||
/** Preserve named read failures and distinguish a missing source from an empty file. */
|
||||
@@ -27,6 +32,9 @@ export const readConfigurationText = async (
|
||||
kind: "unreadable",
|
||||
path,
|
||||
detail: `Configuration ${path} could not be read: ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
...(cause instanceof RegularFileCleanupFailure
|
||||
? { cleanupFailure: cause }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
};
|
||||
@@ -63,7 +71,16 @@ export const readClientConfigurationFiles = async (
|
||||
for (const path of client.configPaths ?? [client.configPath]) {
|
||||
const document = await readConfigurationFile(path, client.format);
|
||||
if (!document.ok) return document;
|
||||
if (document.value !== undefined) documents.push(document.value);
|
||||
if (document.value !== undefined) {
|
||||
const environment = clientRegistrationEnvironment(document.value);
|
||||
if (!environment.ok)
|
||||
return err({
|
||||
kind: "malformed",
|
||||
path,
|
||||
detail: `Configuration ${path} has an invalid REA server environment: ${environment.error.settings.map(({ setting, constraint }) => `${setting}: ${constraint}`).join("; ")}. Repair it before rerunning setup.`,
|
||||
});
|
||||
documents.push(document.value);
|
||||
}
|
||||
}
|
||||
return ok(documents);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { parseEnvironment } from "../config/environment.js";
|
||||
import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import {
|
||||
effectiveClientServer,
|
||||
type ClientConfigurationDocument,
|
||||
} from "./ClientConfigurationDocument.js";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
|
||||
/** Read explicit REA overrides without dropping peers when one value is invalid. */
|
||||
export const clientRegistrationEnvironment = (
|
||||
parsed: ClientConfigurationDocument | undefined,
|
||||
overrides: Readonly<Record<string, string>> = {},
|
||||
): Result<Readonly<Record<string, string>>, ConfigurationError> => {
|
||||
const key =
|
||||
parsed?.dialect === "opencode" || parsed?.dialect === "opencode_v2"
|
||||
? "environment"
|
||||
: "env";
|
||||
const registration =
|
||||
parsed === undefined
|
||||
? undefined
|
||||
: effectiveClientServer(parsed, PRODUCT_IDENTITY.mcpServerKey);
|
||||
const raw =
|
||||
typeof registration === "object" && registration !== null
|
||||
? Reflect.get(registration, key)
|
||||
: undefined;
|
||||
const existing = z
|
||||
.record(z.string(), z.string())
|
||||
.safeParse(raw === undefined ? {} : raw);
|
||||
if (!existing.success)
|
||||
return err(
|
||||
new ConfigurationError("Invalid REA server environment", {
|
||||
settings: existing.error.issues.map((issue) => ({
|
||||
setting: [key, ...issue.path].join("."),
|
||||
constraint:
|
||||
"Server environment must be an object containing only string values",
|
||||
})),
|
||||
}),
|
||||
);
|
||||
// Validate known settings with the runtime's parser, but retain the original
|
||||
// string overrides, including unknown client-specific keys and no defaults.
|
||||
const environment = { ...existing.data, ...overrides };
|
||||
const validated = parseEnvironment(environment);
|
||||
if (!validated.ok) return validated;
|
||||
return ok(
|
||||
Object.fromEntries(
|
||||
Object.entries(environment).sort(([left], [right]) =>
|
||||
left.localeCompare(right),
|
||||
),
|
||||
),
|
||||
);
|
||||
};
|
||||
@@ -5,27 +5,22 @@ import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { jsonValueSchema } from "../domain/jsonValue.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { compareBundles } from "../domain/bundleComparison.js";
|
||||
import { EvidenceLedger } from "./investigation/EvidenceLedger.js";
|
||||
import {
|
||||
readEvidenceBundle,
|
||||
writeEvidenceBundle,
|
||||
} from "./EvidenceBundleFiles.js";
|
||||
|
||||
/** Validate and merge one bundle using the session Evidence ledger. */
|
||||
/** Validate a standalone bundle and report its complete record/revision counts. */
|
||||
export const importEvidenceBundleCommand = async (
|
||||
path: string,
|
||||
): Promise<Result<JsonValue, AnalysisError>> => {
|
||||
const loaded = await readEvidenceBundle(path);
|
||||
if (!loaded.ok) return loaded;
|
||||
const ledger = createLedger();
|
||||
const imported = ledger.import(loaded.value);
|
||||
return imported.ok
|
||||
? ok({
|
||||
imported: imported.value.recordsAdded,
|
||||
unknowns_added: imported.value.unknownsAdded,
|
||||
total: ledger.export().records.length,
|
||||
})
|
||||
: imported;
|
||||
return ok({
|
||||
imported: loaded.value.records.length,
|
||||
unknowns_added: loaded.value.unknowns.length,
|
||||
total: loaded.value.records.length,
|
||||
});
|
||||
};
|
||||
|
||||
/** Validate a source bundle and atomically export canonical bytes. */
|
||||
@@ -64,8 +59,6 @@ export const compareEvidenceBundlesCommand = async (input: {
|
||||
}
|
||||
};
|
||||
|
||||
const createLedger = (): EvidenceLedger => new EvidenceLedger();
|
||||
|
||||
const projectWrite = (
|
||||
bundle: EvidenceBundle,
|
||||
written: Awaited<ReturnType<typeof writeEvidenceBundle>>,
|
||||
|
||||
@@ -5,6 +5,12 @@ import type {
|
||||
} from "../domain/binaryTargetTypes.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
/** Observed ABI declarations and coverage of their source tables. */
|
||||
export interface MipsAbiFlagsInspection {
|
||||
readonly abiFlags: MipsAbiFlags | null;
|
||||
readonly limitations: readonly string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Read ELF32 MIPS ABI declarations from the caller's already-open file.
|
||||
* Inspect both typed tables (not section names), including records beyond the
|
||||
@@ -15,7 +21,7 @@ export const readMipsElfAbiFlags = async (
|
||||
handle: FileHandle,
|
||||
metadata: MipsElfMetadata,
|
||||
checkCancelled: () => void,
|
||||
): Promise<Result<MipsAbiFlags | null, string>> => {
|
||||
): Promise<Result<MipsAbiFlagsInspection, string>> => {
|
||||
checkCancelled();
|
||||
if (metadata.elfClass !== 32)
|
||||
return err("MIPS ABI record inspection currently supports ELF32 only");
|
||||
@@ -81,6 +87,7 @@ export const readMipsElfAbiFlags = async (
|
||||
dataOffset: 4,
|
||||
sizeOffset: 16,
|
||||
recordType: 0x70000003, // PT_MIPS_ABIFLAGS
|
||||
optional: false,
|
||||
},
|
||||
{
|
||||
label: "section header table",
|
||||
@@ -92,6 +99,7 @@ export const readMipsElfAbiFlags = async (
|
||||
dataOffset: 16,
|
||||
sizeOffset: 20,
|
||||
recordType: 0x7000002a, // SHT_MIPS_ABIFLAGS
|
||||
optional: true,
|
||||
},
|
||||
];
|
||||
// Extended numbering needs a separate section-zero reader. Refuse it rather
|
||||
@@ -99,6 +107,7 @@ export const readMipsElfAbiFlags = async (
|
||||
if (u16(bytes, 44) === 0xffff || u16(bytes, 48) >= 0xff00)
|
||||
return err("extended or reserved MIPS ELF table numbering is unsupported");
|
||||
let record: Buffer | undefined;
|
||||
const limitations: string[] = [];
|
||||
for (const table of tables) {
|
||||
if (table.count === 0) {
|
||||
if (table.offset !== 0)
|
||||
@@ -107,16 +116,27 @@ export const readMipsElfAbiFlags = async (
|
||||
}
|
||||
if (table.offset < 52 || table.stride !== table.entrySize)
|
||||
return err(`invalid MIPS ${table.label} offset or entry size`);
|
||||
const count = table.optional
|
||||
? Math.min(
|
||||
table.count,
|
||||
Math.max(0, Math.floor((fileSize - table.offset) / table.entrySize)),
|
||||
)
|
||||
: table.count;
|
||||
if (count < table.count)
|
||||
limitations.push(
|
||||
`MIPS ${table.label} at file offset ${table.offset} extends beyond the captured file: examined ${count} of ${table.count} complete headers. ABI declarations in unavailable headers remain unknown.`,
|
||||
);
|
||||
if (count === 0) continue;
|
||||
// Counts are bounded by the ELF16 fields and strides by their exact ELF32
|
||||
// structure sizes. Never allocate according to an untrusted section size.
|
||||
const contents = await read(
|
||||
table.offset,
|
||||
table.count * table.entrySize,
|
||||
count * table.entrySize,
|
||||
table.label,
|
||||
);
|
||||
if (!contents.ok) return contents;
|
||||
let found = false;
|
||||
for (let i = 0; i < table.count; i += 1) {
|
||||
for (let i = 0; i < count; i += 1) {
|
||||
checkCancelled();
|
||||
const position = i * table.entrySize;
|
||||
if (u32(contents.value, position + table.typeOffset) !== table.recordType)
|
||||
@@ -134,10 +154,10 @@ export const readMipsElfAbiFlags = async (
|
||||
record = candidate.value;
|
||||
}
|
||||
}
|
||||
if (record === undefined) return ok(null);
|
||||
if (record === undefined) return ok({ abiFlags: null, limitations });
|
||||
// Preserve declarations verbatim, including unknown versions/values. Provider
|
||||
// admission, not parsing, decides which interpretations have been verified.
|
||||
return ok({
|
||||
const abiFlags: MipsAbiFlags = {
|
||||
version: u16(record, 0),
|
||||
isaLevel: record.readUInt8(2),
|
||||
isaRevision: record.readUInt8(3),
|
||||
@@ -149,5 +169,6 @@ export const readMipsElfAbiFlags = async (
|
||||
ases: u32(record, 12),
|
||||
flags1: u32(record, 16),
|
||||
flags2: u32(record, 20),
|
||||
});
|
||||
};
|
||||
return ok({ abiFlags, limitations });
|
||||
};
|
||||
|
||||
@@ -8,14 +8,24 @@ import {
|
||||
type HistoricalSourceGraphInput,
|
||||
} from "../domain/referenceSourceGraph.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js";
|
||||
import { readReferenceSource } from "../reference/ReferenceSourceReader.js";
|
||||
import type {
|
||||
ReferenceSourceEntryKind,
|
||||
ReferenceSourceRead,
|
||||
ReferenceSourceReaderError,
|
||||
} from "../reference/ReferenceSourceReaderTypes.js";
|
||||
import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js";
|
||||
import { readReferenceSourceVcs } from "./ReferenceSourceVcsAdapter.js";
|
||||
import {
|
||||
type ReferenceSourceImportError,
|
||||
type ReferenceSourceImportOptions,
|
||||
} from "./ReferenceSourceImportTypes.js";
|
||||
import { prepareReferenceSourceImport } from "./ReferenceSourceImportPolicy.js";
|
||||
import {
|
||||
prepareReferenceSourceImport,
|
||||
type PreparedReferenceSourceImport,
|
||||
} from "./ReferenceSourceImportPolicy.js";
|
||||
|
||||
const failure = (
|
||||
code: ReferenceSourceImportError["code"],
|
||||
@@ -26,8 +36,12 @@ const failure = (
|
||||
message,
|
||||
});
|
||||
|
||||
const cancelled = (): ReferenceSourceImportError =>
|
||||
failure("cancelled", "Reference source import cancelled");
|
||||
const cancelled = (
|
||||
partial?: ReferenceSourceRead,
|
||||
): ReferenceSourceImportError => ({
|
||||
...failure("cancelled", "Reference source import cancelled"),
|
||||
...(partial === undefined ? {} : { partial }),
|
||||
});
|
||||
|
||||
const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true;
|
||||
|
||||
@@ -115,9 +129,48 @@ const sortExclusions = (
|
||||
[...exclusions].sort((left, right) => {
|
||||
const byPath = compareUnicodeCodePoints(left.path, right.path);
|
||||
if (byPath !== 0) return byPath;
|
||||
return compareUnicodeCodePoints(left.reason, right.reason);
|
||||
const byReason = compareUnicodeCodePoints(left.reason, right.reason);
|
||||
if (byReason !== 0) return byReason;
|
||||
return compareUnicodeCodePoints(
|
||||
"pattern" in left ? left.pattern : "",
|
||||
"pattern" in right ? right.pattern : "",
|
||||
);
|
||||
});
|
||||
|
||||
const createShouldExclude =
|
||||
(
|
||||
exclusions: HistoricalSourceGraphInput["exclusions"],
|
||||
secrets: PreparedReferenceSourceImport["secrets"],
|
||||
ignored: PreparedReferenceSourceImport["ignored"],
|
||||
): ((path: string, kind: ReferenceSourceEntryKind) => boolean) =>
|
||||
(path, kind) => {
|
||||
const patternPath = kind === "directory" ? `${path}/` : path;
|
||||
const secretMatch = secrets.test(patternPath);
|
||||
if (secretMatch.ignored) {
|
||||
if (!secretMatch.rule)
|
||||
throw new Error(`Ignored secret path has no matching rule: ${path}`);
|
||||
exclusions.push({
|
||||
path,
|
||||
reason: "configured-secret",
|
||||
pattern: secretMatch.rule.pattern,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
const match = ignored.test(patternPath);
|
||||
if (!match.ignored) return false;
|
||||
if (!match.rule)
|
||||
throw new Error(`Ignored path has no matching rule: ${path}`);
|
||||
const reason = match.rule.mark;
|
||||
if (
|
||||
reason !== "project-ignored" &&
|
||||
reason !== "default-ignored" &&
|
||||
reason !== "caller-excluded"
|
||||
)
|
||||
throw new Error(`Ignored path has unknown rule origin: ${path}`);
|
||||
exclusions.push({ path, reason, pattern: match.rule.pattern });
|
||||
return true;
|
||||
};
|
||||
|
||||
/**
|
||||
* Import a reference source directory into a committed historical source graph.
|
||||
*
|
||||
@@ -127,43 +180,41 @@ const sortExclusions = (
|
||||
*/
|
||||
export const importReferenceSource = async (
|
||||
options: ReferenceSourceImportOptions,
|
||||
resources: ArtifactResourceScope,
|
||||
): Promise<Result<HistoricalSourceGraph, ReferenceSourceImportError>> => {
|
||||
if (isAborted(options.signal)) return err(cancelled());
|
||||
const prepared = await prepareReferenceSourceImport(options);
|
||||
const prepared = await prepareReferenceSourceImport(options, resources);
|
||||
if (!prepared.ok) return prepared;
|
||||
const { ignored, root, secrets } = prepared.value;
|
||||
if (isAborted(options.signal)) return err(cancelled());
|
||||
|
||||
const exclusions: HistoricalSourceGraphInput["exclusions"] = [];
|
||||
const shouldExclude = (path: string): boolean => {
|
||||
if (secrets.ignores(path)) {
|
||||
exclusions.push({ path, reason: "configured-secret" });
|
||||
return true;
|
||||
}
|
||||
if (ignored.ignores(path)) {
|
||||
exclusions.push({ path, reason: "caller-excluded" });
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
const shouldExclude = createShouldExclude(exclusions, secrets, ignored);
|
||||
|
||||
const [readResult, vcs] = await Promise.all([
|
||||
readReferenceSource(root, {
|
||||
const [readResult, vcsResult] = await Promise.all([
|
||||
readReferenceSource(root, resources, {
|
||||
...(options.signal === undefined ? {} : { signal: options.signal }),
|
||||
shouldExclude,
|
||||
}),
|
||||
readReferenceSourceVcs(root, options.signal),
|
||||
readReferenceSourceVcs(root, resources, options.signal),
|
||||
]);
|
||||
|
||||
if (!readResult.ok) {
|
||||
const error = readResult.error;
|
||||
if (error.code === "cancelled") return err(cancelled());
|
||||
if (error.code === "unsupported")
|
||||
return err(failure("unsupported", error.message));
|
||||
return err(failure("io", error.message));
|
||||
}
|
||||
if (!readResult.ok)
|
||||
return err(
|
||||
combineReadFailures(
|
||||
readResult.error,
|
||||
vcsResult.ok ? undefined : vcsResult.error,
|
||||
readResult.error.partial,
|
||||
),
|
||||
);
|
||||
if (!vcsResult.ok)
|
||||
return err(
|
||||
combineReadFailures(vcsResult.error, undefined, readResult.value),
|
||||
);
|
||||
|
||||
if (isAborted(options.signal)) return err(cancelled());
|
||||
const vcs = vcsResult.value;
|
||||
|
||||
if (isAborted(options.signal)) return err(cancelled(readResult.value));
|
||||
|
||||
const read = readResult.value;
|
||||
const filePaths = new Set(
|
||||
@@ -175,7 +226,7 @@ export const importReferenceSource = async (
|
||||
const { entries, relationships, parseFailures, limitations } =
|
||||
parseReferenceSourceEntries(read, filePaths, options.signal);
|
||||
|
||||
if (isAborted(options.signal)) return err(cancelled());
|
||||
if (isAborted(options.signal)) return err(cancelled(read));
|
||||
|
||||
const uniqueRelationships = deduplicateRelationships(relationships);
|
||||
const uniqueFailures = normalizeHistoricalSourceParseFailures(parseFailures);
|
||||
@@ -221,3 +272,45 @@ export const importReferenceSource = async (
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const combineReadFailures = (
|
||||
primary: ReferenceSourceReaderError,
|
||||
other: ReferenceSourceReaderError | undefined,
|
||||
partial: ReferenceSourceRead | undefined,
|
||||
): ReferenceSourceImportError => {
|
||||
const code =
|
||||
primary.code === "cancelled" || other?.code === "cancelled"
|
||||
? "cancelled"
|
||||
: primary.code;
|
||||
const cleanup = mergeCleanup(primary.cleanup, other?.cleanup);
|
||||
const message =
|
||||
other === undefined
|
||||
? primary.message
|
||||
: `${primary.message}; additional reference metadata failure: ${other.message}`;
|
||||
const cause =
|
||||
other === undefined
|
||||
? primary.cause
|
||||
: new AggregateError(
|
||||
[primary, other],
|
||||
"Reference source inventory and metadata both failed",
|
||||
{ cause: primary },
|
||||
);
|
||||
return {
|
||||
...failure(code, message),
|
||||
...(cleanup === undefined ? {} : { cleanup }),
|
||||
...(partial === undefined ? {} : { partial }),
|
||||
...(cause === undefined ? {} : { cause }),
|
||||
};
|
||||
};
|
||||
|
||||
const mergeCleanup = (
|
||||
first: AnalysisCleanupObservation | undefined,
|
||||
second: AnalysisCleanupObservation | undefined,
|
||||
): AnalysisCleanupObservation | undefined => {
|
||||
if (first === undefined) return second;
|
||||
if (second === undefined) return first;
|
||||
return {
|
||||
reason: `${first.reason}; ${second.reason}`,
|
||||
resources: [...new Set([...first.resources, ...second.resources])],
|
||||
};
|
||||
};
|
||||
|
||||
@@ -2,14 +2,21 @@ import { lstat, realpath, stat } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import ignore from "ignore";
|
||||
import { readRegularFileText } from "./RegularFileRead.js";
|
||||
import {
|
||||
readRegularFileText,
|
||||
RegularFileCleanupFailure,
|
||||
retryRegularFileCleanup,
|
||||
} from "./RegularFileRead.js";
|
||||
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
DEFAULT_REFERENCE_SOURCE_IGNORE_PATTERNS,
|
||||
type ReferenceSourceImportError,
|
||||
type ReferenceSourceImportOptions,
|
||||
} from "./ReferenceSourceImportTypes.js";
|
||||
import { rootFilesystemFailure } from "../reference/ReferenceSourceReaderErrors.js";
|
||||
|
||||
/** Validated inputs ready for filesystem traversal. */
|
||||
export interface PreparedReferenceSourceImport {
|
||||
@@ -27,29 +34,57 @@ const failure = (
|
||||
message,
|
||||
});
|
||||
|
||||
const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true;
|
||||
|
||||
const addMarkedPatterns = (
|
||||
matcher: ReturnType<typeof ignore>,
|
||||
patterns: readonly string[],
|
||||
mark: "project-ignored" | "default-ignored" | "caller-excluded",
|
||||
): void => {
|
||||
for (const pattern of patterns)
|
||||
for (const line of pattern.split(/\r?\n/u))
|
||||
matcher.add({ pattern: line, mark });
|
||||
};
|
||||
|
||||
const resolveRoot = async (
|
||||
requestedRoot: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<string, ReferenceSourceImportError>> => {
|
||||
if (isAborted(signal))
|
||||
return err(failure("cancelled", "Reference source import cancelled"));
|
||||
try {
|
||||
if (!(await stat(requestedRoot)).isDirectory())
|
||||
const metadata = await stat(requestedRoot);
|
||||
if (isAborted(signal))
|
||||
return err(failure("cancelled", "Reference source import cancelled"));
|
||||
if (!metadata.isDirectory())
|
||||
return err(
|
||||
failure("invalid-root", "Reference source root is not a directory"),
|
||||
failure(
|
||||
"invalid-root",
|
||||
`Reference source root is not a directory: ${requestedRoot}`,
|
||||
),
|
||||
);
|
||||
const canonicalRoot = await realpath(resolve(requestedRoot));
|
||||
if (isAborted(signal))
|
||||
return err(failure("cancelled", "Reference source import cancelled"));
|
||||
return ok(canonicalRoot);
|
||||
} catch (cause: unknown) {
|
||||
void cause;
|
||||
return err(
|
||||
failure("invalid-root", "Reference source root could not be resolved"),
|
||||
if (isAborted(signal))
|
||||
return err(failure("cancelled", "Reference source import cancelled"));
|
||||
const rootFailure = rootFilesystemFailure(
|
||||
cause,
|
||||
"Reference source root could not be resolved",
|
||||
);
|
||||
if (rootFailure === undefined) throw cause;
|
||||
return err({ ...failure(rootFailure.code, rootFailure.message), cause });
|
||||
}
|
||||
};
|
||||
|
||||
const buildIgnored = async (
|
||||
root: string,
|
||||
excludePaths: readonly string[],
|
||||
resources: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ReturnType<typeof ignore>> => {
|
||||
): Promise<Result<ReturnType<typeof ignore>, ReferenceSourceImportError>> => {
|
||||
const ignored = ignore();
|
||||
const policyPath = join(root, ".gitignore");
|
||||
let present = true;
|
||||
@@ -68,39 +103,89 @@ const buildIgnored = async (
|
||||
present = false;
|
||||
}
|
||||
signal?.throwIfAborted();
|
||||
if (present) ignored.add(await readRegularFileText(policyPath, { signal }));
|
||||
ignored.add([...DEFAULT_REFERENCE_SOURCE_IGNORE_PATTERNS]);
|
||||
for (const path of excludePaths) {
|
||||
ignored.add(path);
|
||||
ignored.add(`${path}/`);
|
||||
if (present) {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readRegularFileText(policyPath, { signal });
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof RegularFileCleanupFailure)) throw cause;
|
||||
const retry = await retryRegularFileCleanup(cause, resources);
|
||||
if (retry.cleanup !== undefined) {
|
||||
const primary =
|
||||
cause.outcome.kind === "failed"
|
||||
? cause.outcome.cause
|
||||
: cause.cleanupCause;
|
||||
return err({
|
||||
...failure(
|
||||
signal?.aborted === true ? "cancelled" : "io",
|
||||
`Reference source ignore policy could not be read at ${policyPath}: ${errorMessage(primary)}`,
|
||||
),
|
||||
cleanup: retry.cleanup,
|
||||
cause: primary,
|
||||
});
|
||||
}
|
||||
if (retry.outcome.kind === "failed") throw retry.outcome.cause;
|
||||
text = retry.outcome.value;
|
||||
}
|
||||
addMarkedPatterns(ignored, [text], "project-ignored");
|
||||
}
|
||||
return ignored;
|
||||
addMarkedPatterns(
|
||||
ignored,
|
||||
DEFAULT_REFERENCE_SOURCE_IGNORE_PATTERNS,
|
||||
"default-ignored",
|
||||
);
|
||||
for (const path of excludePaths) {
|
||||
addMarkedPatterns(ignored, [path, `${path}/`], "caller-excluded");
|
||||
}
|
||||
return ok(ignored);
|
||||
};
|
||||
|
||||
/** Resolve the caller-selected directory and build path filters. */
|
||||
export const prepareReferenceSourceImport = async (
|
||||
options: ReferenceSourceImportOptions,
|
||||
resources: ArtifactResourceScope,
|
||||
): Promise<
|
||||
Result<PreparedReferenceSourceImport, ReferenceSourceImportError>
|
||||
> => {
|
||||
const root = await resolveRoot(options.root);
|
||||
if (!root.ok) return root;
|
||||
try {
|
||||
return ok({
|
||||
root: root.value,
|
||||
ignored: await buildIgnored(
|
||||
root.value,
|
||||
options.excludePaths ?? [],
|
||||
options.signal,
|
||||
),
|
||||
secrets: ignore().add([...options.policy.secretPatterns]),
|
||||
return await resources.run(async () => {
|
||||
const root = await resolveRoot(options.root, options.signal);
|
||||
if (!root.ok) return root;
|
||||
try {
|
||||
const ignored = await buildIgnored(
|
||||
root.value,
|
||||
options.excludePaths ?? [],
|
||||
resources,
|
||||
options.signal,
|
||||
);
|
||||
if (!ignored.ok) return ignored;
|
||||
return ok({
|
||||
root: root.value,
|
||||
ignored: ignored.value,
|
||||
secrets: ignore().add([...options.policy.secretPatterns]),
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
return err({
|
||||
...failure(
|
||||
options.signal?.aborted === true ? "cancelled" : "io",
|
||||
`Reference source ignore policy could not be read at ${join(root.value, ".gitignore")}: ${errorMessage(cause)}`,
|
||||
),
|
||||
cause,
|
||||
});
|
||||
}
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
failure(
|
||||
options.signal?.aborted === true ? "cancelled" : "io",
|
||||
`Reference source ignore policy could not be read at ${join(root.value, ".gitignore")}: ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
if (!(cause instanceof ArtifactReaderFailure)) throw cause;
|
||||
return err({
|
||||
...failure(
|
||||
cause.reason === "cancelled" ? "cancelled" : "io",
|
||||
cause.message,
|
||||
),
|
||||
);
|
||||
...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }),
|
||||
cause,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const errorMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
projectReferenceSourceImportError,
|
||||
type ReferenceSourceImportError,
|
||||
} from "./ReferenceSourceImportTypes.js";
|
||||
import type { ReferenceSourceRead } from "../reference/ReferenceSourceReaderTypes.js";
|
||||
|
||||
describe("reference-source import error projection", () => {
|
||||
it("preserves primary evidence with category-specific recovery guidance", () => {
|
||||
const expectedCategories = {
|
||||
cancelled: "cancelled",
|
||||
"invalid-root": "invalid_input",
|
||||
unsupported: "unsupported_host",
|
||||
io: "execution_failure",
|
||||
parse: "execution_failure",
|
||||
} as const;
|
||||
for (const [code, category] of Object.entries(expectedCategories)) {
|
||||
const projected = projectReferenceSourceImportError({
|
||||
tag: "reference-source-import",
|
||||
code: code as ReferenceSourceImportError["code"],
|
||||
message: "EIO while reading /selected/source/src/main.ts",
|
||||
});
|
||||
expect(projected.category).toBe(category);
|
||||
expect(projected.message).toContain(
|
||||
"EIO while reading /selected/source/src/main.ts",
|
||||
);
|
||||
expect(projected.message).toMatch(
|
||||
/try again|when ready|Check that|REA on Linux/u,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves cancellation, partial facts, and cleanup diagnostics", () => {
|
||||
const bytes = Buffer.from("export const observed = true;\n");
|
||||
const partial: ReferenceSourceRead = {
|
||||
root: "/selected/source",
|
||||
entries: [
|
||||
{
|
||||
status: "read",
|
||||
kind: "file",
|
||||
path: "src/main.ts",
|
||||
bytes,
|
||||
size: bytes.length,
|
||||
},
|
||||
],
|
||||
bytesRead: bytes.length,
|
||||
limitations: [],
|
||||
};
|
||||
const projected = projectReferenceSourceImportError({
|
||||
tag: "reference-source-import",
|
||||
code: "cancelled",
|
||||
message: "Traversal cancelled after reading src/main.ts",
|
||||
cleanup: {
|
||||
reason: "directory close rejected",
|
||||
resources: ["/selected/source"],
|
||||
},
|
||||
partial,
|
||||
});
|
||||
|
||||
expect(projected).toMatchObject({
|
||||
category: "cancelled",
|
||||
message: expect.stringContaining(
|
||||
"Traversal cancelled after reading src/main.ts",
|
||||
),
|
||||
cleanup: {
|
||||
reason: "directory close rejected",
|
||||
resources: ["/selected/source"],
|
||||
},
|
||||
partial: {
|
||||
root: "/selected/source",
|
||||
entries: [{ path: "src/main.ts", bytes }],
|
||||
},
|
||||
});
|
||||
expect(projected.message).toContain("Cleanup failed during import");
|
||||
expect(projected.message).toContain("directory close rejected");
|
||||
expect(projected.message).toContain("/selected/source");
|
||||
expect(projected.message).not.toContain("remains unconfirmed");
|
||||
});
|
||||
});
|
||||
@@ -1,44 +1,67 @@
|
||||
import type { ReferenceSourcePolicy } from "../domain/referenceSourcePolicy.js";
|
||||
import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js";
|
||||
import type { ReferenceSourceRead } from "../reference/ReferenceSourceReaderTypes.js";
|
||||
|
||||
/** Typed expected failure returned by historical-source imports. */
|
||||
export interface ReferenceSourceImportError {
|
||||
readonly tag: "reference-source-import";
|
||||
readonly code: "cancelled" | "invalid-root" | "unsupported" | "io" | "parse";
|
||||
readonly message: string;
|
||||
readonly cleanup?: AnalysisCleanupObservation;
|
||||
readonly partial?: ReferenceSourceRead;
|
||||
readonly cause?: unknown;
|
||||
}
|
||||
|
||||
const IMPORT_FAILURE_GUIDANCE = {
|
||||
cancelled: {
|
||||
category: "cancelled",
|
||||
message:
|
||||
"Reference source import was cancelled. Start it again when ready.",
|
||||
},
|
||||
"invalid-root": {
|
||||
category: "invalid_input",
|
||||
message:
|
||||
"Reference source directory could not be opened. Check that the path exists, is readable, and points to a directory.",
|
||||
},
|
||||
unsupported: {
|
||||
category: "unsupported_host",
|
||||
message:
|
||||
"Safe no-follow file opens are unavailable on this host. Import the source tree with REA on Linux (including WSL) or macOS.",
|
||||
},
|
||||
io: {
|
||||
category: "execution_failure",
|
||||
message:
|
||||
"Reference source files could not be read. Check directory permissions and try again.",
|
||||
},
|
||||
parse: {
|
||||
category: "execution_failure",
|
||||
message:
|
||||
"Reference source could not be indexed. Check that the source tree is readable, then try again.",
|
||||
},
|
||||
} satisfies Record<
|
||||
ReferenceSourceImportError["code"],
|
||||
{ readonly category: string; readonly message: string }
|
||||
>;
|
||||
|
||||
/** Safe CLI projection for a historical-source import failure. */
|
||||
export const projectReferenceSourceImportError = (
|
||||
error: ReferenceSourceImportError,
|
||||
): Readonly<{ category: string; message: string }> => {
|
||||
if (error.code === "cancelled")
|
||||
return {
|
||||
category: "cancelled",
|
||||
message:
|
||||
"Reference source import was cancelled. Start it again when ready.",
|
||||
};
|
||||
if (error.code === "invalid-root")
|
||||
return {
|
||||
category: "invalid_input",
|
||||
message:
|
||||
"Reference source directory could not be opened. Check that the path exists, is readable, and points to a directory.",
|
||||
};
|
||||
if (error.code === "unsupported")
|
||||
return {
|
||||
category: "unsupported_host",
|
||||
message:
|
||||
"Safe no-follow file opens are unavailable on this host. Import the source tree with REA on Linux (including WSL) or macOS.",
|
||||
};
|
||||
if (error.code === "io")
|
||||
return {
|
||||
category: "execution_failure",
|
||||
message:
|
||||
"Reference source files could not be read. Check directory permissions and try again.",
|
||||
};
|
||||
): Readonly<{
|
||||
category: string;
|
||||
message: string;
|
||||
cleanup?: AnalysisCleanupObservation;
|
||||
partial?: ReferenceSourceRead;
|
||||
}> => {
|
||||
const primary = IMPORT_FAILURE_GUIDANCE[error.code];
|
||||
const cleanupMessage =
|
||||
error.cleanup === undefined
|
||||
? ""
|
||||
: ` Cleanup failed during import for ${error.cleanup.resources.join(", ")}: ${error.cleanup.reason}.`;
|
||||
return {
|
||||
category: "execution_failure",
|
||||
message:
|
||||
"Reference source could not be indexed. Check that the source tree is readable, then try again.",
|
||||
category: primary.category,
|
||||
message: `${error.message} ${primary.message}${cleanupMessage}`,
|
||||
...(error.cleanup === undefined ? {} : { cleanup: error.cleanup }),
|
||||
...(error.partial === undefined ? {} : { partial: error.partial }),
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -1,7 +1,15 @@
|
||||
import fs from "node:fs";
|
||||
import { lstat } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { readRegularFile, readRegularFileText } from "./RegularFileRead.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
readRegularFile,
|
||||
RegularFileCleanupFailure,
|
||||
retryRegularFileCleanup,
|
||||
} from "./RegularFileRead.js";
|
||||
import type { ReferenceSourceReaderError } from "../reference/ReferenceSourceReaderTypes.js";
|
||||
|
||||
export type ReferenceSourceVcsInfo =
|
||||
| {
|
||||
@@ -19,40 +27,125 @@ export type ReferenceSourceVcsInfo =
|
||||
*/
|
||||
export const readReferenceSourceVcs = async (
|
||||
root: string,
|
||||
resources: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ReferenceSourceVcsInfo> => {
|
||||
if (isAborted(signal)) return { kind: "unknown", head: null, dirty: null };
|
||||
): Promise<Result<ReferenceSourceVcsInfo, ReferenceSourceReaderError>> => {
|
||||
try {
|
||||
await lstat(join(root, ".git"));
|
||||
return await resources.run(async () => {
|
||||
if (isAborted(signal)) return ok(unknownVcs());
|
||||
try {
|
||||
await lstat(join(root, ".git"));
|
||||
} catch (cause: unknown) {
|
||||
return ok(
|
||||
errorCode(cause) === "ENOENT"
|
||||
? { kind: "none", head: null, dirty: null }
|
||||
: unknownVcs(),
|
||||
);
|
||||
}
|
||||
try {
|
||||
const head = await resolveSourceHead(root, resources, signal);
|
||||
if (isAborted(signal)) return ok(unknownVcs());
|
||||
return ok({ kind: "git", head, dirty: null });
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
return ok(unknownVcs());
|
||||
}
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
return errorCode(cause) === "ENOENT"
|
||||
? { kind: "none", head: null, dirty: null }
|
||||
: { kind: "unknown", head: null, dirty: null };
|
||||
}
|
||||
try {
|
||||
const head = await resolveSourceHead(root, signal);
|
||||
if (isAborted(signal)) return { kind: "unknown", head: null, dirty: null };
|
||||
return { kind: "git", head, dirty: null };
|
||||
} catch (cause: unknown) {
|
||||
// Unresolvable refs mean VCS state is unknown, not absent.
|
||||
void cause;
|
||||
return { kind: "unknown", head: null, dirty: null };
|
||||
if (!(cause instanceof ArtifactReaderFailure)) throw cause;
|
||||
return err({
|
||||
tag: "reference-source-reader",
|
||||
code: cause.reason === "cancelled" ? "cancelled" : "io",
|
||||
message: cause.message,
|
||||
...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }),
|
||||
cause,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const unknownVcs = (): ReferenceSourceVcsInfo => ({
|
||||
kind: "unknown",
|
||||
head: null,
|
||||
dirty: null,
|
||||
});
|
||||
|
||||
const resolveSourceHead = async (
|
||||
root: string,
|
||||
resources: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string> => {
|
||||
// isomorphic-git is loaded on first use so CLI and MCP startup skip it.
|
||||
const { resolveRef } = await import("isomorphic-git");
|
||||
let failedRead: { readonly cause: unknown } | undefined;
|
||||
const cleanupFailures: ArtifactReaderFailure[] = [];
|
||||
const readMetadataBytes = async (path: string): Promise<Buffer> => {
|
||||
try {
|
||||
return await readRegularFile(path, { signal });
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof RegularFileCleanupFailure)) throw cause;
|
||||
const retry = await retryRegularFileCleanup(cause, resources);
|
||||
if (
|
||||
retry.outcome.kind === "failed" &&
|
||||
errorCode(retry.outcome.cause) !== "ENOENT"
|
||||
)
|
||||
failedRead ??= { cause: retry.outcome.cause };
|
||||
if (
|
||||
retry.cleanup === undefined &&
|
||||
retry.outcome.kind === "failed" &&
|
||||
!isAborted(signal)
|
||||
)
|
||||
throw retry.outcome.cause;
|
||||
const primary =
|
||||
retry.outcome.kind === "failed"
|
||||
? retry.outcome.cause
|
||||
: cause.cleanupCause;
|
||||
throw new ArtifactReaderFailure(
|
||||
isAborted(signal) ? "cancelled" : "io",
|
||||
`Git metadata could not be read safely: ${errorMessage(primary)}`,
|
||||
{
|
||||
cause: primary,
|
||||
...(retry.cleanup === undefined ? {} : { cleanup: retry.cleanup }),
|
||||
},
|
||||
);
|
||||
}
|
||||
};
|
||||
const readMetadataText = async (path: string): Promise<string> =>
|
||||
(await readMetadataBytes(path)).toString("utf8");
|
||||
const throwCleanupFailures = (primary: unknown): never => {
|
||||
const observations = cleanupFailures.flatMap((failure) =>
|
||||
failure.cleanup === undefined ? [] : [failure.cleanup],
|
||||
);
|
||||
throw new ArtifactReaderFailure(
|
||||
isAborted(signal) ? "cancelled" : "io",
|
||||
`Git metadata could not be read safely: ${errorMessage(primary)}`,
|
||||
{
|
||||
cause: new AggregateError(
|
||||
[primary, ...cleanupFailures],
|
||||
"Git metadata reads and cleanup failed",
|
||||
{ cause: primary },
|
||||
),
|
||||
...(observations.length === 0
|
||||
? {}
|
||||
: {
|
||||
cleanup: {
|
||||
reason: observations.map(({ reason }) => reason).join("; "),
|
||||
resources: [
|
||||
...new Set(
|
||||
observations.flatMap(({ resources }) => resources),
|
||||
),
|
||||
],
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
};
|
||||
const referenceFs = {
|
||||
...fs,
|
||||
promises: {
|
||||
...fs.promises,
|
||||
// isomorphic-git treats every read failure as absence. Preserve real
|
||||
// failures so a readable packed ref cannot hide an unreadable loose ref.
|
||||
// Let isomorphic-git continue from a failed loose ref to packed-refs,
|
||||
// then report the original read failure after resolution. This preserves
|
||||
// both observations when packed metadata also fails cleanup.
|
||||
readFile: async (
|
||||
path: string | undefined,
|
||||
options:
|
||||
@@ -65,14 +158,16 @@ const resolveSourceHead = async (
|
||||
if (path === undefined)
|
||||
throw new TypeError("Git filesystem read requires a path");
|
||||
try {
|
||||
const bytes = await readRegularFile(path, { signal });
|
||||
const bytes = await readMetadataBytes(path);
|
||||
const encoding =
|
||||
typeof options === "string" ? options : options?.encoding;
|
||||
return encoding === undefined || encoding === null
|
||||
? bytes
|
||||
: bytes.toString(encoding);
|
||||
} catch (cause: unknown) {
|
||||
if (errorCode(cause) !== "ENOENT") failedRead ??= { cause };
|
||||
if (cause instanceof ArtifactReaderFailure) {
|
||||
cleanupFailures.push(cause);
|
||||
} else if (errorCode(cause) !== "ENOENT") failedRead ??= { cause };
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
@@ -82,17 +177,24 @@ const resolveSourceHead = async (
|
||||
readonly gitdir?: string;
|
||||
readonly ref: string;
|
||||
}) => {
|
||||
const head = await resolveRef({ fs: referenceFs, dir: root, ...options });
|
||||
let head: string;
|
||||
try {
|
||||
head = await resolveRef({ fs: referenceFs, dir: root, ...options });
|
||||
} catch (cause: unknown) {
|
||||
if (cleanupFailures.length > 0)
|
||||
throwCleanupFailures(failedRead?.cause ?? cause);
|
||||
if (failedRead !== undefined) throw failedRead.cause;
|
||||
throw cause;
|
||||
}
|
||||
if (cleanupFailures.length > 0)
|
||||
throwCleanupFailures(failedRead?.cause ?? cleanupFailures[0]?.cause);
|
||||
signal?.throwIfAborted();
|
||||
if (failedRead !== undefined) throw failedRead.cause;
|
||||
return head;
|
||||
};
|
||||
const dotgit = join(root, ".git");
|
||||
if (!(await lstat(dotgit)).isFile()) return readRef({ ref: "HEAD" });
|
||||
const pointer = (await readRegularFileText(dotgit, { signal })).replace(
|
||||
/\r?\n$/u,
|
||||
"",
|
||||
);
|
||||
const pointer = (await readMetadataText(dotgit)).replace(/\r?\n$/u, "");
|
||||
if (!pointer.startsWith("gitdir: ") || pointer.length === 8)
|
||||
throw new Error("Invalid Git directory pointer");
|
||||
const gitdir = resolve(root, pointer.slice(8));
|
||||
@@ -100,17 +202,16 @@ const resolveSourceHead = async (
|
||||
try {
|
||||
commonDirectory = resolve(
|
||||
gitdir,
|
||||
(
|
||||
await readRegularFileText(join(gitdir, "commondir"), { signal })
|
||||
).replace(/\r?\n$/u, ""),
|
||||
(await readMetadataText(join(gitdir, "commondir"))).replace(
|
||||
/\r?\n$/u,
|
||||
"",
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (errorCode(cause) !== "ENOENT") throw cause;
|
||||
return readRef({ gitdir, ref: "HEAD" });
|
||||
}
|
||||
const head = (
|
||||
await readRegularFileText(join(gitdir, "HEAD"), { signal })
|
||||
).trim();
|
||||
const head = (await readMetadataText(join(gitdir, "HEAD"))).trim();
|
||||
if (!head.startsWith("ref: ")) return readRef({ gitdir, ref: "HEAD" });
|
||||
const ref = head.slice(5);
|
||||
const privateRef = ["refs/bisect/", "refs/rewritten/", "refs/worktree/"].some(
|
||||
@@ -127,4 +228,7 @@ const errorCode = (cause: unknown): string | undefined =>
|
||||
? String(cause.code)
|
||||
: undefined;
|
||||
|
||||
const errorMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
|
||||
const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true;
|
||||
|
||||
@@ -2,11 +2,15 @@ import type { Stats } from "node:fs";
|
||||
import { lstat, stat, type FileHandle } from "node:fs/promises";
|
||||
|
||||
import {
|
||||
RegularFileAdmissionFailure,
|
||||
openRegularFile,
|
||||
sameRegularFileState,
|
||||
RegularFileChangedError,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { readBoundedFileBytes } from "../process/BoundedFileBytes.js";
|
||||
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js";
|
||||
|
||||
/** Filesystem admission policy for one selected regular input. */
|
||||
export interface RegularFileReadOptions {
|
||||
@@ -14,6 +18,51 @@ export interface RegularFileReadOptions {
|
||||
readonly symlinks?: "follow" | "reject";
|
||||
}
|
||||
|
||||
export type RegularFileReadOutcome<Value> =
|
||||
| { readonly kind: "completed"; readonly value: Value }
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
|
||||
/** A close failed after reading; this error transfers the exact handle owner. */
|
||||
export class RegularFileCleanupFailure<Value = unknown> extends Error {
|
||||
constructor(
|
||||
readonly path: string,
|
||||
readonly owner: OwnedFileHandle,
|
||||
readonly outcome: RegularFileReadOutcome<Value>,
|
||||
readonly cleanupCause: unknown,
|
||||
) {
|
||||
super(`Regular file cleanup failed: ${path}`, {
|
||||
cause: outcome.kind === "failed" ? outcome.cause : cleanupCause,
|
||||
});
|
||||
this.name = "RegularFileCleanupFailure";
|
||||
}
|
||||
}
|
||||
|
||||
/** Retry a failed close through its retained owner and report any remaining cleanup. */
|
||||
export const retryRegularFileCleanup = async <Value>(
|
||||
failure: RegularFileCleanupFailure<Value>,
|
||||
resources: ArtifactResourceScope,
|
||||
): Promise<{
|
||||
readonly outcome: RegularFileReadOutcome<Value>;
|
||||
readonly cleanup?: AnalysisCleanupObservation;
|
||||
}> => {
|
||||
const released = await resources.release({
|
||||
kind: "file-handle",
|
||||
handle: failure.owner,
|
||||
resource: failure.path,
|
||||
});
|
||||
if (released.kind === "released") return { outcome: failure.outcome };
|
||||
return {
|
||||
outcome: failure.outcome,
|
||||
cleanup: {
|
||||
reason: `${failureMessage(failure.cleanupCause)}; retry: ${failureMessage(released.cause)}`,
|
||||
resources: [failure.path],
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const failureMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
|
||||
/** Admit one regular-file handle, retain it through the consumer, and always close it. */
|
||||
export const withRegularFile = async <Value>(
|
||||
path: string,
|
||||
@@ -21,10 +70,23 @@ export const withRegularFile = async <Value>(
|
||||
options: RegularFileReadOptions = {},
|
||||
): Promise<Value> => {
|
||||
const { signal } = options;
|
||||
const handle = await openRegularFile(path, {
|
||||
symlinks: options.symlinks ?? "follow",
|
||||
signal,
|
||||
});
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await openRegularFile(path, {
|
||||
symlinks: options.symlinks ?? "follow",
|
||||
signal,
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof RegularFileAdmissionFailure)) throw cause;
|
||||
throw new RegularFileCleanupFailure(
|
||||
path,
|
||||
cause.owner,
|
||||
{ kind: "failed", cause: cause.cause },
|
||||
cause.cleanupCause,
|
||||
);
|
||||
}
|
||||
const owner = new OwnedFileHandle(handle);
|
||||
let outcome: RegularFileReadOutcome<Value>;
|
||||
try {
|
||||
const stats = await handle.stat();
|
||||
signal?.throwIfAborted();
|
||||
@@ -39,10 +101,17 @@ export const withRegularFile = async <Value>(
|
||||
!sameRegularFileState(stats, currentPath)
|
||||
)
|
||||
throw new RegularFileChangedError(path);
|
||||
return value;
|
||||
} finally {
|
||||
await handle.close();
|
||||
outcome = { kind: "completed", value };
|
||||
} catch (cause: unknown) {
|
||||
outcome = { kind: "failed", cause };
|
||||
}
|
||||
try {
|
||||
await owner.close();
|
||||
} catch (cleanupCause: unknown) {
|
||||
throw new RegularFileCleanupFailure(path, owner, outcome, cleanupCause);
|
||||
}
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
return outcome.value;
|
||||
};
|
||||
|
||||
/** Admit a regular file without waiting for a pipe, then read its verified handle. */
|
||||
@@ -69,4 +138,21 @@ export const readRegularFile = (
|
||||
export const readRegularFileText = async (
|
||||
path: string,
|
||||
options: RegularFileReadOptions = {},
|
||||
): Promise<string> => (await readRegularFile(path, options)).toString("utf8");
|
||||
): Promise<string> => {
|
||||
try {
|
||||
return (await readRegularFile(path, options)).toString("utf8");
|
||||
} catch (cause: unknown) {
|
||||
if (
|
||||
cause instanceof RegularFileCleanupFailure &&
|
||||
cause.outcome.kind === "completed" &&
|
||||
Buffer.isBuffer(cause.outcome.value)
|
||||
)
|
||||
throw new RegularFileCleanupFailure(
|
||||
cause.path,
|
||||
cause.owner,
|
||||
{ kind: "completed", value: cause.outcome.value.toString("utf8") },
|
||||
cause.cleanupCause,
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import { SUPPORTED_NODE_VERSION_PROSE } from "../domain/runtimeVersion.js";
|
||||
import { FakeSetupHost, options } from "./Setup.fixture.js";
|
||||
import { runSetup } from "./Setup.js";
|
||||
@@ -107,6 +110,76 @@ describe("setup workflow", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("setup failure cleanup projection", () => {
|
||||
it("projects a configuration failure together with failed host cleanup", async () => {
|
||||
const configPath = "/fixture/home/vscode/settings.json";
|
||||
const primary = new ConfigurationError("Client config could not be read", {
|
||||
settings: [
|
||||
{
|
||||
setting: configPath,
|
||||
constraint: "The selected config could not be read.",
|
||||
},
|
||||
],
|
||||
cleanup: {
|
||||
reason: "The admitted config handle remains open.",
|
||||
resources: [configPath],
|
||||
},
|
||||
});
|
||||
class SetupHostWithCleanupFailure extends FakeSetupHost {
|
||||
close = async (): Promise<string> =>
|
||||
"Setup could not release its client configuration resources.";
|
||||
}
|
||||
const host = new SetupHostWithCleanupFailure();
|
||||
host.clients = [{ name: "codex", configPath }];
|
||||
host.inspectClientConfiguration = async () => {
|
||||
throw primary;
|
||||
};
|
||||
|
||||
const failure = await runSetup(
|
||||
{ ...options(true), clientIds: ["codex"] },
|
||||
host,
|
||||
).then(
|
||||
() => undefined,
|
||||
(cause: unknown) => cause,
|
||||
);
|
||||
expect(failure).toBeInstanceOf(AnalysisError);
|
||||
if (!(failure instanceof AnalysisError))
|
||||
throw new Error("Expected setup to preserve a typed analysis failure");
|
||||
|
||||
const projected = projectAnalysisError(failure);
|
||||
expect(projected).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
cleanup: "incomplete",
|
||||
resources: [configPath],
|
||||
diagnostics: {
|
||||
primary_error: {
|
||||
code: "cleanup_incomplete",
|
||||
details: { resources: [configPath] },
|
||||
},
|
||||
cleanup_error: {
|
||||
code: "cleanup_incomplete",
|
||||
details: { resources: [configPath] },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(projected.details).toMatchObject({
|
||||
diagnostics: {
|
||||
primary_error: { message: expect.stringContaining("selected config") },
|
||||
cleanup_error: {
|
||||
details: {
|
||||
diagnostics: {
|
||||
reason:
|
||||
"Setup could not release its client configuration resources.",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("setup scoped readiness", () => {
|
||||
it("limits Hopper host checks to workflows that request Hopper", async () => {
|
||||
const platformHost = new FakeSetupHost("win32");
|
||||
|
||||
@@ -23,6 +23,9 @@ import {
|
||||
hostRemediation,
|
||||
systemSetupHost,
|
||||
} from "./SetupHost.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
|
||||
/** Discover, approve, and apply setup actions idempotently. */
|
||||
export const runSetup = async (
|
||||
@@ -48,6 +51,17 @@ export const runSetup = async (
|
||||
}
|
||||
|
||||
if (threw) {
|
||||
if (cleanupFailure !== undefined && primaryFailure instanceof AnalysisError)
|
||||
throw analysisErrorWithCleanupFailure(
|
||||
primaryFailure,
|
||||
new ProviderCleanupError(
|
||||
"setup",
|
||||
primaryFailure.cleanupResources,
|
||||
{ reason: cleanupFailure },
|
||||
{ operation: "setup" },
|
||||
),
|
||||
"setup",
|
||||
);
|
||||
if (cleanupFailure !== undefined)
|
||||
throw new AggregateError(
|
||||
[primaryFailure, new Error(cleanupFailure)],
|
||||
|
||||
@@ -11,18 +11,22 @@ import {
|
||||
serializeClientConfiguration,
|
||||
withClientServers,
|
||||
type ClientConfigurationDocument,
|
||||
type ClientRegistrationDialect,
|
||||
} from "./ClientConfigurationDocument.js";
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import { copyFile, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { dirname } from "node:path";
|
||||
import writeFileAtomic from "write-file-atomic";
|
||||
import { clientRegistrationEnvironment } from "./ClientRegistrationEnvironment.js";
|
||||
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js";
|
||||
import { MCP_STARTUP_POLICY } from "../mcpStartupPolicy.js";
|
||||
import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js";
|
||||
import { readRegularFileText } from "./RegularFileRead.js";
|
||||
import {
|
||||
readRegularFileText,
|
||||
RegularFileCleanupFailure,
|
||||
} from "./RegularFileRead.js";
|
||||
import type { ClientConfigurationFileError } from "./ClientConfigurationFiles.js";
|
||||
import type {
|
||||
ClientConfigurationInspection,
|
||||
ClientConfigurationResult,
|
||||
@@ -38,25 +42,37 @@ export const configureClientConfiguration = (
|
||||
): Promise<ClientConfigurationResult> => {
|
||||
if (client.configPathError !== undefined)
|
||||
return Promise.resolve({ status: "failed", reason: "path" });
|
||||
if (client.format === undefined || client.format === "unsupported")
|
||||
return Promise.resolve({ status: "failed", reason: "readback" });
|
||||
return configureClientDocument(client, environment, command, client.format);
|
||||
return configureClientDocument(client, environment, command);
|
||||
};
|
||||
|
||||
const configureClientDocument = async (
|
||||
client: SetupClient,
|
||||
environment: SetupProviderEnvironment,
|
||||
command: readonly string[],
|
||||
format: NonNullable<SetupClient["format"]>,
|
||||
): Promise<ClientConfigurationResult> => {
|
||||
const format = client.format;
|
||||
if (format === undefined || format === "unsupported")
|
||||
return { status: "failed", reason: "readback" };
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) return { status: "failed", reason: "readback" };
|
||||
if (!files.ok) {
|
||||
throwCleanupFailure(files.error);
|
||||
return { status: "failed", reason: "readback" };
|
||||
}
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
undefined,
|
||||
clientConfigurationDesired(client, environment, command, client.format),
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
environment,
|
||||
command,
|
||||
files.value.at(-1),
|
||||
),
|
||||
);
|
||||
if (!policy.ok || policy.value !== undefined)
|
||||
if (!policy.ok) {
|
||||
throwCleanupFailure(policy.error);
|
||||
return { status: "failed", reason: "readback" };
|
||||
}
|
||||
if (policy.value !== undefined)
|
||||
return { status: "failed", reason: "readback" };
|
||||
const transactionPath = await resolveClientConfigTransactionPath(
|
||||
client.configPath,
|
||||
@@ -67,6 +83,7 @@ const configureClientDocument = async (
|
||||
try {
|
||||
original = await readRegularFileText(transactionPath);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure) throw cause;
|
||||
if (!isMissing(cause)) return { status: "failed", reason: "readback" };
|
||||
}
|
||||
let parsed: ClientConfigurationDocument;
|
||||
@@ -78,11 +95,16 @@ const configureClientDocument = async (
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
parsed,
|
||||
clientConfigurationDesired(client, environment, command, parsed.dialect),
|
||||
clientConfigurationDesired(client, environment, command, parsed),
|
||||
);
|
||||
if (!policy.ok || policy.value !== undefined)
|
||||
if (!policy.ok) {
|
||||
throwCleanupFailure(policy.error);
|
||||
return { status: "failed", reason: "readback" };
|
||||
}
|
||||
if (policy.value !== undefined)
|
||||
return { status: "failed", reason: "readback" };
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure) throw cause;
|
||||
// Malformed existing configuration fails the readback gate.
|
||||
void cause;
|
||||
return { status: "failed", reason: "readback" };
|
||||
@@ -91,9 +113,36 @@ const configureClientDocument = async (
|
||||
client,
|
||||
environment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
parsed,
|
||||
);
|
||||
if (registrationCurrent(parsed, desired)) return { status: "unchanged" };
|
||||
return persistClientConfiguration({
|
||||
client,
|
||||
transactionPath,
|
||||
format,
|
||||
original,
|
||||
parsed,
|
||||
desired,
|
||||
});
|
||||
};
|
||||
|
||||
interface ClientConfigurationUpdate {
|
||||
readonly client: SetupClient;
|
||||
readonly transactionPath: string;
|
||||
readonly format: NonNullable<SetupClient["format"]>;
|
||||
readonly original: string | undefined;
|
||||
readonly parsed: ClientConfigurationDocument;
|
||||
readonly desired: unknown;
|
||||
}
|
||||
|
||||
const persistClientConfiguration = async ({
|
||||
client,
|
||||
transactionPath,
|
||||
format,
|
||||
original,
|
||||
parsed,
|
||||
desired,
|
||||
}: ClientConfigurationUpdate): Promise<ClientConfigurationResult> => {
|
||||
const backupPath =
|
||||
original === undefined ? undefined : `${client.configPath}.rea.backup`;
|
||||
if (
|
||||
@@ -158,6 +207,7 @@ const configureClientDocument = async (
|
||||
return { status: "failed", reason: "readback" };
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure) throw cause;
|
||||
// Readback failure restores the transaction before reporting.
|
||||
void cause;
|
||||
await restoreConfig(transactionPath, original);
|
||||
@@ -178,21 +228,22 @@ export const clientConfigurationAligned = async (
|
||||
if (client.configPathError !== undefined) return false;
|
||||
try {
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) return false;
|
||||
if (!files.ok) {
|
||||
throwCleanupFailure(files.error);
|
||||
return false;
|
||||
}
|
||||
const original = await readRegularFileText(client.configPath);
|
||||
const parsed = parseClientConfiguration(original, client.format);
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
parsed,
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
),
|
||||
clientConfigurationDesired(client, providerEnvironment, command, parsed),
|
||||
);
|
||||
if (!policy.ok) {
|
||||
throwCleanupFailure(policy.error);
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
policy.ok &&
|
||||
policy.value === undefined &&
|
||||
registrationCurrent(
|
||||
parsed,
|
||||
@@ -200,11 +251,12 @@ export const clientConfigurationAligned = async (
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
parsed,
|
||||
),
|
||||
)
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure) throw cause;
|
||||
// Unreadable configuration is treated as not aligned so setup repairs it.
|
||||
void cause;
|
||||
return false;
|
||||
@@ -221,7 +273,13 @@ export const inspectClientConfiguration = async (
|
||||
return { status: "invalid", remediation: client.configPathError };
|
||||
if (client.format === "unsupported") return { status: "already_current" };
|
||||
const files = await readClientConfigurationFiles(client);
|
||||
if (!files.ok) return { status: "invalid", remediation: files.error.detail };
|
||||
if (!files.ok) {
|
||||
throwCleanupFailure(files.error);
|
||||
return {
|
||||
status: "invalid",
|
||||
remediation: files.error.detail,
|
||||
};
|
||||
}
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
undefined,
|
||||
@@ -229,11 +287,16 @@ export const inspectClientConfiguration = async (
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
client.format,
|
||||
files.value.at(-1),
|
||||
),
|
||||
);
|
||||
if (!policy.ok)
|
||||
return { status: "invalid", remediation: policy.error.detail };
|
||||
if (!policy.ok) {
|
||||
throwCleanupFailure(policy.error);
|
||||
return {
|
||||
status: "invalid",
|
||||
remediation: policy.error.detail,
|
||||
};
|
||||
}
|
||||
if (policy.value !== undefined)
|
||||
return { status: "invalid", remediation: policy.value };
|
||||
const transactionPath = await resolveClientConfigTransactionPath(
|
||||
@@ -249,11 +312,11 @@ export const inspectClientConfiguration = async (
|
||||
try {
|
||||
original = await readRegularFileText(transactionPath);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure) throw cause;
|
||||
if (isMissing(cause)) return { status: "create" };
|
||||
return {
|
||||
status: "invalid",
|
||||
remediation:
|
||||
"The configuration file could not be read. Check its permissions before rerunning setup.",
|
||||
remediation: `The configuration file could not be read. Check its permissions before rerunning setup. ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
};
|
||||
}
|
||||
try {
|
||||
@@ -261,26 +324,27 @@ export const inspectClientConfiguration = async (
|
||||
const policy = await readClientPolicyBlock(
|
||||
client,
|
||||
parsed,
|
||||
clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
),
|
||||
clientConfigurationDesired(client, providerEnvironment, command, parsed),
|
||||
);
|
||||
if (!policy.ok)
|
||||
return { status: "invalid", remediation: policy.error.detail };
|
||||
if (!policy.ok) {
|
||||
throwCleanupFailure(policy.error);
|
||||
return {
|
||||
status: "invalid",
|
||||
remediation: policy.error.detail,
|
||||
};
|
||||
}
|
||||
if (policy.value !== undefined)
|
||||
return { status: "invalid", remediation: policy.value };
|
||||
const desired = clientConfigurationDesired(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
parsed.dialect,
|
||||
parsed,
|
||||
);
|
||||
if (registrationCurrent(parsed, desired))
|
||||
return { status: "already_current" };
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileCleanupFailure) throw cause;
|
||||
// Malformed configuration is reported with the invalid remediation.
|
||||
void cause;
|
||||
return {
|
||||
@@ -297,6 +361,11 @@ export const inspectClientConfiguration = async (
|
||||
|
||||
const isMissing = (cause: unknown): boolean =>
|
||||
cause instanceof Error && "code" in cause && cause.code === "ENOENT";
|
||||
|
||||
const throwCleanupFailure = (error: ClientConfigurationFileError): void => {
|
||||
if (error.cleanupFailure !== undefined) throw error.cleanupFailure;
|
||||
};
|
||||
|
||||
const preserveConfigBackup = async (
|
||||
source: string,
|
||||
destination: string,
|
||||
@@ -338,17 +407,20 @@ const clientConfigurationDesired = (
|
||||
client: SetupClient,
|
||||
providerEnvironment: SetupProviderEnvironment,
|
||||
command: readonly string[],
|
||||
format: ClientRegistrationDialect | undefined,
|
||||
parsed?: ClientConfigurationDocument,
|
||||
) => {
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(providerEnvironment).sort(([left], [right]) =>
|
||||
left.localeCompare(right),
|
||||
),
|
||||
const format = parsed?.dialect ?? client.format ?? "json";
|
||||
// Preserve explicitly configured server settings, never the ambient process
|
||||
// environment. Newly discovered provider paths win only for their own keys.
|
||||
const environment = clientRegistrationEnvironment(
|
||||
parsed,
|
||||
providerEnvironment,
|
||||
);
|
||||
if (!environment.ok) throw environment.error;
|
||||
const registration = clientRegistrationEntry(
|
||||
format ?? "json",
|
||||
format,
|
||||
command.length === 0 ? [PRODUCT_IDENTITY.cliBinary, "mcp"] : command,
|
||||
environment,
|
||||
environment.value,
|
||||
);
|
||||
return {
|
||||
...registration,
|
||||
|
||||
+110
-12
@@ -1,6 +1,9 @@
|
||||
import { access } from "node:fs/promises";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import { ConfigurationError } from "../domain/configurationErrors.js";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js";
|
||||
import {
|
||||
@@ -30,6 +33,10 @@ import {
|
||||
inspectClientConfiguration,
|
||||
} from "./SetupClientConfiguration.js";
|
||||
import { setupInstallFailure } from "./SetupInstallFailure.js";
|
||||
import {
|
||||
RegularFileCleanupFailure,
|
||||
retryRegularFileCleanup,
|
||||
} from "./RegularFileRead.js";
|
||||
import { providerRegistrationEnvironment } from "./SetupRegistrationEnvironment.js";
|
||||
import type {
|
||||
SetupHost,
|
||||
@@ -98,6 +105,7 @@ export const systemSetupHost = (
|
||||
platform === "darwin"
|
||||
? systemMacHopperInstallHost(homeDirectory, environment)
|
||||
: undefined;
|
||||
const resources = new ArtifactResourceScope();
|
||||
return {
|
||||
platform,
|
||||
homeDirectory,
|
||||
@@ -110,7 +118,7 @@ export const systemSetupHost = (
|
||||
nodeVersion: process.versions.node,
|
||||
macosVersion: () => doctorHost.macosVersion(),
|
||||
linuxDistribution: readLinuxDistribution,
|
||||
close: () => macHopperHost?.close() ?? Promise.resolve(undefined),
|
||||
close: () => closeSetupResources(resources, macHopperHost),
|
||||
initialSetupState: async (
|
||||
scope?: DoctorScope,
|
||||
): Promise<SetupInitialState> => {
|
||||
@@ -162,17 +170,7 @@ export const systemSetupHost = (
|
||||
detectedClients: () => detectClients(homeDirectory, platform, environment),
|
||||
supportedClients: () =>
|
||||
Promise.resolve(supportedClients(homeDirectory, platform, environment)),
|
||||
configureClient: (client, providerEnvironment, command) =>
|
||||
client.format === "unsupported"
|
||||
? Promise.resolve({ status: "skipped" })
|
||||
: configureClientConfiguration(client, providerEnvironment, command),
|
||||
clientNeedsConfigure: (client, providerEnvironment, command) =>
|
||||
client.format === "unsupported"
|
||||
? Promise.resolve(false)
|
||||
: clientConfigurationAligned(client, providerEnvironment, command).then(
|
||||
(aligned) => !aligned,
|
||||
),
|
||||
inspectClientConfiguration: inspectClientConfiguration,
|
||||
...clientConfigurationOperations(resources),
|
||||
skillNeedsInstall: (clientIds) =>
|
||||
canonicalSkillNeedsInstall(
|
||||
homeDirectory,
|
||||
@@ -186,6 +184,106 @@ export const systemSetupHost = (
|
||||
};
|
||||
};
|
||||
|
||||
const clientConfigurationOperations = (resources: ArtifactResourceScope) => ({
|
||||
configureClient: (
|
||||
client: SetupClient,
|
||||
providerEnvironment: SetupProviderEnvironment,
|
||||
command: readonly string[],
|
||||
) =>
|
||||
runClientConfigurationOperation(resources, async () =>
|
||||
client.format === "unsupported"
|
||||
? { status: "skipped" as const }
|
||||
: configureClientConfiguration(client, providerEnvironment, command),
|
||||
),
|
||||
clientNeedsConfigure: (
|
||||
client: SetupClient,
|
||||
providerEnvironment: SetupProviderEnvironment,
|
||||
command: readonly string[],
|
||||
) =>
|
||||
runClientConfigurationOperation(resources, async () =>
|
||||
client.format === "unsupported"
|
||||
? false
|
||||
: !(await clientConfigurationAligned(
|
||||
client,
|
||||
providerEnvironment,
|
||||
command,
|
||||
)),
|
||||
),
|
||||
inspectClientConfiguration: (
|
||||
client: SetupClient,
|
||||
providerEnvironment: SetupProviderEnvironment,
|
||||
command: readonly string[],
|
||||
) =>
|
||||
runClientConfigurationOperation(resources, () =>
|
||||
inspectClientConfiguration(client, providerEnvironment, command),
|
||||
),
|
||||
});
|
||||
|
||||
const closeSetupResources = async (
|
||||
resources: ArtifactResourceScope,
|
||||
macHopperHost: ReturnType<typeof systemMacHopperInstallHost> | undefined,
|
||||
): Promise<string | undefined> => {
|
||||
const [resourceCleanup, hopperCleanup] = await Promise.allSettled([
|
||||
resources.close(),
|
||||
Promise.resolve().then(() => macHopperHost?.close()),
|
||||
]);
|
||||
const failures: string[] = [];
|
||||
if (hopperCleanup.status === "rejected")
|
||||
failures.push(
|
||||
`Hopper host cleanup failed: ${errorMessage(hopperCleanup.reason)}`,
|
||||
);
|
||||
else if (hopperCleanup.value !== undefined)
|
||||
failures.push(hopperCleanup.value);
|
||||
if (resourceCleanup.status === "rejected")
|
||||
failures.push(errorMessage(resourceCleanup.reason));
|
||||
return failures.length === 0 ? undefined : failures.join("; ");
|
||||
};
|
||||
|
||||
const runClientConfigurationOperation = async <Value>(
|
||||
resources: ArtifactResourceScope,
|
||||
operation: () => Promise<Value>,
|
||||
): Promise<Value> => {
|
||||
try {
|
||||
return await resources.run(async () => {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof RegularFileCleanupFailure)) throw cause;
|
||||
const cleanup = await retryRegularFileCleanup(cause, resources);
|
||||
const primary =
|
||||
cleanup.outcome.kind === "failed"
|
||||
? cleanup.outcome.cause
|
||||
: cause.cleanupCause;
|
||||
throw configurationCleanupError(
|
||||
`Could not safely access client configuration at ${cause.path}: ${errorMessage(primary)}`,
|
||||
cause,
|
||||
cleanup.cleanup,
|
||||
);
|
||||
}
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure)
|
||||
throw configurationCleanupError(cause.message, cause, cause.cleanup);
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
const configurationCleanupError = (
|
||||
message: string,
|
||||
cause: unknown,
|
||||
cleanup:
|
||||
| { readonly reason: string; readonly resources: readonly string[] }
|
||||
| undefined,
|
||||
): ConfigurationError =>
|
||||
new ConfigurationError(message, {
|
||||
cause,
|
||||
settings: [{ setting: "client configuration", constraint: message }],
|
||||
...(cleanup === undefined ? {} : { cleanup }),
|
||||
});
|
||||
|
||||
const errorMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
|
||||
/** Detect supported agents from their config files or stable installation markers. */
|
||||
export const detectClients = async (
|
||||
home: string,
|
||||
|
||||
@@ -312,8 +312,8 @@ const clientConfigurationDetail = (
|
||||
.map(([name, value]) => `${name}=${value}`);
|
||||
const environmentDetail =
|
||||
entries.length === 0
|
||||
? " No additional environment variables."
|
||||
: ` Environment: ${entries.join(", ")}.`;
|
||||
? " No additional provider environment variables. Existing server environment overrides are retained."
|
||||
: ` Detected provider environment: ${entries.join(", ")}. Existing server environment overrides are retained; detected values replace matching keys.`;
|
||||
const hopperDetail = installHopper
|
||||
? environment.HOPPER_LAUNCHER_PATH === undefined
|
||||
? " If this plan installs Hopper, REA will add HOPPER_LAUNCHER_PATH using the launcher path reported by that installation."
|
||||
|
||||
@@ -10,6 +10,7 @@ import { clientSkillDirectories } from "./SupportedClients.js";
|
||||
|
||||
export const MANAGED_SKILL_FILES = [
|
||||
"SKILL.md",
|
||||
"references/connection-and-recovery.md",
|
||||
"references/native-and-artifacts.md",
|
||||
"references/javascript-applications.md",
|
||||
"references/android-applications.md",
|
||||
|
||||
@@ -4,6 +4,7 @@ import { isAbsolute } from "node:path";
|
||||
import { z } from "zod";
|
||||
|
||||
import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
|
||||
import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
|
||||
import { publishWebScripts } from "../browser/assets/PublishWebScripts.js";
|
||||
import { selectScriptCapture } from "../browser/assets/ScriptCaptureAdapters.js";
|
||||
import {
|
||||
@@ -14,9 +15,11 @@ import {
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../domain/evidence.js";
|
||||
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
|
||||
import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js";
|
||||
import { jsonValueSchema } from "../domain/jsonValue.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { safeParseJson } from "../domain/safeJson.js";
|
||||
import {
|
||||
@@ -26,7 +29,11 @@ import {
|
||||
import { WebScriptExportError } from "../domain/webScriptExportError.js";
|
||||
import type { ExecutionOptions } from "./AnalysisProvider.js";
|
||||
import { WEB_SCRIPT_EXPORT_PROVIDER as PROVIDER } from "./InvestigationProviders.js";
|
||||
import { readRegularFile } from "./RegularFileRead.js";
|
||||
import {
|
||||
readRegularFile,
|
||||
RegularFileCleanupFailure,
|
||||
retryRegularFileCleanup,
|
||||
} from "./RegularFileRead.js";
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
RegularFileChangedError,
|
||||
@@ -34,14 +41,29 @@ import {
|
||||
|
||||
const OPERATION = "export_web_scripts";
|
||||
|
||||
interface CaptureRead {
|
||||
readonly bytes: Buffer;
|
||||
readonly cleanup?: {
|
||||
readonly reason: string;
|
||||
readonly incomplete: boolean;
|
||||
readonly cause: unknown;
|
||||
};
|
||||
}
|
||||
|
||||
interface PreparedCapture {
|
||||
readonly bytes: Buffer;
|
||||
readonly loaded: ReturnType<typeof selectScriptCapture>;
|
||||
}
|
||||
|
||||
/** Export one local capture through the shared CLI/MCP application workflow. */
|
||||
export const exportWebScripts = async (
|
||||
rawInput: unknown,
|
||||
resources: ArtifactResourceScope,
|
||||
options: ExecutionOptions = {},
|
||||
): Promise<Result<Evidence, AnalysisError>> => {
|
||||
const input = exportWebScriptsInputSchema.safeParse(rawInput);
|
||||
return input.success
|
||||
? exportWebScriptsValidated(input.data, options)
|
||||
? exportWebScriptsValidated(input.data, resources, options)
|
||||
: err(
|
||||
analysisInputErrorFromIssues(OPERATION, input.error.issues, rawInput),
|
||||
);
|
||||
@@ -50,6 +72,7 @@ export const exportWebScripts = async (
|
||||
/** Publish input already parsed by a named adapter contract. */
|
||||
export const exportWebScriptsValidated = async (
|
||||
input: ExportWebScriptsInput,
|
||||
resources: ArtifactResourceScope,
|
||||
options: ExecutionOptions = {},
|
||||
): Promise<Result<Evidence, AnalysisError>> => {
|
||||
if (!isAbsolute(input.capture_path) || !isAbsolute(input.output_directory))
|
||||
@@ -65,19 +88,19 @@ export const exportWebScriptsValidated = async (
|
||||
);
|
||||
try {
|
||||
options.signal?.throwIfAborted();
|
||||
const read = await readCapture(input.capture_path, options.signal);
|
||||
if (!read.ok) return read;
|
||||
const bytes = read.value;
|
||||
const loaded = parseCapture(bytes);
|
||||
if (!loaded.ok) return loaded;
|
||||
options.signal?.throwIfAborted();
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
const result = await publishWebScripts(
|
||||
input,
|
||||
loaded.value,
|
||||
sha256,
|
||||
const prepared = await prepareCapture(
|
||||
input.capture_path,
|
||||
resources,
|
||||
options.signal,
|
||||
);
|
||||
if (!prepared.ok) return prepared;
|
||||
const { bytes, loaded } = prepared.value;
|
||||
options.signal?.throwIfAborted();
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
const result = await publishWebScripts(input, loaded, sha256, {
|
||||
resources,
|
||||
...(options.signal === undefined ? {} : { signal: options.signal }),
|
||||
});
|
||||
return ok(
|
||||
createEvidence(
|
||||
{ path: input.capture_path, sha256, format: "file" },
|
||||
@@ -96,55 +119,109 @@ export const exportWebScriptsValidated = async (
|
||||
limitations: result.limitations,
|
||||
locations: [{ kind: "artifact-path", path: result.manifest.path }],
|
||||
evidenceLinks:
|
||||
loaded.value.sourceEvidenceId === null
|
||||
? []
|
||||
: [loaded.value.sourceEvidenceId],
|
||||
loaded.sourceEvidenceId === null ? [] : [loaded.sourceEvidenceId],
|
||||
},
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof WebScriptExportError) return err(cause);
|
||||
if (options.signal?.aborted === true)
|
||||
return err(new AnalysisCancelledError(OPERATION));
|
||||
if (cause instanceof ArtifactReaderFailure)
|
||||
return err(
|
||||
new WebScriptExportError(
|
||||
cause.reason,
|
||||
input.output_directory,
|
||||
cause.message,
|
||||
),
|
||||
);
|
||||
if (cause instanceof z.ZodError)
|
||||
return err(new AnalysisOutputError(OPERATION, cause.message, { cause }));
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
"code" in cause &&
|
||||
typeof cause.code === "string"
|
||||
)
|
||||
return err(
|
||||
new WebScriptExportError(
|
||||
"io",
|
||||
`${input.capture_path} → ${input.output_directory}`,
|
||||
cause.message,
|
||||
),
|
||||
);
|
||||
return err(mapExportFailure(cause, input, options.signal));
|
||||
}
|
||||
};
|
||||
|
||||
const prepareCapture = async (
|
||||
path: string,
|
||||
resources: ArtifactResourceScope,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<Result<PreparedCapture, AnalysisError>> => {
|
||||
const read = await resources.run(() => readCapture(path, resources, signal));
|
||||
if (!read.ok) return read;
|
||||
const cleanup = read.value.cleanup;
|
||||
const loaded = parseCapture(
|
||||
read.value.bytes,
|
||||
cleanup === undefined || cleanup.incomplete
|
||||
? undefined
|
||||
: { cause: cleanup.cause },
|
||||
);
|
||||
if (!loaded.ok)
|
||||
return err(
|
||||
new ProviderAdapterError(PROVIDER.id, OPERATION, {
|
||||
cause,
|
||||
diagnostics: {
|
||||
capture_path: input.capture_path,
|
||||
output_directory: input.output_directory,
|
||||
error_name: cause instanceof Error ? cause.name : "UnknownError",
|
||||
error_message:
|
||||
cause instanceof Error
|
||||
? cause.message
|
||||
: typeof cause === "string"
|
||||
? cause
|
||||
: "Unknown script export failure",
|
||||
},
|
||||
}),
|
||||
cleanup?.incomplete === true
|
||||
? incompleteCaptureCleanup(loaded.error, path, cleanup)
|
||||
: loaded.error,
|
||||
);
|
||||
if (cleanup !== undefined) {
|
||||
const primary = new WebScriptExportError(
|
||||
"io",
|
||||
path,
|
||||
`Capture handle close failed: ${cleanup.reason}`,
|
||||
{ cause: cleanup.cause },
|
||||
);
|
||||
return err(
|
||||
cleanup.incomplete
|
||||
? incompleteCaptureCleanup(primary, path, cleanup)
|
||||
: primary,
|
||||
);
|
||||
}
|
||||
return ok({ bytes: read.value.bytes, loaded: loaded.value });
|
||||
};
|
||||
|
||||
const mapExportFailure = (
|
||||
cause: unknown,
|
||||
input: ExportWebScriptsInput,
|
||||
signal: AbortSignal | undefined,
|
||||
): AnalysisError => {
|
||||
if (cause instanceof WebScriptExportError) return cause;
|
||||
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
|
||||
return analysisErrorWithCleanupFailure(
|
||||
new WebScriptExportError(
|
||||
cause.reason,
|
||||
input.output_directory,
|
||||
cause.message,
|
||||
{ cause },
|
||||
),
|
||||
new ProviderCleanupError(
|
||||
PROVIDER.id,
|
||||
cause.cleanup.resources,
|
||||
{ reason: cause.cleanup.reason },
|
||||
{ cause, operation: OPERATION },
|
||||
),
|
||||
OPERATION,
|
||||
);
|
||||
if (signal?.aborted === true)
|
||||
return new AnalysisCancelledError(OPERATION, { cause });
|
||||
if (cause instanceof ArtifactReaderFailure)
|
||||
return new WebScriptExportError(
|
||||
cause.reason,
|
||||
input.output_directory,
|
||||
cause.message,
|
||||
{ cause },
|
||||
);
|
||||
if (cause instanceof z.ZodError)
|
||||
return new AnalysisOutputError(OPERATION, cause.message, { cause });
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
"code" in cause &&
|
||||
typeof cause.code === "string"
|
||||
)
|
||||
return new WebScriptExportError(
|
||||
"io",
|
||||
`${input.capture_path} → ${input.output_directory}`,
|
||||
cause.message,
|
||||
{ cause },
|
||||
);
|
||||
return new ProviderAdapterError(PROVIDER.id, OPERATION, {
|
||||
cause,
|
||||
diagnostics: {
|
||||
capture_path: input.capture_path,
|
||||
output_directory: input.output_directory,
|
||||
error_name: cause instanceof Error ? cause.name : "UnknownError",
|
||||
error_message:
|
||||
cause instanceof Error
|
||||
? cause.message
|
||||
: typeof cause === "string"
|
||||
? cause
|
||||
: "Unknown script export failure",
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -153,45 +230,115 @@ export const exportWebScriptsValidated = async (
|
||||
*/
|
||||
const readCapture = async (
|
||||
path: string,
|
||||
resources: ArtifactResourceScope,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<Result<Buffer, AnalysisError>> => {
|
||||
): Promise<Result<CaptureRead, AnalysisError>> => {
|
||||
try {
|
||||
return ok(await readRegularFile(path, { signal }));
|
||||
return ok({ bytes: await readRegularFile(path, { signal }) });
|
||||
} catch (cause: unknown) {
|
||||
const code =
|
||||
cause instanceof Error && "code" in cause ? String(cause.code) : "";
|
||||
if (code === "EACCES" || code === "EPERM")
|
||||
return err(
|
||||
new AnalysisAccessDeniedError(OPERATION, path, code, { cause }),
|
||||
if (cause instanceof RegularFileCleanupFailure) {
|
||||
const retry = await retryRegularFileCleanup(cause, resources);
|
||||
const cleanup = {
|
||||
reason: retry.cleanup?.reason ?? errorMessage(cause.cleanupCause),
|
||||
incomplete: retry.cleanup !== undefined,
|
||||
cause: cause.cleanupCause,
|
||||
};
|
||||
if (cause.outcome.kind === "completed")
|
||||
return ok({ bytes: cause.outcome.value as Buffer, cleanup });
|
||||
const primary = captureReadFailure(
|
||||
path,
|
||||
cause.outcome.cause,
|
||||
signal,
|
||||
retry.cleanup === undefined ? { cause: cause.cleanupCause } : undefined,
|
||||
);
|
||||
if (
|
||||
cause instanceof NonRegularFileReadError ||
|
||||
cause instanceof RegularFileChangedError ||
|
||||
code === "ENOENT" ||
|
||||
code === "ENOTDIR" ||
|
||||
code === "EISDIR" ||
|
||||
code === "ENXIO"
|
||||
)
|
||||
return err(
|
||||
new AnalysisInputError(OPERATION, { cause }, [
|
||||
{
|
||||
path: ["capture_path"],
|
||||
reason: "invalid_value",
|
||||
message:
|
||||
cause instanceof NonRegularFileReadError && code !== "EISDIR"
|
||||
? cause.message
|
||||
: code === "EISDIR"
|
||||
? `Selected capture is a directory, not a file: ${path}`
|
||||
: `Selected capture could not be read (${code}): ${path}`,
|
||||
},
|
||||
]),
|
||||
retry.cleanup === undefined
|
||||
? primary
|
||||
: incompleteCaptureCleanup(primary, path, cleanup),
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
return err(captureReadFailure(path, cause, signal));
|
||||
}
|
||||
};
|
||||
|
||||
const captureReadFailure = (
|
||||
path: string,
|
||||
cause: unknown,
|
||||
signal: AbortSignal | undefined,
|
||||
closeFailure?: { readonly cause: unknown },
|
||||
): AnalysisError => {
|
||||
const causeWithClose =
|
||||
closeFailure === undefined
|
||||
? cause
|
||||
: new AggregateError(
|
||||
[cause, closeFailure.cause],
|
||||
"Capture read and file-handle close both failed",
|
||||
{ cause },
|
||||
);
|
||||
if (signal?.aborted === true)
|
||||
return new AnalysisCancelledError(OPERATION, { cause: causeWithClose });
|
||||
const code =
|
||||
cause instanceof Error && "code" in cause ? String(cause.code) : "";
|
||||
if (code === "EACCES" || code === "EPERM")
|
||||
return new AnalysisAccessDeniedError(OPERATION, path, code, {
|
||||
cause: causeWithClose,
|
||||
});
|
||||
if (
|
||||
cause instanceof NonRegularFileReadError ||
|
||||
cause instanceof RegularFileChangedError ||
|
||||
code === "ENOENT" ||
|
||||
code === "ENOTDIR" ||
|
||||
code === "EISDIR" ||
|
||||
code === "ENXIO"
|
||||
)
|
||||
return new AnalysisInputError(OPERATION, { cause: causeWithClose }, [
|
||||
{
|
||||
path: ["capture_path"],
|
||||
reason: "invalid_value",
|
||||
message:
|
||||
cause instanceof NonRegularFileReadError && code !== "EISDIR"
|
||||
? cause.message
|
||||
: code === "EISDIR"
|
||||
? `Selected capture is a directory, not a file: ${path}`
|
||||
: `Selected capture could not be read (${code}): ${path}`,
|
||||
},
|
||||
]);
|
||||
if (code !== "")
|
||||
return new WebScriptExportError("io", path, errorMessage(cause), {
|
||||
cause: causeWithClose,
|
||||
});
|
||||
return new ProviderAdapterError(PROVIDER.id, OPERATION, {
|
||||
cause: causeWithClose,
|
||||
diagnostics: {
|
||||
capture_path: path,
|
||||
error_name: cause instanceof Error ? cause.name : "UnknownError",
|
||||
error_message: cause instanceof Error ? cause.message : String(cause),
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const incompleteCaptureCleanup = (
|
||||
primary: AnalysisError,
|
||||
path: string,
|
||||
cleanup: NonNullable<CaptureRead["cleanup"]>,
|
||||
): AnalysisError =>
|
||||
analysisErrorWithCleanupFailure(
|
||||
primary,
|
||||
new ProviderCleanupError(
|
||||
PROVIDER.id,
|
||||
[path],
|
||||
{ reason: cleanup.reason },
|
||||
{ cause: cleanup.cause, operation: OPERATION },
|
||||
),
|
||||
OPERATION,
|
||||
);
|
||||
|
||||
const errorMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
|
||||
const parseCapture = (
|
||||
bytes: Buffer,
|
||||
closeFailure?: { readonly cause: unknown },
|
||||
): Result<ReturnType<typeof selectScriptCapture>, AnalysisError> => {
|
||||
try {
|
||||
const text = new TextDecoder("utf-8", {
|
||||
@@ -202,8 +349,16 @@ const parseCapture = (
|
||||
if (!json.ok) throw new TypeError(json.error);
|
||||
return ok(selectScriptCapture(json.value));
|
||||
} catch (cause: unknown) {
|
||||
const causeWithClose =
|
||||
closeFailure === undefined
|
||||
? cause
|
||||
: new AggregateError(
|
||||
[cause, closeFailure.cause],
|
||||
"Capture parse and file-handle close both failed",
|
||||
{ cause },
|
||||
);
|
||||
return err(
|
||||
new AnalysisInputError(OPERATION, { cause }, [
|
||||
new AnalysisInputError(OPERATION, { cause: causeWithClose }, [
|
||||
{
|
||||
path: ["capture_path"],
|
||||
reason: "invalid_format",
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
} from "../../../tests/fixtures/analysisView.js";
|
||||
import {
|
||||
inspectAnalysisView,
|
||||
inspectAnalysisViewValidated,
|
||||
summarizeRetainedAnalysis,
|
||||
} from "./AnalysisViewService.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
@@ -58,6 +59,35 @@ it.each(["/fixtures/unknown.exe", ""])(
|
||||
},
|
||||
);
|
||||
|
||||
it("reports inline Evidence identity failures at the complete source path", () => {
|
||||
const parent = analysisViewLayoutEvidence();
|
||||
const input = {
|
||||
source: {
|
||||
kind: "inline",
|
||||
evidence: { ...parent, evidence_id: `ev_${"0".repeat(64)}` },
|
||||
},
|
||||
view: { kind: "summary" },
|
||||
} as const;
|
||||
|
||||
for (const result of [
|
||||
inspectAnalysisView(input),
|
||||
inspectAnalysisViewValidated(input),
|
||||
])
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
_tag: "AnalysisInputError",
|
||||
issues: [
|
||||
{
|
||||
path: ["source", "evidence", "evidence_id"],
|
||||
reason: "invalid_value",
|
||||
message: "Evidence semantic identifier does not match its record",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("projects a small native view from an authenticated >10 MiB retained dossier", () => {
|
||||
const original = functionDossierSchema.parse(ghidraFunctionDossier());
|
||||
const parent = createEvidence(
|
||||
|
||||
@@ -132,8 +132,11 @@ const resolveAnalysisViewSource = (
|
||||
return err(
|
||||
analysisInputErrorFromIssues(
|
||||
INSPECT_ANALYSIS_VIEW_OPERATION,
|
||||
cause.issues,
|
||||
source.evidence,
|
||||
cause.issues.map((issue) => ({
|
||||
...issue,
|
||||
path: ["source", "evidence", ...issue.path],
|
||||
})),
|
||||
{ source },
|
||||
{ cause },
|
||||
),
|
||||
);
|
||||
|
||||
@@ -4,7 +4,7 @@ import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
describe("JavaScript analysis cancellation before publication", () => {
|
||||
it("interrupts semantic graph commitment before result validation", async () => {
|
||||
|
||||
@@ -10,7 +10,7 @@ import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascri
|
||||
import { matchJavaScriptApplicationVersions } from "../../domain/javascript/javascriptApplicationVersionKeys.js";
|
||||
import { findApplicationFeatureSeeds } from "../../domain/javascript/javascriptFeatureSeed.js";
|
||||
import { traceApplicationFeatureInputSchema } from "../../domain/javascript/javascriptFeatureTraceSchemas.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
const analyzeSource = async (
|
||||
source: string,
|
||||
|
||||
@@ -5,7 +5,7 @@ import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
it.each([
|
||||
{
|
||||
|
||||
@@ -4,9 +4,11 @@ import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
import { scanCanonicalArtifactInventory } from "../../../tests/fixtures/artifactInventory.js";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { projectAnalysisError } from "../../domain/analysisErrorProjection.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
describe("JavaScript application failure diagnostics", () => {
|
||||
it("identifies the rejected result field in caller-visible diagnostics", async () => {
|
||||
@@ -153,3 +155,41 @@ describe("JavaScript application failure diagnostics", () => {
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("JavaScript application cleanup diagnostics", () => {
|
||||
it("preserves cleanup and a complete inventory when translating reader failures", async () => {
|
||||
const inputPath = await createTestTempDirectory("rea-js-cleanup-failure-");
|
||||
await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
|
||||
const inventory = await scanCanonicalArtifactInventory(inputPath);
|
||||
const cleanup = {
|
||||
reason: "reader remains open",
|
||||
resources: [`JavaScript artifact reader for ${inputPath}`],
|
||||
};
|
||||
const partialObservation = {
|
||||
kind: "artifact-inventory" as const,
|
||||
inventory,
|
||||
};
|
||||
const failure = new ArtifactReaderFailure("io", "Reader cleanup failed", {
|
||||
cleanup,
|
||||
partialObservation,
|
||||
});
|
||||
|
||||
const result = await analyzeJavaScriptApplication(
|
||||
{ input_path: inputPath, format: "directory" },
|
||||
{
|
||||
progress: {
|
||||
report: async () => {
|
||||
throw failure;
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if (result.ok) throw new Error("Expected reader failure");
|
||||
expect(result.error).toMatchObject({
|
||||
_tag: "ArtifactOperationError",
|
||||
cleanup,
|
||||
partialObservation,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import {
|
||||
analyzeJavaScriptApplicationInputSchema,
|
||||
parseOwnedJavaScriptApplicationAnalysisSteps,
|
||||
@@ -32,6 +33,7 @@ const OPERATION = "analyze_javascript_application" as const;
|
||||
/** Statically analyze one local JavaScript/Electron application. */
|
||||
export const analyzeJavaScriptApplication = async (
|
||||
rawInput: unknown,
|
||||
scope: ArtifactResourceScope,
|
||||
options: ExecutionOptions = {},
|
||||
): Promise<Result<Evidence, AnalysisError>> => {
|
||||
const parsed = analyzeJavaScriptApplicationInputSchema.safeParse(rawInput);
|
||||
@@ -39,12 +41,13 @@ export const analyzeJavaScriptApplication = async (
|
||||
return err(
|
||||
analysisInputErrorFromIssues(OPERATION, parsed.error.issues, rawInput),
|
||||
);
|
||||
return analyzeJavaScriptApplicationValidated(parsed.data, options);
|
||||
return analyzeJavaScriptApplicationValidated(parsed.data, scope, options);
|
||||
};
|
||||
|
||||
/** Analyze input already parsed by a trusted adapter boundary. */
|
||||
export const analyzeJavaScriptApplicationValidated = async (
|
||||
input: z.output<typeof analyzeJavaScriptApplicationInputSchema>,
|
||||
scope: ArtifactResourceScope,
|
||||
options: ExecutionOptions = {},
|
||||
): Promise<Result<Evidence, AnalysisError>> => {
|
||||
try {
|
||||
@@ -62,6 +65,7 @@ export const analyzeJavaScriptApplicationValidated = async (
|
||||
format: input.format,
|
||||
integrity_policy: input.integrity_policy,
|
||||
},
|
||||
scope,
|
||||
options.signal,
|
||||
options.progress,
|
||||
);
|
||||
@@ -114,14 +118,7 @@ export const analyzeJavaScriptApplicationValidated = async (
|
||||
)
|
||||
return err(cause);
|
||||
if (cause instanceof ArtifactReaderFailure)
|
||||
return err(
|
||||
new ArtifactOperationError(
|
||||
OPERATION,
|
||||
cause.reason,
|
||||
cause.details,
|
||||
cause.message,
|
||||
),
|
||||
);
|
||||
return err(artifactFailureToOperationError(cause));
|
||||
if (cause instanceof z.ZodError)
|
||||
return err(
|
||||
new AnalysisOutputError(OPERATION, describeResultSchemaFailure(cause), {
|
||||
@@ -153,6 +150,23 @@ export const analyzeJavaScriptApplicationValidated = async (
|
||||
}
|
||||
};
|
||||
|
||||
const artifactFailureToOperationError = (
|
||||
cause: ArtifactReaderFailure,
|
||||
): ArtifactOperationError =>
|
||||
new ArtifactOperationError(
|
||||
OPERATION,
|
||||
cause.reason,
|
||||
cause.details,
|
||||
cause.message,
|
||||
{
|
||||
cause,
|
||||
...(cause.cleanup === undefined ? {} : { cleanup: cause.cleanup }),
|
||||
...(cause.partialObservation === undefined
|
||||
? {}
|
||||
: { partialObservation: cause.partialObservation }),
|
||||
},
|
||||
);
|
||||
|
||||
const describeResultSchemaFailure = (cause: z.ZodError): string => {
|
||||
const issue = cause.issues[0];
|
||||
if (issue === undefined) return "Result schema rejected an unknown issue";
|
||||
|
||||
@@ -7,7 +7,7 @@ import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirect
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { javaScriptSemanticQueryResultSchema } from "../../domain/javascript/javascriptSemanticQuerySchemas.js";
|
||||
import { queryJavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticQuery.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
it.each([
|
||||
{
|
||||
|
||||
@@ -5,7 +5,7 @@ import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
const graphIdentities = async (source: string) => {
|
||||
const inputPath = await createTestTempDirectory("rea-js-template-literal-");
|
||||
|
||||
@@ -7,7 +7,7 @@ import { canonicalJson } from "../../domain/comparisonSemantics.js";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
type ApplicationAnalysis = ReturnType<
|
||||
typeof javascriptApplicationAnalysisResultSchema.parse
|
||||
|
||||
@@ -7,7 +7,6 @@ import type { ApplicationVersionComparisonResult } from "../../domain/javascript
|
||||
import type { JavaScriptExportShapeComparisonResult } from "../../domain/javascript/javascriptExportShapeComparisonSchemas.js";
|
||||
import type { ApplicationFeatureTraceResult } from "../../domain/javascript/javascriptFeatureTraceSchemas.js";
|
||||
import type { JavaScriptSemanticTraceResult } from "../../domain/javascript/javascriptSemanticTraceSchemas.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import type { SourceToBundleComparisonResult } from "../../domain/javascript/sourceToBundleComparisonSchemas.js";
|
||||
import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "../InvestigationProviders.js";
|
||||
|
||||
@@ -20,7 +19,7 @@ export const createApplicationFeatureTraceEvidence = (
|
||||
predicateType: "rea.application-feature-trace",
|
||||
operation: "trace_application_feature",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse(result),
|
||||
result,
|
||||
rawResult: null,
|
||||
confidence: "inferred",
|
||||
authority: "analyst-inference",
|
||||
@@ -38,7 +37,7 @@ export const createJavaScriptSemanticTraceEvidence = (
|
||||
predicateType: "rea.javascript-semantic-trace",
|
||||
operation: "trace_javascript_semantics",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse(result),
|
||||
result,
|
||||
rawResult: null,
|
||||
confidence: "inferred",
|
||||
authority: "analyst-inference",
|
||||
@@ -56,7 +55,7 @@ export const createApplicationVersionComparisonEvidence = (
|
||||
predicateType: "rea.application-version-comparison",
|
||||
operation: "compare_application_versions",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse(result),
|
||||
result,
|
||||
rawResult: null,
|
||||
confidence: "inferred",
|
||||
authority: "analyst-inference",
|
||||
@@ -74,7 +73,7 @@ export const createSourceToBundleComparisonEvidence = (
|
||||
predicateType: "rea.source-to-bundle-comparison",
|
||||
operation: "compare_source_to_bundle",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse(result),
|
||||
result,
|
||||
rawResult: null,
|
||||
confidence: "inferred",
|
||||
authority: "analyst-inference",
|
||||
@@ -92,7 +91,7 @@ export const createJavaScriptExportShapeComparisonEvidence = (
|
||||
predicateType: "rea.javascript-export-shape-comparison",
|
||||
operation: "compare_javascript_export_shapes",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse(result),
|
||||
result,
|
||||
rawResult: null,
|
||||
confidence: "inferred",
|
||||
authority: "analyst-inference",
|
||||
|
||||
@@ -115,9 +115,18 @@ export const addJavaScriptArtifactContainers = (
|
||||
export const addJavaScriptArtifactFiles = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
): void => {
|
||||
const packagesByPath = indexFirstByPath(context.analysis.packages);
|
||||
const jsonModulesByPath = indexFirstByPath(context.analysis.json_modules);
|
||||
const sourceMapsByPath = indexFirstByPath(context.analysis.source_maps);
|
||||
for (const analyzed of context.analysis.files) {
|
||||
const { file } = analyzed;
|
||||
const target = createFileTarget(context, file, analyzed.javascript);
|
||||
const jsonValue = jsonModulesByPath.get(file.path);
|
||||
const target = createFileTarget(
|
||||
context,
|
||||
file,
|
||||
analyzed.javascript,
|
||||
jsonValue,
|
||||
);
|
||||
context.fileNodes.set(file.path, target);
|
||||
if (file.kind === "javascript") context.assetNodes.set(file.path, target);
|
||||
const entry = createAsarEntry(context, file);
|
||||
@@ -163,9 +172,7 @@ export const addJavaScriptArtifactFiles = (
|
||||
? "JavaScript syntax could not be parsed."
|
||||
: `JavaScript text was unavailable: ${file.text.reason}.`,
|
||||
});
|
||||
const packageValue = context.analysis.packages.find(
|
||||
({ path }) => path === file.path,
|
||||
);
|
||||
const packageValue = packagesByPath.get(file.path);
|
||||
if (packageValue !== undefined && packageValue.status !== "included")
|
||||
addUnavailableStaticParseScope(context, {
|
||||
file,
|
||||
@@ -173,9 +180,6 @@ export const addJavaScriptArtifactFiles = (
|
||||
operation: "parse-package-json",
|
||||
limitation: packageValue.limitation,
|
||||
});
|
||||
const jsonValue = context.analysis.json_modules.find(
|
||||
({ path }) => path === file.path,
|
||||
);
|
||||
if (jsonValue !== undefined && jsonValue.status !== "included")
|
||||
addUnavailableStaticParseScope(context, {
|
||||
file,
|
||||
@@ -183,9 +187,7 @@ export const addJavaScriptArtifactFiles = (
|
||||
operation: "parse-json-module",
|
||||
limitation: jsonValue.limitation,
|
||||
});
|
||||
const sourceMap = context.analysis.source_maps.find(
|
||||
({ path }) => path === file.path,
|
||||
);
|
||||
const sourceMap = sourceMapsByPath.get(file.path);
|
||||
if (sourceMap !== undefined && sourceMap.status === "invalid")
|
||||
addUnavailableStaticParseScope(context, {
|
||||
file,
|
||||
@@ -270,11 +272,9 @@ const createFileTarget = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
file: JavaScriptArtifactFile,
|
||||
javascript: JavaScriptModuleArtifactAnalysis["files"][number]["javascript"],
|
||||
json: JavaScriptModuleArtifactAnalysis["json_modules"][number] | undefined,
|
||||
): ApplicationNode => {
|
||||
const kind = artifactFileNodeKind(file.kind);
|
||||
const json = context.analysis.json_modules.find(
|
||||
({ path }) => path === file.path,
|
||||
);
|
||||
return context.accumulator.addNode({
|
||||
kind,
|
||||
identity: {
|
||||
@@ -309,6 +309,16 @@ const createFileTarget = (
|
||||
});
|
||||
};
|
||||
|
||||
const indexFirstByPath = <Value extends { readonly path: string }>(
|
||||
values: readonly Value[],
|
||||
): ReadonlyMap<string, Value> => {
|
||||
const index = new Map<string, Value>();
|
||||
for (const value of values) {
|
||||
if (!index.has(value.path)) index.set(value.path, value);
|
||||
}
|
||||
return index;
|
||||
};
|
||||
|
||||
const createAsarEntry = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
file: JavaScriptArtifactFile,
|
||||
|
||||
@@ -234,7 +234,8 @@ const resolvePackageSpecifier = (
|
||||
if (rawExports === undefined || rawExports === null) {
|
||||
if (subpath === ".")
|
||||
return resolveDirectory(input, directory, metadata, new Set());
|
||||
const candidate = confineCandidate(posix.join(directory, subpath.slice(2)));
|
||||
// Preserve the trailing slash for a package directory request.
|
||||
const candidate = confineCandidate(`${directory}/${subpath.slice(2)}`);
|
||||
return typeof candidate === "string"
|
||||
? resolveCandidate(input, candidate, new Set())
|
||||
: candidate;
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { JavaScriptApplicationGraph } from "../../domain/javascript/javascr
|
||||
import type { JavaScriptSemanticGraph } from "../../domain/javascript/javascriptSemanticGraph.js";
|
||||
import type { ElectronBoundarySummary } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
import type { ArtifactInventoryPartialObservation } from "../../domain/artifactPartialObservation.js";
|
||||
import { analyzeAndProjectJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js";
|
||||
import { readJavaScriptArtifactFiles } from "../../artifacts/javascript/JavaScriptArtifactFiles.js";
|
||||
import { buildImmutableJavaScriptArtifactGraphSteps } from "./JavaScriptArtifactGraphBuilder.js";
|
||||
@@ -19,7 +20,8 @@ import {
|
||||
javascriptArtifactReconstructionInputSchema,
|
||||
type JavaScriptArtifactReconstructionInput,
|
||||
} from "./JavaScriptArtifactReconstructionInput.js";
|
||||
import { scanCanonicalArtifactInventory } from "../../artifacts/inventory/scanCanonical.js";
|
||||
import { scanCanonicalArtifactInventoryInScope } from "../../artifacts/inventory/scanCanonical.js";
|
||||
import type { ArtifactResourceScope } from "../../artifacts/ArtifactResourceScope.js";
|
||||
import { summarizeElectronBoundaries } from "./ElectronBoundaryAnalysis.js";
|
||||
import { createJavaScriptSemanticGraphProjection } from "./JavaScriptSemanticGraphBuilder.js";
|
||||
import type { ProgressReporter } from "../ProgressReporter.js";
|
||||
@@ -56,118 +58,161 @@ export interface JavaScriptArtifactReconstructionResult {
|
||||
/** Reconstruct one local ASAR or extracted directory without executing code. */
|
||||
export const reconstructJavaScriptArtifact = async (
|
||||
rawInput: unknown,
|
||||
scope: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
progress?: ProgressReporter,
|
||||
): Promise<JavaScriptArtifactReconstructionResult> => {
|
||||
const reportPhase = async (phase: string, message: string): Promise<void> => {
|
||||
await progress?.report({ phase, completed: 0, total: 1, message });
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
};
|
||||
const input = javascriptArtifactReconstructionInputSchema.parse(rawInput);
|
||||
abortIfNeeded(signal);
|
||||
const path = await resolveSelectedInput(input.input_path);
|
||||
const format = await resolveFormat(path, input);
|
||||
const snapshot = await scanCanonicalArtifactInventory(path, {
|
||||
signal,
|
||||
integrity: { mode: input.integrity_policy },
|
||||
});
|
||||
if (snapshot.manifest.root_format !== format)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Artifact inventory classified ${path} as ${snapshot.manifest.root_format}, not ${format}`,
|
||||
);
|
||||
const reader = createReader(path, format);
|
||||
try {
|
||||
await reportPhase(
|
||||
"read_javascript_artifacts",
|
||||
"Reading inventoried JavaScript application sources",
|
||||
);
|
||||
const files = await readJavaScriptArtifactFiles(reader, snapshot, signal);
|
||||
await reportPhase(
|
||||
"parse_javascript_sources",
|
||||
`Parsing and projecting ${String(files.files.length)} application source files`,
|
||||
);
|
||||
const semanticProjection = createJavaScriptSemanticGraphProjection();
|
||||
const analysis = await analyzeAndProjectJavaScriptArtifactFiles(
|
||||
files,
|
||||
semanticProjection.projectFileSteps,
|
||||
async (file, completed, total) => {
|
||||
abortIfNeeded(signal);
|
||||
await progress?.report({
|
||||
phase: "parse_javascript_source",
|
||||
completed: 0,
|
||||
total: 1,
|
||||
message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`,
|
||||
});
|
||||
abortIfNeeded(signal);
|
||||
},
|
||||
signal,
|
||||
);
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
abortIfNeeded(signal);
|
||||
await reportPhase(
|
||||
"build_javascript_application_graph",
|
||||
"Constructing application and Electron boundary relationships",
|
||||
);
|
||||
const applicationGraphSteps = buildImmutableJavaScriptArtifactGraphSteps(
|
||||
snapshot,
|
||||
files,
|
||||
analysis,
|
||||
);
|
||||
await reportPhase(
|
||||
"seal_javascript_application_graph",
|
||||
"Sealing the validated application graph",
|
||||
);
|
||||
const graph = await completeJavaScriptAnalysisSteps(
|
||||
applicationGraphSteps,
|
||||
signal,
|
||||
);
|
||||
await reportPhase(
|
||||
"build_javascript_semantic_graph",
|
||||
"Binding and validating static semantic relationships",
|
||||
);
|
||||
const semanticGraphSteps = semanticProjection.finishImmutableSteps(
|
||||
snapshot.manifest.root_sha256,
|
||||
graph,
|
||||
);
|
||||
await reportPhase(
|
||||
"seal_javascript_semantic_graph",
|
||||
"Validating and sealing the semantic graph",
|
||||
);
|
||||
const semanticGraph = await completeJavaScriptAnalysisSteps(
|
||||
semanticGraphSteps,
|
||||
signal,
|
||||
);
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
return {
|
||||
input_path: path,
|
||||
format,
|
||||
root_artifact_sha256: snapshot.manifest.root_sha256,
|
||||
inventory_manifest_id: snapshot.manifest.manifest_id,
|
||||
inventory_graph_sha256: snapshot.manifest.graph_sha256,
|
||||
integrity_contradictions: snapshot.integrity_contradictions,
|
||||
graph,
|
||||
semantic_graph: semanticGraph,
|
||||
electron_summary: summarizeElectronBoundaries(analysis),
|
||||
statistics: {
|
||||
relevant_files: files.files.length,
|
||||
nested_asar_containers: files.containers.length,
|
||||
text_bytes_read: files.text_bytes_read,
|
||||
invalid_utf8_files: files.invalid_utf8_files,
|
||||
parsed_javascript_files: analysis.files.filter(
|
||||
({ javascript }) => javascript !== null,
|
||||
).length,
|
||||
visited_ast_nodes: analysis.visited_ast_nodes,
|
||||
findings: analysis.findings,
|
||||
modules: analysis.modules,
|
||||
parse_failures: analysis.parse_failures,
|
||||
},
|
||||
limitations: analysis.limitations,
|
||||
): Promise<JavaScriptArtifactReconstructionResult> =>
|
||||
scope.run(async () => {
|
||||
const reportPhase = async (
|
||||
phase: string,
|
||||
message: string,
|
||||
): Promise<void> => {
|
||||
await progress?.report({ phase, completed: 0, total: 1, message });
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
};
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
};
|
||||
const input = javascriptArtifactReconstructionInputSchema.parse(rawInput);
|
||||
abortIfNeeded(signal);
|
||||
const path = await resolveSelectedInput(input.input_path);
|
||||
const format = await resolveFormat(path, input);
|
||||
const snapshot = await scanCanonicalArtifactInventoryInScope(path, {
|
||||
resourceScope: scope,
|
||||
signal,
|
||||
integrity: { mode: input.integrity_policy },
|
||||
});
|
||||
if (snapshot.manifest.root_format !== format)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Artifact inventory classified ${path} as ${snapshot.manifest.root_format}, not ${format}`,
|
||||
);
|
||||
const reader = createReader(path, format);
|
||||
let outcome:
|
||||
| {
|
||||
readonly kind: "completed";
|
||||
readonly result: JavaScriptArtifactReconstructionResult;
|
||||
}
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
try {
|
||||
await reportPhase(
|
||||
"read_javascript_artifacts",
|
||||
"Reading inventoried JavaScript application sources",
|
||||
);
|
||||
const files = await readJavaScriptArtifactFiles(
|
||||
reader,
|
||||
snapshot,
|
||||
scope,
|
||||
signal,
|
||||
);
|
||||
await reportPhase(
|
||||
"parse_javascript_sources",
|
||||
`Parsing and projecting ${String(files.files.length)} application source files`,
|
||||
);
|
||||
const semanticProjection = createJavaScriptSemanticGraphProjection();
|
||||
const analysis = await analyzeAndProjectJavaScriptArtifactFiles(
|
||||
files,
|
||||
semanticProjection.projectFileSteps,
|
||||
async (file, completed, total) => {
|
||||
abortIfNeeded(signal);
|
||||
await progress?.report({
|
||||
phase: "parse_javascript_source",
|
||||
completed: 0,
|
||||
total: 1,
|
||||
message: `Parsing and projecting ${file.path} (${String(completed + 1)}/${String(total)})`,
|
||||
});
|
||||
abortIfNeeded(signal);
|
||||
},
|
||||
signal,
|
||||
);
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
abortIfNeeded(signal);
|
||||
await reportPhase(
|
||||
"build_javascript_application_graph",
|
||||
"Constructing application and Electron boundary relationships",
|
||||
);
|
||||
const applicationGraphSteps = buildImmutableJavaScriptArtifactGraphSteps(
|
||||
snapshot,
|
||||
files,
|
||||
analysis,
|
||||
);
|
||||
await reportPhase(
|
||||
"seal_javascript_application_graph",
|
||||
"Sealing the validated application graph",
|
||||
);
|
||||
const graph = await completeJavaScriptAnalysisSteps(
|
||||
applicationGraphSteps,
|
||||
signal,
|
||||
);
|
||||
await reportPhase(
|
||||
"build_javascript_semantic_graph",
|
||||
"Binding and validating static semantic relationships",
|
||||
);
|
||||
const semanticGraphSteps = semanticProjection.finishImmutableSteps(
|
||||
snapshot.manifest.root_sha256,
|
||||
graph,
|
||||
);
|
||||
await reportPhase(
|
||||
"seal_javascript_semantic_graph",
|
||||
"Validating and sealing the semantic graph",
|
||||
);
|
||||
const semanticGraph = await completeJavaScriptAnalysisSteps(
|
||||
semanticGraphSteps,
|
||||
signal,
|
||||
);
|
||||
await checkpointJavaScriptAnalysis(signal);
|
||||
outcome = {
|
||||
kind: "completed",
|
||||
result: {
|
||||
input_path: path,
|
||||
format,
|
||||
root_artifact_sha256: snapshot.manifest.root_sha256,
|
||||
inventory_manifest_id: snapshot.manifest.manifest_id,
|
||||
inventory_graph_sha256: snapshot.manifest.graph_sha256,
|
||||
integrity_contradictions: snapshot.integrity_contradictions,
|
||||
graph,
|
||||
semantic_graph: semanticGraph,
|
||||
electron_summary: summarizeElectronBoundaries(analysis),
|
||||
statistics: {
|
||||
relevant_files: files.files.length,
|
||||
nested_asar_containers: files.containers.length,
|
||||
text_bytes_read: files.text_bytes_read,
|
||||
invalid_utf8_files: files.invalid_utf8_files,
|
||||
parsed_javascript_files: analysis.files.filter(
|
||||
({ javascript }) => javascript !== null,
|
||||
).length,
|
||||
visited_ast_nodes: analysis.visited_ast_nodes,
|
||||
findings: analysis.findings,
|
||||
modules: analysis.modules,
|
||||
parse_failures: analysis.parse_failures,
|
||||
},
|
||||
limitations: analysis.limitations,
|
||||
},
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
outcome = { kind: "failed", cause };
|
||||
}
|
||||
const owner = {
|
||||
kind: "reader" as const,
|
||||
reader,
|
||||
resource: `JavaScript artifact reader for ${path}`,
|
||||
};
|
||||
const cleanupAttempt = await scope.release(owner);
|
||||
if (cleanupAttempt.kind === "failed") {
|
||||
const cleanup = ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupAttempt.cause,
|
||||
owner.resource,
|
||||
);
|
||||
const partialObservation: ArtifactInventoryPartialObservation = {
|
||||
kind: "artifact-inventory",
|
||||
inventory: snapshot,
|
||||
};
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
outcome.kind === "failed" ? outcome.cause : cleanupAttempt.cause,
|
||||
cleanup,
|
||||
partialObservation,
|
||||
);
|
||||
}
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
return outcome.result;
|
||||
});
|
||||
|
||||
const OPERATION = "analyze_javascript_application";
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import { describe, expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { staticPath } from "../../domain/javascript/javascriptStaticAnalysisHelpers.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
const urlCases = [
|
||||
["./preload%20name.cjs", "preload name.cjs", "main.mjs"],
|
||||
|
||||
@@ -7,7 +7,7 @@ import { promisify } from "node:util";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
|
||||
import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
|
||||
const filesFor = (paths: readonly string[]) =>
|
||||
|
||||
@@ -5,7 +5,7 @@ import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { javascriptApplicationAnalysisResultSchema } from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
import { analyzeJavaScriptApplication } from "../../../tests/support/javascriptApplicationScope.js";
|
||||
|
||||
it("classifies overloaded open calls from lexical receiver facts", async () => {
|
||||
const inputPath = await createTestTempDirectory(
|
||||
|
||||
@@ -355,7 +355,7 @@ const semanticNodeFor = (roleKey: string) => {
|
||||
unpacked: false,
|
||||
text: { included: true, value: "" },
|
||||
};
|
||||
const state = createSemanticGraphProjectionState({ nodes: [] });
|
||||
const state = createSemanticGraphProjectionState();
|
||||
return constructSemanticGraphNode(
|
||||
file,
|
||||
{
|
||||
@@ -565,7 +565,7 @@ it("does not mark a file truncated when it exactly fills its budget", () => {
|
||||
it("only records node loss when the budget rejects a new identity", () => {
|
||||
// The truncation signal must record blocked node creation, not a zero
|
||||
// remaining budget: a file using exactly its share drops nothing.
|
||||
const state = createSemanticGraphProjectionState({ nodes: [] });
|
||||
const state = createSemanticGraphProjectionState();
|
||||
state.fileNodeBudget = 2;
|
||||
state.fileNodesDropped = false;
|
||||
const first = addSemanticGraphNode(state, semanticNodeFor("first"));
|
||||
|
||||
@@ -148,7 +148,7 @@ export interface JavaScriptSemanticGraphProjection {
|
||||
/** Accumulate semantic nodes while each source IR is still file-local. */
|
||||
export const createJavaScriptSemanticGraphProjection =
|
||||
(): JavaScriptSemanticGraphProjection => {
|
||||
const state = emptyState({ nodes: [] });
|
||||
const state = emptyState();
|
||||
const fingerprints: JavaScriptSemanticFingerprint[] = [];
|
||||
let truncatedFiles = 0;
|
||||
const projectSourceSteps = function* (
|
||||
|
||||
@@ -28,7 +28,6 @@ export interface SemanticGraphProjectionState {
|
||||
readonly unknowns: Map<string, JavaScriptSemanticGraphUnknown>;
|
||||
readonly evidenceContexts: JavaScriptSemanticEvidenceContextRegistry;
|
||||
readonly roots: Set<string>;
|
||||
readonly applicationNodeIdsByLocation: ReadonlyMap<string, readonly string[]>;
|
||||
/**
|
||||
* Remaining node budget for the file currently being projected.
|
||||
*
|
||||
@@ -69,32 +68,29 @@ export interface SemanticRelationConstructionInput {
|
||||
readonly properties?: Readonly<Record<string, JsonValue>>;
|
||||
}
|
||||
|
||||
/** Create empty projection state with structural application-node mappings. */
|
||||
export const createSemanticGraphProjectionState = (
|
||||
applicationGraph: Pick<JavaScriptApplicationGraph, "nodes">,
|
||||
): SemanticGraphProjectionState => {
|
||||
const index = indexApplicationNodes(applicationGraph.nodes);
|
||||
return {
|
||||
/** Create empty projection state; application-node links are bound at finish. */
|
||||
export const createSemanticGraphProjectionState =
|
||||
(): SemanticGraphProjectionState => ({
|
||||
nodes: new Map(),
|
||||
relations: new Map(),
|
||||
unknowns: new Map(),
|
||||
evidenceContexts: new JavaScriptSemanticEvidenceContextRegistry(),
|
||||
roots: new Set(),
|
||||
applicationNodeIdsByLocation: index.identifiers,
|
||||
};
|
||||
};
|
||||
});
|
||||
|
||||
/** Bind retained semantic identities after all application observations exist. */
|
||||
export const bindSemanticGraphApplicationNodes = (
|
||||
state: SemanticGraphProjectionState,
|
||||
applicationGraph: Pick<JavaScriptApplicationGraph, "nodes">,
|
||||
): void => {
|
||||
const index = indexApplicationNodes(applicationGraph.nodes).identifiers;
|
||||
const applicationNodeIdsByLocation = indexApplicationNodes(
|
||||
applicationGraph.nodes,
|
||||
);
|
||||
for (const [identifier, node] of state.nodes) {
|
||||
state.nodes.set(identifier, {
|
||||
...node,
|
||||
application_node_ids: [
|
||||
...(index.get(
|
||||
...(applicationNodeIdsByLocation.get(
|
||||
applicationLocationKey(
|
||||
node.identity.artifact_sha256,
|
||||
node.identity.module_path,
|
||||
@@ -122,7 +118,7 @@ export const constructSemanticGraphNode = (
|
||||
role_key: input.roleKey,
|
||||
},
|
||||
function_node_id: input.functionNodeId,
|
||||
application_node_ids: matchingApplicationNodeIds(file, input, state),
|
||||
application_node_ids: [],
|
||||
label: input.label === null ? null : javascriptDisplayText(input.label),
|
||||
properties: input.properties ?? {},
|
||||
evidence: observedSemanticEvidence(file, input.location),
|
||||
@@ -247,9 +243,7 @@ export const addSemanticFallbackRoot = (
|
||||
|
||||
const indexApplicationNodes = (
|
||||
nodes: JavaScriptApplicationGraph["nodes"],
|
||||
): {
|
||||
readonly identifiers: ReadonlyMap<string, readonly string[]>;
|
||||
} => {
|
||||
): ReadonlyMap<string, readonly string[]> => {
|
||||
const identifiers = new Map<string, Set<string>>();
|
||||
for (const node of nodes) {
|
||||
for (const observation of node.observations) {
|
||||
@@ -280,19 +274,9 @@ const indexApplicationNodes = (
|
||||
return [key, sorted];
|
||||
}),
|
||||
);
|
||||
return { identifiers: indexed };
|
||||
return indexed;
|
||||
};
|
||||
|
||||
const matchingApplicationNodeIds = (
|
||||
file: JavaScriptArtifactFile,
|
||||
input: SemanticNodeConstructionInput,
|
||||
state: SemanticGraphProjectionState,
|
||||
): string[] => [
|
||||
...(state.applicationNodeIdsByLocation.get(
|
||||
applicationLocationKey(file.sha256, file.path, input.location),
|
||||
) ?? []),
|
||||
];
|
||||
|
||||
const addApplicationNodeIdentifier = (
|
||||
identifiers: Map<string, Set<string>>,
|
||||
key: string,
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
import { ghidraFunctionDossier } from "../../domain/ghidraValues.fixture.js";
|
||||
import { functionDossierSchema } from "../../domain/hopperValues.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import { nativeValueTraceSchema } from "../../domain/native/nativeValueTrace.js";
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import { ok, err } from "../../domain/result.js";
|
||||
|
||||
@@ -102,6 +103,132 @@ const analysis: AnalysisOperationPort = {
|
||||
);
|
||||
},
|
||||
};
|
||||
describe("native value parameter-use node membership", () => {
|
||||
it("retains only parameter uses whose parameter nodes fit the node budget", async () => {
|
||||
const operationCount = 3_000;
|
||||
const parameterUseCount = 12_000;
|
||||
const base = functionDossierSchema.parse(ghidraFunctionDossier());
|
||||
const procedureAddress = base.procedure.address;
|
||||
const largeFlow = {
|
||||
available: true as const,
|
||||
provenance: "ghidra-high-pcode" as const,
|
||||
operations: Array.from({ length: operationCount }, (_, index) => ({
|
||||
...node(`operation-${String(index)}`, procedureAddress, "MULTIEQUAL"),
|
||||
inputs: [scalar, scalar, scalar, scalar],
|
||||
})),
|
||||
def_use: [],
|
||||
effects: [],
|
||||
parameters: [{ ordinal: 0, name: "value", data_type: "/int" }],
|
||||
parameter_uses: Array.from({ length: parameterUseCount }, (_, index) => ({
|
||||
ordinal: 0,
|
||||
use: `operation-${String(index % operationCount)}`,
|
||||
input_index: Math.floor(index / operationCount),
|
||||
})),
|
||||
truncated: false,
|
||||
omitted_operations_lower_bound: 0,
|
||||
known_omitted_inputs: 0,
|
||||
known_omitted_edges: 0,
|
||||
limitations: [],
|
||||
};
|
||||
const largeDossier = {
|
||||
...base,
|
||||
native_value_flow: largeFlow,
|
||||
};
|
||||
const largeAnalysis: AnalysisOperationPort = {
|
||||
execute: async () =>
|
||||
ok(createAnalysisExecution(largeDossier, provider, { subject })),
|
||||
};
|
||||
|
||||
const includedParameter = await traceNativeValues(largeAnalysis, {
|
||||
procedure: procedureAddress,
|
||||
max_nodes: 20_000,
|
||||
max_edges: 40_000,
|
||||
offset: operationCount,
|
||||
limit: 1,
|
||||
});
|
||||
if (!includedParameter.ok) throw includedParameter.error;
|
||||
const includedResult = nativeValueTraceSchema.parse(
|
||||
includedParameter.value,
|
||||
);
|
||||
expect(includedResult).toMatchObject({
|
||||
total_nodes: operationCount + 1,
|
||||
total_edges: parameterUseCount,
|
||||
nodes: [
|
||||
expect.objectContaining({
|
||||
id: `${procedureAddress}/parameter:0`,
|
||||
kind: "parameter",
|
||||
}),
|
||||
],
|
||||
truncated: true,
|
||||
});
|
||||
expect(includedResult.edges).toHaveLength(parameterUseCount);
|
||||
|
||||
const omittedParameter = await traceNativeValues(largeAnalysis, {
|
||||
procedure: procedureAddress,
|
||||
max_nodes: operationCount,
|
||||
max_edges: 40_000,
|
||||
offset: operationCount - 1,
|
||||
limit: 1,
|
||||
});
|
||||
if (!omittedParameter.ok) throw omittedParameter.error;
|
||||
const omittedResult = nativeValueTraceSchema.parse(omittedParameter.value);
|
||||
expect(omittedResult).toMatchObject({
|
||||
total_nodes: operationCount,
|
||||
total_edges: 0,
|
||||
nodes: [expect.objectContaining({ kind: "operation" })],
|
||||
edges: [],
|
||||
truncated: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves operation IDs that collide with generated parameter IDs", async () => {
|
||||
const base = functionDossierSchema.parse(ghidraFunctionDossier());
|
||||
const address = base.procedure.address;
|
||||
const execution = createAnalysisExecution(
|
||||
{
|
||||
...base,
|
||||
native_value_flow: {
|
||||
available: true,
|
||||
provenance: "ghidra-high-pcode",
|
||||
operations: [
|
||||
node("parameter:0", address, "COPY"),
|
||||
node("use", address, "COPY"),
|
||||
],
|
||||
def_use: [],
|
||||
effects: [],
|
||||
parameters: [{ ordinal: 0, name: "value", data_type: "/int" }],
|
||||
parameter_uses: [{ ordinal: 0, use: "use", input_index: 1 }],
|
||||
truncated: false,
|
||||
omitted_operations_lower_bound: 0,
|
||||
known_omitted_inputs: 0,
|
||||
known_omitted_edges: 0,
|
||||
limitations: [],
|
||||
},
|
||||
},
|
||||
provider,
|
||||
{ subject },
|
||||
);
|
||||
const result = await traceNativeValues(
|
||||
{ execute: async () => ok(execution) },
|
||||
{ procedure: address, max_nodes: 2 },
|
||||
);
|
||||
|
||||
if (!result.ok) throw result.error;
|
||||
const parsedResult = nativeValueTraceSchema.parse(result.value);
|
||||
expect(parsedResult).toMatchObject({
|
||||
total_nodes: 2,
|
||||
total_edges: 1,
|
||||
edges: [
|
||||
{
|
||||
source: `${address}/parameter:0`,
|
||||
target: `${address}/use`,
|
||||
kind: "parameter-use",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("bounded native value dependency composition", () => {
|
||||
it("reports work truncation and stable pagination without traversing ambiguous calls", async () => {
|
||||
const bounded = await traceNativeValues(analysis, {
|
||||
|
||||
@@ -21,6 +21,7 @@ export const traceNativeValues = async (
|
||||
): EnhancedResult => {
|
||||
const input = nativeValueTraceInputSchema.parse(parameters);
|
||||
const nodes: typeof nativeValueTraceSchema._output.nodes = [];
|
||||
const nodeIds = new Set<string>();
|
||||
const edges: typeof nativeValueTraceSchema._output.edges = [];
|
||||
const procedures: typeof nativeValueTraceSchema._output.procedures = [];
|
||||
const unknowns: typeof nativeValueTraceSchema._output.unknowns = [];
|
||||
@@ -154,25 +155,28 @@ export const traceNativeValues = async (
|
||||
bytes += size;
|
||||
ids.add(operation.id);
|
||||
nodes.push(node);
|
||||
nodeIds.add(node.id);
|
||||
}
|
||||
for (const parameter of flow.parameters) {
|
||||
if (nodes.length >= input.max_nodes) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
const id = `${address}/parameter:${parameter.ordinal}`;
|
||||
nodes.push({
|
||||
id: `${address}/parameter:${parameter.ordinal}`,
|
||||
id,
|
||||
procedure: address,
|
||||
evidence_id: evidenceId,
|
||||
kind: "parameter",
|
||||
operation: null,
|
||||
parameter,
|
||||
});
|
||||
nodeIds.add(id);
|
||||
}
|
||||
for (const use of flow.parameter_uses) {
|
||||
if (
|
||||
!ids.has(use.use) ||
|
||||
!nodes.some((node) => node.id === `${address}/parameter:${use.ordinal}`)
|
||||
!nodeIds.has(`${address}/parameter:${use.ordinal}`)
|
||||
)
|
||||
continue;
|
||||
if (edges.length >= input.max_edges) {
|
||||
|
||||
@@ -33,10 +33,12 @@ import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { interfaceBuilderLimitsSchema } from "../domain/apple/interfaceBuilderGraph.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { artifactCapabilities } from "./ArtifactProviderMetadata.js";
|
||||
import { ARTIFACT_GRAPH_PROVIDER } from "../application/InvestigationProviders.js";
|
||||
import { createEvidence } from "../domain/evidence.js";
|
||||
import { createArtifactInspection } from "../domain/artifactInspection.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
|
||||
/** Read-only inventory and exclusively owned extraction provider. */
|
||||
export class ArtifactProvider implements AnalysisProvider {
|
||||
@@ -67,6 +69,8 @@ export class ArtifactProvider implements AnalysisProvider {
|
||||
}
|
||||
|
||||
class ArtifactClient implements AnalysisClient {
|
||||
readonly #resourceScope = new ArtifactResourceScope();
|
||||
|
||||
constructor(
|
||||
private readonly target: BinaryTarget,
|
||||
private readonly environment: Readonly<NodeJS.ProcessEnv>,
|
||||
@@ -184,6 +188,7 @@ class ArtifactClient implements AnalysisClient {
|
||||
outputRoot: parsed.output_root,
|
||||
environment: this.environment,
|
||||
integrityPolicy: parsed.integrity_policy,
|
||||
resourceScope: this.#resourceScope,
|
||||
},
|
||||
options?.signal,
|
||||
);
|
||||
@@ -220,7 +225,25 @@ class ArtifactClient implements AnalysisClient {
|
||||
}
|
||||
|
||||
close(): Promise<Result<null, AnalysisError>> {
|
||||
return Promise.resolve(ok(null));
|
||||
return this.#resourceScope.close().then(
|
||||
() => ok(null),
|
||||
(cause: unknown) => {
|
||||
const failure =
|
||||
cause instanceof ArtifactReaderFailure ? cause : undefined;
|
||||
return err(
|
||||
new ProviderCleanupError(
|
||||
ARTIFACT_GRAPH_PROVIDER.id,
|
||||
failure?.cleanup?.resources ?? ["artifact resources"],
|
||||
{
|
||||
reason:
|
||||
failure?.cleanup?.reason ??
|
||||
(cause instanceof Error ? cause.message : String(cause)),
|
||||
},
|
||||
{ cause, operation: "close_binary" },
|
||||
),
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** The active target's kind is outside this operation's supported targets. */
|
||||
@@ -353,6 +376,7 @@ class ArtifactClient implements AnalysisClient {
|
||||
options?: ExecutionOptions,
|
||||
) {
|
||||
return inventoryArtifact(this.target.sourcePath ?? this.target.path, {
|
||||
resourceScope: this.#resourceScope,
|
||||
environment: this.environment,
|
||||
...(options?.signal === undefined ? {} : { signal: options.signal }),
|
||||
integrity: { mode: parsed.integrity_policy },
|
||||
|
||||
@@ -31,6 +31,37 @@ export interface ArtifactEntry {
|
||||
};
|
||||
}
|
||||
|
||||
/** Snapshot mutable entry metadata before the producer later validates open(). */
|
||||
export const copyArtifactEntry = (entry: ArtifactEntry): ArtifactEntry => ({
|
||||
...entry,
|
||||
limitations: [...entry.limitations],
|
||||
...(entry.sourceIdentity === undefined
|
||||
? {}
|
||||
: { sourceIdentity: { ...entry.sourceIdentity } }),
|
||||
});
|
||||
|
||||
/** Compare every portable and adapter-owned fact supplied with an entry. */
|
||||
export const sameArtifactEntry = (
|
||||
entry: ArtifactEntry,
|
||||
produced: ArtifactEntry,
|
||||
): boolean =>
|
||||
entry.path === produced.path &&
|
||||
entry.kind === produced.kind &&
|
||||
entry.declaredSize === produced.declaredSize &&
|
||||
entry.compressedSize === produced.compressedSize &&
|
||||
entry.executable === produced.executable &&
|
||||
entry.encrypted === produced.encrypted &&
|
||||
entry.byteOffset === produced.byteOffset &&
|
||||
entry.declaredSha256 === produced.declaredSha256 &&
|
||||
entry.unpacked === produced.unpacked &&
|
||||
entry.adapterKey === produced.adapterKey &&
|
||||
entry.limitations.length === produced.limitations.length &&
|
||||
entry.limitations.every(
|
||||
(limitation, index) => limitation === produced.limitations[index],
|
||||
) &&
|
||||
entry.sourceIdentity?.device === produced.sourceIdentity?.device &&
|
||||
entry.sourceIdentity?.inode === produced.sourceIdentity?.inode;
|
||||
|
||||
/** Read-only adapter over one directory, archive, or virtual container. */
|
||||
export interface ArtifactReader {
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "file";
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import type { ArtifactReader } from "./ArtifactReader.js";
|
||||
import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
|
||||
const deferred = <T>() => {
|
||||
let resolve: (value: T) => void = () => {};
|
||||
const promise = new Promise<T>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
};
|
||||
|
||||
it("waits for admitted operations and shares concurrent close", async () => {
|
||||
const scope = new ArtifactResourceScope();
|
||||
const started = deferred<void>();
|
||||
const finish = deferred<void>();
|
||||
const operation = scope.run(async () => {
|
||||
started.resolve();
|
||||
await finish.promise;
|
||||
});
|
||||
await started.promise;
|
||||
|
||||
const close = scope.close();
|
||||
expect(scope.close()).toBe(close);
|
||||
await expect(scope.run(async () => undefined)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
message: "Artifact resource scope is closed",
|
||||
});
|
||||
let closed = false;
|
||||
void close.then(() => {
|
||||
closed = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(closed).toBe(false);
|
||||
|
||||
finish.resolve();
|
||||
await operation;
|
||||
await close;
|
||||
expect(closed).toBe(true);
|
||||
});
|
||||
|
||||
it("retains the same failed reader owner and retries before admitting work", async () => {
|
||||
const scope = new ArtifactResourceScope();
|
||||
let closeAttempts = 0;
|
||||
const reader = {
|
||||
async close() {
|
||||
closeAttempts += 1;
|
||||
if (closeAttempts < 4) throw new Error(`close attempt ${closeAttempts}`);
|
||||
},
|
||||
} as unknown as ArtifactReader;
|
||||
const owner = { kind: "reader" as const, resource: "fixture reader", reader };
|
||||
|
||||
expect(await scope.release(owner)).toMatchObject({ kind: "failed" });
|
||||
expect(closeAttempts).toBe(1);
|
||||
|
||||
await expect(scope.run(async () => "must not run")).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
message: "Artifact cleanup remains incomplete: fixture reader",
|
||||
});
|
||||
expect(closeAttempts).toBe(2);
|
||||
await expect(scope.close()).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
expect(closeAttempts).toBe(3);
|
||||
await scope.close();
|
||||
expect(closeAttempts).toBe(4);
|
||||
});
|
||||
@@ -0,0 +1,147 @@
|
||||
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import type { OwnedDirectoryHandle } from "../filesystem/OwnedDirectoryHandle.js";
|
||||
|
||||
import type { ArtifactReader } from "./ArtifactReader.js";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import type { SafeOutputTree } from "./SafeOutputTree.js";
|
||||
|
||||
/** One artifact resource and its format-specific cleanup owner. */
|
||||
export type ArtifactResourceOwner =
|
||||
| {
|
||||
readonly kind: "directory-handle";
|
||||
readonly resource: string;
|
||||
readonly handle: OwnedDirectoryHandle;
|
||||
}
|
||||
| {
|
||||
readonly kind: "reader";
|
||||
readonly resource: string;
|
||||
readonly reader: ArtifactReader;
|
||||
}
|
||||
| {
|
||||
readonly kind: "file-handle";
|
||||
readonly resource: string;
|
||||
readonly handle: OwnedFileHandle;
|
||||
}
|
||||
| {
|
||||
readonly kind: "output-tree";
|
||||
readonly resource: string;
|
||||
readonly tree: SafeOutputTree;
|
||||
};
|
||||
|
||||
export type ArtifactCleanupAttempt =
|
||||
| { readonly kind: "released" }
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
|
||||
/** Caller-owned scope that retains failed artifact cleanup for a later close. */
|
||||
export class ArtifactResourceScope {
|
||||
readonly #pending = new Set<ArtifactResourceOwner>();
|
||||
readonly #active = new Set<Promise<unknown>>();
|
||||
#draining: Promise<void> | undefined;
|
||||
#closed = false;
|
||||
|
||||
/** Admit a resource operation while the scope remains open. */
|
||||
async run<T>(operation: () => Promise<T>): Promise<T> {
|
||||
while (true) {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Artifact resource scope is closed",
|
||||
);
|
||||
if (this.#draining !== undefined) await this.#draining;
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Artifact resource scope is closed",
|
||||
);
|
||||
if (this.#pending.size > 0) {
|
||||
await this.#retryCleanup();
|
||||
continue;
|
||||
}
|
||||
const running = Promise.resolve().then(operation);
|
||||
this.#active.add(running);
|
||||
try {
|
||||
return await running;
|
||||
} finally {
|
||||
this.#active.delete(running);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Attempt cleanup now and retain the same owner if it fails. */
|
||||
async release(owner: ArtifactResourceOwner): Promise<ArtifactCleanupAttempt> {
|
||||
try {
|
||||
await cleanup(owner);
|
||||
this.#pending.delete(owner);
|
||||
return { kind: "released" };
|
||||
} catch (cause: unknown) {
|
||||
this.#pending.add(owner);
|
||||
return { kind: "failed", cause };
|
||||
}
|
||||
}
|
||||
|
||||
/** Retry retained owners without closing resource admission. */
|
||||
#retryCleanup(): Promise<void> {
|
||||
if (this.#draining !== undefined) return this.#draining;
|
||||
this.#draining = this.#drainAndRetry().finally(() => {
|
||||
this.#draining = undefined;
|
||||
});
|
||||
return this.#draining;
|
||||
}
|
||||
|
||||
/** Stop admission, wait for active operations, and retry every retained owner. */
|
||||
close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
return this.#retryCleanup();
|
||||
}
|
||||
|
||||
async #drainAndRetry(): Promise<void> {
|
||||
while (this.#active.size > 0) await Promise.allSettled(this.#active);
|
||||
const failures: {
|
||||
readonly owner: ArtifactResourceOwner;
|
||||
readonly cause: unknown;
|
||||
}[] = [];
|
||||
for (const owner of this.#pending) {
|
||||
try {
|
||||
await cleanup(owner);
|
||||
this.#pending.delete(owner);
|
||||
} catch (cause: unknown) {
|
||||
failures.push({ owner, cause });
|
||||
}
|
||||
}
|
||||
if (failures.length === 0) return;
|
||||
const observations = failures.map(({ owner, cause }) =>
|
||||
ArtifactReaderFailure.cleanupObservation(cause, owner.resource),
|
||||
);
|
||||
const resources = [
|
||||
...new Set(observations.flatMap(({ resources }) => resources)),
|
||||
];
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`Artifact cleanup remains incomplete: ${resources.join(", ")}`,
|
||||
{
|
||||
cause: new AggregateError(
|
||||
failures.map(({ cause }) => cause),
|
||||
"One or more artifact resources remain owned",
|
||||
),
|
||||
cleanup: {
|
||||
reason: observations.map(({ reason }) => reason).join("; "),
|
||||
resources,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const cleanup = async (owner: ArtifactResourceOwner): Promise<void> => {
|
||||
switch (owner.kind) {
|
||||
case "reader":
|
||||
await owner.reader.close();
|
||||
return;
|
||||
case "file-handle":
|
||||
case "directory-handle":
|
||||
await owner.handle.close();
|
||||
return;
|
||||
case "output-tree":
|
||||
await owner.tree.rollback();
|
||||
}
|
||||
};
|
||||
@@ -19,15 +19,19 @@ import { isAbsolute, join, relative, sep } from "node:path";
|
||||
import { Readable, Transform } from "node:stream";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
|
||||
import { getRawHeader, listPackage, statFile, uncache } from "@electron/asar";
|
||||
import { listPackage, statFile, uncache } from "@electron/asar";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js";
|
||||
import { admitAsarHeader } from "./AsarHeader.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
|
||||
/**
|
||||
* Official Electron ASAR adapter with range-streamed member reads.
|
||||
@@ -41,6 +45,13 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
#archiveSize: number | undefined;
|
||||
#headerSize: number | undefined;
|
||||
readonly #entries = new Map<string, AsarEntryState>();
|
||||
readonly #unpackedHandles = new Map<
|
||||
UnpackedFileHandle,
|
||||
{
|
||||
readonly path: string;
|
||||
readonly owner: OwnedFileHandle<UnpackedFileHandle>;
|
||||
}
|
||||
>();
|
||||
#snapshot: { readonly path: string; readonly sha256: string } | undefined;
|
||||
#snapshotRoot: string | undefined;
|
||||
|
||||
@@ -65,6 +76,7 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
try {
|
||||
const admitted = await admitAsarHeader(headerHandle);
|
||||
this.#archiveSize = admitted.archiveBytes;
|
||||
this.#headerSize = admitted.headerBytes;
|
||||
} finally {
|
||||
await headerHandle.close();
|
||||
}
|
||||
@@ -72,7 +84,6 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
paths = listPackage(snapshotPath, { isPack: false }).sort((left, right) =>
|
||||
left.localeCompare(right, "en"),
|
||||
);
|
||||
this.#headerSize = getRawHeader(snapshotPath).headerSize;
|
||||
const archiveMetadata = await lstat(snapshotPath);
|
||||
if (!archiveMetadata.isFile() || archiveMetadata.isSymbolicLink())
|
||||
throw new ArtifactReaderFailure(
|
||||
@@ -156,32 +167,8 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
);
|
||||
const { metadata } = state;
|
||||
const producedEntry = state.entry;
|
||||
if (producedEntry.unpacked) {
|
||||
const handle = await this.#openUnpackedEntry(producedEntry);
|
||||
try {
|
||||
const observed = await handle.stat();
|
||||
abortIfNeeded(signal);
|
||||
if (!observed.isFile())
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`ASAR unpacked entry is not a regular file: ${producedEntry.path}`,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
await handle.close().catch(() => undefined);
|
||||
throw asarFailure(this.path, `read ${producedEntry.path}`, cause);
|
||||
}
|
||||
const source = handle.createReadStream({
|
||||
start: 0,
|
||||
autoClose: true,
|
||||
});
|
||||
return readValidatedAsarEntry(
|
||||
source,
|
||||
undefined,
|
||||
producedEntry.path,
|
||||
this.path,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
if (producedEntry.unpacked)
|
||||
return this.#openUnpackedStream(producedEntry, signal);
|
||||
const archiveSize = this.#archiveSize;
|
||||
const headerSize = this.#headerSize;
|
||||
const offset =
|
||||
@@ -315,22 +302,39 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
let cleanupFailure: ArtifactReaderFailure | undefined;
|
||||
for (const [handle, { path }] of this.#unpackedHandles) {
|
||||
try {
|
||||
await this.#closeUnpackedHandle(handle);
|
||||
} catch (cause: unknown) {
|
||||
cleanupFailure = ArtifactReaderFailure.withCleanup(
|
||||
cleanupFailure ?? cause,
|
||||
ArtifactReaderFailure.cleanupObservation(cause, path),
|
||||
);
|
||||
}
|
||||
}
|
||||
const root = this.#snapshotRoot;
|
||||
if (root !== undefined) {
|
||||
try {
|
||||
await this.removeSnapshot(root);
|
||||
if (this.#snapshotRoot === root) this.#snapshotRoot = undefined;
|
||||
} catch (cause: unknown) {
|
||||
throw ArtifactReaderFailure.withCleanup(cause, {
|
||||
reason: `Could not remove ASAR snapshot: ${root}`,
|
||||
resources: [root],
|
||||
});
|
||||
cleanupFailure = ArtifactReaderFailure.withCleanup(
|
||||
cleanupFailure ?? cause,
|
||||
{
|
||||
reason: `Could not remove ASAR snapshot: ${root}`,
|
||||
resources: [root],
|
||||
},
|
||||
);
|
||||
}
|
||||
this.#snapshotRoot = undefined;
|
||||
}
|
||||
if (this.#snapshot !== undefined) uncache(this.#snapshot.path);
|
||||
this.#snapshot = undefined;
|
||||
this.#resetArchiveState();
|
||||
this.#entries.clear();
|
||||
if (root === undefined || this.#snapshotRoot === undefined) {
|
||||
if (this.#snapshot !== undefined) uncache(this.#snapshot.path);
|
||||
this.#snapshot = undefined;
|
||||
this.#resetArchiveState();
|
||||
this.#entries.clear();
|
||||
}
|
||||
if (cleanupFailure !== undefined) throw cleanupFailure;
|
||||
}
|
||||
|
||||
#snapshotPath(): string {
|
||||
@@ -346,8 +350,12 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
return [];
|
||||
}
|
||||
|
||||
async #openUnpackedEntry(entry: ArtifactEntry): Promise<UnpackedFileHandle> {
|
||||
async #openUnpackedStream(
|
||||
entry: ArtifactEntry,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Readable> {
|
||||
const unpackedRoot = `${this.path}.unpacked`;
|
||||
let handle: UnpackedFileHandle | undefined;
|
||||
try {
|
||||
const rootMetadata = await lstat(unpackedRoot);
|
||||
if (!rootMetadata.isDirectory() || rootMetadata.isSymbolicLink())
|
||||
@@ -374,22 +382,21 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
"path",
|
||||
`ASAR unpacked entry is not a regular file: ${entry.path}`,
|
||||
);
|
||||
const handle = await this.openUnpackedFile(
|
||||
handle = await this.openUnpackedFile(
|
||||
canonical,
|
||||
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
|
||||
);
|
||||
const openedMetadata = await handle
|
||||
.stat()
|
||||
.catch(async (cause: unknown) => {
|
||||
await handle.close().catch(() => undefined);
|
||||
throw cause;
|
||||
});
|
||||
const openedHandle = handle;
|
||||
this.#unpackedHandles.set(openedHandle, {
|
||||
path: canonical,
|
||||
owner: new OwnedFileHandle(openedHandle),
|
||||
});
|
||||
const openedMetadata = await openedHandle.stat();
|
||||
if (
|
||||
!openedMetadata.isFile() ||
|
||||
openedMetadata.dev !== pathMetadata.dev ||
|
||||
openedMetadata.ino !== pathMetadata.ino
|
||||
) {
|
||||
await handle.close();
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`ASAR unpacked entry changed before read: ${entry.path}`,
|
||||
@@ -402,25 +409,46 @@ export class AsarArtifactReader implements ArtifactReader {
|
||||
},
|
||||
);
|
||||
}
|
||||
return handle;
|
||||
abortIfNeeded(signal);
|
||||
const source = readFileHandleChunks(openedHandle, {
|
||||
start: 0,
|
||||
});
|
||||
source.once("close", () => {
|
||||
void this.#closeUnpackedHandle(openedHandle).catch(() => undefined);
|
||||
});
|
||||
return readValidatedAsarEntry(
|
||||
source,
|
||||
undefined,
|
||||
entry.path,
|
||||
this.path,
|
||||
signal,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
if (entry.unpacked && isMissingFile(cause))
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`ASAR unpacked entry bytes are unavailable: ${entry.path}`,
|
||||
{ cause },
|
||||
{
|
||||
logicalPath: entry.path,
|
||||
declaredSha256: entry.declaredSha256,
|
||||
calculatedSha256: null,
|
||||
unpacked: true,
|
||||
},
|
||||
);
|
||||
throw asarFailure(this.path, `read ${entry.path}`, cause);
|
||||
let failure = asarUnpackedEntryFailure(this.path, entry, cause);
|
||||
if (handle !== undefined && this.#unpackedHandles.has(handle)) {
|
||||
try {
|
||||
await this.#closeUnpackedHandle(handle);
|
||||
} catch (cleanupCause: unknown) {
|
||||
failure = ArtifactReaderFailure.withCleanup(
|
||||
failure,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupCause,
|
||||
this.#unpackedHandles.get(handle)?.path ?? entry.path,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
async #closeUnpackedHandle(handle: UnpackedFileHandle): Promise<void> {
|
||||
const owned = this.#unpackedHandles.get(handle);
|
||||
if (owned === undefined) return;
|
||||
await owned.owner.close();
|
||||
this.#unpackedHandles.delete(handle);
|
||||
}
|
||||
|
||||
#resetArchiveState(): void {
|
||||
this.#archiveSize = undefined;
|
||||
this.#headerSize = undefined;
|
||||
@@ -433,43 +461,36 @@ type AsarEntryState = {
|
||||
readonly entry: ArtifactEntry;
|
||||
};
|
||||
type UnpackedFileHandle = {
|
||||
readonly fd: number;
|
||||
stat(): Promise<Stats>;
|
||||
close(): Promise<void>;
|
||||
createReadStream: FileHandle["createReadStream"];
|
||||
read: FileHandle["read"];
|
||||
};
|
||||
type OpenAsarUnpackedFile = (
|
||||
path: string,
|
||||
flags: number,
|
||||
) => Promise<UnpackedFileHandle>;
|
||||
|
||||
const copyArtifactEntry = (entry: ArtifactEntry): ArtifactEntry => ({
|
||||
...entry,
|
||||
limitations: [...entry.limitations],
|
||||
...(entry.sourceIdentity === undefined
|
||||
? {}
|
||||
: { sourceIdentity: { ...entry.sourceIdentity } }),
|
||||
});
|
||||
|
||||
const sameArtifactEntry = (
|
||||
const asarUnpackedEntryFailure = (
|
||||
archivePath: string,
|
||||
entry: ArtifactEntry,
|
||||
produced: ArtifactEntry,
|
||||
): boolean =>
|
||||
entry.path === produced.path &&
|
||||
entry.kind === produced.kind &&
|
||||
entry.declaredSize === produced.declaredSize &&
|
||||
entry.compressedSize === produced.compressedSize &&
|
||||
entry.executable === produced.executable &&
|
||||
entry.encrypted === produced.encrypted &&
|
||||
entry.byteOffset === produced.byteOffset &&
|
||||
entry.declaredSha256 === produced.declaredSha256 &&
|
||||
entry.unpacked === produced.unpacked &&
|
||||
entry.adapterKey === produced.adapterKey &&
|
||||
entry.limitations.length === produced.limitations.length &&
|
||||
entry.limitations.every(
|
||||
(limitation, index) => limitation === produced.limitations[index],
|
||||
) &&
|
||||
entry.sourceIdentity?.device === produced.sourceIdentity?.device &&
|
||||
entry.sourceIdentity?.inode === produced.sourceIdentity?.inode;
|
||||
cause: unknown,
|
||||
): ArtifactReaderFailure => {
|
||||
if (cause instanceof ArtifactReaderFailure) return cause;
|
||||
if (entry.unpacked && isMissingFile(cause))
|
||||
return new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`ASAR unpacked entry bytes are unavailable: ${entry.path}`,
|
||||
{ cause },
|
||||
{
|
||||
logicalPath: entry.path,
|
||||
declaredSha256: entry.declaredSha256,
|
||||
calculatedSha256: null,
|
||||
unpacked: true,
|
||||
},
|
||||
);
|
||||
return asarFailure(archivePath, `read ${entry.path}`, cause);
|
||||
};
|
||||
|
||||
const isAsarFileMetadata = (
|
||||
metadata: ReturnType<typeof statFile>,
|
||||
|
||||
@@ -1,23 +1,33 @@
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, opendir, realpath } from "node:fs/promises";
|
||||
import { constants, type Stats } from "node:fs";
|
||||
import {
|
||||
lstat,
|
||||
open,
|
||||
opendir,
|
||||
realpath,
|
||||
type FileHandle,
|
||||
} from "node:fs/promises";
|
||||
import { isAbsolute, join, relative, sep } from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import { Readable } from "node:stream";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
import { OwnedDirectoryHandle } from "../filesystem/OwnedDirectoryHandle.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
|
||||
/** Non-symlink-following directory reader. */
|
||||
export class DirectoryArtifactReader implements ArtifactReader {
|
||||
readonly format = "directory" as const;
|
||||
readonly #rootPromise: Promise<string>;
|
||||
#rootPromise: Promise<string> | undefined;
|
||||
readonly #rootPath: string;
|
||||
readonly #directories = new Set<OwnedDirectoryHandle>();
|
||||
readonly #files = new Map<OwnedFileHandle, string>();
|
||||
|
||||
constructor(root: string) {
|
||||
this.#rootPath = root;
|
||||
this.#rootPromise = realpath(root);
|
||||
}
|
||||
|
||||
/** Enumerate entries, optionally restricting descent while retaining directories. */
|
||||
@@ -25,9 +35,7 @@ export class DirectoryArtifactReader implements ArtifactReader {
|
||||
signal?: AbortSignal,
|
||||
descendInto?: (path: string) => boolean,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
const root = await this.#rootPromise.catch((cause: unknown) => {
|
||||
throw directoryIoFailure("resolve artifact root", this.#rootPath, cause);
|
||||
});
|
||||
const root = await this.#resolveRoot();
|
||||
const pending = [root];
|
||||
while (pending.length > 0) {
|
||||
abortIfNeeded(signal);
|
||||
@@ -36,62 +44,100 @@ export class DirectoryArtifactReader implements ArtifactReader {
|
||||
await withDirectoryIoContext("inspect directory", directory, () =>
|
||||
assertContainedDirectory(root, directory),
|
||||
);
|
||||
const handle = await opendir(directory).catch((cause: unknown) => {
|
||||
throw directoryIoFailure("open directory", directory, cause);
|
||||
});
|
||||
const directories: string[] = [];
|
||||
try {
|
||||
for await (const child of handle) {
|
||||
abortIfNeeded(signal);
|
||||
const absolute = join(directory, child.name);
|
||||
const metadata = await lstat(absolute).catch((cause: unknown) => {
|
||||
throw directoryIoFailure("inspect entry", absolute, cause);
|
||||
});
|
||||
const path = relative(root, absolute).split(sep).join("/");
|
||||
const kind = metadata.isSymbolicLink()
|
||||
? "symlink"
|
||||
: metadata.isDirectory()
|
||||
? "directory"
|
||||
: metadata.isFile()
|
||||
? "file"
|
||||
: undefined;
|
||||
if (kind === undefined) continue;
|
||||
yield {
|
||||
path,
|
||||
kind,
|
||||
declaredSize: kind === "file" ? metadata.size : null,
|
||||
compressedSize: null,
|
||||
executable: (metadata.mode & 0o111) !== 0,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations:
|
||||
kind === "symlink"
|
||||
? ["Symlink target was not followed or disclosed."]
|
||||
: [],
|
||||
adapterKey: absolute,
|
||||
...(kind === "file"
|
||||
? {
|
||||
sourceIdentity: { device: metadata.dev, inode: metadata.ino },
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
if (kind === "directory" && (descendInto?.(path) ?? true))
|
||||
directories.push(absolute);
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
throw directoryIoFailure("read directory", directory, cause);
|
||||
} finally {
|
||||
// best-effort cleanup: directory-handle close must not mask traversal.
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
for await (const entry of this.#readDirectory(
|
||||
directory,
|
||||
pending,
|
||||
signal,
|
||||
descendInto,
|
||||
))
|
||||
yield entry;
|
||||
await withDirectoryIoContext("inspect directory", directory, () =>
|
||||
assertContainedDirectory(root, directory),
|
||||
);
|
||||
directories.reverse();
|
||||
pending.push(...directories);
|
||||
}
|
||||
}
|
||||
|
||||
async #resolveRoot(): Promise<string> {
|
||||
return (this.#rootPromise ??= realpath(this.#rootPath)).catch(
|
||||
(cause: unknown) => {
|
||||
throw directoryIoFailure(
|
||||
"resolve artifact root",
|
||||
this.#rootPath,
|
||||
cause,
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async *#readDirectory(
|
||||
directory: string,
|
||||
pending: string[],
|
||||
signal?: AbortSignal,
|
||||
descendInto?: (path: string) => boolean,
|
||||
): AsyncIterable<ArtifactEntry> {
|
||||
const root = await this.#resolveRoot();
|
||||
const handle = new OwnedDirectoryHandle(
|
||||
await opendir(directory).catch((cause: unknown) => {
|
||||
throw directoryIoFailure("open directory", directory, cause);
|
||||
}),
|
||||
);
|
||||
this.#directories.add(handle);
|
||||
const childDirectories: string[] = [];
|
||||
let traversal:
|
||||
| { readonly kind: "complete" }
|
||||
| { readonly kind: "failed"; readonly cause: unknown } = {
|
||||
kind: "complete",
|
||||
};
|
||||
try {
|
||||
for (;;) {
|
||||
const child = await handle.handle.read();
|
||||
if (child === null) break;
|
||||
abortIfNeeded(signal);
|
||||
const absolute = join(directory, child.name);
|
||||
const metadata = await lstat(absolute).catch((cause: unknown) => {
|
||||
throw directoryIoFailure("inspect entry", absolute, cause);
|
||||
});
|
||||
const path = relative(root, absolute).split(sep).join("/");
|
||||
const entry = artifactEntry(path, absolute, metadata);
|
||||
if (entry === undefined) continue;
|
||||
yield entry;
|
||||
if (entry.kind === "directory" && (descendInto?.(path) ?? true))
|
||||
childDirectories.push(absolute);
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
traversal = {
|
||||
kind: "failed",
|
||||
cause:
|
||||
cause instanceof ArtifactReaderFailure
|
||||
? cause
|
||||
: directoryIoFailure("read directory", directory, cause),
|
||||
};
|
||||
} finally {
|
||||
await this.#finishDirectory(handle, directory, traversal);
|
||||
}
|
||||
if (traversal.kind === "failed") throw traversal.cause;
|
||||
childDirectories.reverse();
|
||||
pending.push(...childDirectories);
|
||||
}
|
||||
|
||||
async #finishDirectory(
|
||||
handle: OwnedDirectoryHandle,
|
||||
path: string,
|
||||
traversal:
|
||||
| { readonly kind: "complete" }
|
||||
| { readonly kind: "failed"; readonly cause: unknown },
|
||||
): Promise<void> {
|
||||
try {
|
||||
await handle.close();
|
||||
this.#directories.delete(handle);
|
||||
} catch (cause: unknown) {
|
||||
const closeFailure = directoryIoFailure("close directory", path, cause);
|
||||
if (traversal.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
traversal.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(closeFailure, path),
|
||||
);
|
||||
throw closeFailure;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -102,14 +148,16 @@ export class DirectoryArtifactReader implements ArtifactReader {
|
||||
"format",
|
||||
"Only regular files can be opened",
|
||||
);
|
||||
const handle = await open(
|
||||
const fileHandle = await open(
|
||||
entry.adapterKey,
|
||||
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
|
||||
).catch((cause: unknown) => {
|
||||
throw directoryIoFailure("open entry", entry.adapterKey, cause);
|
||||
});
|
||||
const owner = new OwnedFileHandle(fileHandle);
|
||||
this.#files.set(owner, entry.adapterKey);
|
||||
try {
|
||||
const observed = await handle.stat().catch((cause: unknown) => {
|
||||
const observed = await owner.handle.stat().catch((cause: unknown) => {
|
||||
throw directoryIoFailure(
|
||||
"inspect opened entry",
|
||||
entry.adapterKey,
|
||||
@@ -127,16 +175,76 @@ export class DirectoryArtifactReader implements ArtifactReader {
|
||||
`Directory entry changed before open: ${entry.path}`,
|
||||
);
|
||||
}
|
||||
return handle.createReadStream({ autoClose: true });
|
||||
abortIfNeeded(signal);
|
||||
const source = Readable.from(
|
||||
mapFileReadCancellation(owner.handle, signal),
|
||||
{ objectMode: false },
|
||||
);
|
||||
source.once("close", () => {
|
||||
void this.#closeFile(owner).catch(() => undefined);
|
||||
});
|
||||
return source;
|
||||
} catch (cause: unknown) {
|
||||
// best-effort cleanup: file-handle close must not mask the open failure.
|
||||
await handle.close().catch(() => undefined);
|
||||
try {
|
||||
await this.#closeFile(owner);
|
||||
} catch (closeCause: unknown) {
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
directoryIoFailure(
|
||||
"close opened entry",
|
||||
entry.adapterKey,
|
||||
closeCause,
|
||||
),
|
||||
entry.adapterKey,
|
||||
),
|
||||
);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
return Promise.resolve();
|
||||
async close(): Promise<void> {
|
||||
const failures: unknown[] = [];
|
||||
for (const [owner, path] of this.#files) {
|
||||
try {
|
||||
await this.#closeFile(owner);
|
||||
} catch (cause: unknown) {
|
||||
failures.push(directoryIoFailure("close opened entry", path, cause));
|
||||
}
|
||||
}
|
||||
for (const owner of this.#directories) {
|
||||
try {
|
||||
await owner.close();
|
||||
this.#directories.delete(owner);
|
||||
} catch (cause: unknown) {
|
||||
failures.push(
|
||||
directoryIoFailure("close directory", owner.handle.path, cause),
|
||||
);
|
||||
}
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
const reason = failures.map(failureMessage).join("; ");
|
||||
throw new ArtifactReaderFailure(
|
||||
"io",
|
||||
`Directory artifact cleanup failed: ${reason}`,
|
||||
{
|
||||
cause: new AggregateError(failures),
|
||||
cleanup: {
|
||||
reason,
|
||||
resources: [
|
||||
...this.#files.values(),
|
||||
...[...this.#directories].map((owner) => owner.handle.path),
|
||||
],
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async #closeFile(owner: OwnedFileHandle<FileHandle>): Promise<void> {
|
||||
await owner.close();
|
||||
this.#files.delete(owner);
|
||||
}
|
||||
|
||||
provenance(): readonly [] {
|
||||
@@ -144,6 +252,42 @@ export class DirectoryArtifactReader implements ArtifactReader {
|
||||
}
|
||||
}
|
||||
|
||||
const artifactEntry = (
|
||||
path: string,
|
||||
absolute: string,
|
||||
metadata: Stats,
|
||||
): ArtifactEntry | undefined => {
|
||||
const kind = metadata.isSymbolicLink()
|
||||
? "symlink"
|
||||
: metadata.isDirectory()
|
||||
? "directory"
|
||||
: metadata.isFile()
|
||||
? "file"
|
||||
: undefined;
|
||||
if (kind === undefined) return undefined;
|
||||
return {
|
||||
path,
|
||||
kind,
|
||||
declaredSize: kind === "file" ? metadata.size : null,
|
||||
compressedSize: null,
|
||||
executable: (metadata.mode & 0o111) !== 0,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations:
|
||||
kind === "symlink"
|
||||
? ["Symlink target was not followed or disclosed."]
|
||||
: [],
|
||||
adapterKey: absolute,
|
||||
...(kind === "file"
|
||||
? {
|
||||
sourceIdentity: { device: metadata.dev, inode: metadata.ino },
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
|
||||
const directoryIoFailure = (
|
||||
operation: string,
|
||||
path: string,
|
||||
@@ -157,13 +301,38 @@ const directoryIoFailure = (
|
||||
typeof cause.code === "string"
|
||||
? ` (${cause.code})`
|
||||
: "";
|
||||
const detail = failureMessage(cause);
|
||||
return new ArtifactReaderFailure(
|
||||
"io",
|
||||
`Could not ${operation} at ${path}${code}`,
|
||||
`Could not ${operation} at ${path}${code}: ${detail}`,
|
||||
{ cause },
|
||||
);
|
||||
};
|
||||
|
||||
const failureMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
|
||||
const mapFileReadCancellation = async function* (
|
||||
handle: FileHandle,
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<Buffer> {
|
||||
try {
|
||||
for await (const chunk of readFileHandleChunks(handle, { signal })) {
|
||||
abortIfNeeded(signal);
|
||||
yield chunk;
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
} catch (cause: unknown) {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"Directory artifact read cancelled",
|
||||
{ cause },
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
const withDirectoryIoContext = async <T>(
|
||||
operation: string,
|
||||
path: string,
|
||||
|
||||
@@ -1,41 +1,84 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { describe, expect, it, onTestFinished } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { CPU, fatImage, machoImage } from "./apple/MachoImage.fixture.js";
|
||||
import { MachOSliceArtifactReader } from "./MachOSliceArtifactReader.js";
|
||||
import { ok } from "../domain/result.js";
|
||||
import { openRegularFile } from "../filesystem/RegularFile.js";
|
||||
import type { NativeCommandRunner } from "../native/CommandRunner.js";
|
||||
|
||||
/**
|
||||
* Read a 32-bit FAT table directly. The host's /bin/ls changes between macOS
|
||||
* releases, so the expected slice layout comes from the file itself.
|
||||
*/
|
||||
const fatSlices = async (
|
||||
path: string,
|
||||
): Promise<readonly (readonly [number, number])[]> => {
|
||||
const bytes = await readFile(path);
|
||||
expect(bytes.readUInt32BE(0)).toBe(0xca_fe_ba_be);
|
||||
return Array.from({ length: bytes.readUInt32BE(4) }, (_, index) => {
|
||||
const entry = 8 + index * 20;
|
||||
return [bytes.readUInt32BE(entry + 8), bytes.readUInt32BE(entry + 12)];
|
||||
describe("MachOSliceArtifactReader", () => {
|
||||
it("keeps a borrowed source usable after a cancelled slice read", async () => {
|
||||
const root = await createTestTempDirectory("rea-slices-cancelled-");
|
||||
const binary = join(root, "fat");
|
||||
const x86 = machoImage({ cpu: CPU.x86_64 });
|
||||
const arm = Buffer.concat([
|
||||
Buffer.from(machoImage({ cpu: CPU.arm64 })),
|
||||
Buffer.alloc(160_000, 0x5a),
|
||||
]);
|
||||
await writeFile(
|
||||
binary,
|
||||
fatImage([
|
||||
{ cpu: CPU.x86_64, bytes: x86 },
|
||||
{ cpu: CPU.arm64, bytes: arm },
|
||||
]),
|
||||
);
|
||||
const handle = await openRegularFile(binary, { symlinks: "reject" });
|
||||
const source = { handle, initial: await handle.stat() };
|
||||
const runner: NativeCommandRunner = {
|
||||
run: () =>
|
||||
Promise.resolve(
|
||||
ok({
|
||||
tool: "lipo",
|
||||
executable: "/usr/bin/lipo",
|
||||
executableSha256: "1".repeat(64),
|
||||
toolVersion: null,
|
||||
versionReason: "fixture",
|
||||
arguments: ["-detailed_info", binary],
|
||||
stdout: `architecture x86_64\n cputype 16777223\n cpusubtype 3\n offset 4096\n size ${String(x86.length)}\n align 2^12 (4096)\narchitecture arm64\n cputype 16777228\n cpusubtype 0\n offset 8192\n size ${String(arm.length)}\n align 2^12 (4096)\n`,
|
||||
stderr: "",
|
||||
stdoutBytes: 1,
|
||||
stderrBytes: 0,
|
||||
exitCode: 0,
|
||||
signal: null,
|
||||
}),
|
||||
),
|
||||
};
|
||||
const reader = new MachOSliceArtifactReader(binary, {}, runner, source);
|
||||
onTestFinished(async () => {
|
||||
await reader.close();
|
||||
await handle.close();
|
||||
});
|
||||
|
||||
const entries = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
const armEntry = entries.find((entry) => entry.path === "slices/arm64");
|
||||
if (armEntry === undefined) throw new Error("Expected an arm64 slice");
|
||||
|
||||
const controller = new AbortController();
|
||||
const cancelledRead = await reader.open(armEntry, controller.signal);
|
||||
const iterator = cancelledRead[Symbol.asyncIterator]();
|
||||
const firstChunk = await iterator.next();
|
||||
expect(firstChunk.value).toHaveLength(64 * 1024);
|
||||
controller.abort();
|
||||
await expect(iterator.next()).rejects.toMatchObject({
|
||||
reason: "cancelled",
|
||||
});
|
||||
|
||||
const verifiedEntries = [];
|
||||
for await (const entry of reader.entries()) verifiedEntries.push(entry);
|
||||
const verifiedArmEntry = verifiedEntries.find(
|
||||
(entry) => entry.path === "slices/arm64",
|
||||
);
|
||||
if (verifiedArmEntry === undefined)
|
||||
throw new Error("Expected arm64 after re-verifying the borrowed source");
|
||||
const verifiedChunks: Buffer[] = [];
|
||||
const verifiedRead = await reader.open(verifiedArmEntry);
|
||||
for await (const chunk of verifiedRead)
|
||||
verifiedChunks.push(Buffer.from(chunk));
|
||||
expect(Buffer.concat(verifiedChunks)).toEqual(arm);
|
||||
});
|
||||
};
|
||||
|
||||
describe("Mach-O slice reader with system lipo", () => {
|
||||
it.skipIf(process.platform !== "darwin")(
|
||||
"checks real /bin/ls lipo CPU symbols and ptrauth capabilities against the FAT table",
|
||||
async () => {
|
||||
const reader = new MachOSliceArtifactReader("/bin/ls", {});
|
||||
const entries = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
expect(entries.map(({ path }) => path)).toEqual([
|
||||
"slices/x86_64",
|
||||
"slices/arm64e",
|
||||
]);
|
||||
expect(
|
||||
entries.map(({ byteOffset, declaredSize }) => [
|
||||
byteOffset,
|
||||
declaredSize,
|
||||
]),
|
||||
).toEqual(await fatSlices("/bin/ls"));
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -12,25 +12,33 @@ import {
|
||||
type LipoArchitecture,
|
||||
} from "../native/parsers/lipo.js";
|
||||
import type { MachoSlice } from "../domain/apple/dylibResolution.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { readMachoImage, type ReadAt } from "./apple/MachoLoadCommandReader.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
RegularFileAdmissionFailure,
|
||||
openRegularFile,
|
||||
sameRegularFileState,
|
||||
type StableRegularFileDescriptor,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
|
||||
/** Read-only universal Mach-O slice reader backed by native lipo metadata. */
|
||||
export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
readonly format = "file" as const;
|
||||
#command: ArtifactCommand | undefined;
|
||||
#source: StableRegularFileDescriptor | undefined;
|
||||
#ownedSource: OwnedFileHandle | undefined;
|
||||
#pendingSource: Promise<StableRegularFileDescriptor> | undefined;
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
readonly #ownsSource: boolean;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
@@ -48,6 +56,7 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
this.#entries.clear();
|
||||
const source = await this.#ensureSource();
|
||||
await this.#verifySource(source);
|
||||
const captured = await this.runner.run(
|
||||
@@ -127,7 +136,7 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
"integrity",
|
||||
`lipo reported an out-of-bounds Mach-O slice: ${architecture.name}`,
|
||||
);
|
||||
yield {
|
||||
const entry: ArtifactEntry = {
|
||||
path: `slices/${architecture.name}`,
|
||||
kind: "slice",
|
||||
declaredSize: sliceSize,
|
||||
@@ -163,6 +172,8 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
],
|
||||
adapterKey: `${String(offset)}:${String(sliceSize)}`,
|
||||
};
|
||||
this.#entries.set(entry.adapterKey, copyArtifactEntry(entry));
|
||||
yield entry;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,14 +182,19 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure("cancelled", "Mach-O slice read cancelled"),
|
||||
);
|
||||
const [offsetText, sizeText] = entry.adapterKey.split(":");
|
||||
const offset = parseSliceKeyInteger(offsetText);
|
||||
const size = parseSliceKeyInteger(sizeText);
|
||||
const produced = this.#entries.get(entry.adapterKey);
|
||||
if (produced === undefined || !sameArtifactEntry(entry, produced))
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Mach-O slice entry metadata does not match this reader's inventory",
|
||||
);
|
||||
const offset = produced.byteOffset;
|
||||
const size = produced.declaredSize;
|
||||
if (
|
||||
!Number.isSafeInteger(offset) ||
|
||||
!Number.isSafeInteger(size) ||
|
||||
offset === null ||
|
||||
size === null ||
|
||||
!Number.isSafeInteger(offset) ||
|
||||
!Number.isSafeInteger(size) ||
|
||||
offset < 0 ||
|
||||
size <= 0
|
||||
)
|
||||
@@ -196,10 +212,9 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
"integrity",
|
||||
`Mach-O slice range is outside the artifact: ${entry.path}`,
|
||||
);
|
||||
const sourceStream = source.handle.createReadStream({
|
||||
const sourceStream = readFileHandleChunks(source.handle, {
|
||||
start: offset,
|
||||
end: offset + size - 1,
|
||||
autoClose: false,
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
});
|
||||
const verifySource = () => this.#verifySource(source);
|
||||
@@ -272,23 +287,51 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
}
|
||||
|
||||
async #acquireSource(): Promise<StableRegularFileDescriptor> {
|
||||
const handle = await openRegularFile(this.path, { symlinks: "reject" });
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await openRegularFile(this.path, { symlinks: "reject" });
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof RegularFileAdmissionFailure)) throw cause;
|
||||
this.#ownedSource = cause.owner;
|
||||
const primary =
|
||||
cause.cause instanceof NonRegularFileReadError
|
||||
? new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Universal Mach-O source is not a regular file: ${this.path}`,
|
||||
{ cause: cause.cause },
|
||||
)
|
||||
: cause.cause;
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
primary,
|
||||
ArtifactReaderFailure.cleanupObservation(cause.cleanupCause, this.path),
|
||||
);
|
||||
}
|
||||
const owner = new OwnedFileHandle(handle);
|
||||
this.#ownedSource = owner;
|
||||
try {
|
||||
const initial = await handle.stat();
|
||||
const source = { handle, initial };
|
||||
this.#source = source;
|
||||
return source;
|
||||
} catch (cause: unknown) {
|
||||
await handle.close().catch(() => undefined);
|
||||
try {
|
||||
await owner.close();
|
||||
this.#ownedSource = undefined;
|
||||
} catch (cleanupCause: unknown) {
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cause,
|
||||
ArtifactReaderFailure.cleanupObservation(cleanupCause, this.path),
|
||||
);
|
||||
}
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
async #closeSource(): Promise<void> {
|
||||
await this.#pendingSource?.catch(() => undefined);
|
||||
if (!this.#ownsSource || this.#source === undefined) return;
|
||||
const source = this.#source;
|
||||
await source.handle.close();
|
||||
if (!this.#ownsSource || this.#ownedSource === undefined) return;
|
||||
await this.#ownedSource.close();
|
||||
this.#ownedSource = undefined;
|
||||
this.#source = undefined;
|
||||
}
|
||||
|
||||
@@ -357,9 +400,3 @@ const lipoMatchesSlice = (
|
||||
const lipoNameMatches = (reported: string, observed: string): boolean =>
|
||||
reported === observed ||
|
||||
(observed === "arm64e" && /^arm64e\.[A-Za-z0-9_]+$/u.test(reported));
|
||||
|
||||
const parseSliceKeyInteger = (value: string | undefined): number | null => {
|
||||
if (value === undefined || !/^\d+$/u.test(value)) return null;
|
||||
const parsed = Number(value);
|
||||
return Number.isSafeInteger(parsed) ? parsed : null;
|
||||
};
|
||||
|
||||
@@ -111,7 +111,9 @@ const createSystemHost = (
|
||||
/** Read-only macOS DMG adapter that owns attachment and reverse-order detach. */
|
||||
export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
readonly format = "file" as const;
|
||||
private readonly host: NativeDmgHost;
|
||||
readonly #provenance: ArtifactCommand[] = [];
|
||||
#initialization: Promise<void> | undefined;
|
||||
#directory: DirectoryArtifactReader | undefined;
|
||||
#devices: string[] = [];
|
||||
#inventoryConfirmedDevices = new Set<string>();
|
||||
@@ -121,33 +123,24 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
#attachOutputReturned = false;
|
||||
#attachMayHaveMounted = false;
|
||||
#ownershipUncertainty: string | undefined;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
|
||||
private constructor(
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
private readonly host: NativeDmgHost,
|
||||
) {}
|
||||
|
||||
/** Verify and attach one image beneath an exclusively owned temporary root. */
|
||||
static async create(
|
||||
path: string,
|
||||
environment: Readonly<NodeJS.ProcessEnv>,
|
||||
signal?: AbortSignal,
|
||||
host?: NativeDmgHost,
|
||||
): Promise<NativeDmgArtifactReader> {
|
||||
) {
|
||||
if (process.platform !== "darwin" && host === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Native DMG traversal is available only on macOS",
|
||||
);
|
||||
const reader = new NativeDmgArtifactReader(
|
||||
path,
|
||||
host ?? createSystemHost(environment),
|
||||
);
|
||||
await reader.attach(signal);
|
||||
return reader;
|
||||
this.host = host ?? createSystemHost(environment);
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
await this.#ensureAttached(signal);
|
||||
if (this.#directory === undefined)
|
||||
throw new ArtifactReaderFailure("unavailable", "DMG is not attached");
|
||||
const prefix = basename(this.path);
|
||||
@@ -155,11 +148,10 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
yield { ...entry, path: `${prefix}/${entry.path}` };
|
||||
}
|
||||
|
||||
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
await this.#ensureAttached(signal);
|
||||
if (this.#directory === undefined)
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure("unavailable", "DMG is not attached"),
|
||||
);
|
||||
throw new ArtifactReaderFailure("unavailable", "DMG is not attached");
|
||||
return this.#directory.open(entry, signal);
|
||||
}
|
||||
|
||||
@@ -167,7 +159,19 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
return structuredClone(this.#provenance);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
this.#closePromise ??= this.#closeOwnedResources().catch(
|
||||
(cause: unknown) => {
|
||||
this.#closePromise = undefined;
|
||||
throw cause;
|
||||
},
|
||||
);
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #closeOwnedResources(): Promise<void> {
|
||||
await this.#initialization?.catch(() => undefined);
|
||||
let detachFailure = await this.#reconcileAttachState();
|
||||
let remainingDevices: string[] = [];
|
||||
for (const device of [...this.#devices].reverse()) {
|
||||
@@ -281,7 +285,16 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
}
|
||||
}
|
||||
|
||||
async attach(signal?: AbortSignal): Promise<void> {
|
||||
async #ensureAttached(signal?: AbortSignal): Promise<void> {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure("unavailable", "DMG reader is closed");
|
||||
this.#initialization ??= this.#attach(signal);
|
||||
await this.#initialization;
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure("unavailable", "DMG reader is closed");
|
||||
}
|
||||
|
||||
async #attach(signal?: AbortSignal): Promise<void> {
|
||||
await runChecked(this.host, ["verify", this.path], signal);
|
||||
this.#provenance.push(command(["verify", this.path], ["read"]));
|
||||
const baseline = await this.#readInfo();
|
||||
@@ -361,20 +374,6 @@ export class NativeDmgArtifactReader implements ArtifactReader {
|
||||
this.#attachMayHaveMounted = false;
|
||||
this.#ownershipUncertainty = undefined;
|
||||
}
|
||||
let cleanupFailure: unknown;
|
||||
try {
|
||||
await this.close();
|
||||
} catch (cleanupCause: unknown) {
|
||||
cleanupFailure = cleanupCause;
|
||||
}
|
||||
if (cleanupFailure !== undefined)
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupFailure,
|
||||
`DMG mount root ${this.#mountRoot ?? this.path}`,
|
||||
),
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
|
||||
it("shares one in-flight successful close between concurrent owners", async () => {
|
||||
let resolveClose: (() => void) | undefined;
|
||||
const file = {
|
||||
fd: 17,
|
||||
close: vi.fn(
|
||||
() =>
|
||||
new Promise<void>((resolve) => {
|
||||
file.fd = -1;
|
||||
resolveClose = resolve;
|
||||
}),
|
||||
),
|
||||
};
|
||||
const owner = new OwnedFileHandle(file);
|
||||
|
||||
const first = owner.close();
|
||||
const second = owner.close();
|
||||
await Promise.resolve();
|
||||
expect(file.close).toHaveBeenCalledTimes(1);
|
||||
resolveClose?.();
|
||||
|
||||
await expect(Promise.all([first, second])).resolves.toEqual([
|
||||
undefined,
|
||||
undefined,
|
||||
]);
|
||||
await expect(owner.close()).resolves.toBeUndefined();
|
||||
expect(file.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shares one in-flight failed close and keeps its unconfirmed result sticky", async () => {
|
||||
const failure = new Error("native close failed");
|
||||
let rejectClose: (() => void) | undefined;
|
||||
const file = {
|
||||
fd: 17,
|
||||
close: vi.fn(
|
||||
() =>
|
||||
new Promise<void>((_resolve, reject) => {
|
||||
file.fd = -1;
|
||||
rejectClose = () => reject(failure);
|
||||
}),
|
||||
),
|
||||
};
|
||||
const owner = new OwnedFileHandle(file);
|
||||
|
||||
const first = owner.close();
|
||||
const second = owner.close();
|
||||
await Promise.resolve();
|
||||
expect(file.close).toHaveBeenCalledTimes(1);
|
||||
rejectClose?.();
|
||||
|
||||
await expect(Promise.all([first, second])).rejects.toBe(failure);
|
||||
await expect(owner.close()).rejects.toBe(failure);
|
||||
expect(file.close).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import type { Stats } from "node:fs";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { afterEach, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
openRegularFile,
|
||||
RegularFileAdmissionFailure,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
|
||||
const { mockOpen } = vi.hoisted(() => ({ mockOpen: vi.fn() }));
|
||||
|
||||
vi.mock("node:fs/promises", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("node:fs/promises")>()),
|
||||
open: mockOpen,
|
||||
}));
|
||||
|
||||
afterEach(() => mockOpen.mockReset());
|
||||
|
||||
it.each(["cancellation after open", "non-regular metadata", "stat failure"])(
|
||||
"retains the exact owner when close fails after %s",
|
||||
async (failureCase) => {
|
||||
const closeFailure = new Error("native close failed");
|
||||
const statFailure = new Error("descriptor stat failed");
|
||||
const cancellation = new Error("cancelled after open");
|
||||
const controller = new AbortController();
|
||||
const file = {
|
||||
fd: 17,
|
||||
stat: vi.fn(async () => {
|
||||
if (failureCase === "cancellation after open") {
|
||||
controller.abort(cancellation);
|
||||
return { isFile: () => true } as Stats;
|
||||
}
|
||||
if (failureCase === "non-regular metadata")
|
||||
return {
|
||||
isFile: () => false,
|
||||
isDirectory: () => false,
|
||||
} as Stats;
|
||||
throw statFailure;
|
||||
}),
|
||||
close: vi.fn(async () => {
|
||||
file.fd = -1;
|
||||
throw closeFailure;
|
||||
}),
|
||||
};
|
||||
mockOpen.mockResolvedValue(file as unknown as FileHandle);
|
||||
|
||||
const cause = await openRegularFile("/selected/input", {
|
||||
symlinks: "reject",
|
||||
signal: controller.signal,
|
||||
}).catch((failure: unknown) => failure);
|
||||
|
||||
expect(cause).toBeInstanceOf(RegularFileAdmissionFailure);
|
||||
if (!(cause instanceof RegularFileAdmissionFailure))
|
||||
throw new Error("Expected an admission failure that retains its owner");
|
||||
expect(cause.owner.handle).toBe(file);
|
||||
expect(cause.cleanupCause).toBe(closeFailure);
|
||||
expect(file.close).toHaveBeenCalledTimes(1);
|
||||
await expect(cause.owner.close()).rejects.toBe(closeFailure);
|
||||
expect(file.close).toHaveBeenCalledTimes(1);
|
||||
|
||||
if (failureCase === "non-regular metadata")
|
||||
expect(cause.cause).toBeInstanceOf(NonRegularFileReadError);
|
||||
else if (failureCase === "stat failure")
|
||||
expect(cause.cause).toBe(statFailure);
|
||||
else expect(cause.cause).toBe(cancellation);
|
||||
},
|
||||
);
|
||||
+326
-145
@@ -1,16 +1,15 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import {
|
||||
lstat,
|
||||
mkdir,
|
||||
open,
|
||||
readdir,
|
||||
realpath,
|
||||
rm,
|
||||
type FileHandle,
|
||||
} from "node:fs/promises";
|
||||
import { basename, dirname, isAbsolute, join, resolve } from "node:path";
|
||||
import type { Readable } from "node:stream";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
|
||||
import {
|
||||
@@ -19,6 +18,20 @@ import {
|
||||
normalizeArtifactPath,
|
||||
} from "./ArtifactPaths.js";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import { SafeOutputTreeCreationFailure } from "./SafeOutputTreeCreationFailure.js";
|
||||
import { hashOutputFile } from "./SafeOutputTreeReadback.js";
|
||||
import { removeOwnedTree } from "./SafeOutputTreeCleanup.js";
|
||||
import {
|
||||
assertHandleIdentity,
|
||||
assertFilePathIdentity,
|
||||
assertPathIdentity,
|
||||
isAbsent,
|
||||
readDirectoryIdentity,
|
||||
readFileIdentity,
|
||||
replacedDirectory,
|
||||
type DirectoryIdentity,
|
||||
type FileIdentity,
|
||||
} from "./SafeOutputTreeIdentity.js";
|
||||
|
||||
/** One file durably written to an operation-owned output tree. */
|
||||
export interface SafeOutputFile {
|
||||
@@ -36,10 +49,21 @@ export type SafeOutputCleanup =
|
||||
readonly residualPaths: readonly [string, ...string[]];
|
||||
};
|
||||
|
||||
/** Symlink-resistant materialization in an exclusively owned, initially absent tree. */
|
||||
/**
|
||||
* Materialize files in a tree created by this operation.
|
||||
*
|
||||
* Path identities are revalidated around operations, but Node has no portable
|
||||
* descriptor-relative traversal; a syscall-boundary pathname race remains.
|
||||
*/
|
||||
export class SafeOutputTree {
|
||||
readonly #registry = new ArtifactPathRegistry();
|
||||
readonly #outputRoot: string;
|
||||
#rootIdentity: DirectoryIdentity | undefined;
|
||||
readonly #nestedDirectories = new Map<string, DirectoryIdentity>();
|
||||
readonly #unknownDirectories = new Set<string>();
|
||||
readonly #ownedFiles = new Map<string, FileIdentity>();
|
||||
readonly #openFiles = new Map<string, OwnedFileHandle>();
|
||||
readonly #openHandles = new Set<OwnedFileHandle>();
|
||||
#published = false;
|
||||
#cleanup: SafeOutputCleanup = {
|
||||
status: "not-required",
|
||||
@@ -91,41 +115,52 @@ export class SafeOutputTree {
|
||||
);
|
||||
throw cause;
|
||||
});
|
||||
const tree = new SafeOutputTree(canonicalOutput, platform);
|
||||
try {
|
||||
const rootIdentity = await readDirectoryIdentity(canonicalOutput);
|
||||
tree.#rootIdentity = rootIdentity;
|
||||
if (platform !== "win32") {
|
||||
const stagingHandle = await open(
|
||||
canonicalOutput,
|
||||
constants.O_RDONLY | constants.O_DIRECTORY,
|
||||
const stagingHandle = tree.#trackHandle(
|
||||
await open(
|
||||
canonicalOutput,
|
||||
constants.O_RDONLY | constants.O_DIRECTORY,
|
||||
),
|
||||
);
|
||||
let setup:
|
||||
| { readonly kind: "ready" }
|
||||
| {
|
||||
readonly kind: "failed";
|
||||
readonly cause: unknown;
|
||||
} = { kind: "ready" };
|
||||
try {
|
||||
await stagingHandle.chmod(0o700);
|
||||
} finally {
|
||||
await stagingHandle.close();
|
||||
await assertHandleIdentity(
|
||||
stagingHandle.handle,
|
||||
rootIdentity,
|
||||
canonicalOutput,
|
||||
);
|
||||
await stagingHandle.handle.chmod(0o700);
|
||||
} catch (cause: unknown) {
|
||||
setup = { kind: "failed", cause };
|
||||
}
|
||||
try {
|
||||
await tree.#closeHandle(stagingHandle);
|
||||
} catch (closeCause: unknown) {
|
||||
if (setup.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
setup.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
closeCause,
|
||||
canonicalOutput,
|
||||
),
|
||||
);
|
||||
throw closeCause;
|
||||
}
|
||||
if (setup.kind === "failed") throw setup.cause;
|
||||
}
|
||||
return new SafeOutputTree(canonicalOutput, platform);
|
||||
await assertPathIdentity(canonicalOutput, rootIdentity);
|
||||
return tree;
|
||||
} catch (cause: unknown) {
|
||||
let removalFailure: unknown;
|
||||
try {
|
||||
await rm(canonicalOutput, { recursive: true, force: true });
|
||||
} catch (cleanupCause: unknown) {
|
||||
removalFailure = cleanupCause;
|
||||
}
|
||||
let absent = false;
|
||||
try {
|
||||
absent = await isAbsent(canonicalOutput);
|
||||
} catch (cleanupCause: unknown) {
|
||||
removalFailure ??= cleanupCause;
|
||||
}
|
||||
if (!absent)
|
||||
throw ArtifactReaderFailure.withCleanup(cause, {
|
||||
reason:
|
||||
removalFailure instanceof Error
|
||||
? removalFailure.message
|
||||
: "Extraction output root remains after setup failure",
|
||||
resources: [canonicalOutput],
|
||||
});
|
||||
throw cause;
|
||||
throw new SafeOutputTreeCreationFailure(cause, tree);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,6 +168,11 @@ export class SafeOutputTree {
|
||||
return this.#outputRoot;
|
||||
}
|
||||
|
||||
/** The output was durably committed, even if descriptor cleanup later failed. */
|
||||
get published(): boolean {
|
||||
return this.#published;
|
||||
}
|
||||
|
||||
get cleanup(): SafeOutputCleanup {
|
||||
return structuredClone(this.#cleanup);
|
||||
}
|
||||
@@ -153,7 +193,11 @@ export class SafeOutputTree {
|
||||
);
|
||||
const path = normalizeArtifactPath(relativePath);
|
||||
this.#registry.add(path, "file");
|
||||
const destination = await this.#prepareParent(path);
|
||||
const lineage = await this.#prepareParent(path);
|
||||
const parent = lineage.at(-1);
|
||||
if (parent === undefined) throw replacedDirectory(this.#outputRoot);
|
||||
const fileName = path.slice(path.lastIndexOf("/") + 1);
|
||||
const destination = join(parent.path, fileName);
|
||||
const handle = await open(
|
||||
destination,
|
||||
constants.O_CREAT |
|
||||
@@ -162,57 +206,69 @@ export class SafeOutputTree {
|
||||
constants.O_NOFOLLOW,
|
||||
0o600,
|
||||
).catch(async (cause: unknown) => {
|
||||
await throwIfDestinationCaseCollision(
|
||||
dirname(destination),
|
||||
path,
|
||||
cause,
|
||||
);
|
||||
await throwIfDestinationCaseCollision(parent.path, path, cause);
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Could not exclusively create extraction path: ${path}`,
|
||||
{ cause },
|
||||
);
|
||||
});
|
||||
const ownedHandle = this.#trackHandle(handle);
|
||||
this.#openFiles.set(destination, ownedHandle);
|
||||
const fileIdentity = await readFileIdentity(
|
||||
ownedHandle.handle,
|
||||
destination,
|
||||
);
|
||||
await assertFilePathIdentity(destination, fileIdentity);
|
||||
this.#ownedFiles.set(destination, fileIdentity);
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
try {
|
||||
for await (const raw of source) {
|
||||
abortIfNeeded(signal);
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
if (chunk.length > expected.bytes - bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Extracted content exceeds the inventoried size: ${path}`,
|
||||
);
|
||||
bytes += chunk.length;
|
||||
hash.update(chunk);
|
||||
await writeAll(handle, chunk);
|
||||
}
|
||||
if (bytes !== expected.bytes)
|
||||
await this.#assertLineage(lineage);
|
||||
for await (const raw of source) {
|
||||
abortIfNeeded(signal);
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
if (chunk.length > expected.bytes - bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Extracted content size disagrees with inventory: ${path}`,
|
||||
`Extracted content exceeds the inventoried size: ${path}`,
|
||||
);
|
||||
const sha256 = hash.digest("hex");
|
||||
if (sha256 !== expected.sha256)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Extracted content disagrees with inventory: ${path}`,
|
||||
);
|
||||
await handle.sync();
|
||||
await handle.close();
|
||||
const readback = await hashFile(destination, bytes, signal);
|
||||
if (readback.sha256 !== sha256 || readback.bytes !== bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Durable readback verification failed: ${path}`,
|
||||
);
|
||||
return { relativePath: path, sha256, bytesWritten: bytes };
|
||||
} catch (cause: unknown) {
|
||||
// best-effort cleanup: file-handle close must not mask the write failure.
|
||||
await handle.close().catch(() => undefined);
|
||||
throw cause;
|
||||
bytes += chunk.length;
|
||||
hash.update(chunk);
|
||||
await writeAll(ownedHandle.handle, chunk);
|
||||
}
|
||||
if (bytes !== expected.bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Extracted content size disagrees with inventory: ${path}`,
|
||||
);
|
||||
const sha256 = hash.digest("hex");
|
||||
if (sha256 !== expected.sha256)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Extracted content disagrees with inventory: ${path}`,
|
||||
);
|
||||
await this.#assertLineage(lineage);
|
||||
await assertFilePathIdentity(destination, fileIdentity);
|
||||
await ownedHandle.handle.sync();
|
||||
await this.#closeFileHandle(destination, ownedHandle);
|
||||
const readbackHandle = this.#trackHandle(
|
||||
await open(destination, constants.O_RDONLY | constants.O_NOFOLLOW),
|
||||
);
|
||||
const readback = await hashOutputFile(
|
||||
readbackHandle,
|
||||
destination,
|
||||
fileIdentity,
|
||||
{ maximum: bytes, signal },
|
||||
);
|
||||
await this.#closeHandle(readbackHandle);
|
||||
await this.#assertLineage(lineage);
|
||||
await assertFilePathIdentity(destination, fileIdentity);
|
||||
if (readback.sha256 !== sha256 || readback.bytes !== bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Durable readback verification failed: ${path}`,
|
||||
);
|
||||
return { relativePath: path, sha256, bytesWritten: bytes };
|
||||
} catch (cause: unknown) {
|
||||
try {
|
||||
source.destroy();
|
||||
@@ -226,45 +282,107 @@ export class SafeOutputTree {
|
||||
/** Sync the owned output tree and prevent further writes through this instance. */
|
||||
async commit(): Promise<void> {
|
||||
this.#assertWritable();
|
||||
await this.#assertOwnedDirectories();
|
||||
// Windows has no directory fsync; file contents are already synced in write().
|
||||
if (this.platform === "win32") {
|
||||
this.#published = true;
|
||||
return;
|
||||
}
|
||||
const parent = await open(
|
||||
dirname(this.#outputRoot),
|
||||
constants.O_RDONLY | constants.O_DIRECTORY,
|
||||
);
|
||||
let output: FileHandle | undefined;
|
||||
try {
|
||||
output = await open(
|
||||
this.#outputRoot,
|
||||
const parent = this.#trackHandle(
|
||||
await open(
|
||||
dirname(this.#outputRoot),
|
||||
constants.O_RDONLY | constants.O_DIRECTORY,
|
||||
),
|
||||
);
|
||||
let output: OwnedFileHandle | undefined;
|
||||
let sync:
|
||||
| { readonly kind: "synced" }
|
||||
| {
|
||||
readonly kind: "failed";
|
||||
readonly cause: unknown;
|
||||
} = { kind: "synced" };
|
||||
try {
|
||||
output = this.#trackHandle(
|
||||
await open(
|
||||
this.#outputRoot,
|
||||
constants.O_RDONLY | constants.O_DIRECTORY,
|
||||
),
|
||||
);
|
||||
await output.sync();
|
||||
await parent.sync();
|
||||
await assertHandleIdentity(
|
||||
output.handle,
|
||||
this.#requireRootIdentity(),
|
||||
this.#outputRoot,
|
||||
);
|
||||
await output.handle.sync();
|
||||
await parent.handle.sync();
|
||||
this.#published = true;
|
||||
} catch (cause: unknown) {
|
||||
sync = { kind: "failed", cause };
|
||||
}
|
||||
const closeResults = await Promise.allSettled(
|
||||
[parent, output]
|
||||
.filter(isDefined)
|
||||
.map((handle) => this.#closeHandle(handle)),
|
||||
);
|
||||
const closeFailures = closeResults
|
||||
.filter(isRejected)
|
||||
.map(({ reason }) => reason);
|
||||
if (closeFailures.length > 0) {
|
||||
const closeFailure = new AggregateError(
|
||||
closeFailures,
|
||||
"Extraction output directory handles could not be closed",
|
||||
);
|
||||
const cleanup = ArtifactReaderFailure.cleanupObservation(
|
||||
closeFailure,
|
||||
this.#outputRoot,
|
||||
);
|
||||
if (sync.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Could not durably sync extraction output",
|
||||
{ cause: sync.cause },
|
||||
),
|
||||
cleanup,
|
||||
);
|
||||
throw ArtifactReaderFailure.withCleanup(closeFailure, cleanup);
|
||||
}
|
||||
if (sync.kind === "failed")
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Could not durably sync extraction output",
|
||||
{ cause },
|
||||
{ cause: sync.cause },
|
||||
);
|
||||
} finally {
|
||||
await Promise.allSettled([parent.close(), output?.close()]);
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove only this operation's unsealed tree and verify absence. */
|
||||
async rollback(): Promise<SafeOutputCleanup> {
|
||||
if (this.#published) return structuredClone(this.#cleanup);
|
||||
if (this.#published) {
|
||||
await this.#closePendingHandles();
|
||||
return structuredClone(this.#cleanup);
|
||||
}
|
||||
if (this.#cleanup.status === "complete")
|
||||
return structuredClone(this.#cleanup);
|
||||
this.#cleanup = {
|
||||
status: "incomplete",
|
||||
residualPaths: [basename(this.#outputRoot)],
|
||||
};
|
||||
let removalFailure: unknown;
|
||||
try {
|
||||
await rm(this.#outputRoot, { recursive: true, force: true });
|
||||
await this.#admitPendingFiles();
|
||||
await this.#closePendingHandles();
|
||||
await this.#assertOwnedDirectories();
|
||||
await removeOwnedTree({
|
||||
outputRoot: this.#outputRoot,
|
||||
rootIdentity: this.#requireRootIdentity(),
|
||||
directories: this.#nestedDirectories,
|
||||
files: this.#ownedFiles,
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
removalFailure = cause;
|
||||
}
|
||||
const absent = await isAbsent(this.#outputRoot);
|
||||
const absent =
|
||||
removalFailure === undefined && (await isAbsent(this.#outputRoot));
|
||||
this.#cleanup = absent
|
||||
? { status: "complete", residualPaths: [] }
|
||||
: {
|
||||
@@ -289,36 +407,81 @@ export class SafeOutputTree {
|
||||
return structuredClone(this.#cleanup);
|
||||
}
|
||||
|
||||
async #prepareParent(relativePath: string): Promise<string> {
|
||||
async #prepareParent(
|
||||
relativePath: string,
|
||||
): Promise<readonly DirectoryLineageEntry[]> {
|
||||
const parts = relativePath.split("/");
|
||||
const fileName = parts.pop();
|
||||
if (fileName === undefined)
|
||||
if (parts.pop() === undefined)
|
||||
throw new ArtifactReaderFailure("path", "Invalid extraction path");
|
||||
let current = this.#outputRoot;
|
||||
let logicalParent = "";
|
||||
const lineage: DirectoryLineageEntry[] = [
|
||||
{ path: this.#outputRoot, identity: this.#requireRootIdentity() },
|
||||
];
|
||||
if (parts.length === 0)
|
||||
await assertPathIdentity(this.#outputRoot, this.#requireRootIdentity());
|
||||
for (const part of parts) {
|
||||
const parentDirectory = current;
|
||||
const parent = lineage.at(-1);
|
||||
if (parent === undefined) throw replacedDirectory(this.#outputRoot);
|
||||
const logicalPath =
|
||||
logicalParent.length === 0 ? part : `${logicalParent}/${part}`;
|
||||
current = join(current, part);
|
||||
await assertPathIdentity(parent.path, parent.identity);
|
||||
let created = true;
|
||||
await mkdir(current, { mode: 0o700 }).catch((cause: unknown) => {
|
||||
if (!isAlreadyExists(cause)) throw cause;
|
||||
created = false;
|
||||
});
|
||||
if (!created)
|
||||
await throwIfDestinationCaseCollision(parentDirectory, logicalPath);
|
||||
const metadata = await lstat(current);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink()) {
|
||||
await throwIfDestinationCaseCollision(parentDirectory, logicalPath);
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Unsafe extraction parent: ${relativePath}`,
|
||||
);
|
||||
if (!created) {
|
||||
await throwIfDestinationCaseCollision(parent.path, logicalPath);
|
||||
const identity = this.#nestedDirectories.get(current);
|
||||
if (identity === undefined) throw replacedDirectory(current);
|
||||
await assertPathIdentity(current, identity);
|
||||
} else {
|
||||
try {
|
||||
const identity = await readDirectoryIdentity(current);
|
||||
this.#nestedDirectories.set(current, identity);
|
||||
} catch (cause: unknown) {
|
||||
this.#unknownDirectories.add(current);
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
await assertPathIdentity(parent.path, parent.identity);
|
||||
const identity = this.#nestedDirectories.get(current);
|
||||
if (identity === undefined) throw replacedDirectory(current);
|
||||
await assertPathIdentity(current, identity);
|
||||
lineage.push({ path: current, identity });
|
||||
logicalParent = logicalPath;
|
||||
}
|
||||
return join(current, fileName);
|
||||
return lineage;
|
||||
}
|
||||
|
||||
async #assertLineage(
|
||||
lineage: readonly DirectoryLineageEntry[],
|
||||
): Promise<void> {
|
||||
for (const { path, identity } of lineage)
|
||||
await assertPathIdentity(path, identity);
|
||||
}
|
||||
|
||||
async #assertOwnedDirectories(): Promise<void> {
|
||||
const rootIdentity = this.#requireRootIdentity();
|
||||
await assertPathIdentity(this.#outputRoot, rootIdentity);
|
||||
for (const [path, identity] of this.#nestedDirectories)
|
||||
await assertPathIdentity(path, identity);
|
||||
if (this.#unknownDirectories.size > 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Extraction directory ownership is unknown: ${[...this.#unknownDirectories].join(", ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
#requireRootIdentity(): DirectoryIdentity {
|
||||
if (this.#rootIdentity === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Extraction output root identity is unknown: ${this.#outputRoot}`,
|
||||
);
|
||||
return this.#rootIdentity;
|
||||
}
|
||||
|
||||
#assertWritable(): void {
|
||||
@@ -327,33 +490,63 @@ export class SafeOutputTree {
|
||||
"integrity",
|
||||
"Extraction tree is already committed",
|
||||
);
|
||||
if (this.#cleanup.status !== "not-required")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Extraction tree cleanup has already started",
|
||||
);
|
||||
}
|
||||
|
||||
#trackHandle(handle: FileHandle): OwnedFileHandle {
|
||||
const owner = new OwnedFileHandle(handle);
|
||||
this.#openHandles.add(owner);
|
||||
return owner;
|
||||
}
|
||||
|
||||
async #closeHandle(owner: OwnedFileHandle): Promise<void> {
|
||||
await owner.close();
|
||||
this.#openHandles.delete(owner);
|
||||
}
|
||||
|
||||
async #closePendingHandles(): Promise<void> {
|
||||
const failures: unknown[] = [];
|
||||
for (const owner of this.#openHandles) {
|
||||
try {
|
||||
await this.#closeHandle(owner);
|
||||
} catch (cause: unknown) {
|
||||
failures.push(cause);
|
||||
}
|
||||
}
|
||||
if (failures.length > 0)
|
||||
throw new AggregateError(failures, "Output tree handles remain open");
|
||||
}
|
||||
|
||||
async #closeFileHandle(path: string, owner: OwnedFileHandle): Promise<void> {
|
||||
await this.#closeHandle(owner);
|
||||
this.#openFiles.delete(path);
|
||||
}
|
||||
|
||||
async #admitPendingFiles(): Promise<void> {
|
||||
for (const [path, owner] of this.#openFiles) {
|
||||
let identity = this.#ownedFiles.get(path);
|
||||
if (identity === undefined) {
|
||||
identity = await readFileIdentity(owner.handle, path);
|
||||
await assertFilePathIdentity(path, identity);
|
||||
this.#ownedFiles.set(path, identity);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const hashFile = async (
|
||||
path: string,
|
||||
maximum: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ readonly sha256: string; readonly bytes: number }> => {
|
||||
const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
try {
|
||||
for await (const raw of handle.createReadStream({ autoClose: false })) {
|
||||
abortIfNeeded(signal);
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
if (bytes > maximum)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Readback exceeded the bytes written",
|
||||
);
|
||||
hash.update(chunk);
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
return { sha256: hash.digest("hex"), bytes };
|
||||
const isDefined = <T>(value: T | undefined): value is T => value !== undefined;
|
||||
|
||||
const isRejected = (
|
||||
result: PromiseSettledResult<void>,
|
||||
): result is PromiseRejectedResult => result.status === "rejected";
|
||||
|
||||
type DirectoryLineageEntry = {
|
||||
readonly path: string;
|
||||
readonly identity: DirectoryIdentity;
|
||||
};
|
||||
|
||||
const writeAll = async (
|
||||
@@ -376,17 +569,13 @@ const writeAll = async (
|
||||
}
|
||||
};
|
||||
|
||||
const isAbsent = async (path: string): Promise<boolean> =>
|
||||
lstat(path).then(
|
||||
() => false,
|
||||
(cause: unknown) => {
|
||||
if (isNotFound(cause)) return true;
|
||||
throw cause;
|
||||
},
|
||||
);
|
||||
|
||||
const isNotFound = (cause: unknown): boolean =>
|
||||
cause instanceof Error && "code" in cause && cause.code === "ENOENT";
|
||||
const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"Artifact extraction cancelled",
|
||||
);
|
||||
};
|
||||
|
||||
const isAlreadyExists = (cause: unknown): boolean =>
|
||||
cause instanceof Error && "code" in cause && cause.code === "EEXIST";
|
||||
@@ -407,11 +596,3 @@ const throwIfDestinationCaseCollision = async (
|
||||
cause === undefined ? undefined : { cause },
|
||||
);
|
||||
};
|
||||
|
||||
const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"Artifact extraction cancelled",
|
||||
);
|
||||
};
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { readdir, rmdir, unlink } from "node:fs/promises";
|
||||
import { dirname, sep } from "node:path";
|
||||
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import {
|
||||
assertFilePathIdentity,
|
||||
assertPathIdentity,
|
||||
replacedDirectory,
|
||||
type DirectoryIdentity,
|
||||
type FileIdentity,
|
||||
} from "./SafeOutputTreeIdentity.js";
|
||||
|
||||
/** Remove only tracked files and empty directories, preserving successful progress. */
|
||||
export const removeOwnedTree = async (input: {
|
||||
readonly outputRoot: string;
|
||||
readonly rootIdentity: DirectoryIdentity;
|
||||
readonly directories: Map<string, DirectoryIdentity>;
|
||||
readonly files: Map<string, FileIdentity>;
|
||||
}): Promise<void> => {
|
||||
const { outputRoot, rootIdentity, directories, files } = input;
|
||||
const filesByDirectory = new Map<string, [string, FileIdentity][]>();
|
||||
for (const entry of files) {
|
||||
const directory = dirname(entry[0]);
|
||||
const group = filesByDirectory.get(directory) ?? [];
|
||||
group.push(entry);
|
||||
filesByDirectory.set(directory, group);
|
||||
}
|
||||
const postorder = [...directories].sort(
|
||||
([left], [right]) =>
|
||||
right.slice(outputRoot.length).split(sep).length -
|
||||
left.slice(outputRoot.length).split(sep).length,
|
||||
);
|
||||
for (const [path, identity] of postorder) {
|
||||
await removeFiles(path, identity, filesByDirectory.get(path) ?? [], files);
|
||||
const parentPath = dirname(path);
|
||||
const parentIdentity =
|
||||
parentPath === outputRoot ? rootIdentity : directories.get(parentPath);
|
||||
if (parentIdentity === undefined) throw replacedDirectory(parentPath);
|
||||
await assertPathIdentity(parentPath, parentIdentity);
|
||||
await assertPathIdentity(path, identity);
|
||||
await requireEmpty(path);
|
||||
await rmdir(path);
|
||||
directories.delete(path);
|
||||
}
|
||||
await removeFiles(
|
||||
outputRoot,
|
||||
rootIdentity,
|
||||
filesByDirectory.get(outputRoot) ?? [],
|
||||
files,
|
||||
);
|
||||
await assertPathIdentity(outputRoot, rootIdentity);
|
||||
await requireEmpty(outputRoot);
|
||||
await rmdir(outputRoot);
|
||||
};
|
||||
|
||||
const removeFiles = async (
|
||||
directory: string,
|
||||
identity: DirectoryIdentity,
|
||||
filesInDirectory: readonly (readonly [string, FileIdentity])[],
|
||||
retainedFiles: Map<string, FileIdentity>,
|
||||
): Promise<void> => {
|
||||
for (const [path, fileIdentity] of filesInDirectory) {
|
||||
await assertPathIdentity(directory, identity);
|
||||
await assertFilePathIdentity(path, fileIdentity);
|
||||
await unlink(path);
|
||||
retainedFiles.delete(path);
|
||||
}
|
||||
};
|
||||
|
||||
const requireEmpty = async (directory: string): Promise<void> => {
|
||||
if ((await readdir(directory)).length > 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Unowned entries remain in extraction directory: ${directory}`,
|
||||
);
|
||||
};
|
||||
@@ -0,0 +1,31 @@
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import type { SafeOutputTree } from "./SafeOutputTree.js";
|
||||
|
||||
/** Setup failed after this tree was acquired; callers still own this exact tree. */
|
||||
export class SafeOutputTreeCreationFailure extends ArtifactReaderFailure {
|
||||
readonly tree: SafeOutputTree;
|
||||
|
||||
constructor(cause: unknown, tree: SafeOutputTree) {
|
||||
const primary =
|
||||
cause instanceof ArtifactReaderFailure
|
||||
? cause
|
||||
: new ArtifactReaderFailure("io", errorMessage(cause), { cause });
|
||||
super(
|
||||
primary.reason,
|
||||
primary.message,
|
||||
{
|
||||
cause: primary,
|
||||
...(primary.cleanup === undefined ? {} : { cleanup: primary.cleanup }),
|
||||
...(primary.partialObservation === undefined
|
||||
? {}
|
||||
: { partialObservation: primary.partialObservation }),
|
||||
},
|
||||
primary.details,
|
||||
);
|
||||
this.name = "SafeOutputTreeCreationFailure";
|
||||
this.tree = tree;
|
||||
}
|
||||
}
|
||||
|
||||
const errorMessage = (cause: unknown): string =>
|
||||
cause instanceof Error ? cause.message : String(cause);
|
||||
@@ -0,0 +1,90 @@
|
||||
import type { BigIntStats } from "node:fs";
|
||||
import { lstat } from "node:fs/promises";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
export type DirectoryIdentity = Pick<BigIntStats, "dev" | "ino">;
|
||||
export type FileIdentity = DirectoryIdentity;
|
||||
|
||||
export const readDirectoryIdentity = async (
|
||||
path: string,
|
||||
): Promise<DirectoryIdentity> => {
|
||||
const metadata = await lstat(path, { bigint: true });
|
||||
if (!metadata.isDirectory()) throw replacedDirectory(path);
|
||||
return { dev: metadata.dev, ino: metadata.ino };
|
||||
};
|
||||
|
||||
export const assertPathIdentity = async (
|
||||
path: string,
|
||||
expected: DirectoryIdentity,
|
||||
): Promise<void> => {
|
||||
const metadata = await lstat(path, { bigint: true });
|
||||
if (
|
||||
!metadata.isDirectory() ||
|
||||
metadata.dev !== expected.dev ||
|
||||
metadata.ino !== expected.ino
|
||||
)
|
||||
throw replacedDirectory(path);
|
||||
};
|
||||
|
||||
export const assertHandleIdentity = async (
|
||||
handle: FileHandle,
|
||||
expected: DirectoryIdentity,
|
||||
path: string,
|
||||
): Promise<void> => {
|
||||
const metadata = await handle.stat({ bigint: true });
|
||||
if (
|
||||
!metadata.isDirectory() ||
|
||||
metadata.dev !== expected.dev ||
|
||||
metadata.ino !== expected.ino
|
||||
)
|
||||
throw replacedDirectory(path);
|
||||
};
|
||||
|
||||
export const readFileIdentity = async (
|
||||
handle: FileHandle,
|
||||
path: string,
|
||||
): Promise<FileIdentity> => {
|
||||
const metadata = await handle.stat({ bigint: true });
|
||||
if (!metadata.isFile())
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Extraction file identity changed: ${path}`,
|
||||
);
|
||||
return { dev: metadata.dev, ino: metadata.ino };
|
||||
};
|
||||
|
||||
export const assertFilePathIdentity = async (
|
||||
path: string,
|
||||
expected: FileIdentity,
|
||||
): Promise<void> => {
|
||||
const metadata = await lstat(path, { bigint: true });
|
||||
if (
|
||||
!metadata.isFile() ||
|
||||
metadata.dev !== expected.dev ||
|
||||
metadata.ino !== expected.ino
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Extraction file identity changed: ${path}`,
|
||||
);
|
||||
};
|
||||
|
||||
export const isAbsent = async (path: string): Promise<boolean> =>
|
||||
lstat(path).then(
|
||||
() => false,
|
||||
(cause: unknown) => {
|
||||
if (isNotFound(cause)) return true;
|
||||
throw cause;
|
||||
},
|
||||
);
|
||||
|
||||
export const replacedDirectory = (path: string): ArtifactReaderFailure =>
|
||||
new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Extraction directory identity changed: ${path}`,
|
||||
);
|
||||
|
||||
const isNotFound = (cause: unknown): boolean =>
|
||||
cause instanceof Error && "code" in cause && cause.code === "ENOENT";
|
||||
@@ -0,0 +1,62 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import {
|
||||
assertFilePathIdentity,
|
||||
readFileIdentity,
|
||||
type FileIdentity,
|
||||
} from "./SafeOutputTreeIdentity.js";
|
||||
|
||||
/** Hash one retained readback descriptor and confirm it still names the output file. */
|
||||
export const hashOutputFile = async (
|
||||
owner: OwnedFileHandle,
|
||||
path: string,
|
||||
identity: FileIdentity,
|
||||
options: {
|
||||
readonly maximum: number;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
},
|
||||
): Promise<{ readonly sha256: string; readonly bytes: number }> => {
|
||||
const admittedIdentity = await readFileIdentity(owner.handle, path);
|
||||
await assertFilePathIdentity(path, identity);
|
||||
if (
|
||||
admittedIdentity.dev !== identity.dev ||
|
||||
admittedIdentity.ino !== identity.ino
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Extraction file identity changed: ${path}`,
|
||||
);
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
try {
|
||||
for await (const raw of readFileHandleChunks(owner.handle, options)) {
|
||||
if (options.signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"Artifact extraction cancelled",
|
||||
);
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
if (bytes > options.maximum)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Readback exceeded the bytes written",
|
||||
);
|
||||
hash.update(chunk);
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
if (options.signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"Artifact extraction cancelled",
|
||||
{ cause },
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
await assertFilePathIdentity(path, identity);
|
||||
return { sha256: hash.digest("hex"), bytes };
|
||||
};
|
||||
@@ -6,12 +6,16 @@ import { Reader, ZipReader, type Entry, type FileEntry } from "@zip.js/zip.js";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
|
||||
import { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
RegularFileAdmissionFailure,
|
||||
openRegularFile,
|
||||
sameRegularFileState,
|
||||
type StableRegularFileDescriptor,
|
||||
@@ -19,6 +23,7 @@ import {
|
||||
|
||||
class NodeFileReader extends Reader<string> {
|
||||
#handle: FileHandle | undefined;
|
||||
#owner: OwnedFileHandle | undefined;
|
||||
#initPromise: Promise<void> | undefined;
|
||||
readonly #ownsHandle: boolean;
|
||||
|
||||
@@ -38,10 +43,30 @@ class NodeFileReader extends Reader<string> {
|
||||
|
||||
async #initialize(): Promise<void> {
|
||||
try {
|
||||
this.#handle =
|
||||
const handle =
|
||||
this.admitted?.handle ??
|
||||
(await openRegularFile(this.path, { symlinks: "reject" }));
|
||||
this.#handle = handle;
|
||||
if (this.#ownsHandle) this.#owner = new OwnedFileHandle(handle);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof RegularFileAdmissionFailure) {
|
||||
this.#owner = cause.owner;
|
||||
const primary =
|
||||
cause.cause instanceof NonRegularFileReadError
|
||||
? new ArtifactReaderFailure(
|
||||
"format",
|
||||
`ZIP source is not a regular file: ${this.path}`,
|
||||
{ cause: cause.cause },
|
||||
)
|
||||
: cause.cause;
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
primary,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cause.cleanupCause,
|
||||
this.path,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (cause instanceof NonRegularFileReadError)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
@@ -86,8 +111,9 @@ class NodeFileReader extends Reader<string> {
|
||||
async closeHandle(): Promise<void> {
|
||||
if (!this.#ownsHandle) return;
|
||||
await this.#initPromise?.catch(() => undefined);
|
||||
if (this.#handle === undefined) return;
|
||||
await this.#handle.close();
|
||||
if (this.#owner === undefined) return;
|
||||
await this.#owner.close();
|
||||
this.#owner = undefined;
|
||||
this.#handle = undefined;
|
||||
}
|
||||
}
|
||||
@@ -97,7 +123,10 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
readonly format: ZipPackageFormat;
|
||||
readonly #source: NodeFileReader;
|
||||
readonly #reader: ZipReader<string>;
|
||||
readonly #entries = new Map<string, Entry>();
|
||||
readonly #entries = new Map<
|
||||
string,
|
||||
{ readonly source: Entry; readonly metadata: ArtifactEntry }
|
||||
>();
|
||||
|
||||
/** Optionally bound each metadata read before allocating its backing buffer. */
|
||||
constructor(
|
||||
@@ -120,39 +149,38 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
for await (const entry of this.#reader.getEntriesGenerator()) {
|
||||
abortIfNeeded(signal);
|
||||
this.#entries.set(entry.filename, entry);
|
||||
const symlink = isSymlink(entry);
|
||||
yield {
|
||||
path: entry.filename,
|
||||
kind: entry.directory ? "directory" : symlink ? "symlink" : "file",
|
||||
declaredSize: entry.directory ? null : entry.uncompressedSize,
|
||||
compressedSize: entry.directory ? null : entry.compressedSize,
|
||||
executable: entry.executable,
|
||||
encrypted: entry.encrypted,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: symlink ? ["Archive symlink target was not read."] : [],
|
||||
adapterKey: entry.filename,
|
||||
};
|
||||
const metadata = projectZipEntry(entry);
|
||||
this.#entries.set(entry.filename, {
|
||||
source: entry,
|
||||
metadata: copyArtifactEntry(metadata),
|
||||
});
|
||||
yield metadata;
|
||||
}
|
||||
}
|
||||
|
||||
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
abortIfNeeded(signal);
|
||||
const stored = this.#entries.get(entry.adapterKey);
|
||||
if (stored === undefined || stored.directory || isSymlink(stored))
|
||||
if (stored === undefined || !sameArtifactEntry(entry, stored.metadata))
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"ZIP entry metadata does not match this reader's inventory",
|
||||
),
|
||||
);
|
||||
const source = stored.source;
|
||||
if (source.directory || isSymlink(source))
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure("format", "ZIP entry is not a regular file"),
|
||||
);
|
||||
if (stored.encrypted)
|
||||
if (source.encrypted)
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Encrypted ZIP entry is unsupported",
|
||||
),
|
||||
);
|
||||
return Promise.resolve(extractStream(stored, signal));
|
||||
return Promise.resolve(extractStream(source, signal));
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
@@ -166,6 +194,23 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
}
|
||||
}
|
||||
|
||||
const projectZipEntry = (entry: Entry): ArtifactEntry => {
|
||||
const symlink = isSymlink(entry);
|
||||
return {
|
||||
path: entry.filename,
|
||||
kind: entry.directory ? "directory" : symlink ? "symlink" : "file",
|
||||
declaredSize: entry.directory ? null : entry.uncompressedSize,
|
||||
compressedSize: entry.directory ? null : entry.compressedSize,
|
||||
executable: entry.executable,
|
||||
encrypted: entry.encrypted,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: symlink ? ["Archive symlink target was not read."] : [],
|
||||
adapterKey: entry.filename,
|
||||
};
|
||||
};
|
||||
|
||||
const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => {
|
||||
const output = new PassThrough();
|
||||
const controller = new AbortController();
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
} from "../../domain/propertyListKeys.js";
|
||||
import { decodeNibArchive, type NibArchiveDocument } from "./NibArchive.js";
|
||||
import {
|
||||
estimatePropertyListDecodeBytes,
|
||||
preflightPropertyListDecode,
|
||||
InterfaceBuilderDecodeBudget,
|
||||
InterfaceBuilderDecodeBudgetExceeded,
|
||||
} from "./InterfaceBuilderDecodeBudget.js";
|
||||
@@ -47,7 +47,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
|
||||
const omittedArchives: Array<{ path: string; reason: string }> = [];
|
||||
let unreportedOmittedArchives = 0;
|
||||
const incompleteHierarchies = new Map<string, number>();
|
||||
const prototypeKeyOmissions = new Map<string, number>();
|
||||
const prototypeKeyOmissions = new Map<string, number | null>();
|
||||
let omitted = 0;
|
||||
let attempted = 0;
|
||||
let aggregateInputBytes = 0;
|
||||
@@ -93,6 +93,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
|
||||
bytes.subarray(0, 10).toString("ascii") === "NIBArchive";
|
||||
let nib: ReturnType<typeof projectNibArchive> | null = null;
|
||||
let xmlText: string | undefined;
|
||||
let binaryPrototypeKeyObserved = false;
|
||||
if (isNibArchive) {
|
||||
const documentBudget = new InterfaceBuilderDecodeBudget(
|
||||
decodeBudget.remainingBytes,
|
||||
@@ -109,23 +110,25 @@ export const analyzeInterfaceBuilderBundle = async (input: {
|
||||
} else {
|
||||
if (bytes.subarray(0, 8).toString("ascii") !== "bplist00")
|
||||
xmlText = decodeXmlPlistText(bytes);
|
||||
const decodedReservation = estimatePropertyListDecodeBytes(
|
||||
const decodedReservation = preflightPropertyListDecode(
|
||||
bytes,
|
||||
decodeBudget.remainingBytes,
|
||||
xmlText,
|
||||
);
|
||||
decodeBudget.reserve(
|
||||
decodedReservation,
|
||||
decodedReservation.estimatedBytes,
|
||||
"plist representation exceeds the aggregate Interface Builder decode budget",
|
||||
);
|
||||
binaryPrototypeKeyObserved =
|
||||
decodedReservation.binaryPrototypeKeyObserved;
|
||||
}
|
||||
const { value: raw, omittedPrototypeKeys } =
|
||||
nib === null
|
||||
? decodePlist(bytes, xmlText)
|
||||
? decodePlist(bytes, xmlText, binaryPrototypeKeyObserved)
|
||||
: { value: nib.raw, omittedPrototypeKeys: 0 };
|
||||
if (nib !== null && nib.omitted > 0)
|
||||
incompleteHierarchies.set(entry.path, nib.omitted);
|
||||
if (omittedPrototypeKeys > 0)
|
||||
if (omittedPrototypeKeys !== 0)
|
||||
prototypeKeyOmissions.set(entry.path, omittedPrototypeKeys);
|
||||
const documentHash = createHash("sha256").update(bytes).digest("hex");
|
||||
documents.push({
|
||||
@@ -226,7 +229,7 @@ const finalizeHierarchyCoverage = (
|
||||
result: ReturnType<typeof buildInterfaceBuilderAnalysis>,
|
||||
counts: {
|
||||
incompleteHierarchies: ReadonlyMap<string, number>;
|
||||
prototypeKeyOmissions: ReadonlyMap<string, number>;
|
||||
prototypeKeyOmissions: ReadonlyMap<string, number | null>;
|
||||
invalid: readonly string[];
|
||||
omittedArchives: readonly { path: string; reason: string }[];
|
||||
unreportedOmittedArchives: number;
|
||||
@@ -641,11 +644,18 @@ const readEntry = async (
|
||||
/** Project decoded plist data and dates while retaining omitted-key coverage. */
|
||||
const decodePlist = (
|
||||
bytes: Buffer,
|
||||
xmlText?: string,
|
||||
): { readonly value: JsonValue; readonly omittedPrototypeKeys: number } => {
|
||||
const { value, omittedPrototypeKeys } =
|
||||
bytes.subarray(0, 8).toString("ascii") === "bplist00"
|
||||
? { value: parseBinary(bytes), omittedPrototypeKeys: 0 }
|
||||
: parseXmlPropertyList(xmlText ?? decodeXmlPlistText(bytes));
|
||||
xmlText: string | undefined,
|
||||
binaryPrototypeKeyObserved: boolean,
|
||||
): {
|
||||
readonly value: JsonValue;
|
||||
readonly omittedPrototypeKeys: number | null;
|
||||
} => {
|
||||
const binary = bytes.subarray(0, 8).toString("ascii") === "bplist00";
|
||||
const { value, omittedPrototypeKeys } = binary
|
||||
? {
|
||||
value: parseBinary(bytes),
|
||||
omittedPrototypeKeys: binaryPrototypeKeyObserved ? null : 0,
|
||||
}
|
||||
: parseXmlPropertyList(xmlText ?? decodeXmlPlistText(bytes));
|
||||
return { value: projectPlistValue(value).value, omittedPrototypeKeys };
|
||||
};
|
||||
|
||||
@@ -89,21 +89,24 @@ export class InterfaceBuilderDecodeBudget {
|
||||
}
|
||||
|
||||
/** Preflight the representation created by the pinned plist decoder. */
|
||||
export const estimatePropertyListDecodeBytes = (
|
||||
export const preflightPropertyListDecode = (
|
||||
bytes: Buffer,
|
||||
maxBytes: number,
|
||||
xmlText?: string,
|
||||
): number => {
|
||||
const estimate =
|
||||
): PlistDecodePreflight => {
|
||||
const expansion =
|
||||
bytes.subarray(0, 8).toString("ascii") === "bplist00"
|
||||
? estimateBinaryPlistExpansion(bytes, maxBytes)
|
||||
: estimateXmlPlistExpansion(bytes, maxBytes, xmlText);
|
||||
if (estimate > maxBytes)
|
||||
: {
|
||||
estimatedBytes: estimateXmlPlistExpansion(bytes, maxBytes, xmlText),
|
||||
binaryPrototypeKeyObserved: false,
|
||||
};
|
||||
if (expansion.estimatedBytes > maxBytes)
|
||||
throw new InterfaceBuilderDecodeBudgetExceeded(
|
||||
"aggregate_decode_budget_exhausted",
|
||||
"plist representation exceeds the aggregate Interface Builder decode budget",
|
||||
);
|
||||
return estimate;
|
||||
return expansion;
|
||||
};
|
||||
|
||||
const estimateXmlPlistExpansion = (
|
||||
@@ -173,6 +176,14 @@ export interface BinaryPlistExpansionOptions {
|
||||
readonly bounds?: "strict" | "decoder";
|
||||
}
|
||||
|
||||
/** Preflight size and binary-key observations; XML keys are inspected by decode. */
|
||||
export interface PlistDecodePreflight {
|
||||
readonly estimatedBytes: number;
|
||||
readonly binaryPrototypeKeyObserved: boolean;
|
||||
}
|
||||
|
||||
const PROTOTYPE_KEY = "__proto__";
|
||||
|
||||
/**
|
||||
* Count every expanded binary-plist reference before calling plist.parseBinary,
|
||||
* which recurses on each reference and copies shared containers. Malformed
|
||||
@@ -186,7 +197,7 @@ export const estimateBinaryPlistExpansion = (
|
||||
fail = interfaceBuilderBudgetFailure,
|
||||
bounds = "strict",
|
||||
}: BinaryPlistExpansionOptions = {},
|
||||
): number => {
|
||||
): PlistDecodePreflight => {
|
||||
if (bytes.length < 40)
|
||||
throw new TypeError("binary plist trailer is truncated");
|
||||
const trailer = bytes.length - 32;
|
||||
@@ -218,13 +229,17 @@ export const estimateBinaryPlistExpansion = (
|
||||
)
|
||||
throw new TypeError("binary plist trailer or object table is invalid");
|
||||
let estimate = bytes.length * 2 + objectCount * 16;
|
||||
let binaryPrototypeKeyObserved = false;
|
||||
if (estimate > maxBytes)
|
||||
throw fail("expansion", `binary plist object table exceeds ${budget}`);
|
||||
|
||||
const active = new Set<number>();
|
||||
const pending: Array<{ object: number; depth: number; exit: boolean }> = [
|
||||
{ object: topObject, depth: 0, exit: false },
|
||||
];
|
||||
const pending: Array<{
|
||||
object: number;
|
||||
depth: number;
|
||||
exit: boolean;
|
||||
dictionaryKey: boolean;
|
||||
}> = [{ object: topObject, depth: 0, exit: false, dictionaryKey: false }];
|
||||
let pendingObjectCount = 1;
|
||||
while (pending.length > 0) {
|
||||
const entry = pending.pop();
|
||||
@@ -269,6 +284,26 @@ export const estimateBinaryPlistExpansion = (
|
||||
const byteLength = type === 6 ? size * 2 : size;
|
||||
if (cursor + byteLength > objectsEnd)
|
||||
throw new TypeError("binary plist scalar object is truncated");
|
||||
if (
|
||||
!binaryPrototypeKeyObserved &&
|
||||
entry.dictionaryKey &&
|
||||
(type === 5 || type === 6) &&
|
||||
size === PROTOTYPE_KEY.length
|
||||
) {
|
||||
let matches = true;
|
||||
for (let index = 0; index < PROTOTYPE_KEY.length; index += 1) {
|
||||
const codeUnit =
|
||||
type === 5
|
||||
? (bytes[cursor + index] ?? 0)
|
||||
: ((bytes[cursor + index * 2] ?? 0) << 8) |
|
||||
(bytes[cursor + index * 2 + 1] ?? 0);
|
||||
if (codeUnit !== PROTOTYPE_KEY.charCodeAt(index)) {
|
||||
matches = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (matches) binaryPrototypeKeyObserved = true;
|
||||
}
|
||||
estimate += type === 4 ? size + 4 * Math.ceil(size / 3) : byteLength * 2;
|
||||
if (estimate > maxBytes)
|
||||
throw fail(
|
||||
@@ -289,7 +324,12 @@ export const estimateBinaryPlistExpansion = (
|
||||
`binary plist reference expansion exceeds ${budget}`,
|
||||
);
|
||||
active.add(entry.object);
|
||||
pending.push({ object: entry.object, depth: entry.depth, exit: true });
|
||||
pending.push({
|
||||
object: entry.object,
|
||||
depth: entry.depth,
|
||||
exit: true,
|
||||
dictionaryKey: false,
|
||||
});
|
||||
for (let index = childCount - 1; index >= 0; index -= 1) {
|
||||
const reference = readInteger(
|
||||
cursor + index * referenceSize,
|
||||
@@ -297,9 +337,14 @@ export const estimateBinaryPlistExpansion = (
|
||||
);
|
||||
if (reference < 0 || reference >= objectCount)
|
||||
throw new TypeError("binary plist reference is invalid");
|
||||
pending.push({ object: reference, depth: entry.depth + 1, exit: false });
|
||||
pending.push({
|
||||
object: reference,
|
||||
depth: entry.depth + 1,
|
||||
exit: false,
|
||||
dictionaryKey: type === 13 && index < size,
|
||||
});
|
||||
}
|
||||
pendingObjectCount += childCount;
|
||||
}
|
||||
return estimate;
|
||||
return { estimatedBytes: estimate, binaryPrototypeKeyObserved };
|
||||
};
|
||||
|
||||
@@ -65,6 +65,40 @@ describe("inert keyed archive decoding", () => {
|
||||
"1 dictionary entry keyed __proto__ was omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
it("reports a binary archive dictionary keyed __proto__ as omitted", () => {
|
||||
const placeholder = "proto_key";
|
||||
const encoded = Buffer.from(
|
||||
buildBinary({
|
||||
...archive,
|
||||
[placeholder]: "hidden",
|
||||
$objects: ["$null", "value"],
|
||||
}),
|
||||
);
|
||||
const start = encoded.indexOf(placeholder);
|
||||
expect(start).toBeGreaterThanOrEqual(0);
|
||||
expect(encoded.indexOf(placeholder, start + placeholder.length)).toBe(-1);
|
||||
Buffer.from("__proto__").copy(encoded, start);
|
||||
|
||||
const graph = decodeKeyedArchiveBytes(encoded, { offset: 0, limit: 2 });
|
||||
expect(graph.limitations).toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
it("does not report a binary __proto__ value as an omitted key", () => {
|
||||
const graph = decodeKeyedArchiveBytes(
|
||||
Buffer.from(
|
||||
buildBinary({
|
||||
...archive,
|
||||
note: "__proto__",
|
||||
$objects: ["$null", "value"],
|
||||
}),
|
||||
),
|
||||
{ offset: 0, limit: 2 },
|
||||
);
|
||||
expect(graph.limitations).not.toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
it("rejects missing roots, malformed plist, and non-keyed archives", () => {
|
||||
expect(() =>
|
||||
decodeKeyedArchiveBytes(Buffer.from("bplist00bad"), {
|
||||
@@ -133,6 +167,20 @@ const plistArray = (references: readonly number[]): Buffer => {
|
||||
};
|
||||
const plistString = (value: string): Buffer =>
|
||||
Buffer.concat([Buffer.from([0x50 | value.length]), Buffer.from(value)]);
|
||||
const plistUtf16String = (value: string): Buffer => {
|
||||
const bytes = Buffer.alloc(1 + value.length * 2);
|
||||
bytes[0] = 0x60 | value.length;
|
||||
for (let index = 0; index < value.length; index += 1)
|
||||
bytes.writeUInt16BE(value.charCodeAt(index), 1 + index * 2);
|
||||
return bytes;
|
||||
};
|
||||
const plistExtendedUtf16String = (value: string): Buffer => {
|
||||
const bytes = Buffer.alloc(3 + value.length * 2);
|
||||
bytes.set([0x6f, 0x10, value.length]);
|
||||
for (let index = 0; index < value.length; index += 1)
|
||||
bytes.writeUInt16BE(value.charCodeAt(index), 3 + index * 2);
|
||||
return bytes;
|
||||
};
|
||||
// {"$archiver": "NSKeyedArchiver", "$top": {"root": UID 1}, "$objects": object 8}
|
||||
const keyedRoot = (payload: readonly Buffer[]): Buffer =>
|
||||
binaryPlist([
|
||||
@@ -187,4 +235,91 @@ describe("binary plist reference preflight", () => {
|
||||
[2, "$null"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("recognizes a shared key object used by separate dictionaries", () => {
|
||||
const graph = decodeKeyedArchiveBytes(
|
||||
keyedRoot([
|
||||
plistArray([9, 12]),
|
||||
Buffer.from([0xd1, 0, 10, 0, 11]),
|
||||
plistString("__proto__"),
|
||||
plistString("first"),
|
||||
Buffer.from([0xd1, 0, 10, 0, 13]),
|
||||
plistString("second"),
|
||||
]),
|
||||
selection,
|
||||
);
|
||||
expect(graph.limitations).toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
|
||||
it("reports binary omission presence without claiming a count", () => {
|
||||
const placeholder = "proto_key";
|
||||
const nested = Buffer.from(
|
||||
buildBinary({
|
||||
...archive,
|
||||
[placeholder]: { [placeholder]: "hidden", visible: true },
|
||||
$objects: ["$null", "value"],
|
||||
}),
|
||||
);
|
||||
let index = nested.indexOf(placeholder);
|
||||
while (index !== -1) {
|
||||
Buffer.from("__proto__").copy(nested, index);
|
||||
index = nested.indexOf(placeholder, index + placeholder.length);
|
||||
}
|
||||
const nestedGraph = decodeKeyedArchiveBytes(nested, selection);
|
||||
expect(nestedGraph.limitations).toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
|
||||
const shadowedValue = keyedRoot([
|
||||
plistArray([9]),
|
||||
Buffer.from([0xd2, 0, 10, 0, 10, 0, 11, 0, 12]),
|
||||
plistString("shadow"),
|
||||
Buffer.from([0xd1, 0, 13, 0, 14]),
|
||||
plistString("replacement"),
|
||||
plistString(placeholder),
|
||||
plistString("hidden"),
|
||||
]);
|
||||
const keyOffset = shadowedValue.indexOf(Buffer.from(placeholder));
|
||||
expect(keyOffset).toBeGreaterThanOrEqual(0);
|
||||
Buffer.from("__proto__").copy(shadowedValue, keyOffset);
|
||||
const shadowedGraph = decodeKeyedArchiveBytes(shadowedValue, selection);
|
||||
expect(shadowedGraph.limitations).toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
expect(shadowedGraph.limitations).not.toContain(
|
||||
"1 dictionary entry keyed __proto__ was omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
|
||||
it("detects UTF-16 binary dictionary keys", () => {
|
||||
const graph = decodeKeyedArchiveBytes(
|
||||
keyedRoot([
|
||||
plistArray([9]),
|
||||
Buffer.from([0xd1, 0, 10, 0, 11]),
|
||||
plistUtf16String("__proto__"),
|
||||
plistString("hidden"),
|
||||
]),
|
||||
selection,
|
||||
);
|
||||
expect(graph.limitations).toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
|
||||
it("detects extended-length UTF-16 binary dictionary keys", () => {
|
||||
const graph = decodeKeyedArchiveBytes(
|
||||
keyedRoot([
|
||||
plistArray([9]),
|
||||
Buffer.from([0xd1, 0, 10, 0, 11]),
|
||||
plistExtendedUtf16String("__proto__"),
|
||||
plistString("hidden"),
|
||||
]),
|
||||
selection,
|
||||
);
|
||||
expect(graph.limitations).toContain(
|
||||
"Dictionary entries keyed __proto__ were omitted because REA results cannot represent that key.",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,7 +6,6 @@ import {
|
||||
AnalysisInputError,
|
||||
AnalysisUnsupportedTargetError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { projectPlistValue } from "../../domain/apple/plistValue.js";
|
||||
import { createPlistNumberProjection } from "../../domain/apple/plistNumbers.js";
|
||||
import {
|
||||
@@ -51,20 +50,27 @@ export const decodeKeyedArchiveBytes = (
|
||||
// plist.parseBinary recurses on every object reference and copies shared
|
||||
// containers, so a few hundred bytes can exhaust memory or the stack.
|
||||
// RangeError reports a resource limit; a reference cycle is malformed.
|
||||
if (binary)
|
||||
estimateBinaryPlistExpansion(bytes, MAX_DECODE_BYTES, {
|
||||
budget: "the keyed archive decode budget",
|
||||
fail: (kind, message) =>
|
||||
kind === "cycle"
|
||||
? new TypeError(
|
||||
"binary plist object references form a cycle, which a property list cannot represent",
|
||||
)
|
||||
: new RangeError(message),
|
||||
bounds: "decoder",
|
||||
});
|
||||
const parsed = binary
|
||||
? { value: parseBinary(bytes), omittedPrototypeKeys: 0 }
|
||||
: parseXmlPropertyList(xmlText ?? "");
|
||||
const binaryExpansion = binary
|
||||
? estimateBinaryPlistExpansion(bytes, MAX_DECODE_BYTES, {
|
||||
budget: "the keyed archive decode budget",
|
||||
fail: (kind, message) =>
|
||||
kind === "cycle"
|
||||
? new TypeError(
|
||||
"binary plist object references form a cycle, which a property list cannot represent",
|
||||
)
|
||||
: new RangeError(message),
|
||||
bounds: "decoder",
|
||||
})
|
||||
: undefined;
|
||||
const parsed =
|
||||
binaryExpansion !== undefined
|
||||
? {
|
||||
value: parseBinary(bytes),
|
||||
omittedPrototypeKeys: binaryExpansion.binaryPrototypeKeyObserved
|
||||
? null
|
||||
: 0,
|
||||
}
|
||||
: parseXmlPropertyList(xmlText ?? "");
|
||||
const { value: plistValue, unknownRealCount } = projectPlistValue(
|
||||
parsed.value,
|
||||
);
|
||||
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
|
||||
import { SafeOutputTree } from "../SafeOutputTree.js";
|
||||
import { SafeOutputTreeCreationFailure } from "../SafeOutputTreeCreationFailure.js";
|
||||
import { ZipArtifactReader } from "../ZipArtifactReader.js";
|
||||
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
|
||||
import {
|
||||
@@ -26,9 +27,13 @@ import {
|
||||
type IntegrityContradiction,
|
||||
} from "../../domain/artifactGraph.js";
|
||||
import { AnalysisUnsupportedTargetError } from "../../domain/analysisErrorCore.js";
|
||||
import {
|
||||
type ArtifactResourceOwner,
|
||||
type ArtifactResourceScope,
|
||||
} from "../ArtifactResourceScope.js";
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
import { scanArtifactInventory } from "../inventory/ArtifactInventory.js";
|
||||
import { scanArtifactInventoryInScope } from "../inventory/ArtifactInventory.js";
|
||||
import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrityPolicy.js";
|
||||
|
||||
/** Local extraction input with the output root chosen by the adapter. */
|
||||
@@ -38,23 +43,36 @@ export interface ArtifactExtractionInput {
|
||||
readonly outputRoot: string;
|
||||
readonly environment: Readonly<NodeJS.ProcessEnv>;
|
||||
readonly integrityPolicy: ArtifactIntegrityPolicyName;
|
||||
readonly resourceScope: ArtifactResourceScope;
|
||||
}
|
||||
|
||||
/** Extract every regular inventory occurrence into an exclusively owned absent root. */
|
||||
export const extractArtifact = async (
|
||||
input: ArtifactExtractionInput,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArtifactExtractionResult> =>
|
||||
input.resourceScope.run(() => extractArtifactInScope(input, signal));
|
||||
|
||||
const extractArtifactInScope = async (
|
||||
input: ArtifactExtractionInput,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArtifactExtractionResult> => {
|
||||
// Cancellation wins over every later refusal, including unsupported formats.
|
||||
abortIfNeeded(signal);
|
||||
const sourcePath = await realpath(input.inputPath);
|
||||
await requireExtractableFormat(sourcePath, input);
|
||||
const snapshot = await scanArtifactInventory(sourcePath, {
|
||||
const snapshot = await scanArtifactInventoryInScope(sourcePath, {
|
||||
resourceScope: input.resourceScope,
|
||||
signal,
|
||||
environment: input.environment,
|
||||
integrity: { mode: input.integrityPolicy },
|
||||
});
|
||||
return materializeArtifactInventory(input, sourcePath, snapshot, signal);
|
||||
return materializeArtifactInventoryInScope(
|
||||
input,
|
||||
sourcePath,
|
||||
snapshot,
|
||||
signal,
|
||||
);
|
||||
};
|
||||
|
||||
/** Materialize a scanned inventory, verifying every current occurrence against it. */
|
||||
@@ -63,6 +81,17 @@ export const materializeArtifactInventory = async (
|
||||
sourcePath: string,
|
||||
snapshot: ArtifactInventorySnapshot,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArtifactExtractionResult> => {
|
||||
return input.resourceScope.run(() =>
|
||||
materializeArtifactInventoryInScope(input, sourcePath, snapshot, signal),
|
||||
);
|
||||
};
|
||||
|
||||
const materializeArtifactInventoryInScope = async (
|
||||
input: ArtifactExtractionInput,
|
||||
sourcePath: string,
|
||||
snapshot: ArtifactInventorySnapshot,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArtifactExtractionResult> => {
|
||||
abortIfNeeded(signal);
|
||||
const selectedOccurrences = snapshot.occurrences.filter(
|
||||
@@ -90,20 +119,14 @@ export const materializeArtifactInventory = async (
|
||||
const selectedIds = new Set(
|
||||
activeOccurrences.map(({ occurrence_id: id }) => id),
|
||||
);
|
||||
const occurrences = new Map<string, ArtifactOccurrence>();
|
||||
const neededNodes = new Set<string>();
|
||||
collectOccurrences(
|
||||
snapshot.occurrences,
|
||||
selectedIds,
|
||||
occurrences,
|
||||
neededNodes,
|
||||
const neededNodes = new Set(
|
||||
activeOccurrences.map(({ artifact_id: id }) => id),
|
||||
);
|
||||
const nodes = new Map<string, ArtifactNode>();
|
||||
collectNodes(snapshot.nodes, neededNodes, nodes);
|
||||
const inventory: LoadedInventory = {
|
||||
for (const node of snapshot.nodes)
|
||||
if (neededNodes.has(node.artifact_id)) nodes.set(node.artifact_id, node);
|
||||
const inventory: ExtractionInventory = {
|
||||
manifest: snapshot.manifest,
|
||||
occurrences,
|
||||
nodes,
|
||||
integrityContradictions: snapshot.integrity_contradictions.filter(
|
||||
({ occurrence_id: id }) => selectedIds.has(id),
|
||||
),
|
||||
@@ -144,7 +167,7 @@ export const materializeArtifactInventory = async (
|
||||
"format",
|
||||
`Selected occurrence is not an extractable regular child file: ${occurrence.logical_path} (${occurrence.occurrence_id})`,
|
||||
);
|
||||
const node = inventory.nodes.get(occurrence.artifact_id);
|
||||
const node = nodes.get(occurrence.artifact_id);
|
||||
if (node === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
@@ -183,7 +206,7 @@ const materializeSelection = async ({
|
||||
}: {
|
||||
readonly input: ArtifactExtractionInput;
|
||||
readonly sourcePath: string;
|
||||
readonly inventory: LoadedInventory;
|
||||
readonly inventory: ExtractionInventory;
|
||||
readonly selected: readonly SelectedOccurrence[];
|
||||
readonly signal: AbortSignal | undefined;
|
||||
}): Promise<ArtifactExtractionResult> => {
|
||||
@@ -191,114 +214,150 @@ const materializeSelection = async ({
|
||||
selected.map((item) => [item.occurrence.logical_path, item]),
|
||||
);
|
||||
const reader = await createReader(sourcePath, input);
|
||||
const readerOwner: ArtifactResourceOwner = {
|
||||
kind: "reader" as const,
|
||||
reader,
|
||||
resource: `artifact reader for ${sourcePath}`,
|
||||
};
|
||||
let localReaderOwner: ArtifactResourceOwner | undefined = readerOwner;
|
||||
let output: SafeOutputTree | undefined;
|
||||
let readerCloseAttempted = false;
|
||||
let readerClosed = false;
|
||||
let completed: ArtifactExtractionResult | undefined;
|
||||
const extracted: ExtractedOccurrence[] = [];
|
||||
try {
|
||||
output = await SafeOutputTree.create(input.outputRoot);
|
||||
const registry = new ArtifactPathRegistry();
|
||||
for await (const entry of reader.entries(signal)) {
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
registry.add(path, entry.kind);
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await reader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
stream,
|
||||
{
|
||||
sha256: selectedItem.node.sha256,
|
||||
bytes: selectedItem.node.size,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
extracted.push({
|
||||
artifact_id: selectedItem.node.artifact_id,
|
||||
relative_path: written.relativePath,
|
||||
sha256: written.sha256,
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
byPath.delete(path);
|
||||
try {
|
||||
output = await SafeOutputTree.create(input.outputRoot);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof SafeOutputTreeCreationFailure) output = cause.tree;
|
||||
throw cause;
|
||||
}
|
||||
if (byPath.size > 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Inventoried regular artifact entries were not materialized: ${[...byPath.keys()].sort(compareUnicodeCodePoints).join(", ")}`,
|
||||
await writeSelectedEntries({ reader, output, byPath, extracted, signal });
|
||||
localReaderOwner = undefined;
|
||||
const closeAttempt = await input.resourceScope.release(readerOwner);
|
||||
if (closeAttempt.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
closeAttempt.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
closeAttempt.cause,
|
||||
readerOwner.resource,
|
||||
),
|
||||
);
|
||||
readerCloseAttempted = true;
|
||||
await reader.close();
|
||||
readerClosed = true;
|
||||
extracted.sort((left, right) =>
|
||||
compareUnicodeCodePoints(left.relative_path, right.relative_path),
|
||||
);
|
||||
const result = createExtractionResult(
|
||||
input,
|
||||
inventory,
|
||||
selected,
|
||||
extracted,
|
||||
);
|
||||
completed = createExtractionResult(input, inventory, selected, extracted);
|
||||
await output.commit();
|
||||
return result;
|
||||
return completed;
|
||||
} catch (cause: unknown) {
|
||||
const cleanupFailures: {
|
||||
readonly cause: unknown;
|
||||
readonly resource: string;
|
||||
}[] = [];
|
||||
if (readerCloseAttempted && !readerClosed)
|
||||
cleanupFailures.push({
|
||||
cause,
|
||||
resource: `artifact reader for ${sourcePath}`,
|
||||
});
|
||||
if (!readerCloseAttempted) {
|
||||
try {
|
||||
readerCloseAttempted = true;
|
||||
await reader.close();
|
||||
} catch (cleanupCause: unknown) {
|
||||
cleanupFailures.push({
|
||||
cause: cleanupCause,
|
||||
resource: `artifact reader for ${sourcePath}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (output !== undefined) {
|
||||
try {
|
||||
await output.rollback();
|
||||
} catch (cleanupCause: unknown) {
|
||||
cleanupFailures.push({
|
||||
cause: cleanupCause,
|
||||
resource: input.outputRoot,
|
||||
});
|
||||
}
|
||||
}
|
||||
const cleanupFailures = await releaseExtractionOwners(
|
||||
input.resourceScope,
|
||||
localReaderOwner,
|
||||
output,
|
||||
input.outputRoot,
|
||||
);
|
||||
if (cleanupFailures.length > 0) {
|
||||
const observations = cleanupFailures.map(
|
||||
({ cause: cleanupCause, resource }) =>
|
||||
ArtifactReaderFailure.cleanupObservation(cleanupCause, resource),
|
||||
);
|
||||
throw ArtifactReaderFailure.withCleanup(cause, {
|
||||
reason: observations.map(({ reason }) => reason).join("; "),
|
||||
resources: [
|
||||
...new Set(observations.flatMap(({ resources }) => resources)),
|
||||
],
|
||||
});
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cause,
|
||||
{
|
||||
reason: observations.map(({ reason }) => reason).join("; "),
|
||||
resources: [
|
||||
...new Set(observations.flatMap(({ resources }) => resources)),
|
||||
],
|
||||
},
|
||||
output?.published === true && completed !== undefined
|
||||
? { kind: "artifact-extraction", extraction: completed }
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
if (output?.published === true && completed !== undefined) return completed;
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
const writeSelectedEntries = async ({
|
||||
reader,
|
||||
output,
|
||||
byPath,
|
||||
extracted,
|
||||
signal,
|
||||
}: {
|
||||
readonly reader: ArtifactReader;
|
||||
readonly output: SafeOutputTree;
|
||||
readonly byPath: Map<string, SelectedOccurrence>;
|
||||
readonly extracted: ExtractedOccurrence[];
|
||||
readonly signal: AbortSignal | undefined;
|
||||
}): Promise<void> => {
|
||||
const registry = new ArtifactPathRegistry();
|
||||
for await (const entry of reader.entries(signal)) {
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
registry.add(path, entry.kind);
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await reader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
stream,
|
||||
{
|
||||
sha256: selectedItem.node.sha256,
|
||||
bytes: selectedItem.node.size,
|
||||
},
|
||||
signal,
|
||||
);
|
||||
extracted.push({
|
||||
artifact_id: selectedItem.node.artifact_id,
|
||||
relative_path: written.relativePath,
|
||||
sha256: written.sha256,
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
byPath.delete(path);
|
||||
}
|
||||
if (byPath.size > 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Inventoried regular artifact entries were not materialized: ${[...byPath.keys()].sort(compareUnicodeCodePoints).join(", ")}`,
|
||||
);
|
||||
};
|
||||
|
||||
const releaseExtractionOwners = async (
|
||||
resourceScope: ArtifactResourceScope,
|
||||
readerOwner: ArtifactResourceOwner | undefined,
|
||||
output: SafeOutputTree | undefined,
|
||||
outputRoot: string,
|
||||
): Promise<{ readonly cause: unknown; readonly resource: string }[]> => {
|
||||
const failures: { readonly cause: unknown; readonly resource: string }[] = [];
|
||||
if (readerOwner !== undefined) {
|
||||
const attempt = await resourceScope.release(readerOwner);
|
||||
if (attempt.kind === "failed")
|
||||
failures.push({ cause: attempt.cause, resource: readerOwner.resource });
|
||||
}
|
||||
if (output !== undefined) {
|
||||
const owner: ArtifactResourceOwner = {
|
||||
kind: "output-tree",
|
||||
tree: output,
|
||||
resource: outputRoot,
|
||||
};
|
||||
const attempt = await resourceScope.release(owner);
|
||||
if (attempt.kind === "failed")
|
||||
failures.push({ cause: attempt.cause, resource: owner.resource });
|
||||
}
|
||||
return failures;
|
||||
};
|
||||
|
||||
const createExtractionResult = (
|
||||
input: ArtifactExtractionInput,
|
||||
inventory: LoadedInventory,
|
||||
inventory: ExtractionInventory,
|
||||
selected: readonly SelectedOccurrence[],
|
||||
extracted: readonly ExtractedOccurrence[],
|
||||
): ArtifactExtractionResult => {
|
||||
@@ -333,35 +392,11 @@ const createExtractionResult = (
|
||||
});
|
||||
};
|
||||
|
||||
interface LoadedInventory {
|
||||
interface ExtractionInventory {
|
||||
readonly manifest: ArtifactGraphManifest;
|
||||
readonly occurrences: ReadonlyMap<string, ArtifactOccurrence>;
|
||||
readonly nodes: ReadonlyMap<string, ArtifactNode>;
|
||||
readonly integrityContradictions: readonly IntegrityContradiction[];
|
||||
}
|
||||
|
||||
const collectOccurrences = (
|
||||
items: readonly ArtifactOccurrence[],
|
||||
selected: ReadonlySet<string>,
|
||||
output: Map<string, ArtifactOccurrence>,
|
||||
neededNodes: Set<string>,
|
||||
): void => {
|
||||
for (const item of items) {
|
||||
if (!selected.has(item.occurrence_id)) continue;
|
||||
output.set(item.occurrence_id, item);
|
||||
if (item.artifact_id !== null) neededNodes.add(item.artifact_id);
|
||||
}
|
||||
};
|
||||
|
||||
const collectNodes = (
|
||||
items: readonly ArtifactNode[],
|
||||
selected: ReadonlySet<string>,
|
||||
output: Map<string, ArtifactNode>,
|
||||
): void => {
|
||||
for (const item of items)
|
||||
if (selected.has(item.artifact_id)) output.set(item.artifact_id, item);
|
||||
};
|
||||
|
||||
const ZIP_FORMATS = ["ipa", "apk", "msix", "appx", "zip"] as const;
|
||||
|
||||
const isZipFormat = (
|
||||
|
||||
@@ -5,14 +5,14 @@ import {
|
||||
type ArtifactInventoryResult,
|
||||
} from "../../domain/artifactGraph.js";
|
||||
import { abortIfNeeded } from "../ArtifactHash.js";
|
||||
import { scanCanonicalArtifactInventory } from "./scanCanonical.js";
|
||||
import { scanCanonicalArtifactInventoryInScope } from "./scanCanonical.js";
|
||||
import type { ArtifactInventoryOptions } from "./types.js";
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
|
||||
/** Inventory one local artifact and return every graph collection inline. */
|
||||
export const inventoryArtifact = async (
|
||||
inputPath: string,
|
||||
options: ArtifactInventoryOptions = {},
|
||||
options: ArtifactInventoryOptions,
|
||||
): Promise<ArtifactInventoryResult> => {
|
||||
const snapshot = await scanArtifactInventory(inputPath, options);
|
||||
return artifactInventoryResultSchema.parse(snapshot);
|
||||
@@ -21,9 +21,17 @@ export const inventoryArtifact = async (
|
||||
/** Scan an artifact once and retain the complete immutable graph for projection. */
|
||||
export const scanArtifactInventory = async (
|
||||
inputPath: string,
|
||||
options: ArtifactInventoryOptions = {},
|
||||
options: ArtifactInventoryOptions,
|
||||
): Promise<ArtifactInventorySnapshot> =>
|
||||
options.resourceScope.run(() =>
|
||||
scanArtifactInventoryInScope(inputPath, options),
|
||||
);
|
||||
|
||||
export const scanArtifactInventoryInScope = async (
|
||||
inputPath: string,
|
||||
options: ArtifactInventoryOptions,
|
||||
): Promise<ArtifactInventorySnapshot> => {
|
||||
abortIfNeeded(options.signal);
|
||||
const path = await realpath(inputPath);
|
||||
return scanCanonicalArtifactInventory(path, options);
|
||||
return scanCanonicalArtifactInventoryInScope(path, options);
|
||||
};
|
||||
|
||||
@@ -4,7 +4,7 @@ import { join } from "node:path";
|
||||
import { TextReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { inventoryArtifact } from "./ArtifactInventory.js";
|
||||
import { inventoryArtifact } from "../../../tests/fixtures/artifactInventory.js";
|
||||
import { compareArtifacts } from "../../domain/artifactComparison.js";
|
||||
import { projectAppleApplication } from "../../domain/apple/appleApplication.js";
|
||||
import { createEvidence } from "../../domain/evidence.js";
|
||||
|
||||
@@ -1,11 +1,8 @@
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
openRegularFile,
|
||||
sameRegularFileState,
|
||||
} from "../../filesystem/RegularFile.js";
|
||||
import { sameRegularFileState } from "../../filesystem/RegularFile.js";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
|
||||
import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
|
||||
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
|
||||
import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";
|
||||
@@ -15,8 +12,13 @@ import {
|
||||
zipPackageFormatForPath,
|
||||
} from "../../domain/zipPackageFormat.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { type ArtifactResourceOwner } from "../ArtifactResourceScope.js";
|
||||
import type { HashResult } from "../ArtifactHash.js";
|
||||
import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js";
|
||||
import {
|
||||
hashStableRootArtifactHandle,
|
||||
openRootArtifact,
|
||||
} from "./hashStableRootArtifact.js";
|
||||
import type { ArtifactInventoryOptions } from "./types.js";
|
||||
|
||||
interface RootClassification {
|
||||
readonly format: ArtifactOccurrence["artifact_format"];
|
||||
@@ -24,27 +26,47 @@ interface RootClassification {
|
||||
}
|
||||
|
||||
type RootInventoryClassification = RootClassification & {
|
||||
readonly rootSource: StableRegularFileDescriptor | undefined;
|
||||
readonly rootSource: RootInventorySource | undefined;
|
||||
};
|
||||
|
||||
/** The admitted descriptor and the single owner that can confirm its close. */
|
||||
export interface RootInventorySource extends StableRegularFileDescriptor {
|
||||
readonly owner: OwnedFileHandle;
|
||||
}
|
||||
|
||||
/** Classify and hash one file root through the same stable open descriptor. */
|
||||
export const classifyAndHashRoot = async (
|
||||
path: string,
|
||||
directory: boolean,
|
||||
expectedMetadata: Stats,
|
||||
signal?: AbortSignal,
|
||||
options: Pick<ArtifactInventoryOptions, "resourceScope" | "signal">,
|
||||
): Promise<RootClassification> => {
|
||||
const result = await classifyAndHashRootForInventory(
|
||||
path,
|
||||
directory,
|
||||
expectedMetadata,
|
||||
signal,
|
||||
);
|
||||
try {
|
||||
return options.resourceScope.run(async () => {
|
||||
const result = await classifyAndHashRootForInventory(
|
||||
path,
|
||||
directory,
|
||||
expectedMetadata,
|
||||
options,
|
||||
);
|
||||
const rootSource = result.rootSource;
|
||||
if (rootSource !== undefined) {
|
||||
const owner: ArtifactResourceOwner = {
|
||||
kind: "file-handle",
|
||||
handle: rootSource.owner,
|
||||
resource: `root artifact descriptor for ${path}`,
|
||||
};
|
||||
const cleanupAttempt = await options.resourceScope.release(owner);
|
||||
if (cleanupAttempt.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cleanupAttempt.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupAttempt.cause,
|
||||
owner.resource,
|
||||
),
|
||||
);
|
||||
}
|
||||
return { format: result.format, digest: result.digest };
|
||||
} finally {
|
||||
await result.rootSource?.handle.close();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
/** Classify a root and retain its admitted ZIP descriptor for child inventory. */
|
||||
@@ -52,12 +74,27 @@ export const classifyAndHashRootForInventory = async (
|
||||
path: string,
|
||||
directory: boolean,
|
||||
expectedMetadata: Stats,
|
||||
signal?: AbortSignal,
|
||||
options: Pick<ArtifactInventoryOptions, "resourceScope" | "signal">,
|
||||
): Promise<RootInventoryClassification> => {
|
||||
if (directory)
|
||||
return { format: "directory", digest: null, rootSource: undefined };
|
||||
const handle = await openRootFile(path, signal);
|
||||
let transferHandle = false;
|
||||
const ownedHandle = await openRootArtifact(
|
||||
path,
|
||||
options.resourceScope,
|
||||
options.signal,
|
||||
);
|
||||
const handle = ownedHandle.handle;
|
||||
const owner: ArtifactResourceOwner = {
|
||||
kind: "file-handle",
|
||||
handle: ownedHandle,
|
||||
resource: `root artifact descriptor for ${path}`,
|
||||
};
|
||||
let outcome:
|
||||
| {
|
||||
readonly kind: "completed";
|
||||
readonly value: RootInventoryClassification;
|
||||
}
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
try {
|
||||
const initial = await handle.stat();
|
||||
if (!sameRegularFileState(expectedMetadata, initial))
|
||||
@@ -70,18 +107,35 @@ export const classifyAndHashRootForInventory = async (
|
||||
path,
|
||||
handle,
|
||||
initial,
|
||||
signal,
|
||||
options.signal,
|
||||
);
|
||||
const retainSource = isZipFormat(format) || format === "mach-o-universal";
|
||||
if (retainSource) transferHandle = true;
|
||||
return {
|
||||
format,
|
||||
digest,
|
||||
rootSource: retainSource ? { handle, initial } : undefined,
|
||||
outcome = {
|
||||
kind: "completed",
|
||||
value: {
|
||||
format,
|
||||
digest,
|
||||
rootSource: retainSource
|
||||
? { handle, initial, owner: ownedHandle }
|
||||
: undefined,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
if (!transferHandle) await handle.close();
|
||||
} catch (cause: unknown) {
|
||||
outcome = { kind: "failed", cause };
|
||||
}
|
||||
if (outcome.kind === "failed" || outcome.value.rootSource === undefined) {
|
||||
const cleanupAttempt = await options.resourceScope.release(owner);
|
||||
if (cleanupAttempt.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
outcome.kind === "failed" ? outcome.cause : cleanupAttempt.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupAttempt.cause,
|
||||
owner.resource,
|
||||
),
|
||||
);
|
||||
}
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
return outcome.value;
|
||||
};
|
||||
|
||||
const isZipFormat = (format: ArtifactOccurrence["artifact_format"]): boolean =>
|
||||
@@ -91,23 +145,6 @@ const isZipFormat = (format: ArtifactOccurrence["artifact_format"]): boolean =>
|
||||
format === "msix" ||
|
||||
format === "appx";
|
||||
|
||||
const openRootFile = async (
|
||||
path: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FileHandle> => {
|
||||
try {
|
||||
return await openRegularFile(path, { symlinks: "reject", signal });
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof NonRegularFileReadError)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Artifact root is not a regular file: ${path}`,
|
||||
{ cause },
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
const classifyRootFormat = async (
|
||||
path: string,
|
||||
handle: FileHandle,
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
import { lstat } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { readFileHandleChunks } from "../../filesystem/readFileHandleChunks.js";
|
||||
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import {
|
||||
NonRegularFileReadError,
|
||||
RegularFileAdmissionFailure,
|
||||
openRegularFile,
|
||||
sameRegularFileState,
|
||||
} from "../../filesystem/RegularFile.js";
|
||||
@@ -13,33 +16,79 @@ import {
|
||||
hashReadable,
|
||||
type HashResult,
|
||||
} from "../ArtifactHash.js";
|
||||
import type { ArtifactResourceScope } from "../ArtifactResourceScope.js";
|
||||
|
||||
/** Admit a root descriptor without losing ownership when validation cleanup fails. */
|
||||
export const openRootArtifact = async (
|
||||
path: string,
|
||||
resourceScope: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<OwnedFileHandle> => {
|
||||
try {
|
||||
return new OwnedFileHandle(
|
||||
await openRegularFile(path, { symlinks: "reject", signal }),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
const primary =
|
||||
cause instanceof RegularFileAdmissionFailure ? cause.cause : cause;
|
||||
const failure =
|
||||
primary instanceof NonRegularFileReadError
|
||||
? new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Artifact root is not a regular file: ${path}`,
|
||||
{ cause: primary },
|
||||
)
|
||||
: primary;
|
||||
if (cause instanceof RegularFileAdmissionFailure) {
|
||||
const resource = `root artifact descriptor for ${path}`;
|
||||
const cleanup = await resourceScope.release({
|
||||
kind: "file-handle",
|
||||
resource,
|
||||
handle: cause.owner,
|
||||
});
|
||||
if (cleanup.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
failure,
|
||||
ArtifactReaderFailure.cleanupObservation(cleanup.cause, resource),
|
||||
);
|
||||
}
|
||||
throw failure;
|
||||
}
|
||||
};
|
||||
|
||||
/** Hash one stable regular root file from the same descriptor that was checked. */
|
||||
export const hashStableRootArtifact = async (
|
||||
path: string,
|
||||
resourceScope: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<HashResult> => {
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await openRegularFile(path, {
|
||||
symlinks: "reject",
|
||||
signal,
|
||||
});
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof NonRegularFileReadError)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Artifact root is not a regular file: ${path}`,
|
||||
{ cause },
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
const owner = await openRootArtifact(path, resourceScope, signal);
|
||||
const handle = owner.handle;
|
||||
let outcome:
|
||||
| { readonly kind: "completed"; readonly value: HashResult }
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
try {
|
||||
const initial = await handle.stat();
|
||||
return await hashStableRootArtifactHandle(path, handle, initial, signal);
|
||||
} finally {
|
||||
await handle.close();
|
||||
outcome = {
|
||||
kind: "completed",
|
||||
value: await hashStableRootArtifactHandle(path, handle, initial, signal),
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
outcome = { kind: "failed", cause };
|
||||
}
|
||||
const resource = `root artifact descriptor for ${path}`;
|
||||
const cleanup = await resourceScope.release({
|
||||
kind: "file-handle",
|
||||
resource,
|
||||
handle: owner,
|
||||
});
|
||||
if (cleanup.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
outcome.kind === "failed" ? outcome.cause : cleanup.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(cleanup.cause, resource),
|
||||
);
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
return outcome.value;
|
||||
};
|
||||
|
||||
/** Hash and revalidate the same open descriptor used for root classification. */
|
||||
@@ -52,7 +101,7 @@ export const hashStableRootArtifactHandle = async (
|
||||
const digest = await hashReadable(
|
||||
// Read at most the admitted extent plus one byte to detect growth without
|
||||
// following an actively appended file indefinitely.
|
||||
handle.createReadStream({ autoClose: false, start: 0, end: initial.size }),
|
||||
readFileHandleChunks(handle, { start: 0, end: initial.size }),
|
||||
signal,
|
||||
);
|
||||
abortIfNeeded(signal);
|
||||
|
||||
@@ -8,13 +8,12 @@ import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
|
||||
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js";
|
||||
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
|
||||
|
||||
export const createReader = async (
|
||||
export const createReader = (
|
||||
path: string,
|
||||
format: ArtifactOccurrence["artifact_format"],
|
||||
environment: Readonly<NodeJS.ProcessEnv>,
|
||||
signal?: AbortSignal,
|
||||
rootSource?: StableRegularFileDescriptor,
|
||||
): Promise<ArtifactReader | undefined> => {
|
||||
): ArtifactReader | undefined => {
|
||||
switch (format) {
|
||||
case "directory":
|
||||
return new DirectoryArtifactReader(path);
|
||||
@@ -32,7 +31,7 @@ export const createReader = async (
|
||||
: undefined;
|
||||
case "dmg":
|
||||
if (process.platform !== "darwin") return undefined;
|
||||
return NativeDmgArtifactReader.create(path, environment, signal);
|
||||
return new NativeDmgArtifactReader(path, environment);
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -5,12 +5,13 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
|
||||
import { lstat } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
|
||||
import type { RootInventorySource } from "./classify.js";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactReader,
|
||||
} from "../ArtifactReader.js";
|
||||
import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js";
|
||||
import {
|
||||
artifactInventoryResultSchema,
|
||||
type ArtifactInventoryResult,
|
||||
@@ -50,7 +51,16 @@ import {
|
||||
|
||||
export const scanCanonicalArtifactInventory = async (
|
||||
path: string,
|
||||
options: ArtifactInventoryOptions = {},
|
||||
options: ArtifactInventoryOptions,
|
||||
readerFactory: typeof createReader = createReader,
|
||||
): Promise<ArtifactInventorySnapshot> =>
|
||||
options.resourceScope.run(() =>
|
||||
scanCanonicalArtifactInventoryInScope(path, options, readerFactory),
|
||||
);
|
||||
|
||||
export const scanCanonicalArtifactInventoryInScope = async (
|
||||
path: string,
|
||||
options: ArtifactInventoryOptions,
|
||||
readerFactory: typeof createReader = createReader,
|
||||
): Promise<ArtifactInventorySnapshot> => {
|
||||
const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
|
||||
@@ -63,28 +73,22 @@ export const scanCanonicalArtifactInventory = async (
|
||||
path,
|
||||
metadata.isDirectory(),
|
||||
metadata,
|
||||
options.signal,
|
||||
options,
|
||||
);
|
||||
let reader: ArtifactReader | undefined;
|
||||
let readerCreationFailure: { readonly cause: unknown } | undefined;
|
||||
try {
|
||||
reader = await readerFactory(
|
||||
reader = readerFactory(
|
||||
path,
|
||||
rootFormat,
|
||||
options.environment ?? {},
|
||||
options.signal,
|
||||
rootSource,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
readerCreationFailure = { cause };
|
||||
}
|
||||
const ownedReaders: ArtifactReader[] = reader === undefined ? [] : [reader];
|
||||
let outcome:
|
||||
| {
|
||||
readonly kind: "completed";
|
||||
readonly snapshot: ArtifactInventorySnapshot;
|
||||
}
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
let outcome: InventoryScanOutcome;
|
||||
try {
|
||||
if (readerCreationFailure !== undefined) throw readerCreationFailure.cause;
|
||||
if (reader instanceof AsarArtifactReader)
|
||||
@@ -103,6 +107,7 @@ export const scanCanonicalArtifactInventory = async (
|
||||
rootFormat,
|
||||
rootDigest,
|
||||
rootSource,
|
||||
resourceScope: options.resourceScope,
|
||||
reader,
|
||||
signal: options.signal,
|
||||
nodes,
|
||||
@@ -114,54 +119,76 @@ export const scanCanonicalArtifactInventory = async (
|
||||
} catch (cause: unknown) {
|
||||
outcome = { kind: "failed", cause };
|
||||
}
|
||||
let cleanupFailure: ArtifactReaderFailure | undefined;
|
||||
for (const owned of ownedReaders.reverse()) {
|
||||
try {
|
||||
await owned.close();
|
||||
} catch (cleanupCause: unknown) {
|
||||
const cleanup = ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupCause,
|
||||
`artifact reader for ${path}`,
|
||||
);
|
||||
cleanupFailure = ArtifactReaderFailure.withCleanup(
|
||||
cleanupFailure ??
|
||||
(outcome.kind === "failed" ? outcome.cause : cleanupCause),
|
||||
cleanup,
|
||||
outcome.kind === "completed"
|
||||
? { kind: "artifact-inventory", inventory: outcome.snapshot }
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (rootSource !== undefined) {
|
||||
try {
|
||||
await rootSource.handle.close();
|
||||
} catch (cleanupCause: unknown) {
|
||||
const cleanup = ArtifactReaderFailure.cleanupObservation(
|
||||
cleanupCause,
|
||||
`root artifact descriptor for ${path}`,
|
||||
);
|
||||
cleanupFailure = ArtifactReaderFailure.withCleanup(
|
||||
cleanupFailure ??
|
||||
(outcome.kind === "failed" ? outcome.cause : cleanupCause),
|
||||
cleanup,
|
||||
outcome.kind === "completed"
|
||||
? { kind: "artifact-inventory", inventory: outcome.snapshot }
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
}
|
||||
const cleanupFailure = await cleanupInventoryOwners({
|
||||
path,
|
||||
options,
|
||||
rootSource,
|
||||
ownedReaders,
|
||||
outcome,
|
||||
});
|
||||
if (cleanupFailure !== undefined) throw cleanupFailure;
|
||||
if (outcome.kind === "failed") throw outcome.cause;
|
||||
return outcome.snapshot;
|
||||
};
|
||||
|
||||
type InventoryScanOutcome =
|
||||
| {
|
||||
readonly kind: "completed";
|
||||
readonly snapshot: ArtifactInventorySnapshot;
|
||||
}
|
||||
| { readonly kind: "failed"; readonly cause: unknown };
|
||||
|
||||
const cleanupInventoryOwners = async ({
|
||||
path,
|
||||
options,
|
||||
rootSource,
|
||||
ownedReaders,
|
||||
outcome,
|
||||
}: {
|
||||
readonly path: string;
|
||||
readonly options: ArtifactInventoryOptions;
|
||||
readonly rootSource: RootInventorySource | undefined;
|
||||
readonly ownedReaders: readonly ArtifactReader[];
|
||||
readonly outcome: InventoryScanOutcome;
|
||||
}): Promise<ArtifactReaderFailure | undefined> => {
|
||||
let cleanupFailure: ArtifactReaderFailure | undefined;
|
||||
const owners: ArtifactResourceOwner[] = [...ownedReaders]
|
||||
.reverse()
|
||||
.map((reader) => ({
|
||||
kind: "reader" as const,
|
||||
reader,
|
||||
resource: `artifact reader for ${path}`,
|
||||
}));
|
||||
if (rootSource !== undefined)
|
||||
owners.push({
|
||||
kind: "file-handle",
|
||||
handle: rootSource.owner,
|
||||
resource: `root artifact descriptor for ${path}`,
|
||||
});
|
||||
for (const owner of owners) {
|
||||
const attempt = await options.resourceScope.release(owner);
|
||||
if (attempt.kind === "released") continue;
|
||||
const primary =
|
||||
cleanupFailure ??
|
||||
(outcome.kind === "failed" ? outcome.cause : attempt.cause);
|
||||
cleanupFailure = ArtifactReaderFailure.withCleanup(
|
||||
primary,
|
||||
ArtifactReaderFailure.cleanupObservation(attempt.cause, owner.resource),
|
||||
outcome.kind === "completed"
|
||||
? { kind: "artifact-inventory", inventory: outcome.snapshot }
|
||||
: undefined,
|
||||
);
|
||||
}
|
||||
return cleanupFailure;
|
||||
};
|
||||
|
||||
interface SnapshotBuildInput {
|
||||
readonly resourceScope: ArtifactInventoryOptions["resourceScope"];
|
||||
readonly path: string;
|
||||
readonly metadata: Stats;
|
||||
readonly rootFormat: ArtifactOccurrence["artifact_format"];
|
||||
readonly rootDigest: HashResult | null;
|
||||
readonly rootSource: StableRegularFileDescriptor | undefined;
|
||||
readonly rootSource: RootInventorySource | undefined;
|
||||
readonly reader: ArtifactReader | undefined;
|
||||
readonly signal: AbortSignal | undefined;
|
||||
readonly nodes: Map<string, ArtifactNode>;
|
||||
@@ -218,7 +245,13 @@ const buildInventorySnapshot = async (
|
||||
ordinal,
|
||||
}));
|
||||
|
||||
await verifyRootDigest(path, rootDigest, rootSource, signal);
|
||||
await verifyRootDigest(
|
||||
path,
|
||||
rootDigest,
|
||||
rootSource,
|
||||
input.resourceScope,
|
||||
signal,
|
||||
);
|
||||
|
||||
const graphSha256 = artifactGraphDigest({
|
||||
nodes: orderedNodes,
|
||||
@@ -293,13 +326,14 @@ const buildIntegrityContradictions = (
|
||||
const verifyRootDigest = async (
|
||||
path: string,
|
||||
rootDigest: HashResult | null,
|
||||
rootSource: StableRegularFileDescriptor | undefined,
|
||||
rootSource: RootInventorySource | undefined,
|
||||
resourceScope: ArtifactInventoryOptions["resourceScope"],
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<void> => {
|
||||
if (rootDigest === null) return;
|
||||
const verified =
|
||||
rootSource === undefined
|
||||
? await hashStableRootArtifact(path, signal)
|
||||
? await hashStableRootArtifact(path, resourceScope, signal)
|
||||
: await hashStableRootArtifactHandle(
|
||||
path,
|
||||
rootSource.handle,
|
||||
|
||||
@@ -120,6 +120,7 @@ const visitArtifactEntries = async (
|
||||
iterator: currentReader.entries(context.signal)[Symbol.asyncIterator](),
|
||||
},
|
||||
];
|
||||
let failure: { readonly cause: unknown } | undefined;
|
||||
try {
|
||||
while (stack.length > 0) {
|
||||
const frame = stack.at(-1);
|
||||
@@ -194,10 +195,27 @@ const visitArtifactEntries = async (
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch (cause: unknown) {
|
||||
failure = { cause };
|
||||
} finally {
|
||||
// Readers remain owned by the inventory until its complete observation is built.
|
||||
for (const { iterator } of stack) await iterator.return?.();
|
||||
for (const { iterator, prefix, reader } of stack.reverse()) {
|
||||
try {
|
||||
await iterator.return?.();
|
||||
} catch (cause: unknown) {
|
||||
failure = {
|
||||
cause: ArtifactReaderFailure.withCleanup(
|
||||
failure === undefined ? cause : failure.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cause,
|
||||
`artifact traversal for ${prefix || reader.format}`,
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failure !== undefined) throw failure.cause;
|
||||
};
|
||||
|
||||
const digestArtifactEntry = async (
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { ArtifactResourceScope } from "../ArtifactResourceScope.js";
|
||||
|
||||
/** Resolved integrity behavior admitted to the artifact scanner. */
|
||||
export type ArtifactIntegrityPolicy =
|
||||
| { readonly mode: "fail" }
|
||||
@@ -9,6 +11,7 @@ export const STRICT_INTEGRITY_POLICY: ArtifactIntegrityPolicy = {
|
||||
|
||||
/** Options shared by artifact inventory scans. */
|
||||
export interface ArtifactInventoryOptions {
|
||||
readonly resourceScope: ArtifactResourceScope;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly integrity?: ArtifactIntegrityPolicy | undefined;
|
||||
readonly environment?: Readonly<NodeJS.ProcessEnv> | undefined;
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { Readable } from "node:stream";
|
||||
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import type { ArtifactResourceScope } from "../ArtifactResourceScope.js";
|
||||
import {
|
||||
ArtifactPathRegistry,
|
||||
normalizeArtifactPath,
|
||||
@@ -39,6 +40,7 @@ interface ExpectedContainer {
|
||||
}
|
||||
|
||||
interface ReadContext {
|
||||
readonly resources: ArtifactResourceScope;
|
||||
readonly expected: ReadonlyMap<string, ExpectedFile>;
|
||||
readonly expectedContainers: ReadonlyMap<string, ExpectedContainer>;
|
||||
readonly registry: ArtifactPathRegistry;
|
||||
@@ -59,6 +61,7 @@ interface ReadTextInput {
|
||||
export const readJavaScriptArtifactFiles = async (
|
||||
reader: ArtifactReader,
|
||||
snapshot: ArtifactInventorySnapshot,
|
||||
resources: ArtifactResourceScope,
|
||||
signal?: AbortSignal,
|
||||
): Promise<JavaScriptArtifactFileSet> => {
|
||||
if (reader instanceof AsarArtifactReader) {
|
||||
@@ -79,6 +82,7 @@ export const readJavaScriptArtifactFiles = async (
|
||||
const inventory = expectedInventory(snapshot);
|
||||
const expected = inventory.files;
|
||||
const context: ReadContext = {
|
||||
resources,
|
||||
expected,
|
||||
expectedContainers: inventory.containers,
|
||||
registry: new ArtifactPathRegistry(),
|
||||
@@ -122,7 +126,10 @@ const visitReader = async (
|
||||
iterator: reader.entries(context.signal)[Symbol.asyncIterator](),
|
||||
},
|
||||
];
|
||||
const ownedReaders: ArtifactReader[] = [];
|
||||
const ownedReaders: {
|
||||
readonly reader: ArtifactReader;
|
||||
readonly path: string;
|
||||
}[] = [];
|
||||
let failure: { readonly cause: unknown } | undefined;
|
||||
try {
|
||||
while (stack.length > 0) {
|
||||
@@ -174,7 +181,7 @@ const visitReader = async (
|
||||
);
|
||||
context.containers.push(inventory);
|
||||
const nested = new AsarArtifactReader(entry.adapterKey);
|
||||
ownedReaders.push(nested);
|
||||
ownedReaders.push({ reader: nested, path });
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: path,
|
||||
@@ -205,24 +212,33 @@ const visitReader = async (
|
||||
} catch (cause: unknown) {
|
||||
failure = { cause };
|
||||
} finally {
|
||||
for (const { iterator } of stack) {
|
||||
for (const { iterator, prefix, reader } of stack.reverse()) {
|
||||
try {
|
||||
await iterator.return?.();
|
||||
} catch (cause: unknown) {
|
||||
failure ??= { cause };
|
||||
failure = {
|
||||
cause: ArtifactReaderFailure.withCleanup(
|
||||
failure === undefined ? cause : failure.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cause,
|
||||
`JavaScript artifact traversal for ${prefix || reader.format}`,
|
||||
),
|
||||
),
|
||||
};
|
||||
}
|
||||
}
|
||||
for (const owned of ownedReaders.reverse()) {
|
||||
try {
|
||||
await owned.close();
|
||||
} catch (cause: unknown) {
|
||||
const resource = `nested ASAR reader for ${owned.path}`;
|
||||
const cleanup = await context.resources.release({
|
||||
kind: "reader",
|
||||
reader: owned.reader,
|
||||
resource,
|
||||
});
|
||||
if (cleanup.kind === "failed") {
|
||||
failure = {
|
||||
cause: ArtifactReaderFailure.withCleanup(
|
||||
failure?.cause ?? cause,
|
||||
ArtifactReaderFailure.cleanupObservation(
|
||||
cause,
|
||||
`nested ASAR reader for ${prefix}`,
|
||||
),
|
||||
failure === undefined ? cleanup.cause : failure.cause,
|
||||
ArtifactReaderFailure.cleanupObservation(cleanup.cause, resource),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -147,6 +147,53 @@ describe("browserScenarioSchema", () => {
|
||||
expect(secrets?.redact("prefix-suffix")).toBe("[REDACTED:login_input]");
|
||||
});
|
||||
|
||||
it("does not leave a declared secret inside its redaction marker", () => {
|
||||
const secrets = BrowserScenarioSecrets.resolve(
|
||||
browserScenarioSchema.parse({
|
||||
...baseScenario(),
|
||||
secrets: [
|
||||
...baseScenario().secrets,
|
||||
{
|
||||
secret_id: "password",
|
||||
environment_variable: "REA_TEST_SHORT_PASSWORD",
|
||||
},
|
||||
],
|
||||
}),
|
||||
{
|
||||
REA_TEST_SESSION: "prefix",
|
||||
REA_TEST_PASSWORD: "prefix-suffix",
|
||||
REA_TEST_COOKIE: "cookie",
|
||||
REA_TEST_SHORT_PASSWORD: "pass",
|
||||
},
|
||||
);
|
||||
expect(secrets?.redact("seen pass in the page")).toBe(
|
||||
"seen [REDACTED] in the page",
|
||||
);
|
||||
|
||||
const bytes = BrowserScenarioSecrets.resolve(
|
||||
browserScenarioSchema.parse({
|
||||
...baseScenario(),
|
||||
secrets: [
|
||||
...baseScenario().secrets,
|
||||
{
|
||||
secret_id: "password",
|
||||
environment_variable: "REA_TEST_LONG_SECRET",
|
||||
},
|
||||
{ secret_id: "pin", environment_variable: "REA_TEST_PIN" },
|
||||
],
|
||||
}),
|
||||
{
|
||||
REA_TEST_SESSION: "prefix",
|
||||
REA_TEST_PASSWORD: "prefix-suffix",
|
||||
REA_TEST_COOKIE: "cookie",
|
||||
REA_TEST_LONG_SECRET: "session-secret-value",
|
||||
REA_TEST_PIN: "pass",
|
||||
},
|
||||
)?.redactBytes(Buffer.from("body session-secret-value tail"));
|
||||
expect(bytes?.toString("utf8")).toBe("body [REDACTED] tail");
|
||||
expect(bytes?.includes(Buffer.from("pass"))).toBe(false);
|
||||
});
|
||||
|
||||
it("does not resolve inherited environment properties as secret values", () => {
|
||||
const scenario = browserScenarioSchema.parse(baseScenario());
|
||||
const environment = Object.assign(
|
||||
|
||||
@@ -20,6 +20,76 @@ const encodedSecretValues = (secret: string): ReadonlySet<string> =>
|
||||
new URLSearchParams([["value", secret]]).toString().slice("value=".length),
|
||||
]);
|
||||
|
||||
type SecretSpellings = "raw" | "url" | "bytes";
|
||||
|
||||
const secretSpellings = (
|
||||
secret: string,
|
||||
spellings: SecretSpellings,
|
||||
): readonly string[] => {
|
||||
if (secret === "") return [];
|
||||
if (spellings === "raw") return [secret];
|
||||
const values = [...encodedSecretValues(secret)];
|
||||
if (spellings === "bytes") values.push(JSON.stringify(secret).slice(1, -1));
|
||||
return [...new Set(values)].filter((value) => value !== "");
|
||||
};
|
||||
|
||||
const declaredLiterals = (
|
||||
values: ReadonlyMap<string, string>,
|
||||
spellings: SecretSpellings,
|
||||
): readonly string[] => [
|
||||
...new Set(
|
||||
[...values.values()].flatMap((secret) =>
|
||||
secretSpellings(secret, spellings),
|
||||
),
|
||||
),
|
||||
];
|
||||
|
||||
/** Pick a marker that does not itself contain a declared secret spelling. */
|
||||
const redactionMarker = (id: string, literals: readonly string[]): string =>
|
||||
[`${REDACTION_PREFIX}${id}]`, "[REDACTED]", "…", ""].find((candidate) =>
|
||||
literals.every((literal) => !candidate.includes(literal)),
|
||||
) ?? "";
|
||||
|
||||
const replaceSecretLiterals = (
|
||||
value: string,
|
||||
values: ReadonlyMap<string, string>,
|
||||
spellings: SecretSpellings,
|
||||
literals: readonly string[],
|
||||
): string => {
|
||||
const replacements = [...values]
|
||||
.flatMap(([id, secret]) =>
|
||||
secretSpellings(secret, spellings).map((text) => ({ id, text })),
|
||||
)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.text.length - left.text.length ||
|
||||
(left.id < right.id ? -1 : left.id > right.id ? 1 : 0),
|
||||
);
|
||||
let output = value;
|
||||
for (const replacement of replacements)
|
||||
output = output.replaceAll(
|
||||
replacement.text,
|
||||
redactionMarker(replacement.id, literals),
|
||||
);
|
||||
return output;
|
||||
};
|
||||
|
||||
const secretNeedles = (
|
||||
values: ReadonlyMap<string, string>,
|
||||
literals: readonly string[],
|
||||
): Array<{
|
||||
readonly id: string;
|
||||
readonly needle: Buffer;
|
||||
readonly marker: Buffer;
|
||||
}> =>
|
||||
[...values].flatMap(([id, secret]) =>
|
||||
secretSpellings(secret, "bytes").map((text) => ({
|
||||
id,
|
||||
needle: Buffer.from(text),
|
||||
marker: Buffer.from(redactionMarker(id, literals)),
|
||||
})),
|
||||
);
|
||||
|
||||
/** Resolved secret values kept only for one in-memory scenario session. */
|
||||
export class BrowserScenarioSecrets {
|
||||
private constructor(private readonly values: ReadonlyMap<string, string>) {}
|
||||
@@ -57,38 +127,19 @@ export class BrowserScenarioSecrets {
|
||||
}
|
||||
|
||||
redact(value: string): string {
|
||||
let output = value;
|
||||
const replacements = [...this.values].sort(
|
||||
([leftId, left], [rightId, right]) =>
|
||||
right.length - left.length ||
|
||||
(leftId < rightId ? -1 : leftId > rightId ? 1 : 0),
|
||||
);
|
||||
for (const [id, secret] of replacements)
|
||||
if (secret !== "")
|
||||
output = output.replaceAll(secret, `${REDACTION_PREFIX}${id}]`);
|
||||
return output;
|
||||
const literals = declaredLiterals(this.values, "raw");
|
||||
const output = replaceSecretLiterals(value, this.values, "raw", literals);
|
||||
return literals.some((literal) => output.includes(literal)) ? "" : output;
|
||||
}
|
||||
|
||||
/** Replace declared UTF-8, URI/form, and JSON-escaped secrets without decoding binary data. */
|
||||
redactBytes(value: Buffer): Buffer {
|
||||
const candidates = [...this.values]
|
||||
.flatMap(([id, secret]) => {
|
||||
if (secret === "") return [];
|
||||
return [
|
||||
...new Set([
|
||||
...encodedSecretValues(secret),
|
||||
JSON.stringify(secret).slice(1, -1),
|
||||
]),
|
||||
].map((text) => {
|
||||
const needle = Buffer.from(text);
|
||||
return {
|
||||
id,
|
||||
needle,
|
||||
marker: Buffer.from(`${REDACTION_PREFIX}${id}]`),
|
||||
index: value.indexOf(needle),
|
||||
};
|
||||
});
|
||||
})
|
||||
const literals = declaredLiterals(this.values, "bytes");
|
||||
const candidates = secretNeedles(this.values, literals)
|
||||
.map((candidate) => ({
|
||||
...candidate,
|
||||
index: value.indexOf(candidate.needle),
|
||||
}))
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.needle.length - left.needle.length ||
|
||||
@@ -113,23 +164,23 @@ export class BrowserScenarioSecrets {
|
||||
}
|
||||
if (parts.length === 0) return value;
|
||||
parts.push(value.subarray(offset));
|
||||
return Buffer.concat(parts);
|
||||
const redacted = Buffer.concat(parts);
|
||||
return literals.some((literal) => redacted.includes(Buffer.from(literal)))
|
||||
? Buffer.alloc(0)
|
||||
: redacted;
|
||||
}
|
||||
|
||||
/** Remove only declared secret values from one URL and report that action. */
|
||||
sanitizeUrl(value: string): SanitizedBrowserUrl {
|
||||
let output = value;
|
||||
const replacements = [...this.values].sort(
|
||||
([leftId, left], [rightId, right]) =>
|
||||
right.length - left.length ||
|
||||
(leftId < rightId ? -1 : leftId > rightId ? 1 : 0),
|
||||
);
|
||||
for (const [id, secret] of replacements) {
|
||||
if (secret === "") continue;
|
||||
const marker = `${REDACTION_PREFIX}${id}]`;
|
||||
for (const candidate of encodedSecretValues(secret))
|
||||
output = output.replaceAll(candidate, marker);
|
||||
}
|
||||
const literals = declaredLiterals(this.values, "url");
|
||||
const output = replaceSecretLiterals(value, this.values, "url", literals);
|
||||
if (literals.some((literal) => output.includes(literal)))
|
||||
return {
|
||||
url: "",
|
||||
origin: null,
|
||||
query_parameter_names: [],
|
||||
redacted: true,
|
||||
};
|
||||
const sanitized = sanitizeBrowserUrl(output);
|
||||
return {
|
||||
...sanitized,
|
||||
|
||||
@@ -277,3 +277,332 @@ it("preserves case-sensitive SVG attributes and uses the actual href", () => {
|
||||
`${origin}/actual`,
|
||||
);
|
||||
});
|
||||
|
||||
interface SvgSnapshotNode {
|
||||
readonly name: string;
|
||||
readonly parent: number;
|
||||
readonly attributes?: readonly (readonly [string, string])[];
|
||||
}
|
||||
|
||||
const svgDocument = (
|
||||
elements: readonly SvgSnapshotNode[],
|
||||
baseUrl = `${origin}/screens/assets/`,
|
||||
) => {
|
||||
const strings = [documentUrl, baseUrl];
|
||||
const intern = (value: string): number => {
|
||||
const existing = strings.indexOf(value);
|
||||
if (existing >= 0) return existing;
|
||||
strings.push(value);
|
||||
return strings.length - 1;
|
||||
};
|
||||
const nodeType = [9];
|
||||
const nodeName = [intern("#document")];
|
||||
const nodeValue = [intern("")];
|
||||
const parentIndex = [-1];
|
||||
const attributes: number[][] = [[]];
|
||||
for (const element of elements) {
|
||||
nodeType.push(1);
|
||||
nodeName.push(intern(element.name));
|
||||
nodeValue.push(intern(""));
|
||||
parentIndex.push(element.parent);
|
||||
const encoded: number[] = [];
|
||||
for (const [name, value] of element.attributes ?? [])
|
||||
encoded.push(intern(name), intern(value));
|
||||
attributes.push(encoded);
|
||||
}
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
documentURL: 0,
|
||||
baseURL: 1,
|
||||
nodes: { nodeType, nodeName, nodeValue, parentIndex, attributes },
|
||||
},
|
||||
],
|
||||
};
|
||||
};
|
||||
|
||||
const captureSvg = (value: ReturnType<typeof svgDocument>) =>
|
||||
captureDom(value, new Set([origin]), input);
|
||||
|
||||
const urlFor = (
|
||||
result: ReturnType<typeof captureSvg>,
|
||||
nodeIndex: number,
|
||||
): (typeof result.urls)[number] | undefined =>
|
||||
result.urls.find((url) => url.node_index === nodeIndex);
|
||||
|
||||
describe("SVG xlink:href destinations", () => {
|
||||
it("reports an SVG anchor that only has xlink:href, keeping the first value", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "svg", parent: 0 },
|
||||
{ name: "g", parent: 1 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 2,
|
||||
attributes: [
|
||||
["xlink:href", "/xl-s2h"],
|
||||
["xlink:href", "/later"],
|
||||
],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.nodes[3]?.attribute_names).toEqual(["xlink:href"]);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 3,
|
||||
attribute: "href",
|
||||
url: `${origin}/xl-s2h`,
|
||||
destination_scope: "approved",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["xlink:href", "href"],
|
||||
["href", "xlink:href"],
|
||||
] as const)(
|
||||
"uses href when an SVG anchor also has xlink:href (%s then %s)",
|
||||
(first, second) => {
|
||||
const valueFor = (name: string): string =>
|
||||
name === "href" ? "/href-wins" : "/xlink-loses";
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "svg", parent: 0 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
[first, valueFor(first)],
|
||||
[second, valueFor(second)],
|
||||
],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 2,
|
||||
attribute: "href",
|
||||
url: `${origin}/href-wins`,
|
||||
destination_scope: "approved",
|
||||
},
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not report xlink:href on an HTML anchor or a MathML element", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{
|
||||
name: "A",
|
||||
parent: 0,
|
||||
attributes: [["xlink:href", "/xl-html"]],
|
||||
},
|
||||
{ name: "math", parent: 0 },
|
||||
{
|
||||
name: "mrow",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "/xl-math"]],
|
||||
},
|
||||
{ name: "svg", parent: 0 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 4,
|
||||
attributes: [["xlink:href", "/xl-s2h"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.nodes[1]?.attribute_names).toEqual(["xlink:href"]);
|
||||
expect(result.nodes[3]?.attribute_names).toEqual(["xlink:href"]);
|
||||
expect(urlFor(result, 1)).toBeUndefined();
|
||||
expect(urlFor(result, 3)).toBeUndefined();
|
||||
expect(urlFor(result, 5)?.url).toBe(`${origin}/xl-s2h`);
|
||||
});
|
||||
|
||||
it("classifies an SVG xlink:href outside the allowed origins as outside_policy", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "svg", parent: 0 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "https://cdn.example.test/out"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 2,
|
||||
attribute: "href",
|
||||
url: null,
|
||||
destination_scope: "outside_policy",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("resolves SVG image and use xlink:href against the document base", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "svg", parent: 0 },
|
||||
{ name: "image", parent: 1, attributes: [["xlink:href", "guide"]] },
|
||||
{ name: "use", parent: 1, attributes: [["xlink:href", "icon.svg"]] },
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 2,
|
||||
attribute: "href",
|
||||
url: `${origin}/screens/assets/guide`,
|
||||
destination_scope: "approved",
|
||||
},
|
||||
{
|
||||
node_index: 3,
|
||||
attribute: "href",
|
||||
url: `${origin}/screens/assets/icon.svg`,
|
||||
destination_scope: "approved",
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("SVG xlink:href context", () => {
|
||||
it("keeps forward ancestor context local to each document", () => {
|
||||
const svg = svgDocument([
|
||||
{ name: "a", parent: 2, attributes: [["xlink:href", "/svg-link"]] },
|
||||
{ name: "g", parent: 3 },
|
||||
{ name: "svg", parent: 0 },
|
||||
]);
|
||||
const html = svgDocument([
|
||||
{ name: "A", parent: 2, attributes: [["xlink:href", "/html-link"]] },
|
||||
{ name: "DIV", parent: 3 },
|
||||
{ name: "BODY", parent: 0 },
|
||||
]);
|
||||
const offset = svg.strings.length;
|
||||
const result = captureSvg({
|
||||
strings: [...svg.strings, ...html.strings],
|
||||
documents: [
|
||||
...svg.documents,
|
||||
...html.documents.map((document) => ({
|
||||
...document,
|
||||
documentURL: document.documentURL + offset,
|
||||
baseURL: document.baseURL + offset,
|
||||
nodes: {
|
||||
...document.nodes,
|
||||
nodeName: document.nodes.nodeName.map((index) => index + offset),
|
||||
nodeValue: document.nodes.nodeValue.map((index) => index + offset),
|
||||
attributes: document.nodes.attributes.map((attributes) =>
|
||||
attributes.map((index) => index + offset),
|
||||
),
|
||||
},
|
||||
})),
|
||||
],
|
||||
});
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 1,
|
||||
attribute: "href",
|
||||
url: `${origin}/svg-link`,
|
||||
destination_scope: "approved",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops SVG context at foreignObject and resumes it for a nested svg", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "svg", parent: 0 },
|
||||
{ name: "foreignObject", parent: 1 },
|
||||
{
|
||||
name: "A",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "/xl-html-fo"]],
|
||||
},
|
||||
{ name: "svg", parent: 2 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 4,
|
||||
attributes: [["xlink:href", "/xl-nested"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(urlFor(result, 3)).toBeUndefined();
|
||||
expect(urlFor(result, 5)).toMatchObject({
|
||||
attribute: "href",
|
||||
url: `${origin}/xl-nested`,
|
||||
destination_scope: "approved",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps an empty href ahead of xlink:href", () => {
|
||||
const baseUrl = `${origin}/screens/assets/`;
|
||||
const result = captureSvg(
|
||||
svgDocument(
|
||||
[
|
||||
{ name: "svg", parent: 0 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["href", ""],
|
||||
["xlink:href", "/xlink-should-lose"],
|
||||
],
|
||||
},
|
||||
],
|
||||
baseUrl,
|
||||
),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 2,
|
||||
attribute: "href",
|
||||
url: baseUrl,
|
||||
destination_scope: "approved",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("classifies a non-http SVG xlink:href as unsupported", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "svg", parent: 0 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "javascript:void(0)"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 2,
|
||||
attribute: "href",
|
||||
url: null,
|
||||
destination_scope: "unsupported",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not treat a cyclic or dangling parent chain as SVG", () => {
|
||||
const cyclic = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "g", parent: 2 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "/cycle"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
const dangling = captureSvg(
|
||||
svgDocument([
|
||||
{
|
||||
name: "a",
|
||||
parent: 99,
|
||||
attributes: [["xlink:href", "/missing-parent"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(cyclic.urls).toEqual([]);
|
||||
expect(dangling.urls).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -171,6 +171,7 @@ export const captureDom = (
|
||||
? documentNodes.attributes
|
||||
: [];
|
||||
const baseIndex = nodes.length;
|
||||
const svgContexts = new Map<number, boolean>();
|
||||
total += nodeTypes.length;
|
||||
for (let index = 0; index < nodeTypes.length; index += 1) {
|
||||
const attributeIndexes = numberArray(attributes[index]);
|
||||
@@ -205,6 +206,13 @@ export const captureDom = (
|
||||
nodeIndex,
|
||||
nodeName,
|
||||
allowedOrigins,
|
||||
svgElement: elementInSvgContext(
|
||||
index,
|
||||
nodeNames,
|
||||
parents,
|
||||
strings,
|
||||
svgContexts,
|
||||
),
|
||||
});
|
||||
for (const url of metadata.urls) {
|
||||
urls.push(url);
|
||||
@@ -375,8 +383,44 @@ interface DomMetadataOptions {
|
||||
readonly nodeIndex: number;
|
||||
readonly nodeName: string;
|
||||
readonly allowedOrigins: ReadonlySet<string>;
|
||||
readonly svgElement: boolean;
|
||||
}
|
||||
|
||||
// DOMSnapshot carries no namespace URI. Chrome's HTML parser records SVG
|
||||
// local names (`svg`, `foreignObject`). Descendants of foreignObject are HTML
|
||||
// until a nested `svg`.
|
||||
const elementInSvgContext = (
|
||||
index: number,
|
||||
nodeNames: readonly number[],
|
||||
parents: readonly number[],
|
||||
strings: readonly string[],
|
||||
contexts: Map<number, boolean>,
|
||||
): boolean => {
|
||||
let current = index;
|
||||
const seen = new Set<number>();
|
||||
let inSvg = false;
|
||||
while (current >= 0 && current < nodeNames.length && !seen.has(current)) {
|
||||
const cached = contexts.get(current);
|
||||
if (cached !== undefined) {
|
||||
inSvg = cached;
|
||||
break;
|
||||
}
|
||||
seen.add(current);
|
||||
if (indexedString(strings, nodeNames[current]) === "svg") {
|
||||
inSvg = true;
|
||||
break;
|
||||
}
|
||||
const parent = Math.trunc(parents[current] ?? -1);
|
||||
if (parent < 0 || parent === current) break;
|
||||
if (indexedString(strings, nodeNames[parent]) === "foreignObject") break;
|
||||
current = parent;
|
||||
}
|
||||
// Cache the entire examined chain, including forward references and cycles,
|
||||
// so shared ancestors are examined once per document.
|
||||
for (const node of seen) contexts.set(node, inSvg);
|
||||
return inSvg;
|
||||
};
|
||||
|
||||
const domMetadata = (
|
||||
options: DomMetadataOptions,
|
||||
): {
|
||||
@@ -391,6 +435,7 @@ const domMetadata = (
|
||||
nodeIndex,
|
||||
nodeName,
|
||||
allowedOrigins,
|
||||
svgElement,
|
||||
} = options;
|
||||
const pairs = new Map<string, string>();
|
||||
for (let index = 0; index + 1 < attributes.length; index += 2) {
|
||||
@@ -402,7 +447,13 @@ const domMetadata = (
|
||||
}
|
||||
const urls: WebPageInspection["metadata"]["dom_urls"] = [];
|
||||
for (const attribute of domUrlAttributes) {
|
||||
const value = pairs.get(attribute);
|
||||
// Chrome reads SVG xlink:href only when href itself is absent, and an
|
||||
// empty href still wins. Report the fallback as href.
|
||||
const value =
|
||||
pairs.get(attribute) ??
|
||||
(attribute === "href" && svgElement
|
||||
? pairs.get("xlink:href")
|
||||
: undefined);
|
||||
if (value === undefined) continue;
|
||||
const tagName = nodeName.toLowerCase();
|
||||
// Chrome strips HTML whitespace for FORM.action, while submit controls
|
||||
|
||||
@@ -27,8 +27,7 @@ export const ingestResponseBodyShape = (
|
||||
invalidResponseBodyShape(state, requestId);
|
||||
return;
|
||||
}
|
||||
const inferred = inferBodyShape(decoded);
|
||||
updateResponseBodyShape(state, requestId, inferred.shape);
|
||||
updateResponseBodyShape(state, requestId, inferJsonShape(decoded));
|
||||
};
|
||||
|
||||
export const requestBodyShape = (
|
||||
@@ -40,10 +39,10 @@ export const requestBodyShape = (
|
||||
const body = cdpStringValue(request.postData);
|
||||
if (!isJsonContentType(recordValue(request.headers)) || body === undefined)
|
||||
return { status: "unavailable", request: null, response: null };
|
||||
const inferred = inferBodyShape(body);
|
||||
const shape = inferJsonShape(body);
|
||||
return {
|
||||
status: inferred.shape === null ? "unavailable" : "included",
|
||||
request: inferred.shape,
|
||||
status: shape === null ? "unavailable" : "included",
|
||||
request: shape,
|
||||
response: null,
|
||||
};
|
||||
};
|
||||
@@ -68,10 +67,6 @@ export const updateResponseBodyShape = (
|
||||
});
|
||||
};
|
||||
|
||||
const inferBodyShape = (text: string): { readonly shape: JsonShape | null } => {
|
||||
return { shape: inferJsonShape(text) };
|
||||
};
|
||||
|
||||
const invalidResponseBodyShape = (
|
||||
state: CdpCaptureEventsState,
|
||||
requestId: string,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user