Compare commits
76
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7946c3ad4 | ||
|
|
0ade49ae90 | ||
|
|
a1ed4232de | ||
|
|
c9aa907495 | ||
|
|
a7fdb8b626 | ||
|
|
938d7adf70 | ||
|
|
c701376777 | ||
|
|
2af12a18a3 | ||
|
|
5e1afe454f | ||
|
|
a9e00b83f3 | ||
|
|
5f93b4b125 | ||
|
|
ae44e01c04 | ||
|
|
f1f0a986fd | ||
|
|
46d195e8a4 | ||
|
|
9833c94656 | ||
|
|
6e6ff7bd2e | ||
|
|
9455e7c8bc | ||
|
|
e6134056cf | ||
|
|
6dda47c98c | ||
|
|
b49beb42d7 | ||
|
|
a4d77f0ff0 | ||
|
|
82a1a525d7 | ||
|
|
e8b92a40c0 | ||
|
|
d52f66d62a | ||
|
|
e2f03786f4 | ||
|
|
5a55d5b1c6 | ||
|
|
ef30343154 | ||
|
|
38bbb656da | ||
|
|
89252f1f35 | ||
|
|
5547c1e697 | ||
|
|
cf96ceb211 | ||
|
|
a079af297b | ||
|
|
79e5a3923d | ||
|
|
25fb6d32a3 | ||
|
|
40e18b2e37 | ||
|
|
23578197cc | ||
|
|
64d2878882 | ||
|
|
bfbfc7330a | ||
|
|
31570a22fb | ||
|
|
3dcb732da3 | ||
|
|
6a7650e93c | ||
|
|
170600541b | ||
|
|
68b9fa489b | ||
|
|
3e9d38ccd8 | ||
|
|
c47693bb23 | ||
|
|
1d27061a65 | ||
|
|
d9a39e5b48 | ||
|
|
3664a4a57e | ||
|
|
8a31508c89 | ||
|
|
f5009ed55f | ||
|
|
60825fb447 | ||
|
|
c5a16b7e21 | ||
|
|
5dfd9905ed | ||
|
|
5ccb9241be | ||
|
|
be14aa0de9 | ||
|
|
8ee7d23cae | ||
|
|
9b38f3a0af | ||
|
|
c541052ca9 | ||
|
|
3910bedae8 | ||
|
|
949347a148 | ||
|
|
da18453d4d | ||
|
|
b196c5ac8f | ||
|
|
ef0bf91de9 | ||
|
|
300682d4ba | ||
|
|
15b42b5825 | ||
|
|
116d722339 | ||
|
|
31a642f73d | ||
|
|
5389b233b8 | ||
|
|
79e686f6a2 | ||
|
|
f3e41f1e0c | ||
|
|
ce40fe6ded | ||
|
|
bb58eeef27 | ||
|
|
7857d2d329 | ||
|
|
667bf164f9 | ||
|
|
daf504b62f | ||
|
|
54dd1a7604 |
@@ -28,7 +28,7 @@ intentionally not included. -->
|
||||
- Provider, bridge, target-format, or platform compatibility:
|
||||
- Evidence, artifact, provenance, or reconstruction contract:
|
||||
- Process execution, authorization, cleanup, or containment impact:
|
||||
- Generated metadata (`docs/product-catalog.json`), package, or installation impact (the regen-generated bot pushes tracked generated files; never hand-edit them):
|
||||
- Generated metadata (`docs/public/product-catalog.json`), package, or installation impact (CI generates and validates ignored catalogs and `skills/`; edit authored instructions in `skill-src/` and never commit build outputs):
|
||||
|
||||
## Evidence and regression coverage
|
||||
|
||||
@@ -59,6 +59,10 @@ relevant real-provider or platform workflow, or explain why it was unavailable.
|
||||
limits, security/privacy/containment implications, and meaningful proof gaps. -->
|
||||
|
||||
- Breaking changes or migration steps:
|
||||
<!-- For ! / BREAKING CHANGE: show a previously valid call/configuration,
|
||||
its new behavior and migration, and why compatibility/deprecation cannot
|
||||
preserve it. Compatible fixes/additions should use patch/minor semantics.
|
||||
See docs/releasing.md. -->
|
||||
- Real Hopper/Ghidra, browser, or OS coverage:
|
||||
- Package or release metadata impact:
|
||||
- Security, privacy, process, or containment review:
|
||||
|
||||
@@ -102,8 +102,8 @@ jobs:
|
||||
with:
|
||||
name: rea-dist
|
||||
path: dist
|
||||
- name: Generate source catalog from downloaded build
|
||||
run: node scripts/generate-mcp-tool-catalog.mjs
|
||||
- name: Generate projections from downloaded build
|
||||
run: npm run build:projections
|
||||
- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -317,8 +317,9 @@ jobs:
|
||||
- run: npm run verify:keyed-archive
|
||||
- run: npm run verify:asset-catalog
|
||||
- run: npm run verify:macos-bundle
|
||||
- name: Native compiled NIB hierarchy regressions
|
||||
run: npm run test:focused -- tests/boundary/filesystem/nibNativeHierarchy.test.ts tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
- run: npm run verify:installer-package
|
||||
- name: Native installer and compiled NIB hierarchy regressions
|
||||
run: npm run test:focused -- tests/boundary/filesystem/installScript.test.ts tests/boundary/filesystem/nibNativeHierarchy.test.ts tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
- name: Real binary metadata and Interface Builder integration
|
||||
run: npm run verify:apple-dispatch && npm run verify:interface-builder
|
||||
- name: Native plist CLI selection regressions
|
||||
@@ -327,6 +328,8 @@ jobs:
|
||||
run: npm run test:focused -- tests/boundary/cli/nativeDemangleCli.test.ts tests/boundary/mcp/nativeDemangleMcp.test.ts
|
||||
- name: Process filesystem snapshot descriptor regressions
|
||||
run: npm run test:focused -- src/process/capture/FilesystemSnapshot.test.ts
|
||||
- name: Native Darwin ownership and process cleanup regressions
|
||||
run: npm run test:focused -- tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/processCaptureLifecycle.test.ts tests/boundary/cli/processCli.test.ts tests/boundary/mcp/sessionMcp.test.ts
|
||||
|
||||
reconstruction-readiness:
|
||||
name: Reconstruction readiness
|
||||
@@ -441,6 +444,11 @@ jobs:
|
||||
tests/boundary/filesystem/safeOutputTree.test.ts
|
||||
tests/boundary/providers/native/artifactProvider.archiveSafety.test.ts
|
||||
tests/boundary/filesystem/setupUnavailableProvider.test.ts
|
||||
tests/boundary/filesystem/windowsNpxRegistration.test.ts
|
||||
src/application/ClientRegistrationIdentity.test.ts
|
||||
tests/boundary/filesystem/webScriptManifestPaths.test.ts
|
||||
tests/acceptance/webModuleCli.test.ts
|
||||
tests/acceptance/webSourceLocationCli.test.ts
|
||||
tests/boundary/cli/referenceSourceImportStatus.test.ts
|
||||
src/cliReferenceSourceImportStatus.test.ts
|
||||
tests/boundary/filesystem/asarInventoryFreshness.test.ts
|
||||
@@ -481,3 +489,13 @@ jobs:
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run docs:check
|
||||
- name: Generation leaves authored files unchanged
|
||||
run: git diff --exit-code
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: generated-docs
|
||||
path: |
|
||||
docs/public/product-catalog.json
|
||||
docs/verification/managed-conformance-*.json
|
||||
skills/reverse-engineer-anything
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -1,75 +0,0 @@
|
||||
name: Regenerate generated metadata
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
paths:
|
||||
- "src/**"
|
||||
- "skills/**/SKILL.md"
|
||||
- "scripts/generate-*.mjs"
|
||||
- "scripts/check-doc-facts.mjs"
|
||||
- "scripts/lib/*.mjs"
|
||||
- "scripts/verify/managed/**"
|
||||
- "scripts/fixtures/managed/**"
|
||||
- "package.json"
|
||||
- "package-lock.json"
|
||||
|
||||
concurrency:
|
||||
group: regen-${{ github.event.pull_request.number }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HUSKY: "0"
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
jobs:
|
||||
regen:
|
||||
name: Regenerate and push tracked generated files
|
||||
# Same-repo PRs only: the default token cannot push to forks.
|
||||
# The bot-actor guard stops looped runs triggered by our own push.
|
||||
if: >-
|
||||
github.actor != 'github-actions[bot]' &&
|
||||
github.event.pull_request.head.repo.full_name == github.repository
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.pull_request.head.ref }}
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run docs:generate
|
||||
- name: Commit regenerated files
|
||||
shell: bash
|
||||
env:
|
||||
HEAD_REF: ${{ github.event.pull_request.head.ref }}
|
||||
run: |
|
||||
generated_paths=(
|
||||
docs/product-catalog.json
|
||||
docs/error-contract.schema.json
|
||||
docs/verification/managed-conformance-manifest.json
|
||||
docs/verification/managed-conformance-ledger.json
|
||||
skills/reverse-engineer-anything/SKILL.md
|
||||
src/generatedPackageMetadata.ts
|
||||
)
|
||||
if git diff --quiet -- "${generated_paths[@]}"; then
|
||||
echo "Generated metadata is already canonical."
|
||||
exit 0
|
||||
fi
|
||||
git diff --check
|
||||
# Refuse to sweep unrelated working-tree changes into the bot commit.
|
||||
unexpected="$(git diff --name-only | grep -vxF -f <(printf '%s\n' "${generated_paths[@]}") || true)"
|
||||
if [[ -n "${unexpected}" ]]; then
|
||||
echo "docs:generate left unexpected tracked changes:" >&2
|
||||
printf '%s\n' "${unexpected}" >&2
|
||||
exit 1
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add -- "${generated_paths[@]}"
|
||||
git commit -m "chore: regenerate generated metadata"
|
||||
git push origin "HEAD:${HEAD_REF}"
|
||||
@@ -84,6 +84,32 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Check out release controller and Git history
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js for checkpoint validation
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Install checkpoint validation dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Validate checkpoint version and ancestry before release creation
|
||||
env:
|
||||
RELEASE_PHASE: ${{ inputs.phase }}
|
||||
RELEASE_BRANCH: ${{ inputs.release_branch }}
|
||||
SOURCE_SHA: ${{ steps.branch.outputs.sha }}
|
||||
run: >-
|
||||
node scripts/verify-release-checkpoint.mjs
|
||||
--phase "$RELEASE_PHASE" --release-branch "$RELEASE_BRANCH"
|
||||
--source-sha "$SOURCE_SHA"
|
||||
|
||||
- uses: googleapis/release-please-action@v4
|
||||
id: release
|
||||
with:
|
||||
@@ -116,6 +142,7 @@ jobs:
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ fromJSON(steps.release.outputs.pr).headBranchName }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Node.js for generated documentation
|
||||
@@ -129,47 +156,23 @@ jobs:
|
||||
if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true'
|
||||
run: npm ci
|
||||
|
||||
- name: Validate prepared release version and migration notes
|
||||
if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true'
|
||||
env:
|
||||
RELEASE_BRANCH: ${{ inputs.release_branch }}
|
||||
SOURCE_SHA: ${{ steps.branch.outputs.sha }}
|
||||
run: >-
|
||||
node scripts/verify-release-checkpoint.mjs
|
||||
--phase prepare --stage candidate --release-branch "$RELEASE_BRANCH"
|
||||
--source-sha "$SOURCE_SHA"
|
||||
|
||||
- name: Regenerate release documentation
|
||||
if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true'
|
||||
run: npm run docs:generate
|
||||
|
||||
- name: Commit canonical release catalog
|
||||
- name: Validate generated release documentation
|
||||
if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
# Use the same identity for the follow-up commit so it also triggers CI.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }}
|
||||
RELEASE_PR_JSON: ${{ steps.release.outputs.pr }}
|
||||
run: |
|
||||
release_pr_branch="$(node --input-type=module -e '
|
||||
const value = JSON.parse(process.env.RELEASE_PR_JSON ?? "");
|
||||
if (
|
||||
typeof value !== "object" ||
|
||||
value === null ||
|
||||
typeof value.headBranchName !== "string" ||
|
||||
value.headBranchName.length === 0
|
||||
) {
|
||||
process.exit(1);
|
||||
}
|
||||
process.stdout.write(value.headBranchName);
|
||||
')"
|
||||
changed_paths="$(git diff --name-only)"
|
||||
if [[ -z "${changed_paths}" ]]; then
|
||||
echo "Release documentation is already canonical."
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${changed_paths}" != "docs/product-catalog.json" ]]; then
|
||||
echo "docs:generate changed unexpected tracked paths:" >&2
|
||||
printf '%s\n' "${changed_paths}" >&2
|
||||
exit 1
|
||||
fi
|
||||
git diff --check
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add -- docs/product-catalog.json
|
||||
git commit -m "chore: normalize release product catalog"
|
||||
gh auth setup-git
|
||||
git push origin "HEAD:${release_pr_branch}"
|
||||
run: npm run docs:check && git diff --exit-code
|
||||
|
||||
publish:
|
||||
name: Publish to npm
|
||||
|
||||
@@ -38,6 +38,7 @@ test-results.xml
|
||||
.turbo/
|
||||
.cache/
|
||||
docs/.vitepress/cache/
|
||||
docs/.vitepress/dist/
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
@@ -51,3 +52,10 @@ Thumbs.db
|
||||
|
||||
# Build-generated MCP tool catalog (regenerated by `npm run build`; never commit)
|
||||
src/generatedMcpToolCatalog.ts
|
||||
|
||||
# Derived catalogs, portable conformance projections, and packaged skill bundles
|
||||
# (regenerated from this checkout by `npm run build:cached`; never commit)
|
||||
/skills/
|
||||
/docs/public/product-catalog.json
|
||||
/docs/verification/managed-conformance-manifest.json
|
||||
/docs/verification/managed-conformance-ledger.json
|
||||
|
||||
@@ -21,10 +21,10 @@ REA is a layered ESM TypeScript application. Dependencies flow inward from pure
|
||||
|
||||
- `src/domain/` owns pure provider-neutral semantics; `src/contracts/` owns caller-visible schemas and the canonical tool inventory.
|
||||
- `src/hopper/`, `src/ghidra/`, `src/ida/`, `src/browser/`, `src/inspector/`, `src/native/`, `src/artifacts/`, and `src/dotnet/` own provider-specific boundaries. Keep provider protocols out of domain and application code.
|
||||
- `src/application/` composes shared CLI/MCP workflows; `src/server/` translates MCP requests; `src/cli.ts` and `src/main.ts` are the CLI and MCP entry points.
|
||||
- `src/process/` owns shared process lifecycle primitives, not provider wire protocols. `bridge/` contains provider-side adapters.
|
||||
- `src/application/` composes shared CLI/MCP workflows, including Evidence provenance, unknowns, and eligible snapshot bindings; `src/server/` translates MCP requests; `src/cli.ts` and `src/main.ts` are the CLI and MCP entry points.
|
||||
- `src/process/` owns shared process lifecycle primitives, not provider wire protocols. Reuse its supervision and identity primitives before adding provider-local lifecycle code; a PID and executable pathname alone do not establish ownership after exit or reuse. `bridge/` contains provider-side adapters.
|
||||
- `tests/` contains unit, composition, boundary, acceptance, and conformance tests. `scripts/verify-*` and capability directories under `scripts/verify/` contain real-toolchain checks.
|
||||
- `docs/product-catalog.json` is generated. Update its source contracts and regenerate it; do not edit it directly.
|
||||
- `docs/public/product-catalog.json`, `docs/verification/managed-conformance-*.json`, and `skills/` are ignored build outputs. Update source contracts and authored instructions in `skill-src/`, then run `npm run build:cached`; never commit derived catalog digests or portable conformance projections.
|
||||
- `src/generatedMcpToolCatalog.ts` is build-generated and gitignored. Never commit it; resolve any trace of it in merges by deleting it and running `npm run build:cached`.
|
||||
|
||||
## Build, Test, and Development Commands
|
||||
@@ -36,7 +36,7 @@ REA is a layered ESM TypeScript application. Dependencies flow inward from pure
|
||||
- `npm run check:changed`: run cached static checks and source tests affected since the branch merge base (default `origin/main`).
|
||||
- `npm run check:fast`: run cached typecheck and lint checks.
|
||||
- `npm run check:pr`: opt into the complete local deterministic gate and generated-document checks for broad changes; CI owns full coverage. Routine iterations need focused tests and relevant checks, not the whole gate each time.
|
||||
- `npm run docs:check`: check committed generated documents; `npm run docs:generate` regenerates them.
|
||||
- `npm run docs:check`: build and validate generated documents for the current checkout; `npm run docs:generate` regenerates them. CI retains ignored outputs as artifacts and does not push snapshot commits to feature branches.
|
||||
- For provider-dependent changes, see [docs/testing.md](docs/testing.md) and run the matching real-provider verification.
|
||||
- Keep each verification lane's prerequisites limited to the claim it checks. Use host-native fixtures for host/provider acceptance; put optional cross-target formats and their external toolchains in a separate lane. Preflight required commands and report the missing dependency and lane clearly.
|
||||
|
||||
@@ -54,9 +54,11 @@ Use ESM TypeScript, two-space indentation, and the committed Oxfmt configuration
|
||||
|
||||
Treat a boundary as a contract between the producer's actual representation and the consumer's required meaning. When implementing or auditing a boundary, trace the value through parsing, normalization, authorization, serialization, and the CLI/MCP result. Establish affected callers from their code paths; similar tool names or workflows do not prove that they share a schema or failure mode.
|
||||
|
||||
Keep portable evidence and scenario validation distinct from host-native execution checks. Absolute filesystem paths, file URLs, and HTTP paths have different semantics; do not substitute one platform's syntax for the domain concept. Interpret provider metadata according to its documented or observed producer behavior. When a transformation loses information, preserve the reported value and an explicit unknown rather than guessing a canonical identity.
|
||||
When changing boundary behavior, inspect adjacent input representations, failure paths, and affected callers, and correct the underlying assumption across those cases.
|
||||
|
||||
Preserve meaningful failure reasons through application and adapter layers. Malformed input is distinct from an unsupported target, unavailable provider, or host operating-system permission denial. Diagnostics should identify the failed constraint and the target or lifecycle request it applies to. Recovery advice must address that reason and point to an available workflow; generic catches must not erase actionable validation details.
|
||||
Keep portable evidence and scenario validation distinct from host-native execution checks. Absolute filesystem paths, file URLs, and HTTP paths have different semantics; do not substitute one platform's syntax for the domain concept. Interpret provider metadata according to its documented or observed producer behavior. Keep values used for identity, provenance, matching, and path resolution separate from display formatting. When normalization loses information, preserve the source value and an explicit unknown; display placeholders must not feed back into lookup or selection.
|
||||
|
||||
Preserve meaningful failure reasons through application and adapter layers. Malformed input is distinct from an unsupported target, unavailable provider, or host operating-system permission denial. Diagnostics should identify the failed constraint and the target or lifecycle request it applies to. Recovery advice must address that reason and point to an available workflow; generic catches must not erase actionable validation details. Preserve collected observations on execution failure independently of whether cleanup succeeds.
|
||||
|
||||
Leave meaningful target, action, capture, and output choices to the agent. A selected operation already expresses intent; do not require approval booleans or repeated permission declarations. Trace each setting to its consumer: remove ignored options and single-value confirmations, derive built-in lifecycle behavior, and supply defaults for omitted optional metadata. Report actual effects and limitations where they help interpret results rather than asking callers to restate them.
|
||||
|
||||
@@ -80,7 +82,7 @@ Let agents compose experiments with ordinary commands, scripts, and local fixtur
|
||||
- Use **observe/capture** only when runtime activity is required, and declare authority and lifecycle effects in the contract.
|
||||
- Extend an existing tool when intent and result contract are unchanged. Add a tool for a distinct analyst outcome or materially different authority.
|
||||
- Keep caller-facing names and results provider-neutral. Put engine-specific behavior in provider adapters and report each provider's exact coverage.
|
||||
- Keep results complete by default. Add a limit only when it follows from a real format, protocol, authority, or resource-safety constraint; explain truncation and unsupported facets. Keep observed, derived, inferred, and unknown results distinct.
|
||||
- Keep results complete by default. Add a limit only when it follows from a real format, protocol, authority, or resource-safety constraint; explain truncation and unsupported facets. Account for representation expansion and products of independently bounded dimensions before allocating or retaining output. Derive facet completeness from what was examined and exhausted, not an empty failure list. Keep observed, derived, inferred, and unknown results distinct.
|
||||
- Include artifact identity, source locations, Evidence references, actionable errors, and relevant limitations when they affect conclusions. Return the evidence needed for the next analysis inline rather than requiring a resource or identifier lookup.
|
||||
- Implement shared application workflows behind CLI and MCP adapters. Update canonical contracts and generated catalog together.
|
||||
|
||||
|
||||
+19
-3
@@ -35,7 +35,7 @@ links, and `npm run docs:preview` to preview that build at `/rea/`.
|
||||
Site navigation lives in `docs/.vitepress/config.ts`. Keep links to guides
|
||||
relative so they work on GitHub and the website; link to repository files
|
||||
outside `docs/` using their full GitHub URLs. Generated reference documents
|
||||
still use `npm run docs:generate`; `docs:build` only builds the website.
|
||||
use `npm run docs:generate`; `docs:build` generates them before building the website.
|
||||
|
||||
Pull requests run `npm run docs:check`. `.github/workflows/pages.yml` is a
|
||||
manual VitePress build and does not publish. The public site at
|
||||
@@ -74,8 +74,24 @@ require the matching real-provider `verify:*` lane.
|
||||
Formatting uses Oxfmt and the committed `.oxfmtrc.json`; generated sources use
|
||||
the same configuration. Pre-commit formats and lints staged files; pre-push runs
|
||||
`check:fast`.
|
||||
`docs:check` checks committed generated metadata. `docs:generate` regenerates
|
||||
those files, and the docs CI lane checks them.
|
||||
`docs:check` builds and validates generated metadata for the current checkout.
|
||||
The product catalog (`docs/public/product-catalog.json`), portable managed
|
||||
conformance projections (`docs/verification/managed-conformance-*.json`), and
|
||||
packaged skill (`skills/`) are ignored build outputs. Edit skill instructions
|
||||
and references in `skill-src/`; the build adds catalog-dependent metadata to
|
||||
the packaged copy without rewriting authored files. The generated manifest
|
||||
commits to that exact packaged skill bundle. It is a portable projection of
|
||||
the deterministic managed verifier, not a record of optional real-provider runs.
|
||||
CI validates these outputs and retains them as artifacts instead of pushing
|
||||
generated-only commits onto feature branches. Reviewed source metadata such as
|
||||
`src/generatedPackageMetadata.ts` and `docs/error-contract.schema.json` remains
|
||||
tracked and checked for freshness. Do not commit ignored generated outputs.
|
||||
The build-generated product catalog contains documented facts and their provider
|
||||
identity, rather than full runtime schema hashes. The managed skill contains
|
||||
instructions and inventory metadata; doctor compares its installed files with
|
||||
the canonical bundle. Schema-only fixes should not change these outputs or the
|
||||
skill commitment in the conformance manifest. Runtime schema identity remains
|
||||
available through doctor and `binary_session`.
|
||||
Real-provider execution remains uncached; deterministic builds use Turbo.
|
||||
|
||||
Local `npm test` runs every deterministic Vitest project without coverage or
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
|
||||
[](https://www.npmjs.com/package/rea-agents)
|
||||
[](https://github.com/morluto/rea/actions/workflows/ci.yml)
|
||||
[](docs/product-catalog.json)
|
||||
[](docs/mcp-contracts.md#generated-catalog)
|
||||
[](https://nodejs.org/)
|
||||
[](https://skills.sh/morluto/rea/reverse-engineer-anything)
|
||||
[](LICENSE)
|
||||
@@ -296,7 +296,7 @@ For exact options, prerequisites and result contracts:
|
||||
- [Readiness and troubleshooting](docs/installation.md#check-readiness-for-your-task): diagnose one agent or analysis engine.
|
||||
- [CLI and Evidence](docs/cli.md): commands, provider selection, snapshots, import/export and exit statuses.
|
||||
- [MCP contracts](docs/mcp-contracts.md) and [agent prompts](docs/mcp-prompts.md): tool results, sessions and guided investigations.
|
||||
- [Tool catalog](docs/product-catalog.json): generated inventory of tools, providers and CLI commands.
|
||||
- [Tool catalog](docs/mcp-contracts.md#generated-catalog): build-generated inventory of tools, providers and CLI commands.
|
||||
- [Roadmap](docs/roadmap.md): planned work and capability trackers.
|
||||
|
||||
Report vulnerabilities through [SECURITY.md](SECURITY.md).
|
||||
@@ -315,12 +315,12 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and checks,
|
||||
|
||||
[Website](https://morluto.github.io/rea/) · [npm](https://www.npmjs.com/package/rea-agents) · [skills.sh](https://skills.sh/morluto/rea/reverse-engineer-anything) · [Issues](https://github.com/morluto/rea/issues) · [Security](SECURITY.md)
|
||||
|
||||
## License
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
## Star history
|
||||
|
||||
🎉 **20,000 GitHub stars — thank you!**
|
||||
|
||||
Thanks to everyone using REA, reporting bugs, testing builds, and contributing fixes.
|
||||
|
||||
<a href="https://www.star-history.com/?repos=morluto%2Frea&type=date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date&theme=dark&legend=top-left" />
|
||||
@@ -332,3 +332,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and checks,
|
||||
## Disclaimer
|
||||
|
||||
REA provides tools for lawful reverse-engineering research, analysis, and reconstruction. You are responsible for obtaining any required authorization and complying with applicable laws. The project does not endorse illegal or unauthorized use.
|
||||
|
||||
## License
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
@@ -336,6 +336,20 @@ npx -y rea-agents@latest compare /absolute/path/to/left-evidence.json /absolute/
|
||||
- [Hopper Disassembler](https://www.hopperapp.com/)
|
||||
- [سياسة الأمان](SECURITY.md)
|
||||
|
||||
## تاريخ النجوم
|
||||
|
||||
🎉 **وصلنا إلى 20,000 نجمة على GitHub، شكرًا لكم!**
|
||||
|
||||
شكرًا لكل من يستخدم REA ويبلّغ عن الأخطاء ويختبر الإصدارات ويساهم في إصلاحها.
|
||||
|
||||
<a href="https://www.star-history.com/?repos=morluto%2Frea&type=date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
<img alt="تاريخ نجوم REA على GitHub" src="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
## الترخيص
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
@@ -366,6 +366,20 @@ REA にホスト型の解析サービスはありません。現在のプロバ
|
||||
|
||||
開発環境、アーキテクチャ、テスト、リリース手順は [CONTRIBUTING.md](CONTRIBUTING.md) を参照してください。
|
||||
|
||||
## スター履歴
|
||||
|
||||
🎉 **GitHub スターが 20,000 を突破しました。ありがとうございます!**
|
||||
|
||||
REA を使い、問題を報告し、ビルドをテストし、修正を届けてくださる皆さんに感謝します。
|
||||
|
||||
<a href="https://www.star-history.com/?repos=morluto%2Frea&type=date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
<img alt="REA の GitHub スター履歴" src="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
## ライセンス
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
@@ -315,6 +315,20 @@ REA에는 호스팅 분석 서비스가 없습니다. 로컬 Unix 소켓을 통
|
||||
|
||||
개발 환경, 아키텍처, 테스트, 릴리스 지침은 [CONTRIBUTING.md](CONTRIBUTING.md)를 참고하세요.
|
||||
|
||||
## 스타 기록
|
||||
|
||||
🎉 **GitHub 스타 20,000개, 감사합니다!**
|
||||
|
||||
REA를 사용하고, 버그를 제보하고, 빌드를 테스트하고, 수정에 기여해 주신 모든 분께 감사드립니다.
|
||||
|
||||
<a href="https://www.star-history.com/?repos=morluto%2Frea&type=date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
<img alt="REA GitHub 스타 기록" src="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
## 라이선스
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
@@ -316,6 +316,20 @@ REA 不提供托管分析服务,而是通过本地 Unix 套接字把操作交
|
||||
|
||||
开发环境、架构、测试和发布说明请参阅 [CONTRIBUTING.md](CONTRIBUTING.md)。
|
||||
|
||||
## Star 历史
|
||||
|
||||
🎉 **GitHub Star 突破 20,000,感谢大家!**
|
||||
|
||||
感谢每一位使用 REA、反馈问题、测试构建和贡献修复的朋友。
|
||||
|
||||
<a href="https://www.star-history.com/?repos=morluto%2Frea&type=date">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date&theme=dark&legend=top-left" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
<img alt="REA GitHub Star 历史" src="https://api.star-history.com/chart?repos=morluto/rea&type=date" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
## 许可证
|
||||
|
||||
[MIT](LICENSE)
|
||||
|
||||
@@ -133,8 +133,6 @@ def project_record(state, secrets):
|
||||
for key, item in value.items():
|
||||
if not isinstance(key, str):
|
||||
raise CaptureFailure("format", "Native record has a non-string dictionary key.", path)
|
||||
if key == "__proto__":
|
||||
raise CaptureFailure("unsupported", "Native object contains a __proto__ member that the current JSON schema boundary cannot preserve.", path)
|
||||
if any(secret in key for secret in secrets):
|
||||
# Validate the omitted subtree too, then discard all of its
|
||||
# coordinates rather than inventing a replacement identity.
|
||||
|
||||
@@ -112,6 +112,31 @@ enum RunTokenReadError: Error {
|
||||
case ambiguousEnvironmentBoundary
|
||||
}
|
||||
|
||||
@_silgen_name("csops")
|
||||
private func csops(
|
||||
_ pid: Int32,
|
||||
_ ops: UInt32,
|
||||
_ addr: UnsafeMutableRawPointer?,
|
||||
_ size: Int
|
||||
) -> Int32
|
||||
|
||||
private let csOpsStatus: UInt32 = 0
|
||||
private let csPlatformBinary: UInt32 = 0x0400_0000
|
||||
|
||||
/**
|
||||
* Whether the target image is an Apple platform binary. Current macOS omits
|
||||
* the environment of platform binaries from KERN_PROCARGS2 for every caller,
|
||||
* so this distinguishes a policy-withheld environment from a process whose
|
||||
* environment is genuinely absent or transiently unreadable.
|
||||
*/
|
||||
func isPlatformBinary(pid: Int32) -> Bool {
|
||||
var flags: UInt32 = 0
|
||||
let result = withUnsafeMutablePointer(to: &flags) { pointer in
|
||||
csops(pid, csOpsStatus, UnsafeMutableRawPointer(pointer), MemoryLayout<UInt32>.size)
|
||||
}
|
||||
return result == 0 && (flags & csPlatformBinary) != 0
|
||||
}
|
||||
|
||||
private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation {
|
||||
var mib = [CTL_KERN, KERN_PROCARGS2, Int32(pid)]
|
||||
var size = 0
|
||||
@@ -133,7 +158,12 @@ private func readRunToken(pid: Int32, pointerSize: Int) -> RunTokenObservation {
|
||||
} catch RunTokenReadError.invalidTokenEncoding {
|
||||
return unavailable(pid: pid, reason: "invalid_run_token_encoding")
|
||||
} catch RunTokenReadError.environmentUnavailable {
|
||||
return unavailable(pid: pid, reason: "environment_unavailable")
|
||||
return unavailable(
|
||||
pid: pid,
|
||||
reason: isPlatformBinary(pid: pid)
|
||||
? "platform_binary_environment_withheld"
|
||||
: "environment_unavailable"
|
||||
)
|
||||
} catch RunTokenReadError.appleVectorUnavailable {
|
||||
return unavailable(pid: pid, reason: "apple_vector_unavailable")
|
||||
} catch RunTokenReadError.ambiguousEnvironmentBoundary {
|
||||
|
||||
@@ -27,7 +27,7 @@ REA_PWNTOOLS_PYTHON=/absolute/isolated-env/bin/python \
|
||||
REA never installs Python, packages or GDB, changes a user init file, launches
|
||||
the selected object as a host process, or requests runtime library resolution. The Python process
|
||||
uses isolated mode and an owned cache. Exact upstream profiles are recorded in
|
||||
[upstream provenance](../third_party/pwntools/README.md).
|
||||
[upstream provenance](https://github.com/morluto/rea/blob/main/third_party/pwntools/README.md).
|
||||
|
||||
## Interpreting results
|
||||
|
||||
|
||||
@@ -398,7 +398,29 @@
|
||||
"calculated_sha256": {
|
||||
"anyOf": [{ "type": "string" }, { "type": "null" }]
|
||||
},
|
||||
"unpacked": { "type": "boolean" }
|
||||
"unpacked": { "type": "boolean" },
|
||||
"checksum_mismatches": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"representation": {
|
||||
"type": "string",
|
||||
"enum": ["stored", "decoded"]
|
||||
},
|
||||
"algorithm": { "type": "string" },
|
||||
"declared": { "type": "string" },
|
||||
"observed": { "type": "string" }
|
||||
},
|
||||
"required": [
|
||||
"representation",
|
||||
"algorithm",
|
||||
"declared",
|
||||
"observed"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"logical_path",
|
||||
|
||||
@@ -24,7 +24,7 @@ their distinct unavailable-provider or capability classification.
|
||||
The host must already provide util-linux `prlimit`, normally `/usr/bin/prlimit`;
|
||||
`REA_EVM_PRLIMIT_COMMAND` can select another absolute executable. REA installs
|
||||
no system tools and changes no user configuration. [Upstream provenance and
|
||||
license](../third_party/evmole/README.md) are packaged.
|
||||
license](https://github.com/morluto/rea/blob/main/third_party/evmole/README.md) are packaged.
|
||||
|
||||
## Evidence semantics
|
||||
|
||||
|
||||
@@ -95,7 +95,7 @@ npx skills add morluto/rea --skill reverse-engineer-anything
|
||||
|
||||
This installs agent instructions and bundled references, not REA MCP
|
||||
registration or analysis engines. Follow the skill's
|
||||
[conditional connection guide](https://github.com/morluto/rea/blob/main/skills/reverse-engineer-anything/SKILL.md#connect-only-when-needed).
|
||||
[conditional connection guide](https://github.com/morluto/rea/blob/main/skill-src/reverse-engineer-anything/SKILL.md#connect-only-when-needed).
|
||||
Working tools can be used immediately. If tools are missing, inspect the current
|
||||
client's registration with `doctor --client codex --json` (substitute its client
|
||||
ID), then plan repairs with `setup --client codex --dry-run --json`. Show and
|
||||
|
||||
@@ -100,17 +100,22 @@ a temporary output file larger than the running engine's string limit, verifies
|
||||
its bytes and digest, and removes it. Use `-- jsonl` for the compact JSONL check.
|
||||
Each check needs space for one output file plus a 1 GiB free-space reserve.
|
||||
|
||||
Other CLI formats, `--token-count`, and MCP transport serialization still
|
||||
assemble whole strings. Field selection remains useful when the caller needs a
|
||||
Other CLI formats and `--token-count` still assemble whole strings.
|
||||
Field selection remains useful when the caller needs a
|
||||
smaller view, for example `--format json --filter-output
|
||||
evidence_id,normalized_result.statistics`. Streaming output does not bound the
|
||||
memory needed to construct the analysis graph itself.
|
||||
The separate large-response transport work is tracked in
|
||||
[#1053](https://github.com/morluto/rea/issues/1053) and
|
||||
MCP prepares the complete repeated response incrementally against the pinned
|
||||
SDK's 10 MiB stdio receive-buffer budget. Oversized results return an actionable
|
||||
transport constraint and the exact same-session Evidence reference. Use
|
||||
`trace_application_feature` to inspect a selected module's relationships, or
|
||||
`export_evidence_bundle` to write the complete canonical bundle without a
|
||||
document-sized allocation. Same-session analysis reads reuse authenticated
|
||||
immutable snapshots; foreign inline Evidence is still parsed and authenticated.
|
||||
See [MCP tool results](mcp-contracts.md#tool-results) for larger client buffers
|
||||
and `REA_MCP_MAX_RESPONSE_BYTES`. Follow-up results remain complete and can also
|
||||
exceed the transport budget. Compact result views are tracked separately in
|
||||
[#1050](https://github.com/morluto/rea/issues/1050).
|
||||
Client framing limits also apply: the pinned Node MCP SDK's stdio transport
|
||||
defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate
|
||||
the complete response, including text and structured Evidence projections.
|
||||
|
||||
## What is reconstructed
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ part of the current tool set.
|
||||
|
||||
This guide describes the implementation and verification of
|
||||
[ADR-0003](adr/0003-managed-code-evidence-and-provider-boundary.md). The canonical
|
||||
tool inventory is [`product-catalog.json`](https://github.com/morluto/rea/blob/main/docs/product-catalog.json).
|
||||
tool inventory is the [build-generated catalog](mcp-contracts.md#generated-catalog).
|
||||
|
||||
## Shipped scope
|
||||
|
||||
|
||||
@@ -1,5 +1,15 @@
|
||||
# MCP runtime contracts
|
||||
|
||||
## Generated catalog
|
||||
|
||||
Run `npm run build:cached` in a source checkout to generate the machine-readable
|
||||
catalog at `docs/public/product-catalog.json`. The [catalog](product-catalog.json)
|
||||
is also included in documentation builds. PR CI retains it with the packaged
|
||||
skill and portable conformance projections in the `generated-docs` artifact.
|
||||
These outputs describe the exact source revision being built; they are not
|
||||
checked-in snapshots. For a running server, `binary_session` remains the
|
||||
authoritative source of catalog identity and tool availability.
|
||||
|
||||
## Identity and discovery
|
||||
|
||||
`binary_session` reports the active package, server, SDK, and negotiated
|
||||
@@ -154,6 +164,26 @@ directly to `compare_functions`, and `inspect_artifact` Evidence to
|
||||
`compare_artifacts`. Use `get_evidence_bundle` when the task needs broader
|
||||
retained session history or an explicit bundle for transfer.
|
||||
|
||||
REA prepares complete MCP results within the pinned stdio client's 10 MiB
|
||||
receive-buffer budget, including both text and structured representations and
|
||||
room for the JSON-RPC envelope. If a result cannot fit, REA returns
|
||||
`resource_constraint` with `details.resource: "transport"` before constructing
|
||||
a document-sized string. Analysis Evidence remains complete in the current
|
||||
session. Its exact reference is reported in
|
||||
`details.reported_limits.evidence_reference`; use it with a focused application
|
||||
workflow, or call `export_evidence_bundle` with a destination path. Complete
|
||||
bundle exports stream canonical JSON into an atomically published file. A broad
|
||||
follow-up or `get_evidence_bundle` can also exceed the response budget; exporting
|
||||
preserves the complete session without sending it through a single MCP frame.
|
||||
|
||||
Clients that explicitly configure a larger receive buffer can set the REA
|
||||
server's `REA_MCP_MAX_RESPONSE_BYTES` environment variable to the same byte
|
||||
count. This setting must be a safe decimal integer at least 10485760; REA
|
||||
reserves 1024 bytes for the envelope. Raising it restores complete inline
|
||||
delivery for responses that fit that buffer and Node's single-string limit.
|
||||
It does not change the client's buffer, analysis coverage, or retained content.
|
||||
Ordinary responses keep their existing complete result contract.
|
||||
|
||||
## Retained application Evidence inputs
|
||||
|
||||
`trace_application_feature`, `trace_javascript_semantics`,
|
||||
@@ -250,6 +280,12 @@ Artifact integrity fails closed by default. A request can explicitly select
|
||||
`integrity_policy=record-and-continue` when the investigation needs verified
|
||||
siblings to continue after a mismatch.
|
||||
|
||||
Strict member failures include `details.checksum_mismatches` for conflicting
|
||||
stored or decoded checksums, preserving each algorithm and declared/observed
|
||||
value. An unsupported checksum algorithm leaves that verification facet unknown;
|
||||
readable member bytes are still hashed and can be extracted, with the declared
|
||||
algorithm and value retained in limitations.
|
||||
|
||||
Contradictory bytes are quarantined from nested expansion and recorded with
|
||||
declared and observed hashes, trust, provenance, path, and unpacked state.
|
||||
Verified siblings continue. Comparisons classify the result as a contradiction
|
||||
|
||||
@@ -102,6 +102,12 @@ run the same owned-process cleanup path. Settlement reports whether the
|
||||
sampled process group quiesced or whether cleanup was needed or unverifiable;
|
||||
sampling cannot prove that every short-lived or detached descendant was seen.
|
||||
|
||||
When the host withholds an unrelated process’s ownership token, REA leaves that
|
||||
process untouched and records its PID and reason in `cleanup.unverified_processes`
|
||||
and process residual unknowns. Successful cleanup verifies the owned group;
|
||||
it does not attribute those unrelated processes. Related or otherwise unexplained
|
||||
unreadable processes still prevent successful cleanup.
|
||||
|
||||
## Compare two captures
|
||||
|
||||
Compare saved capture Evidence with:
|
||||
|
||||
@@ -1,722 +0,0 @@
|
||||
{
|
||||
"package": {
|
||||
"name": "rea-agents",
|
||||
"version": "6.0.0",
|
||||
"sdk": {
|
||||
"server": "2.3.1",
|
||||
"client": "2.3.1",
|
||||
"core": "2.3.1"
|
||||
},
|
||||
"skill_version": "33"
|
||||
},
|
||||
"tools": {
|
||||
"total": 139,
|
||||
"families": [
|
||||
{
|
||||
"id": "direct",
|
||||
"surface": "official-proxy",
|
||||
"count": 41,
|
||||
"tools": [
|
||||
"address_name",
|
||||
"address_to_file_offset",
|
||||
"annotate_native_function",
|
||||
"comment",
|
||||
"current_address",
|
||||
"current_document",
|
||||
"current_procedure",
|
||||
"goto_address",
|
||||
"inline_comment",
|
||||
"inspect_native_data_type",
|
||||
"inspect_native_instruction",
|
||||
"inspect_native_load_image",
|
||||
"list_bookmarks",
|
||||
"list_documents",
|
||||
"list_names",
|
||||
"list_procedures",
|
||||
"list_segments",
|
||||
"list_strings",
|
||||
"next_address",
|
||||
"prev_address",
|
||||
"procedure_address",
|
||||
"procedure_assembly",
|
||||
"procedure_callees",
|
||||
"procedure_callers",
|
||||
"procedure_info",
|
||||
"procedure_pseudo_code",
|
||||
"procedure_references",
|
||||
"read_bytes",
|
||||
"read_function_instructions",
|
||||
"resolve_containing_procedure",
|
||||
"resolve_native_call_targets",
|
||||
"search_procedures",
|
||||
"search_strings",
|
||||
"set_address_name",
|
||||
"set_addresses_names",
|
||||
"set_bookmark",
|
||||
"set_comment",
|
||||
"set_current_document",
|
||||
"set_inline_comment",
|
||||
"unset_bookmark",
|
||||
"xrefs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "enhanced",
|
||||
"surface": "enhanced",
|
||||
"count": 14,
|
||||
"tools": [
|
||||
"analyze_function",
|
||||
"analyze_swift_types",
|
||||
"batch_decompile",
|
||||
"binary_overview",
|
||||
"find_xrefs_to_name",
|
||||
"get_call_graph",
|
||||
"get_objc_classes",
|
||||
"get_objc_protocols",
|
||||
"inspect_native_api",
|
||||
"inspect_native_dispatch_metadata",
|
||||
"trace_call_path",
|
||||
"trace_feature",
|
||||
"trace_native_ui_action",
|
||||
"trace_native_values"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "native",
|
||||
"surface": "native-provider",
|
||||
"count": 10,
|
||||
"tools": [
|
||||
"capture_native_ui_scenario",
|
||||
"demangle_swift",
|
||||
"inspect_binary_layout",
|
||||
"inspect_macho",
|
||||
"inspect_plist",
|
||||
"inspect_recorded_crash",
|
||||
"inspect_signature",
|
||||
"list_architectures",
|
||||
"observe_native_calls",
|
||||
"observe_native_ui"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "artifact",
|
||||
"surface": "artifact-provider",
|
||||
"count": 7,
|
||||
"tools": [
|
||||
"decode_interface_builder",
|
||||
"extract_artifact",
|
||||
"inspect_artifact",
|
||||
"inspect_asset_catalog",
|
||||
"inspect_evm_interface",
|
||||
"inspect_keyed_archive",
|
||||
"trace_dylib_resolution"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "managed",
|
||||
"surface": "managed",
|
||||
"count": 7,
|
||||
"tools": [
|
||||
"compare_managed_members",
|
||||
"import_managed_reconstruction",
|
||||
"inspect_managed_artifact",
|
||||
"inspect_managed_members",
|
||||
"inspect_managed_native_boundaries",
|
||||
"project_managed_application_graph",
|
||||
"verify_managed_native_boundaries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "firmware",
|
||||
"surface": "firmware-provider",
|
||||
"count": 2,
|
||||
"tools": ["extract_firmware", "inspect_firmware_regions"]
|
||||
},
|
||||
{
|
||||
"id": "android",
|
||||
"surface": "android-provider",
|
||||
"count": 5,
|
||||
"tools": [
|
||||
"inspect_android_class",
|
||||
"inspect_android_method",
|
||||
"inspect_android_package",
|
||||
"search_android_classes",
|
||||
"trace_android_references"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "browser",
|
||||
"surface": "browser-provider",
|
||||
"count": 12,
|
||||
"tools": [
|
||||
"analyze_web_bundle",
|
||||
"capture_browser_scenario",
|
||||
"capture_web_screenshot",
|
||||
"compare_web_captures",
|
||||
"compare_web_screenshots",
|
||||
"discover_webmcp_tools",
|
||||
"inspect_web_event_listeners",
|
||||
"inspect_web_network_capture",
|
||||
"inspect_web_page",
|
||||
"list_browser_targets",
|
||||
"observe_web_execution",
|
||||
"observe_web_session"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "electron",
|
||||
"surface": "electron-provider",
|
||||
"count": 5,
|
||||
"tools": [
|
||||
"analyze_javascript_application",
|
||||
"capture_electron_scenario",
|
||||
"inspect_electron_page",
|
||||
"list_electron_targets",
|
||||
"reconcile_javascript_runtime"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "javascript-runtime",
|
||||
"surface": "runtime-provider",
|
||||
"count": 2,
|
||||
"tools": [
|
||||
"list_javascript_runtime_targets",
|
||||
"observe_javascript_runtime"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "application",
|
||||
"surface": "application-workflow",
|
||||
"count": 13,
|
||||
"tools": [
|
||||
"build_reconstruction_obligation_ledger",
|
||||
"compare_application_versions",
|
||||
"compare_javascript_export_shapes",
|
||||
"compare_source_to_bundle",
|
||||
"evaluate_reconstruction_coverage",
|
||||
"export_web_scripts",
|
||||
"project_android_application_graph",
|
||||
"project_apple_application_graph",
|
||||
"recover_javascript_sources",
|
||||
"trace_application_feature",
|
||||
"trace_javascript_semantics",
|
||||
"trace_web_module_imports",
|
||||
"trace_web_source_location"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "session",
|
||||
"surface": "session",
|
||||
"count": 21,
|
||||
"tools": [
|
||||
"binary_session",
|
||||
"build_call_path",
|
||||
"capture_process_scenario",
|
||||
"close_binary",
|
||||
"compare_artifacts",
|
||||
"compare_bundles",
|
||||
"compare_functions",
|
||||
"compare_process_captures",
|
||||
"correlate_static_and_runtime",
|
||||
"export_evidence_bundle",
|
||||
"find_changed_behavior",
|
||||
"get_evidence_bundle",
|
||||
"get_navigation_context",
|
||||
"import_evidence_bundle",
|
||||
"inspect_address_context",
|
||||
"list_unknowns",
|
||||
"open_binary",
|
||||
"record_unknown",
|
||||
"update_unknown",
|
||||
"verify_reconstruction",
|
||||
"verify_unknown_resolution"
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"providers": [
|
||||
{
|
||||
"id": "binwalk",
|
||||
"name": "Binwalk",
|
||||
"version": "3.1.0",
|
||||
"capabilities": ["inspect_firmware_regions"]
|
||||
},
|
||||
{
|
||||
"id": "evmole-interface",
|
||||
"name": "REA EVM interface adapter",
|
||||
"version": "evmole@0.9.3",
|
||||
"capabilities": ["inspect_evm_interface"]
|
||||
},
|
||||
{
|
||||
"id": "ghidra",
|
||||
"name": "Ghidra",
|
||||
"version": null,
|
||||
"capabilities": [
|
||||
"address_name",
|
||||
"address_to_file_offset",
|
||||
"analyze_function",
|
||||
"annotate_native_function",
|
||||
"inspect_native_data_type",
|
||||
"inspect_native_instruction",
|
||||
"inspect_native_load_image",
|
||||
"list_documents",
|
||||
"list_names",
|
||||
"list_procedures",
|
||||
"list_segments",
|
||||
"list_strings",
|
||||
"procedure_address",
|
||||
"procedure_assembly",
|
||||
"procedure_callees",
|
||||
"procedure_callers",
|
||||
"procedure_info",
|
||||
"procedure_pseudo_code",
|
||||
"procedure_references",
|
||||
"read_bytes",
|
||||
"read_function_instructions",
|
||||
"resolve_containing_procedure",
|
||||
"resolve_native_call_targets",
|
||||
"search_procedures",
|
||||
"search_strings",
|
||||
"xrefs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "har-schema/lossless-json",
|
||||
"name": "REA HAR capture adapter",
|
||||
"version": "har-schema@2.0.0;jsonc-parser@3.3.1;lossless-json@4.3.1",
|
||||
"capabilities": ["inspect_web_network_capture"]
|
||||
},
|
||||
{
|
||||
"id": "hopper",
|
||||
"name": "Hopper Disassembler",
|
||||
"version": null,
|
||||
"capabilities": [
|
||||
"address_name",
|
||||
"address_to_file_offset",
|
||||
"analyze_function",
|
||||
"comment",
|
||||
"current_address",
|
||||
"current_document",
|
||||
"current_procedure",
|
||||
"goto_address",
|
||||
"inline_comment",
|
||||
"list_bookmarks",
|
||||
"list_documents",
|
||||
"list_names",
|
||||
"list_procedures",
|
||||
"list_segments",
|
||||
"list_strings",
|
||||
"next_address",
|
||||
"prev_address",
|
||||
"procedure_address",
|
||||
"procedure_assembly",
|
||||
"procedure_callees",
|
||||
"procedure_callers",
|
||||
"procedure_info",
|
||||
"procedure_pseudo_code",
|
||||
"procedure_references",
|
||||
"read_bytes",
|
||||
"read_function_instructions",
|
||||
"resolve_containing_procedure",
|
||||
"search_procedures",
|
||||
"search_strings",
|
||||
"set_address_name",
|
||||
"set_addresses_names",
|
||||
"set_bookmark",
|
||||
"set_comment",
|
||||
"set_current_document",
|
||||
"set_inline_comment",
|
||||
"unset_bookmark",
|
||||
"xrefs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "ida",
|
||||
"name": "IDA Pro MCP adapter",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"analyze_function",
|
||||
"list_procedures",
|
||||
"list_strings",
|
||||
"procedure_address",
|
||||
"procedure_assembly",
|
||||
"procedure_callees",
|
||||
"procedure_callers",
|
||||
"procedure_pseudo_code",
|
||||
"read_function_instructions",
|
||||
"search_procedures",
|
||||
"search_strings",
|
||||
"xrefs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "jadx",
|
||||
"name": "JADX headless",
|
||||
"version": "0.7.1",
|
||||
"capabilities": [
|
||||
"inspect_android_class",
|
||||
"inspect_android_method",
|
||||
"inspect_android_package",
|
||||
"search_android_classes",
|
||||
"trace_android_references"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "mitmproxy-native-tnetstring",
|
||||
"name": "REA offline native mitmproxy adapter",
|
||||
"version": "12.2.3",
|
||||
"capabilities": ["inspect_web_network_capture"]
|
||||
},
|
||||
{
|
||||
"id": "native-macos",
|
||||
"name": "macOS native inspection utilities",
|
||||
"version": null,
|
||||
"capabilities": [
|
||||
"capture_native_ui_scenario",
|
||||
"demangle_swift",
|
||||
"inspect_macho",
|
||||
"inspect_plist",
|
||||
"inspect_signature",
|
||||
"list_architectures",
|
||||
"observe_native_calls",
|
||||
"observe_native_ui"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "pwntools-elf",
|
||||
"name": "REA pwntools ELF adapter",
|
||||
"version": "pwntools@4.15.0;pyelftools@0.33;unicorn@2.1.2",
|
||||
"capabilities": ["inspect_binary_layout", "inspect_recorded_crash"]
|
||||
},
|
||||
{
|
||||
"id": "rea-android-application",
|
||||
"name": "REA Android application workflows",
|
||||
"version": "1",
|
||||
"capabilities": ["project_android_application_graph"]
|
||||
},
|
||||
{
|
||||
"id": "rea-apple-application",
|
||||
"name": "REA Apple application workflows",
|
||||
"version": "1",
|
||||
"capabilities": ["project_apple_application_graph"]
|
||||
},
|
||||
{
|
||||
"id": "rea-artifact-graph",
|
||||
"name": "REA safe artifact graph provider",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"decode_interface_builder",
|
||||
"extract_artifact",
|
||||
"inspect_artifact",
|
||||
"inspect_asset_catalog",
|
||||
"inspect_keyed_archive",
|
||||
"trace_dylib_resolution"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rea-cdp-browser",
|
||||
"name": "REA Chrome DevTools Protocol observation provider",
|
||||
"version": "2",
|
||||
"capabilities": [
|
||||
"analyze_web_bundle",
|
||||
"capture_web_screenshot",
|
||||
"compare_web_captures",
|
||||
"compare_web_screenshots",
|
||||
"discover_webmcp_tools",
|
||||
"inspect_web_event_listeners",
|
||||
"inspect_web_page",
|
||||
"list_browser_targets",
|
||||
"observe_web_execution",
|
||||
"observe_web_session"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rea-cdp-electron",
|
||||
"name": "REA Electron file-page CDP observation provider",
|
||||
"version": "1",
|
||||
"capabilities": ["inspect_electron_page", "list_electron_targets"]
|
||||
},
|
||||
{
|
||||
"id": "rea-dotnet-static",
|
||||
"name": "REA managed static analysis provider",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"inspect_managed_artifact",
|
||||
"inspect_managed_members",
|
||||
"inspect_managed_native_boundaries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rea-dotnet-workflows",
|
||||
"name": "REA managed-code workflows",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"compare_managed_members",
|
||||
"import_managed_reconstruction",
|
||||
"project_managed_application_graph",
|
||||
"verify_managed_native_boundaries"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rea-javascript-application",
|
||||
"name": "REA JavaScript application analyzer",
|
||||
"version": "1",
|
||||
"capabilities": ["analyze_javascript_application"]
|
||||
},
|
||||
{
|
||||
"id": "rea-javascript-application-workflows",
|
||||
"name": "REA JavaScript application workflows",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"build_reconstruction_obligation_ledger",
|
||||
"compare_application_versions",
|
||||
"compare_javascript_export_shapes",
|
||||
"compare_source_to_bundle",
|
||||
"evaluate_reconstruction_coverage",
|
||||
"trace_application_feature",
|
||||
"trace_javascript_semantics"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rea-javascript-runtime-reconciliation",
|
||||
"name": "REA JavaScript runtime reconciliation",
|
||||
"version": "1",
|
||||
"capabilities": ["reconcile_javascript_runtime"]
|
||||
},
|
||||
{
|
||||
"id": "rea-playwright-browser-scenario",
|
||||
"name": "REA Playwright browser scenario capture provider",
|
||||
"version": "1",
|
||||
"capabilities": ["capture_browser_scenario"]
|
||||
},
|
||||
{
|
||||
"id": "rea-playwright-electron-active",
|
||||
"name": "REA Playwright active Electron observation provider",
|
||||
"version": "1",
|
||||
"capabilities": ["capture_electron_scenario"]
|
||||
},
|
||||
{
|
||||
"id": "rea-v8-inspector",
|
||||
"name": "REA passive Node/Electron V8 Inspector provider",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"list_javascript_runtime_targets",
|
||||
"observe_javascript_runtime"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "rea-web-script-export",
|
||||
"name": "REA captured script export",
|
||||
"version": "1",
|
||||
"capabilities": [
|
||||
"export_web_scripts",
|
||||
"trace_web_module_imports",
|
||||
"trace_web_source_location"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "unblob",
|
||||
"name": "Unblob",
|
||||
"version": "26.6.4",
|
||||
"capabilities": ["extract_firmware"]
|
||||
},
|
||||
{
|
||||
"id": "wakaru",
|
||||
"name": "Wakaru",
|
||||
"version": "1.13.0",
|
||||
"capabilities": ["recover_javascript_sources"]
|
||||
}
|
||||
],
|
||||
"setup_clients": [
|
||||
{
|
||||
"id": "claude_code",
|
||||
"display_name": "Claude Code",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "claude_desktop",
|
||||
"display_name": "Claude Desktop",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "codex",
|
||||
"display_name": "Codex",
|
||||
"format": "toml",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "cursor",
|
||||
"display_name": "Cursor",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "gemini_cli",
|
||||
"display_name": "Gemini CLI",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "windsurf",
|
||||
"display_name": "Windsurf",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "devin",
|
||||
"display_name": "Devin",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "opencode",
|
||||
"display_name": "OpenCode",
|
||||
"format": "opencode",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "antigravity",
|
||||
"display_name": "Antigravity",
|
||||
"format": "json",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "copilot_cli",
|
||||
"display_name": "GitHub Copilot CLI",
|
||||
"format": "copilot_cli",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "commandcode",
|
||||
"display_name": "Command Code",
|
||||
"format": "commandcode",
|
||||
"configuration": "managed"
|
||||
},
|
||||
{
|
||||
"id": "vscode",
|
||||
"display_name": "VS Code",
|
||||
"format": "vscode",
|
||||
"configuration": "managed"
|
||||
}
|
||||
],
|
||||
"cli": {
|
||||
"primary_count": 94,
|
||||
"commands": [
|
||||
"address-to-file-offset",
|
||||
"analyze",
|
||||
"analyze-javascript-application",
|
||||
"analyze-web-bundle",
|
||||
"annotate-native-function",
|
||||
"build-reconstruction-obligation-ledger",
|
||||
"capabilities",
|
||||
"capture-browser-scenario",
|
||||
"capture-electron-scenario",
|
||||
"capture-native-ui-scenario",
|
||||
"capture-process",
|
||||
"capture-web-screenshot",
|
||||
"compare",
|
||||
"compare-application-versions",
|
||||
"compare-javascript-export-shapes",
|
||||
"compare-managed-members",
|
||||
"compare-process-captures",
|
||||
"compare-source-to-bundle",
|
||||
"compare-web-captures",
|
||||
"compare-web-screenshots",
|
||||
"decode-interface-builder",
|
||||
"decompile",
|
||||
"demangle-swift",
|
||||
"discover-webmcp-tools",
|
||||
"doctor",
|
||||
"evaluate-reconstruction-coverage",
|
||||
"evidence-export",
|
||||
"evidence-import",
|
||||
"export-web-scripts",
|
||||
"extract-artifact",
|
||||
"extract-firmware",
|
||||
"function",
|
||||
"import-managed-reconstruction",
|
||||
"import-reference-source",
|
||||
"inspect",
|
||||
"inspect-android-class",
|
||||
"inspect-android-method",
|
||||
"inspect-android-package",
|
||||
"inspect-artifact",
|
||||
"inspect-asset-catalog",
|
||||
"inspect-binary-layout",
|
||||
"inspect-electron-page",
|
||||
"inspect-evm-interface",
|
||||
"inspect-firmware-regions",
|
||||
"inspect-keyed-archive",
|
||||
"inspect-macho",
|
||||
"inspect-managed-artifact",
|
||||
"inspect-managed-members",
|
||||
"inspect-managed-native-boundaries",
|
||||
"inspect-native-api",
|
||||
"inspect-native-data-type",
|
||||
"inspect-native-dispatch-metadata",
|
||||
"inspect-native-instruction",
|
||||
"inspect-native-load-image",
|
||||
"inspect-plist",
|
||||
"inspect-recorded-crash",
|
||||
"inspect-signature",
|
||||
"inspect-web-event-listeners",
|
||||
"inspect-web-network-capture",
|
||||
"inspect-web-page",
|
||||
"instructions",
|
||||
"list-architectures",
|
||||
"list-browser-targets",
|
||||
"list-electron-targets",
|
||||
"list-javascript-runtime-targets",
|
||||
"observe-javascript-runtime",
|
||||
"observe-native-calls",
|
||||
"observe-native-ui",
|
||||
"observe-web-execution",
|
||||
"observe-web-session",
|
||||
"project-android-application-graph",
|
||||
"project-apple-application-graph",
|
||||
"project-managed-application-graph",
|
||||
"providers",
|
||||
"read-bytes",
|
||||
"reconcile-javascript-runtime",
|
||||
"recover-javascript-sources",
|
||||
"resolve-native-call-targets",
|
||||
"search",
|
||||
"search-android-classes",
|
||||
"setup",
|
||||
"trace",
|
||||
"trace-android-references",
|
||||
"trace-application-feature",
|
||||
"trace-dylib-resolution",
|
||||
"trace-javascript-semantics",
|
||||
"trace-native-ui-action",
|
||||
"trace-native-values",
|
||||
"trace-web-module-imports",
|
||||
"trace-web-source-location",
|
||||
"uninstall",
|
||||
"update",
|
||||
"verify-managed-native-boundaries",
|
||||
"xrefs"
|
||||
],
|
||||
"aliases": [
|
||||
{
|
||||
"name": "compare-bundles",
|
||||
"target": "compare"
|
||||
}
|
||||
]
|
||||
},
|
||||
"runtime_catalog": {
|
||||
"counts": {
|
||||
"cli_commands": 94,
|
||||
"mcp_tools": 139,
|
||||
"mcp_prompts": 6
|
||||
},
|
||||
"digests": {
|
||||
"tools_sha256": "9ee5392d18f04f25b5c97c1b69a3644a0d81e52c813167a6f140791bc692741d",
|
||||
"prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d",
|
||||
"combined_sha256": "4c5f67d076baf821b5cb6d6fb2cc0f6c274e96b627b85b892577760a6f71ab1a",
|
||||
"providers_sha256": "954b499c76514890062d2b6c936d73cb3a9c45412d522a081125a325e81cdec5"
|
||||
}
|
||||
}
|
||||
}
|
||||
+52
-5
@@ -11,6 +11,32 @@ breaking changes. Record the full source SHA and create `release/VERSION` at
|
||||
that commit. The version is a maintainer choice; Release Please's proposed
|
||||
version must agree before publication.
|
||||
|
||||
`release/VERSION` also sets the expected version for the workflow. Use an exact
|
||||
SemVer such as `release/6.1.0` or `release/6.1.0-rc.1`, without build metadata.
|
||||
The checkpoint validator rejects a candidate whose package, lockfile, manifest,
|
||||
registry metadata or changelog disagrees with that version before creating a tag.
|
||||
|
||||
### Compatibility and version selection
|
||||
|
||||
REA uses the default SemVer strategy: compatible fixes increment patch;
|
||||
compatible capabilities increment minor; incompatible public behavior increments
|
||||
major. Adding tools, internal refactoring and correcting behavior outside the
|
||||
supported contract do not by themselves require a major release.
|
||||
|
||||
The supported public surface includes documented CLI commands/options, MCP tool
|
||||
names and input/result contracts, saved evidence formats and supported runtime
|
||||
requirements. Before adding `!` or a `BREAKING CHANGE` footer, a PR must identify
|
||||
a previously valid call or configuration that will fail or change meaning,
|
||||
explain why compatibility cannot be preserved, and give its migration. Prefer
|
||||
optional additions, compatibility adapters and a documented deprecation period;
|
||||
group deliberate removals into a planned major release. Maintainers review this
|
||||
impact before merging; the bot parses markers and cannot infer compatibility.
|
||||
|
||||
For example, adding an optional inspection tool is a minor change. Rejecting a
|
||||
relative MCP path previously accepted by the public contract is breaking;
|
||||
rejecting an input that the existing contract already prohibited is a fix.
|
||||
Changing the version number alone does not restore compatibility.
|
||||
|
||||
For example, after selecting a reviewed commit for 5.0.0:
|
||||
|
||||
```bash
|
||||
@@ -48,6 +74,24 @@ normalization, then approve that head's blocked runs from the PR page. Updates
|
||||
to the bot branch can create new approval-required runs; approval of an older
|
||||
head does not verify the normalized candidate. Wait for the final head's CI.
|
||||
|
||||
The validator reads the actual ancestry range from the published baseline tag
|
||||
to the selected checkpoint, including visible first-parent Conventional Commits
|
||||
and the PR titles in GitHub's default merge messages. The report preserves the
|
||||
original merge subject alongside the extracted Conventional Commit title.
|
||||
This matters after merging a side-branch release back into a newer main:
|
||||
Release Please's chronological history cutoff can omit unreleased mainline
|
||||
commits. Unreleased breaking markers require a major increment and references
|
||||
in the new release's breaking-change migration section. Other omitted entries
|
||||
are reported for review. Historical notes from an older release cannot satisfy
|
||||
the new release's migration check. This audits declared markers and references;
|
||||
it does not prove API compatibility or the quality of a migration explanation.
|
||||
|
||||
If preparation reports a mismatch or omitted breaking changes, inspect the
|
||||
recorded ancestry report, correct the release PR's version artifacts and notes,
|
||||
regenerate documentation, and review/test its final head. Do not repeatedly
|
||||
prepare over manual corrections. Publication independently checks the merged
|
||||
candidate again before Release Please can create a tag.
|
||||
|
||||
Review the candidate's version, notes, generated metadata, and package
|
||||
contents. Wait for the candidate's CI and relevant real-provider checks.
|
||||
Routine local iterations need focused checks; CI owns full deterministic
|
||||
@@ -100,8 +144,8 @@ before cutting the next checkpoint; otherwise main retains the previous
|
||||
release baseline and can propose an already-published version again.
|
||||
|
||||
After publication, open a PR from the release branch back to main. Preserve
|
||||
main's later implementation changes and resolve generated-file conflicts by
|
||||
regenerating from the combined contracts with the released package version.
|
||||
main's later implementation changes and regenerate ignored build outputs from
|
||||
the combined contracts with the released package version.
|
||||
Review and test this synchronization PR, then use a merge commit so the release
|
||||
tag remains in main's ancestry. Keep the released tag unchanged.
|
||||
|
||||
@@ -119,9 +163,12 @@ git merge-base --is-ancestor rea-agents-5.0.0 origin/main
|
||||
|
||||
Close any superseded rolling release PR. Future releases repeat the checkpoint
|
||||
procedure from main; never resume automatic release-PR refreshes on main pushes.
|
||||
The generated-metadata workflow stays limited to pull requests into main so it
|
||||
does not push commits onto a frozen candidate. Preparation normalizes
|
||||
`docs/product-catalog.json` on the bot pull request.
|
||||
Preparation generates and validates the catalog, portable conformance
|
||||
projections, and packaged skill from the candidate checkout without committing
|
||||
them. The checkpoint validator binds tracked version authority to the source
|
||||
SHA; generated-catalog and package checks validate the derived representation.
|
||||
No generated-metadata workflow pushes follow-up commits onto feature or release
|
||||
branches.
|
||||
|
||||
## Partial publication and retries
|
||||
|
||||
|
||||
+19
-9
@@ -98,6 +98,14 @@ an XML graph golden. `verify:asset-catalog` compiles source-owned colors with
|
||||
`actool`, invokes real `assetutil`, then checks CLI/MCP results, exact catalog
|
||||
digest, every raw metadata field, pagination and malformed input rejection.
|
||||
Neither artifact workflow requires Hopper or launches it. Both run in macOS CI.
|
||||
`verify:installer-package` needs only macOS's `pkgbuild`, `productbuild` and
|
||||
`pkgutil`. It builds a component package with install scripts, a product package
|
||||
around it, and a pbzx (`--compression latest`) package. It checks that
|
||||
`inspect-artifact` expands Scripts and Payload with exact content digests, and
|
||||
that the expanded Payload members equal Apple's `pkgutil --payload-files`
|
||||
listing. It also checks that the product package nests its component, that the
|
||||
pbzx payload is recorded with a limitation instead of being expanded, and that
|
||||
CLI and stdio MCP results agree.
|
||||
`verify:macos-bundle` needs only macOS with Command Line Tools. It compiles a
|
||||
source-owned app with `clang`: a versioned framework, XPC services, an app
|
||||
extension, a login item, a privileged helper, launchd plists, and a helper tool,
|
||||
@@ -425,7 +433,7 @@ behavior, and complete CI evidence before merging.
|
||||
| `npm run test:watch:all` | Changed tests from every project; builds at startup, so rebuild after production edits before relying on compiled tests |
|
||||
| `npm run check:changed` | Cached typecheck/lint and branch-related source feedback |
|
||||
| `npm run check:pr` | Opt-in complete local deterministic gate and generated-file checks |
|
||||
| `npm run docs:check` | Committed generated-document freshness |
|
||||
| `npm run docs:check` | Generated-document validation from current source and build outputs |
|
||||
|
||||
For example:
|
||||
|
||||
@@ -460,19 +468,21 @@ broad changes or diagnosing CI, rather than after every edit. Package/install
|
||||
changes additionally need package verification; provider changes need actual
|
||||
provider evidence.
|
||||
|
||||
Local full-suite Vitest runs use up to two workers and schedule projects one
|
||||
at a time. Process, acceptance and process-global projects serialize their
|
||||
files to prevent competing lifecycle observations. CI retains its two-worker
|
||||
budget.
|
||||
Each Vitest project uses up to two workers, bounded by available host parallelism.
|
||||
Default-group projects can run together. Only `process-boundary` runs in the
|
||||
later sequence group and serializes its files because process-tree observations
|
||||
share host sampling resources. Acceptance and process-global files retain isolated
|
||||
forks but are not serialized; isolation does not imply serial scheduling. CI uses
|
||||
the same per-project worker bound.
|
||||
The pure domain/contracts and recording-port service projects share one worker
|
||||
module context because their tests own no mutable runtime resources. MCP
|
||||
boundary files also share the immutable server module graph while creating and
|
||||
closing independent in-memory sessions. Adapter, composition, acceptance,
|
||||
process-global, and other boundary projects retain per-file isolation.
|
||||
`npm test`, `npm run docs:check`, and `npm run docs:generate` share
|
||||
repository-local locks and fail fast when the same class of command is already
|
||||
running. The `npm test` build is inside that lock. `check:pr` runs its test task
|
||||
before starting generated-document validation.
|
||||
Build and documentation writers use checkout-local locks for their respective
|
||||
output files. `npm test` delegates its build to the build lock; the Vitest suite
|
||||
does not hold a test-command lock. `check:pr` runs its test task before starting
|
||||
generated-document validation.
|
||||
|
||||
Vitest and Node persistent compile caches are deliberately not enabled by
|
||||
default. To evaluate repeated local runs, opt in for both cold and warm
|
||||
|
||||
+12
-1
@@ -80,7 +80,18 @@ or model-authored loop when one call can answer the question.
|
||||
## Implement and evaluate
|
||||
|
||||
- Put shared user workflows in the application layer; keep MCP translation in
|
||||
the server adapter and CLI behavior aligned with the same workflow.
|
||||
the server adapter and CLI behavior aligned with the same workflow. Share
|
||||
Evidence provenance, unknown projection, and eligible snapshot binding through
|
||||
application/session owners. Matching result payloads alone do not establish
|
||||
equivalent retention or replay behavior.
|
||||
- Derive each facet's coverage from examined records, unsupported representations,
|
||||
and exhaustiveness. Keep issues structured until presentation; do not select a
|
||||
facet by matching diagnostic prose or infer completeness from an empty failure
|
||||
list. Preserve available observations independently of cleanup success.
|
||||
- Measure the resource consumed before choosing a limit. Value alternatives,
|
||||
retained records, trace-frame products, output bytes, and temporary-file growth
|
||||
need different accounting. Enforce a justified budget before the costly step
|
||||
and report its effect; an input-byte or graph-node cap does not bound every stage.
|
||||
- Update the canonical contract inventory, output schemas, examples, generated
|
||||
catalog, and relevant docs together.
|
||||
- Verify valid, malformed, boundary, unavailable, cancellation, and partial
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
{
|
||||
"ledger_id": "ecl_132e857a8a93f0b366cd72f9cfa1c338ffd4100460a7834ef69abff936c84472",
|
||||
"records": [
|
||||
{
|
||||
"claim_id": "managed.application-graph-projection",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_b061a5a559acb3748cb9784c637f6693f74faeb564ba2b11d6fc32619e757459"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.decompiler-reconstruction-import",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_13f3401a3aaddc3f6d087ad160fa928374e142cdb9861254359b46865b8c992e"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.dotnet-framework-x86-pinvoke",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_6f026216826ca06e080b77ff608f02634fa945fbd2e151b4cd261c1444f8bbb1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.ilspy-reconstruction-oracle",
|
||||
"status": "unsupported",
|
||||
"evidence_ids": [
|
||||
"ev_65a94c4de03f9c6ebdaff3f381cca377821f349d3ca6dfad47a31cc586fae939"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.malformed-metadata",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_deae4259ea4395623176de40d06c2abcbc2b3a77f0bfb068d76e568d801f45a8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.managed-native-verification",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_0edcbbcaa1191a4e5324a437f739957b77f9ac45e7897724337f2d6d06b3f9d1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.modern-dotnet-anycpu",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_d1af50e8dc04cd589385769efd8758e82095425f6e9c1cc4bf2f530c6b434c93"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.mvid-and-token-drift",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_5493f40bc875535488d789a490361abcc1f76bfbdf52536d09ff127442a8e854"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.not-managed",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_03ce0cc0183ee0710f8120e837565129559ab3bc41927df326066370c2341699"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.operator-local-manifest",
|
||||
"status": "unsupported",
|
||||
"evidence_ids": [
|
||||
"ev_9d3e50da1d372c55f46649e69eae702cbe8f0ba6a81d1e46afc3e1a284c81757"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.operator-manifest-self-test",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_f974f4b9d039cda841894c6173cae86427ecef48f4e7a4d6c73edf7b23af2b2a"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.unicode-obfuscated-identifiers",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_c4f38ce76d8359d92ee8e9442503e47b7913550442349a0df688bd274a3def67"
|
||||
]
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.x64-ready-to-run-native-body",
|
||||
"status": "pass",
|
||||
"evidence_ids": [
|
||||
"ev_3e05ef1d6a796d3d4cb3372fe8e549efec759c59bf09d8ff660059a2e448cbc0"
|
||||
]
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"total": 13,
|
||||
"pass": 11,
|
||||
"fail": 0,
|
||||
"unsupported": 2,
|
||||
"truncated": 0,
|
||||
"unknown": 0,
|
||||
"complete": false
|
||||
}
|
||||
}
|
||||
@@ -1,201 +0,0 @@
|
||||
{
|
||||
"manifest_id": "ecm_8a67e7a3578e49c8d8d22107eab8f1258094dc40041d14deee93189fd0c514b2",
|
||||
"verifier": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
},
|
||||
"environment": {
|
||||
"platform": "portable",
|
||||
"architecture": "portable",
|
||||
"runtime": "node",
|
||||
"runtime_version": "portable"
|
||||
},
|
||||
"skill_digests": [
|
||||
{
|
||||
"skill_id": "reverse-engineer-anything",
|
||||
"sha256": "ef398cf4b5ee7fb5e0bcf42ca787049ec145e1d911ee7951017fee1cb3980d74"
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
{
|
||||
"claim_id": "managed.application-graph-projection",
|
||||
"scenario": {
|
||||
"id": "managed-application-graph-projection",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"f17fe6286cb07c6385e786e0aa877564d44123197f02617dfec70c537a33e4ea"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.decompiler-reconstruction-import",
|
||||
"scenario": {
|
||||
"id": "decompiler-reconstruction-import",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"cead4cc99733e4e56a5fd0cec8a0813bbc24e249aa72a10eca4c47ee3b0e129d"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.dotnet-framework-x86-pinvoke",
|
||||
"scenario": {
|
||||
"id": "dotnet-framework-x86-pinvoke",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"f17fe6286cb07c6385e786e0aa877564d44123197f02617dfec70c537a33e4ea"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.ilspy-reconstruction-oracle",
|
||||
"scenario": {
|
||||
"id": "ilspy-reconstruction-oracle",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.malformed-metadata",
|
||||
"scenario": {
|
||||
"id": "malformed-metadata",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"827e10b6391d29edca48c191e2ddd79f3bc1165e46e4ba2f4edf41ab60f45e78"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.managed-native-verification",
|
||||
"scenario": {
|
||||
"id": "managed-native-verification",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"9999999999999999999999999999999999999999999999999999999999999999",
|
||||
"f17fe6286cb07c6385e786e0aa877564d44123197f02617dfec70c537a33e4ea"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.modern-dotnet-anycpu",
|
||||
"scenario": {
|
||||
"id": "modern-dotnet-anycpu",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"92b588ae43d418117fa108c0004b5d0451f6112238d21bad8d75c0eb1da860f0"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.mvid-and-token-drift",
|
||||
"scenario": {
|
||||
"id": "mvid-and-token-drift",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"276395c671a656ab57d583158443c10de552dccfb37dc97cd013f673c0f86b43",
|
||||
"ef009dab7424fdeba996f2524dbe86fd899c96394498cca0463fcfdb228c5572"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.not-managed",
|
||||
"scenario": {
|
||||
"id": "not-managed",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"de74dab8ab034054a811d4248efcda0bed505187a061a806e274ae87bc453754"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.operator-local-manifest",
|
||||
"scenario": {
|
||||
"id": "operator-local-manifest",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.operator-manifest-self-test",
|
||||
"scenario": {
|
||||
"id": "operator-manifest-self-test",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"cdc11ec7fd0f392e43b9b23971cb41ec49fd0a997fc33c24fcc553cd4386ae37"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.unicode-obfuscated-identifiers",
|
||||
"scenario": {
|
||||
"id": "unicode-obfuscated-identifiers",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"cead4cc99733e4e56a5fd0cec8a0813bbc24e249aa72a10eca4c47ee3b0e129d"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"claim_id": "managed.x64-ready-to-run-native-body",
|
||||
"scenario": {
|
||||
"id": "x64-ready-to-run-native-body",
|
||||
"version": 1
|
||||
},
|
||||
"artifact_sha256s": [
|
||||
"e47402428322cdcb06f561036d7d108528cbc6e262d92a21f869218a20c80bfb"
|
||||
],
|
||||
"provider": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -32,9 +32,9 @@ HAR uses unchanged `har-schema@2.0.0` draft-06 schemas, `ajv@8.20.0`,
|
||||
`lossless-json@4.3.1` numeric representations, pinned in the npm lockfile.
|
||||
Only the HAR 1.2 profile is supported. Unknown extensions remain reported
|
||||
evidence. Duplicate object keys, including equal values and escaped spellings,
|
||||
are rejected before materialization. A `__proto__` member returns an explicit
|
||||
unsupported-boundary error in both formats: the current shared JSON schema
|
||||
cannot preserve this member. Original capture files remain unchanged.
|
||||
are rejected before materialization. Prototype-named members such as
|
||||
`__proto__` are preserved as ordinary own JSON members without prototype
|
||||
mutation. Original capture files remain unchanged.
|
||||
The observed mitmproxy 12.2.3 `SaveHar` profile can emit `postData.text: null`
|
||||
for a missing request body. REA preserves that reported null, omitting the
|
||||
optional field only from its validation copy for this exact producer profile;
|
||||
|
||||
+1
-1
@@ -107,7 +107,7 @@ if [[ "$dry_run" == true ]]; then
|
||||
fi
|
||||
|
||||
printf 'Installing %s@%s...\n' "$PACKAGE" "$version"
|
||||
npm install --global "${prefix_args[@]}" "$PACKAGE@$version" || fail "npm could not install REA. Check registry access and npm permissions, then retry."
|
||||
npm install --global ${prefix_args[@]+"${prefix_args[@]}"} "$PACKAGE@$version" || fail "npm could not install REA. Check registry access and npm permissions, then retry."
|
||||
[[ -x "$install_bin" ]] || fail "npm completed without installing the rea command. Check the npm global bin directory and PATH, then retry."
|
||||
installed_version="$("$install_bin" --version 2>/dev/null | tr -d '[:space:]')" || fail "the installed REA version could not be read. Reinstall the requested version, then retry."
|
||||
[[ "$installed_version" == "$version" ]] || fail "installed version $installed_version does not match $version. Reinstall the requested version, then retry."
|
||||
|
||||
Generated
+1
@@ -17,6 +17,7 @@
|
||||
"@lydell/node-pty": "1.1.0",
|
||||
"@modelcontextprotocol/client": "2.3.1",
|
||||
"@modelcontextprotocol/server": "2.3.1",
|
||||
"@xmldom/xmldom": "0.9.12",
|
||||
"@xterm/addon-serialize": "0.14.0",
|
||||
"@xterm/headless": "6.0.0",
|
||||
"@zip.js/zip.js": "2.15.0",
|
||||
|
||||
+7
-2
@@ -87,7 +87,8 @@
|
||||
"mcp-catalog:check": "node scripts/generate-mcp-tool-catalog.mjs --check",
|
||||
"prebuild": "npm run deps:check && npm run metadata:check",
|
||||
"build": "node scripts/run-exclusive.mjs build npm run build:unlocked",
|
||||
"build:unlocked": "node scripts/clean-build-output.mjs && tsc -p tsconfig.build.json && node scripts/generate-mcp-tool-catalog.mjs",
|
||||
"build:unlocked": "node scripts/clean-build-output.mjs && tsc -p tsconfig.build.json && npm run build:projections",
|
||||
"build:projections": "node scripts/generate-mcp-tool-catalog.mjs && node scripts/generate-skill-metadata.mjs && node scripts/generate-product-catalog.mjs && node scripts/generate-completion-ledger.mjs",
|
||||
"precheck": "npm run deps:check",
|
||||
"check": "turbo run typecheck lint format:check knip metadata:check",
|
||||
"precheck:fast": "npm run deps:check",
|
||||
@@ -141,6 +142,7 @@
|
||||
"verify:apple-dispatch": "npm run build:cached && node scripts/verify/apple/dispatch.mjs",
|
||||
"verify:native-ui": "npm run build:cached && node scripts/verify-native-ui.mjs",
|
||||
"verify:keyed-archive": "npm run build:cached && node scripts/verify/apple/keyed-archive.mjs",
|
||||
"verify:installer-package": "npm run build:cached && node scripts/verify/apple/installer-package.mjs",
|
||||
"verify:macos-bundle": "npm run build:cached && node scripts/verify/apple/macos-bundle.mjs",
|
||||
"verify:ghidra": "npm run build:cached && node scripts/verify-real-ghidra.mjs",
|
||||
"verify:ida": "npm run build:cached && node scripts/verify-real-ida.mjs",
|
||||
@@ -179,11 +181,13 @@
|
||||
"config:print": "node scripts/print-mcp-config.mjs",
|
||||
"evidence:generate": "npm run build:cached && node scripts/generate-completion-ledger.mjs",
|
||||
"evidence:check": "npm run build:cached && node scripts/generate-completion-ledger.mjs --check",
|
||||
"predocs:dev": "npm run docs:generate",
|
||||
"docs:dev": "vitepress dev docs",
|
||||
"predocs:build": "npm run docs:generate",
|
||||
"docs:build": "vitepress build docs",
|
||||
"docs:preview": "vitepress preview docs",
|
||||
"docs:generate": "node scripts/run-exclusive.mjs docs npm run docs:generate:unlocked",
|
||||
"docs:generate:unlocked": "npm run metadata:generate && npm run build:cached && node scripts/generate-skill-metadata.mjs && node scripts/generate-product-catalog.mjs && node scripts/generate-completion-ledger.mjs && node scripts/generate-error-schema.mjs",
|
||||
"docs:generate:unlocked": "npm run metadata:generate && npm run build:cached && node scripts/generate-error-schema.mjs",
|
||||
"docs:check": "node scripts/run-exclusive.mjs docs turbo run docs:check:built",
|
||||
"docs:check:built": "node scripts/generate-package-metadata.mjs --check && node scripts/generate-mcp-tool-catalog.mjs --check && node scripts/generate-skill-metadata.mjs --check && node scripts/generate-product-catalog.mjs --check && node scripts/generate-completion-ledger.mjs --check && node scripts/check-doc-facts.mjs && node scripts/generate-error-schema.mjs --check",
|
||||
"docs:error-schema": "npm run build:cached && node scripts/generate-error-schema.mjs",
|
||||
@@ -229,6 +233,7 @@
|
||||
"@lydell/node-pty": "1.1.0",
|
||||
"@modelcontextprotocol/client": "2.3.1",
|
||||
"@modelcontextprotocol/server": "2.3.1",
|
||||
"@xmldom/xmldom": "0.9.12",
|
||||
"@xterm/addon-serialize": "0.14.0",
|
||||
"@xterm/headless": "6.0.0",
|
||||
"@zip.js/zip.js": "2.15.0",
|
||||
|
||||
@@ -34,11 +34,6 @@
|
||||
"type": "json",
|
||||
"path": "server.json",
|
||||
"jsonpath": "$.packages[0].version"
|
||||
},
|
||||
{
|
||||
"type": "json",
|
||||
"path": "docs/product-catalog.json",
|
||||
"jsonpath": "$.package.version"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ const { stdout } = await exec(
|
||||
},
|
||||
);
|
||||
const verifierOutput = JSON.parse(stdout);
|
||||
// Managed conformance claims are host-independent. Keep the checked-in
|
||||
// Managed conformance claims are host-independent. Keep the generated
|
||||
// commitment reproducible across macOS, Linux, and Windows rather than
|
||||
// embedding the machine that happened to generate it.
|
||||
const portableCompletionReport = {
|
||||
|
||||
@@ -17,7 +17,7 @@ const lock = JSON.parse(
|
||||
await readFile(join(root, "package-lock.json"), "utf8"),
|
||||
);
|
||||
const skill = await readFile(
|
||||
join(root, "skills/reverse-engineer-anything/SKILL.md"),
|
||||
join(root, "skill-src/reverse-engineer-anything/SKILL.md"),
|
||||
"utf8",
|
||||
);
|
||||
const skillVersion = /^\s{2}version:\s*"([^"]+)"\s*$/mu.exec(skill)?.[1];
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
@@ -14,8 +15,10 @@ for (const argument of arguments_)
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const catalog = await createProductCatalog(root);
|
||||
const path = join(root, "docs/public/product-catalog.json");
|
||||
if (!arguments_.has("--check")) await mkdir(dirname(path), { recursive: true });
|
||||
await ensureGeneratedFile({
|
||||
path: join(root, "docs/product-catalog.json"),
|
||||
path,
|
||||
source: await serializeProductCatalog(catalog),
|
||||
check: arguments_.has("--check"),
|
||||
generateCommand: "npm run docs:generate",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { mkdir, readFile, readdir, rm } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
@@ -10,26 +10,69 @@ for (const argument of arguments_)
|
||||
throw new Error(`Unknown skill metadata option: ${argument}`);
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const path = join(root, "skills/reverse-engineer-anything/SKILL.md");
|
||||
const current = await readFile(path, "utf8");
|
||||
const sourceRoot = join(root, "skill-src/reverse-engineer-anything");
|
||||
const outputRoot = join(root, "skills/reverse-engineer-anything");
|
||||
const check = arguments_.has("--check");
|
||||
const current = (
|
||||
await readFile(join(sourceRoot, "SKILL.md"), "utf8")
|
||||
).replaceAll("\r\n", "\n");
|
||||
const { CATALOG_IDENTITY } = await import(
|
||||
`${pathToFileURL(join(root, "dist/catalogIdentity.js")).href}?${String(Date.now())}`
|
||||
);
|
||||
const withCount = current.replace(
|
||||
/^\s{2}tool_count:\s*\d+\s*$/mu,
|
||||
` tool_count: ${String(CATALOG_IDENTITY.counts.mcp_tools)}`,
|
||||
if (/^\s{2}(?:tool_count|catalog_digest):/mu.test(current))
|
||||
throw new Error(
|
||||
"Catalog metadata belongs in the generated skill, not skill-src",
|
||||
);
|
||||
const versionLine = /^ {2}version: "[^"\r\n]+"$/mu;
|
||||
if (!versionLine.test(current))
|
||||
throw new Error("Missing authored skill version");
|
||||
const source = current.replace(
|
||||
versionLine,
|
||||
`$&\n tool_count: ${String(CATALOG_IDENTITY.counts.mcp_tools)}`,
|
||||
);
|
||||
const digestLine = ` catalog_digest: "${CATALOG_IDENTITY.digests.combined_sha256}"`;
|
||||
const source = /^\s{2}catalog_digest:/mu.test(withCount)
|
||||
? withCount.replace(
|
||||
/^\s{2}catalog_digest:\s*"[a-f0-9]{64}"\s*$/mu,
|
||||
digestLine,
|
||||
)
|
||||
: withCount.replace(/^(\s{2}tool_count:\s*\d+\s*)$/mu, `$1\n${digestLine}`);
|
||||
const paths = await filePaths(sourceRoot);
|
||||
// Rebuild this owned output directory so removed references cannot survive a build.
|
||||
if (!check) await rm(outputRoot, { recursive: true, force: true });
|
||||
for (const relativePath of paths) {
|
||||
const path = join(outputRoot, relativePath);
|
||||
if (!check) await mkdir(dirname(path), { recursive: true });
|
||||
await ensureGeneratedFile({
|
||||
path,
|
||||
source:
|
||||
relativePath === "SKILL.md"
|
||||
? source
|
||||
: (await readFile(join(sourceRoot, relativePath), "utf8")).replaceAll(
|
||||
"\r\n",
|
||||
"\n",
|
||||
),
|
||||
check,
|
||||
generateCommand: "npm run docs:generate",
|
||||
});
|
||||
}
|
||||
if (
|
||||
check &&
|
||||
JSON.stringify(await filePaths(outputRoot)) !== JSON.stringify(paths)
|
||||
)
|
||||
throw new Error(
|
||||
"Generated skill file inventory drifted; run npm run docs:generate",
|
||||
);
|
||||
|
||||
await ensureGeneratedFile({
|
||||
path,
|
||||
source,
|
||||
check: arguments_.has("--check"),
|
||||
generateCommand: "npm run docs:generate",
|
||||
});
|
||||
async function filePaths(directory, prefix = "") {
|
||||
const paths = [];
|
||||
for (const entry of (await readdir(directory, { withFileTypes: true })).sort(
|
||||
(left, right) =>
|
||||
left.name < right.name ? -1 : left.name > right.name ? 1 : 0,
|
||||
)) {
|
||||
const relativePath = prefix === "" ? entry.name : `${prefix}/${entry.name}`;
|
||||
if (entry.isDirectory())
|
||||
paths.push(
|
||||
...(await filePaths(join(directory, entry.name), relativePath)),
|
||||
);
|
||||
else if (entry.isFile()) paths.push(relativePath);
|
||||
else
|
||||
throw new Error(
|
||||
`Skill bundle requires regular files or directories: ${relativePath}`,
|
||||
);
|
||||
}
|
||||
return paths;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, readFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { chromium } from "playwright-core";
|
||||
@@ -62,13 +65,21 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) {
|
||||
);
|
||||
assert.equal(bytes.readUInt32BE(16), 2048);
|
||||
assert.equal(bytes.readUInt32BE(20), 1536);
|
||||
await verifyDefaultScreenshotDelivery({
|
||||
entrypoint,
|
||||
environment: env,
|
||||
endpoint,
|
||||
targetId: target.id,
|
||||
screenshot,
|
||||
});
|
||||
const maxBufferSize = 64 * 1024 * 1024;
|
||||
transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env,
|
||||
env: { ...env, REA_MCP_MAX_RESPONSE_BYTES: String(maxBufferSize) },
|
||||
stderr: "pipe",
|
||||
// The SDK defaults to a 10 MiB receive buffer; inline PNG output is larger.
|
||||
maxBufferSize: 64 * 1024 * 1024,
|
||||
maxBufferSize,
|
||||
});
|
||||
client = new Client({ name: "browser-screenshot-real-e2e", version: "1" });
|
||||
await client.connect(transport);
|
||||
@@ -102,6 +113,9 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) {
|
||||
mocked: false,
|
||||
cli: true,
|
||||
stdio_mcp: true,
|
||||
default_buffer_recovery: true,
|
||||
complete_export: true,
|
||||
matched_large_buffer: true,
|
||||
png_bytes: bytes.byteLength,
|
||||
};
|
||||
} finally {
|
||||
@@ -117,3 +131,73 @@ export async function verifyLargeScreenshotE2e(endpoint, origin) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const verifyDefaultScreenshotDelivery = async ({
|
||||
entrypoint,
|
||||
environment,
|
||||
endpoint,
|
||||
targetId,
|
||||
screenshot,
|
||||
}) => {
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-screenshot-delivery-"));
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({
|
||||
name: "browser-screenshot-default-e2e",
|
||||
version: "1",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const constrained = await client.callTool(
|
||||
{
|
||||
name: "capture_web_screenshot",
|
||||
arguments: { cdp_endpoint: endpoint, target_id: targetId },
|
||||
},
|
||||
{ timeout: 60000 },
|
||||
);
|
||||
assert.equal(constrained.isError, true);
|
||||
assert.equal(
|
||||
constrained.structuredContent?.error?.details?.resource,
|
||||
"transport",
|
||||
);
|
||||
const reference =
|
||||
constrained.structuredContent?.error?.details?.reported_limits
|
||||
?.evidence_reference;
|
||||
assert.equal(reference?.kind, "retained-evidence");
|
||||
await client.ping();
|
||||
const exported = await client.callTool({
|
||||
name: "export_evidence_bundle",
|
||||
arguments: { path: join(workspace, "screenshot-evidence.json") },
|
||||
});
|
||||
assert.notEqual(exported.isError, true, JSON.stringify(exported));
|
||||
const bundle = JSON.parse(
|
||||
await readFile(join(workspace, "screenshot-evidence.json"), "utf8"),
|
||||
);
|
||||
const record = bundle.records.find(
|
||||
(record) => record.evidence_id === reference.evidence_id,
|
||||
);
|
||||
assert.ok(
|
||||
record,
|
||||
"Full screenshot Evidence must remain exportable after the delivery constraint",
|
||||
);
|
||||
const retained = webScreenshotSchema.parse(
|
||||
parseEvidence(record).normalized_result,
|
||||
);
|
||||
assert.deepEqual(retained.artifact, screenshot.artifact);
|
||||
await client.ping();
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
try {
|
||||
await transport.close();
|
||||
} finally {
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -77,7 +77,12 @@ export const documentationFactIssues = async (root, catalog) => {
|
||||
requireText(issues, path, content, "MCP-tool_catalog");
|
||||
if (path === "README.md") {
|
||||
requireText(issues, path, content, "(docs/installation.md");
|
||||
requireText(issues, path, content, "(docs/product-catalog.json)");
|
||||
requireText(
|
||||
issues,
|
||||
path,
|
||||
content,
|
||||
"(docs/mcp-contracts.md#generated-catalog)",
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const client of catalog.setup_clients)
|
||||
@@ -99,11 +104,16 @@ export const documentationFactIssues = async (root, catalog) => {
|
||||
requireText(issues, installationPath, installation, client.display_name);
|
||||
|
||||
const agents = await readFile(join(root, "AGENTS.md"), "utf8");
|
||||
requireText(issues, "AGENTS.md", agents, "docs/product-catalog.json");
|
||||
requireText(issues, "AGENTS.md", agents, "docs/public/product-catalog.json");
|
||||
|
||||
const templatePath = ".github/pull_request_template.md";
|
||||
const template = await readFile(join(root, templatePath), "utf8");
|
||||
requireText(issues, templatePath, template, "docs/product-catalog.json");
|
||||
requireText(
|
||||
issues,
|
||||
templatePath,
|
||||
template,
|
||||
"docs/public/product-catalog.json",
|
||||
);
|
||||
|
||||
return issues;
|
||||
};
|
||||
|
||||
@@ -10,7 +10,10 @@ export const withLargeResultMcp = async (
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: environment,
|
||||
env: {
|
||||
...environment,
|
||||
REA_MCP_MAX_RESPONSE_BYTES: String(maxBufferSize),
|
||||
},
|
||||
stderr: "inherit",
|
||||
maxBufferSize,
|
||||
});
|
||||
|
||||
@@ -58,16 +58,16 @@ export const createProductCatalog = async (root) => {
|
||||
runtime_catalog: {
|
||||
counts: sources.catalogIdentity.CATALOG_IDENTITY.counts,
|
||||
digests: {
|
||||
...sources.catalogIdentity.CATALOG_IDENTITY.digests,
|
||||
// Full runtime schema digests belong to server identity, not this facts projection.
|
||||
providers_sha256: providerCatalogDigest(providers),
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/** Stable checked-in representation of the product catalog. */
|
||||
/** Stable build-generated representation of the product catalog. */
|
||||
export const serializeProductCatalog = (catalog) =>
|
||||
formatGeneratedFile(
|
||||
"docs/product-catalog.json",
|
||||
"docs/public/product-catalog.json",
|
||||
JSON.stringify(catalog, null, 2),
|
||||
);
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
import { gt, major, valid } from "semver";
|
||||
import { z } from "zod";
|
||||
|
||||
const shaSchema = z.string().regex(/^[a-f0-9]{40}$/u);
|
||||
const versionSchema = z.object({ version: z.string() });
|
||||
const manifestSchema = z.object({ ".": z.string() });
|
||||
const sectionsSchema = z.object({
|
||||
"changelog-sections": z.array(
|
||||
z.object({ type: z.string(), hidden: z.boolean().optional() }),
|
||||
),
|
||||
});
|
||||
const GITHUB_MERGE_SUBJECT = /^Merge pull request #(\d+) from \S+/u;
|
||||
|
||||
function conventionalTitle(subject, body) {
|
||||
if (!GITHUB_MERGE_SUBJECT.test(subject)) return subject;
|
||||
// GitHub's default merge message puts the PR title after its subject.
|
||||
// Inspect that first nonempty line, not arbitrary examples in commit prose.
|
||||
return (
|
||||
body
|
||||
.split(/\r?\n/u)
|
||||
.slice(1)
|
||||
.find((line) => line.trim() !== "")
|
||||
?.trim() ?? subject
|
||||
);
|
||||
}
|
||||
|
||||
function releaseVersion(value, location) {
|
||||
if (valid(value) !== value || value.includes("+")) {
|
||||
throw new Error(
|
||||
`${location} must contain an exact SemVer without build metadata: ${value}`,
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function readJson(git, sha, path, schema) {
|
||||
const text = await git(["show", `${sha}:${path}`]);
|
||||
let value;
|
||||
try {
|
||||
value = JSON.parse(text);
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) {
|
||||
throw new Error(`${path}@${sha} contains invalid JSON: ${error.message}`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const parsed = schema.safeParse(value);
|
||||
if (!parsed.success) {
|
||||
throw new Error(`${path}@${sha} is malformed: ${parsed.error.message}`);
|
||||
}
|
||||
return parsed.data;
|
||||
}
|
||||
|
||||
function readCommits(text) {
|
||||
if (text.length === 0) return [];
|
||||
const fields = text.split("\0");
|
||||
if (fields.pop() !== "" || fields.length % 3 !== 0) {
|
||||
throw new Error("Git returned malformed checkpoint history records.");
|
||||
}
|
||||
const commits = [];
|
||||
for (let index = 0; index < fields.length; index += 3) {
|
||||
const sha = shaSchema.parse(fields[index]);
|
||||
const subject = z.string().parse(fields[index + 1]);
|
||||
const body = z.string().parse(fields[index + 2]);
|
||||
const title = conventionalTitle(subject, body);
|
||||
const type = /^([a-z]+)(?:\([^)]*\))?!?: /u.exec(title)?.[1];
|
||||
commits.push({
|
||||
sha,
|
||||
subject,
|
||||
...(title !== subject ? { conventionalTitle: title } : {}),
|
||||
type,
|
||||
breaking:
|
||||
/^[a-z]+(?:\([^)]*\))?!: /u.test(title) ||
|
||||
/^BREAKING(?: CHANGE|-CHANGE):\s*\S/mu.test(body),
|
||||
});
|
||||
}
|
||||
return commits;
|
||||
}
|
||||
|
||||
function referencesCommit(text, commit) {
|
||||
if (text.includes(commit.sha.slice(0, 7))) return true;
|
||||
const pullRequest =
|
||||
GITHUB_MERGE_SUBJECT.exec(commit.subject)?.[1] ??
|
||||
/\(#(\d+)\)$/u.exec(commit.subject)?.[1];
|
||||
return (
|
||||
pullRequest !== undefined &&
|
||||
new RegExp(`#${pullRequest}(?!\\d)`, "u").test(text)
|
||||
);
|
||||
}
|
||||
|
||||
function releaseNotes(text, expectedVersion) {
|
||||
const headings = Array.from(text.matchAll(/^## \[([^\]]+)\].*$/gmu));
|
||||
const first = headings[0];
|
||||
if (first?.[1] !== expectedVersion || first.index === undefined) {
|
||||
throw new Error(`CHANGELOG.md must start with release ${expectedVersion}.`);
|
||||
}
|
||||
return text.slice(first.index, headings[1]?.index ?? text.length);
|
||||
}
|
||||
|
||||
function breakingSection(notes) {
|
||||
const heading = /^### [^\n]*BREAKING CHANGES[^\n]*\n/imu.exec(notes);
|
||||
if (heading === null) return "";
|
||||
const body = notes.slice(heading.index + heading[0].length);
|
||||
const end = body.search(/^### /mu);
|
||||
return end < 0 ? body : body.slice(0, end);
|
||||
}
|
||||
|
||||
async function requireAncestor(git, ancestor, descendant, description) {
|
||||
try {
|
||||
await git(["merge-base", "--is-ancestor", ancestor, descendant]);
|
||||
} catch (error) {
|
||||
const failure = z.object({ code: z.literal(1) }).safeParse(error);
|
||||
if (!failure.success) throw error;
|
||||
throw new Error(
|
||||
`${description}: ${ancestor} is not an ancestor of ${descendant}. Sync the released tag into main before cutting a checkpoint, and prepare candidates from that frozen source.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveBaseline(git, tag) {
|
||||
try {
|
||||
return shaSchema.parse(
|
||||
(await git(["rev-parse", "--verify", `${tag}^{commit}`])).trim(),
|
||||
);
|
||||
} catch (error) {
|
||||
const failure = z.object({ code: z.literal(128) }).safeParse(error);
|
||||
if (!failure.success) throw error;
|
||||
throw new Error(
|
||||
`Published baseline ${tag} is unavailable. Fetch release tags and finish the previous release sync before preparing this checkpoint. ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function validateCandidate(git, sha, version, commits) {
|
||||
const versions = [
|
||||
[
|
||||
"package.json",
|
||||
(await readJson(git, sha, "package.json", versionSchema)).version,
|
||||
],
|
||||
[
|
||||
".release-please-manifest.json",
|
||||
(
|
||||
await readJson(
|
||||
git,
|
||||
sha,
|
||||
".release-please-manifest.json",
|
||||
manifestSchema,
|
||||
)
|
||||
)["."],
|
||||
],
|
||||
];
|
||||
const lock = await readJson(
|
||||
git,
|
||||
sha,
|
||||
"package-lock.json",
|
||||
versionSchema.extend({
|
||||
packages: z.object({ "": versionSchema }),
|
||||
}),
|
||||
);
|
||||
versions.push(
|
||||
["package-lock.json", lock.version],
|
||||
["package-lock.json packages.root", lock.packages[""].version],
|
||||
);
|
||||
const server = await readJson(
|
||||
git,
|
||||
sha,
|
||||
"server.json",
|
||||
versionSchema.extend({
|
||||
packages: z.array(
|
||||
z.object({ identifier: z.string(), version: z.string() }),
|
||||
),
|
||||
}),
|
||||
);
|
||||
const npmPackage = server.packages.find(
|
||||
(entry) => entry.identifier === "rea-agents",
|
||||
);
|
||||
if (npmPackage === undefined)
|
||||
throw new Error("server.json is missing the rea-agents package.");
|
||||
versions.push(
|
||||
["server.json", server.version],
|
||||
["server.json rea-agents package", npmPackage.version],
|
||||
);
|
||||
// Catalogs are generated and checked from this candidate by the release job;
|
||||
// they are not version authority stored in the immutable source tree.
|
||||
const generated = await git([
|
||||
"show",
|
||||
`${sha}:src/generatedPackageMetadata.ts`,
|
||||
]);
|
||||
const generatedVersions = Array.from(
|
||||
generated.matchAll(/^\s+version: "([^"]+)"/gmu),
|
||||
);
|
||||
if (generatedVersions.length !== 1)
|
||||
throw new Error(
|
||||
"Generated package metadata must declare one literal version.",
|
||||
);
|
||||
versions.push(["src/generatedPackageMetadata.ts", generatedVersions[0]?.[1]]);
|
||||
for (const [location, actual] of versions) {
|
||||
if (actual !== version) {
|
||||
throw new Error(
|
||||
`${location}@${sha} declares ${actual}; release branch requires ${version}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const notes = releaseNotes(
|
||||
await git(["show", `${sha}:CHANGELOG.md`]),
|
||||
version,
|
||||
);
|
||||
const breakingNotes = breakingSection(notes);
|
||||
const missingBreaking = commits.filter(
|
||||
(commit) => commit.breaking && !referencesCommit(breakingNotes, commit),
|
||||
);
|
||||
if (missingBreaking.length > 0) {
|
||||
throw new Error(
|
||||
`Release ${version} migration notes omit unreleased breaking changes: ${missingBreaking.map((commit) => `${commit.sha} ${commit.subject}`).join("; ")}`,
|
||||
);
|
||||
}
|
||||
return commits.filter((commit) => !referencesCommit(notes, commit));
|
||||
}
|
||||
|
||||
/** Validate a recorded checkpoint and optional candidate using exact Git ancestry. */
|
||||
export async function inspectReleaseCheckpoint(git, options) {
|
||||
const { phase, stage, releaseBranch, sourceSha, candidateSha } = z
|
||||
.object({
|
||||
phase: z.enum(["prepare", "publish"]),
|
||||
stage: z.enum(["source", "candidate"]),
|
||||
releaseBranch: z.string().startsWith("release/"),
|
||||
sourceSha: shaSchema,
|
||||
candidateSha: shaSchema.optional(),
|
||||
})
|
||||
.parse(options);
|
||||
const expectedVersion = releaseVersion(
|
||||
releaseBranch.slice("release/".length),
|
||||
"release/VERSION",
|
||||
);
|
||||
if (phase === "publish" && stage === "candidate") {
|
||||
throw new Error(
|
||||
"Publish validates its merged source SHA; candidate stage applies only to prepare.",
|
||||
);
|
||||
}
|
||||
const applicationSha =
|
||||
phase === "publish"
|
||||
? shaSchema.parse((await git(["rev-parse", `${sourceSha}^1`])).trim())
|
||||
: sourceSha;
|
||||
const manifest = await readJson(
|
||||
git,
|
||||
applicationSha,
|
||||
".release-please-manifest.json",
|
||||
manifestSchema,
|
||||
);
|
||||
const baselineVersion = releaseVersion(
|
||||
manifest["."],
|
||||
"published baseline manifest",
|
||||
);
|
||||
const sourcePackage = await readJson(
|
||||
git,
|
||||
applicationSha,
|
||||
"package.json",
|
||||
versionSchema,
|
||||
);
|
||||
if (sourcePackage.version !== baselineVersion) {
|
||||
throw new Error(
|
||||
`Source package ${sourcePackage.version} and release baseline ${baselineVersion} disagree.`,
|
||||
);
|
||||
}
|
||||
if (!gt(expectedVersion, baselineVersion)) {
|
||||
throw new Error(
|
||||
`Expected release ${expectedVersion} must advance published baseline ${baselineVersion}.`,
|
||||
);
|
||||
}
|
||||
const baselineTag = `refs/tags/rea-agents-${baselineVersion}`;
|
||||
const baselineSha = await resolveBaseline(git, baselineTag);
|
||||
await requireAncestor(
|
||||
git,
|
||||
baselineSha,
|
||||
applicationSha,
|
||||
"Published baseline is outside the checkpoint ancestry",
|
||||
);
|
||||
const configuration = await readJson(
|
||||
git,
|
||||
applicationSha,
|
||||
"release-please-config.json",
|
||||
sectionsSchema,
|
||||
);
|
||||
const visibleTypes = new Set(
|
||||
configuration["changelog-sections"]
|
||||
.filter((section) => !section.hidden)
|
||||
.map((section) => section.type),
|
||||
);
|
||||
const commits = readCommits(
|
||||
await git([
|
||||
"log",
|
||||
"--first-parent",
|
||||
"-z",
|
||||
"--format=%H%x00%s%x00%B",
|
||||
`${baselineSha}..${applicationSha}`,
|
||||
]),
|
||||
).filter((commit) => commit.breaking || visibleTypes.has(commit.type));
|
||||
const breaking = commits.filter((commit) => commit.breaking);
|
||||
if (
|
||||
breaking.length > 0 &&
|
||||
major(baselineVersion) > 0 &&
|
||||
major(expectedVersion) <= major(baselineVersion)
|
||||
) {
|
||||
throw new Error(
|
||||
`Expected release ${expectedVersion} contains unreleased breaking markers since ${baselineVersion}: ${breaking.map((commit) => `${commit.sha} ${commit.subject}`).join("; ")}. Review compatibility or select a major version.`,
|
||||
);
|
||||
}
|
||||
let snapshot = phase === "publish" ? sourceSha : candidateSha;
|
||||
if (stage === "candidate" && snapshot === undefined) {
|
||||
snapshot = shaSchema.parse((await git(["rev-parse", "HEAD"])).trim());
|
||||
}
|
||||
if (phase === "publish" || stage === "candidate") {
|
||||
await requireAncestor(
|
||||
git,
|
||||
applicationSha,
|
||||
snapshot,
|
||||
"Candidate is outside the selected checkpoint ancestry",
|
||||
);
|
||||
}
|
||||
const missingNotes =
|
||||
phase === "publish" || stage === "candidate"
|
||||
? await validateCandidate(git, snapshot, expectedVersion, commits)
|
||||
: [];
|
||||
return {
|
||||
expectedVersion,
|
||||
baselineVersion,
|
||||
baselineSha,
|
||||
sourceSha,
|
||||
applicationSha,
|
||||
commits,
|
||||
missingNotes,
|
||||
};
|
||||
}
|
||||
@@ -1169,7 +1169,7 @@ try {
|
||||
);
|
||||
const ownershipInspection =
|
||||
["/usr/bin/ps", "/bin/ps"].includes(match[1]) &&
|
||||
line.includes('["ps", "-axo", "pid=,ppid=,pgid=,stat=,command="]');
|
||||
line.includes('["ps", "-axo", "pid=,ppid=,pgid=,uid=,stat=,command="]');
|
||||
assert.ok(
|
||||
[process.execPath, python].includes(match[1]) || ownershipInspection,
|
||||
`Unexpected attempted host execution: ${line}`,
|
||||
|
||||
@@ -474,7 +474,9 @@ try {
|
||||
);
|
||||
const ownershipInspection =
|
||||
["/usr/bin/ps", "/bin/ps"].includes(match[1]) &&
|
||||
line.includes('["ps", "-axo", "pid=,ppid=,pgid=,stat=,command="]');
|
||||
line.includes(
|
||||
'["ps", "-axo", "pid=,ppid=,pgid=,uid=,stat=,command="]',
|
||||
);
|
||||
assert.ok(
|
||||
[
|
||||
process.execPath,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { join } from "node:path";
|
||||
import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js";
|
||||
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { verifyPackedBridge } from "./verify-packed-bridge.mjs";
|
||||
@@ -28,6 +29,26 @@ export async function verifyPackagePack({ root, workspace }) {
|
||||
throw new Error("package did not expose both rea command entry points");
|
||||
if (packedManifest.mcpName !== "io.github.morluto/rea")
|
||||
throw new Error("package did not retain its MCP Registry ownership marker");
|
||||
if (packedFiles.some((path) => path.startsWith("package/skill-src/")))
|
||||
throw new Error(
|
||||
"package included authored skill sources instead of only the generated bundle",
|
||||
);
|
||||
const skill = (
|
||||
await exec("tar", [
|
||||
"-xOf",
|
||||
join(root, tarball),
|
||||
"package/skills/reverse-engineer-anything/SKILL.md",
|
||||
])
|
||||
).stdout;
|
||||
if (
|
||||
!skill.includes(
|
||||
` tool_count: ${String(CATALOG_IDENTITY.counts.mcp_tools)}\n`,
|
||||
) ||
|
||||
skill.includes(" catalog_digest:")
|
||||
)
|
||||
throw new Error(
|
||||
"packaged skill metadata did not match the catalog shipped in the package",
|
||||
);
|
||||
if (
|
||||
packedFiles.some(
|
||||
(path) => path.includes("__pycache__") || path.endsWith(".pyc"),
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFile } from "node:child_process";
|
||||
import { parseArgs, promisify } from "node:util";
|
||||
import { inspectReleaseCheckpoint } from "./lib/release-checkpoint.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
async function git(args) {
|
||||
const result = await execFileAsync("git", args, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
return result.stdout;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
phase: { type: "string" },
|
||||
stage: { type: "string", default: "source" },
|
||||
"release-branch": { type: "string" },
|
||||
"source-sha": { type: "string" },
|
||||
},
|
||||
});
|
||||
const report = await inspectReleaseCheckpoint(git, {
|
||||
phase: values.phase,
|
||||
stage: values.stage,
|
||||
releaseBranch: values["release-branch"],
|
||||
sourceSha: values["source-sha"],
|
||||
});
|
||||
process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
if (report.missingNotes.length > 0) {
|
||||
process.stderr.write(
|
||||
`Review ${report.missingNotes.length} additional unreleased mainline entries missing from these release notes; see the ancestry report above.\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await main().catch((error) => {
|
||||
process.stderr.write(
|
||||
`Release checkpoint validation failed: ${error instanceof Error ? error.message : String(error)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -260,14 +260,16 @@ try {
|
||||
);
|
||||
const original = await readFile(path);
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const error = await inspect(
|
||||
mode,
|
||||
{ capture_path: path, format },
|
||||
"unsupported_provider",
|
||||
);
|
||||
assert.ok(
|
||||
JSON.stringify(error).includes("JSON schema boundary cannot preserve"),
|
||||
);
|
||||
const value = await inspect(mode, { capture_path: path, format });
|
||||
const reported =
|
||||
format === "har"
|
||||
? value.container.reported._extension
|
||||
: value.records[0].reported.metadata;
|
||||
assert.equal(Object.hasOwn(reported, "__proto__"), true);
|
||||
assert.deepEqual(reported["__proto__"], { preserved: 7 });
|
||||
assert.equal(Object.getPrototypeOf(reported), Object.prototype);
|
||||
assert.equal(Reflect.get(Object.prototype, "preserved"), undefined);
|
||||
if (format === "har") assert.equal(reported.constructor, "ordinary");
|
||||
assert.deepEqual(await readFile(path), original);
|
||||
cases++;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import {
|
||||
artifactCli,
|
||||
artifactMcpResult,
|
||||
withArtifactMcp,
|
||||
} from "../../lib/artifact-e2e.mjs";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
|
||||
if (process.platform !== "darwin")
|
||||
throw new Error(
|
||||
"Installer package verification requires macOS pkgbuild, productbuild and pkgutil",
|
||||
);
|
||||
for (const tool of [
|
||||
"/usr/bin/pkgbuild",
|
||||
"/usr/bin/productbuild",
|
||||
"/usr/sbin/pkgutil",
|
||||
])
|
||||
try {
|
||||
await exec(tool, ["--help"]);
|
||||
} catch (cause) {
|
||||
if (cause?.code === "ENOENT")
|
||||
throw new Error(`Installer package verification requires ${tool}`, {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-installer-package-"));
|
||||
const sha256 = (text) => createHash("sha256").update(text).digest("hex");
|
||||
|
||||
/** Inventory occurrences keyed by logical path, with each node's content digest. */
|
||||
const inventoryOf = (inspection) => {
|
||||
const inventory = inspection.substeps[0].evidence.normalized_result;
|
||||
const nodes = new Map(
|
||||
inventory.nodes.map((node) => [node.artifact_id, node]),
|
||||
);
|
||||
return {
|
||||
inventory,
|
||||
entries: new Map(
|
||||
inventory.occurrences.map((occurrence) => [
|
||||
occurrence.logical_path,
|
||||
{ ...occurrence, sha256: nodes.get(occurrence.artifact_id)?.sha256 },
|
||||
]),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
/** Apple's Bom listing of a component package's payload, without the root. */
|
||||
const bomPaths = async (path) =>
|
||||
(await exec("/usr/sbin/pkgutil", ["--payload-files", path])).stdout
|
||||
.split("\n")
|
||||
.filter((line) => line.length > 0 && line !== ".")
|
||||
.map((line) => line.replace(/^\.\//u, ""))
|
||||
.sort();
|
||||
|
||||
try {
|
||||
const payload = join(root, "payload");
|
||||
const scripts = join(root, "scripts");
|
||||
await mkdir(join(payload, "usr/local/rea"), { recursive: true });
|
||||
await mkdir(scripts);
|
||||
await writeFile(join(payload, "usr/local/rea/hello.txt"), "hello\n");
|
||||
for (const name of ["preinstall", "postinstall"]) {
|
||||
await writeFile(join(scripts, name), `#!/bin/sh\necho ${name}\n`);
|
||||
await chmod(join(scripts, name), 0o755);
|
||||
}
|
||||
const component = join(root, "Component.pkg");
|
||||
const build = (output, extra) =>
|
||||
exec("/usr/bin/pkgbuild", [
|
||||
"--quiet",
|
||||
"--root",
|
||||
payload,
|
||||
"--identifier",
|
||||
"com.example.rea.installer",
|
||||
"--version",
|
||||
"1.0",
|
||||
"--install-location",
|
||||
"/",
|
||||
...extra,
|
||||
output,
|
||||
]);
|
||||
await build(component, ["--scripts", scripts]);
|
||||
const product = join(root, "Product.pkg");
|
||||
await exec("/usr/bin/productbuild", [
|
||||
"--quiet",
|
||||
"--package",
|
||||
component,
|
||||
product,
|
||||
]);
|
||||
const pbzx = join(root, "Latest.pkg");
|
||||
await build(pbzx, ["--compression", "latest", "--min-os-version", "13.0"]);
|
||||
|
||||
const componentInventory = inventoryOf(
|
||||
await artifactCli("inspect-artifact", component),
|
||||
);
|
||||
assert.equal(componentInventory.inventory.manifest.root_format, "pkg");
|
||||
const payloadPaths = [...componentInventory.entries.keys()]
|
||||
.filter((path) => path.startsWith("Payload/"))
|
||||
.map((path) => path.slice("Payload/".length))
|
||||
.sort();
|
||||
assert.deepEqual(
|
||||
payloadPaths,
|
||||
await bomPaths(component),
|
||||
"Payload members differ from pkgutil",
|
||||
);
|
||||
assert.equal(
|
||||
componentInventory.entries.get("Payload/usr/local/rea/hello.txt")?.sha256,
|
||||
sha256("hello\n"),
|
||||
);
|
||||
for (const name of ["preinstall", "postinstall"]) {
|
||||
const script = componentInventory.entries.get(`Scripts/${name}`);
|
||||
assert.equal(script?.sha256, sha256(`#!/bin/sh\necho ${name}\n`));
|
||||
assert.equal(script?.executable, true);
|
||||
}
|
||||
assert.ok(componentInventory.entries.has("PackageInfo"));
|
||||
assert.ok(componentInventory.entries.has("Bom"));
|
||||
|
||||
const productInventory = inventoryOf(
|
||||
await artifactCli("inspect-artifact", product),
|
||||
);
|
||||
assert.ok(productInventory.entries.has("Distribution"));
|
||||
assert.equal(
|
||||
productInventory.entries.get(
|
||||
"Component.pkg/Payload/usr/local/rea/hello.txt",
|
||||
)?.sha256,
|
||||
sha256("hello\n"),
|
||||
);
|
||||
assert.ok(productInventory.entries.has("Component.pkg/Scripts/postinstall"));
|
||||
|
||||
const pbzxInventory = inventoryOf(
|
||||
await artifactCli("inspect-artifact", pbzx),
|
||||
);
|
||||
const pbzxPayload = pbzxInventory.entries.get("Payload");
|
||||
assert.ok(
|
||||
pbzxPayload?.limitations.some((line) => line.includes("pbzx")),
|
||||
JSON.stringify(pbzxPayload),
|
||||
);
|
||||
assert.ok(
|
||||
![...pbzxInventory.entries.keys()].some((path) =>
|
||||
path.startsWith("Payload/"),
|
||||
),
|
||||
);
|
||||
|
||||
await withArtifactMcp(component, async (client) => {
|
||||
const viaMcp = inventoryOf(
|
||||
await artifactMcpResult(client, "inspect_artifact"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
viaMcp.inventory.manifest,
|
||||
componentInventory.inventory.manifest,
|
||||
);
|
||||
});
|
||||
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
ok: true,
|
||||
mocked: false,
|
||||
cli: true,
|
||||
stdio_mcp: true,
|
||||
payload_members: payloadPaths.length,
|
||||
matches_pkgutil_bom: true,
|
||||
product_nested: true,
|
||||
pbzx_payload: "recorded-not-expanded",
|
||||
})}\n`,
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -79,7 +79,7 @@ export async function verifyCoreInspectionTrace({
|
||||
(line) =>
|
||||
/^execve\("\/(?:usr\/)?bin\/ps",/u.test(line) &&
|
||||
line.includes(
|
||||
'["ps", "-axo", "pid=,ppid=,pgid=,stat=,command="]',
|
||||
'["ps", "-axo", "pid=,ppid=,pgid=,uid=,stat=,command="]',
|
||||
) &&
|
||||
line.endsWith(" = 0"),
|
||||
);
|
||||
@@ -128,7 +128,9 @@ export async function verifyCoreInspectionTrace({
|
||||
const executable = await realpath(match[1]).catch(() => match[1]);
|
||||
const ownership =
|
||||
psPaths.has(match[1]) &&
|
||||
line.includes('["ps", "-axo", "pid=,ppid=,pgid=,stat=,command="]');
|
||||
line.includes(
|
||||
'["ps", "-axo", "pid=,ppid=,pgid=,uid=,stat=,command="]',
|
||||
);
|
||||
const iconv =
|
||||
iconvPaths.has(match[1]) && line.includes('["iconv", "-l"]');
|
||||
const version =
|
||||
|
||||
+7
-2
@@ -3,8 +3,6 @@ name: reverse-engineer-anything
|
||||
description: Reverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.
|
||||
metadata:
|
||||
version: "33"
|
||||
tool_count: 139
|
||||
catalog_digest: "4c5f67d076baf821b5cb6d6fb2cc0f6c274e96b627b85b892577760a6f71ab1a"
|
||||
---
|
||||
|
||||
# REA
|
||||
@@ -146,6 +144,13 @@ Start with the default result and use its inline Evidence and graph context.
|
||||
Do not repeat an identical tool call. Make a focused follow-up only when the
|
||||
returned result leaves a specific question unanswered.
|
||||
|
||||
If MCP reports `resource_constraint` with `details.resource: "transport"`, use
|
||||
its reported same-session Evidence reference with a focused application tool,
|
||||
or call `export_evidence_bundle` to write the complete session to a selected
|
||||
path. The analysis remains complete in the ledger; avoid repeating it merely
|
||||
to request the same oversized response. Complete CLI JSON output also streams
|
||||
without a single MCP frame.
|
||||
|
||||
Every conclusion must distinguish observations, inferences, and unknowns. Cite
|
||||
Evidence IDs, preserve limitations and incomplete coverage, and never imply
|
||||
that static analysis observed execution. Runtime requests execute the declared
|
||||
+8
-1
@@ -2,10 +2,17 @@
|
||||
|
||||
Use `analyze_javascript_application` directly on the operator-supplied ASAR or
|
||||
extracted tree. The complete result, graph, and Evidence context are returned
|
||||
inline. Source-map contents are part of the static analysis. Start from its
|
||||
inline when they fit the MCP frame. Source-map contents are part of the static analysis. Start from its
|
||||
findings, coverage, unknowns, and graph context, then make a focused follow-up
|
||||
only when a specific question remains unanswered.
|
||||
|
||||
An oversized response reports `resource_constraint` and
|
||||
`details.resource: "transport"`. Reuse
|
||||
`details.reported_limits.evidence_reference` with `trace_application_feature`
|
||||
for the module or feature under investigation. `export_evidence_bundle` writes
|
||||
the complete canonical session to a caller-selected file. Both workflows keep
|
||||
the original Evidence and coverage; a broad follow-up can also exceed framing.
|
||||
|
||||
BrowserWindow preferences, preload and contextBridge surfaces, IPC
|
||||
registrations, utility processes, and native binding requests are static syntax
|
||||
observations. Only a unique exact literal IPC channel match supports an inferred
|
||||
+6
-1
@@ -87,7 +87,12 @@ header, depending on the binary and analyzer version.
|
||||
## Packages and extraction
|
||||
|
||||
Use `inspect_artifact` for application bundles, archives, ZIP/APK/IPA/MSIX/AppX,
|
||||
ASAR, or DMG inputs when the artifact graph and findings help answer the
|
||||
ASAR, DMG, or PKG inputs. A PKG inventory includes Distribution, PackageInfo,
|
||||
the pre/postinstall scripts, and the gzip payload's files. Pbzx payloads are
|
||||
recorded only. Checksums declared in the PKG's table of contents are verified;
|
||||
with `integrity_policy: "record-and-continue"`, a mismatched member stays in
|
||||
the graph as `mismatched`, with the declared and observed values, and is not
|
||||
expanded. Inspect it when the artifact graph and findings help answer the
|
||||
question. It returns the complete artifact graph inline. Cite graph manifest
|
||||
IDs when using them.
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import {
|
||||
isOwnedClientRegistrationCommand,
|
||||
npxRegistrationCommand,
|
||||
} from "./ClientRegistrationIdentity.js";
|
||||
import { setupRegistrationCommand } from "./SetupHost.js";
|
||||
|
||||
it("uses a Windows executable for npm-run setup and preserves POSIX launchers", () => {
|
||||
expect(setupRegistrationCommand("win32", true)).toEqual([
|
||||
"cmd.exe",
|
||||
"/d",
|
||||
"/c",
|
||||
"npx",
|
||||
"-y",
|
||||
PRODUCT_IDENTITY.registrationPackageSpecifier,
|
||||
"mcp",
|
||||
]);
|
||||
for (const platform of ["linux", "darwin"] as const)
|
||||
expect(setupRegistrationCommand(platform, true)).toEqual([
|
||||
"npx",
|
||||
"-y",
|
||||
PRODUCT_IDENTITY.registrationPackageSpecifier,
|
||||
"mcp",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(
|
||||
[
|
||||
npxRegistrationCommand("win32"),
|
||||
["cmd", "/c", "npx", "-y", "rea-agents@5.0.0", "mcp"],
|
||||
["npx", "-y", "rea-agents@5.0.0", "mcp"],
|
||||
].map((command) => ({ command })),
|
||||
)("recognizes owned current and legacy launchers: $command", ({ command }) => {
|
||||
expect(isOwnedClientRegistrationCommand(command)).toBe(true);
|
||||
});
|
||||
|
||||
it.each(
|
||||
[
|
||||
["cmd.exe", "/d", "/c", "node", "custom.js", "mcp"],
|
||||
["cmd.exe", "/k", "npx", "-y", "rea-agents@5.0.0", "mcp"],
|
||||
["cmd.exe", "/c", "npx -y rea-agents@5.0.0 mcp"],
|
||||
["cmd.exe", "/c", "npx", "-y", "another-package", "mcp"],
|
||||
["cmd.exe", "/c", "npx", "-y", "rea-agents@5.0.0", "mcp", "&", "other"],
|
||||
["cmd.exe", "/c", "npx", "-y", "rea-agents@5.0.0 & other", "mcp"],
|
||||
].map((command) => ({ command })),
|
||||
)(
|
||||
"does not claim custom shell commands as REA-owned: $command",
|
||||
({ command }) => {
|
||||
expect(isOwnedClientRegistrationCommand(command)).toBe(false);
|
||||
},
|
||||
);
|
||||
@@ -3,6 +3,28 @@ import { valid } from "semver";
|
||||
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
|
||||
/** Resolve a pinned npx launcher that stdio clients can spawn on the host. */
|
||||
export const npxRegistrationCommand = (
|
||||
platform: NodeJS.Platform = process.platform,
|
||||
): readonly string[] => {
|
||||
const command = [
|
||||
"npx",
|
||||
"-y",
|
||||
PRODUCT_IDENTITY.registrationPackageSpecifier,
|
||||
"mcp",
|
||||
];
|
||||
return platform === "win32" ? ["cmd.exe", "/d", "/c", ...command] : command;
|
||||
};
|
||||
|
||||
const unwrapNpxCommand = (command: readonly string[]): readonly string[] => {
|
||||
const executable = command[0]?.toLowerCase();
|
||||
if (executable !== "cmd" && executable !== "cmd.exe") return command;
|
||||
if (command[1]?.toLowerCase() === "/c") return command.slice(2);
|
||||
if (command[1]?.toLowerCase() === "/d" && command[2]?.toLowerCase() === "/c")
|
||||
return command.slice(3);
|
||||
return [];
|
||||
};
|
||||
|
||||
/** Check that one parsed command points to REA's MCP entry point. */
|
||||
export const isOwnedClientRegistrationCommand = (
|
||||
command: readonly string[],
|
||||
@@ -27,15 +49,16 @@ export const isOwnedClientRegistrationCommand = (
|
||||
return true;
|
||||
}
|
||||
|
||||
const npxCommand = unwrapNpxCommand(command);
|
||||
if (
|
||||
command.length !== 4 ||
|
||||
command[0] !== "npx" ||
|
||||
command[1] !== "-y" ||
|
||||
command[3] !== "mcp"
|
||||
npxCommand.length !== 4 ||
|
||||
npxCommand[0] !== "npx" ||
|
||||
npxCommand[1] !== "-y" ||
|
||||
npxCommand[3] !== "mcp"
|
||||
)
|
||||
return false;
|
||||
|
||||
const packageReference = command[2] ?? "";
|
||||
const packageReference = npxCommand[2] ?? "";
|
||||
if (
|
||||
packageReference === PRODUCT_IDENTITY.packageName ||
|
||||
packageReference === PRODUCT_IDENTITY.packageSpecifier ||
|
||||
|
||||
@@ -9,7 +9,10 @@ import { z } from "zod";
|
||||
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { MCP_STARTUP_POLICY } from "../mcpStartupPolicy.js";
|
||||
import { isOwnedClientRegistrationCommand } from "./ClientRegistrationIdentity.js";
|
||||
import {
|
||||
isOwnedClientRegistrationCommand,
|
||||
npxRegistrationCommand,
|
||||
} from "./ClientRegistrationIdentity.js";
|
||||
import { supportedClients } from "./SupportedClients.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
|
||||
@@ -108,7 +111,12 @@ export const readClientRegistrationStatuses = async (
|
||||
client.name,
|
||||
client.configPath,
|
||||
command,
|
||||
registrationAligned(registration, client, currentCommandPath)
|
||||
registrationAligned(
|
||||
registration,
|
||||
client,
|
||||
currentCommandPath,
|
||||
options.platform ?? process.platform,
|
||||
)
|
||||
? "aligned"
|
||||
: "stale",
|
||||
),
|
||||
@@ -132,6 +140,7 @@ const registrationAligned = (
|
||||
registration: z.output<typeof registrationSchema>,
|
||||
client: SetupClient,
|
||||
currentCommandPath: string,
|
||||
platform: NodeJS.Platform,
|
||||
): boolean => {
|
||||
const command = [registration.command, ...registration.args];
|
||||
if (registration.disabled === true || registration.enabled === false)
|
||||
@@ -160,12 +169,10 @@ const registrationAligned = (
|
||||
resolve(command[1] ?? "") === currentCommandPath
|
||||
)
|
||||
return true;
|
||||
const npxCommand = npxRegistrationCommand(platform);
|
||||
if (
|
||||
command.length === 4 &&
|
||||
command[0] === "npx" &&
|
||||
command[1] === "-y" &&
|
||||
command[2] === PRODUCT_IDENTITY.registrationPackageSpecifier &&
|
||||
command[3] === "mcp"
|
||||
command.length === npxCommand.length &&
|
||||
command.every((argument, index) => argument === npxCommand[index])
|
||||
)
|
||||
return true;
|
||||
return (
|
||||
|
||||
@@ -30,11 +30,7 @@ import {
|
||||
snapshotEvidenceForQuery,
|
||||
snapshotMatchesTarget,
|
||||
} from "../domain/analysisSnapshot.js";
|
||||
import {
|
||||
analysisProfileSchema,
|
||||
committedProviderSchema,
|
||||
type AnalysisProfileCommitment,
|
||||
} from "../domain/analysisProfile.js";
|
||||
import type { AnalysisProfileCommitment } from "../domain/analysisProfile.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { AnalysisSnapshot } from "../domain/analysisSnapshot.js";
|
||||
import type {
|
||||
@@ -45,6 +41,10 @@ import {
|
||||
REA_WORKFLOW_PROVIDER,
|
||||
workflowAnalysisProfile,
|
||||
} from "./InvestigationProviders.js";
|
||||
import {
|
||||
createWorkflowEvidence,
|
||||
workflowSnapshotRecord,
|
||||
} from "./WorkflowEvidence.js";
|
||||
import type { AnalysisProviderSelector } from "../contracts/providerSelection.js";
|
||||
import { artifactInspectionResultSchema } from "../domain/artifactInspection.js";
|
||||
|
||||
@@ -268,35 +268,13 @@ const runAnalysis = async (
|
||||
tool !== "trace_native_ui_action" &&
|
||||
snapshotPath !== undefined &&
|
||||
evidence !== undefined &&
|
||||
"analysis_profile" in evidence &&
|
||||
session.allowsSnapshotReplay(tool)
|
||||
) {
|
||||
const recorded = session.recordWorkflowSnapshot({
|
||||
operation: tool,
|
||||
parameters: arguments_,
|
||||
execution: {
|
||||
result: evidence.normalized_result,
|
||||
rawResult: evidence.raw_result,
|
||||
provider: committedProviderSchema.parse(evidence.provider),
|
||||
analysisProfile: analysisProfileSchema.parse(
|
||||
evidence.analysis_profile,
|
||||
),
|
||||
limitations: evidence.limitations,
|
||||
locations: evidence.locations,
|
||||
subject:
|
||||
evidence.subject === null
|
||||
? null
|
||||
: {
|
||||
path: evidence.subject.local_path,
|
||||
sha256: evidence.subject.digest.sha256,
|
||||
format: evidence.subject.format,
|
||||
...(evidence.subject.architecture === null
|
||||
? {}
|
||||
: { architecture: evidence.subject.architecture }),
|
||||
},
|
||||
},
|
||||
});
|
||||
if (!recorded.ok) return cliError(recorded.error);
|
||||
const workflowRecord = workflowSnapshotRecord(evidence, tool);
|
||||
if (workflowRecord !== undefined) {
|
||||
const recorded = session.recordWorkflowSnapshot(workflowRecord);
|
||||
if (!recorded.ok) return cliError(recorded.error);
|
||||
}
|
||||
}
|
||||
if (
|
||||
tool !== "trace_native_ui_action" &&
|
||||
@@ -356,15 +334,12 @@ const executeAnalysisTool = async (input: {
|
||||
signal,
|
||||
);
|
||||
if (!result.ok) return { output: cliError(result.error) };
|
||||
const evidence = createEvidence(input.openedTarget, REA_WORKFLOW_PROVIDER, {
|
||||
const evidence = createWorkflowEvidence({
|
||||
target: input.openedTarget,
|
||||
operation: tool,
|
||||
parameters: input.arguments,
|
||||
result: result.value,
|
||||
...(evidenceProfile === undefined
|
||||
? {}
|
||||
: { analysisProfile: evidenceProfile }),
|
||||
confidence: "derived",
|
||||
limitations: ["Derived by an REA composed workflow."],
|
||||
upstreamProfile: evidenceProfile,
|
||||
});
|
||||
return { output: evidence, evidence };
|
||||
}
|
||||
@@ -493,8 +468,7 @@ const analysisProfileForEvidence = (
|
||||
| DirectAnalysisTool,
|
||||
): AnalysisProfileCommitment | undefined => {
|
||||
if (!isWorkflowEvidenceTool(tool)) return session.analysisProfile(tool);
|
||||
const upstream = session.analysisProfile();
|
||||
return upstream === undefined ? undefined : workflowAnalysisProfile(upstream);
|
||||
return session.analysisProfile();
|
||||
};
|
||||
|
||||
const isWorkflowEvidenceTool = (
|
||||
|
||||
@@ -26,7 +26,8 @@ export const createDoctorHostFixture = (
|
||||
Promise.resolve({
|
||||
version: PRODUCT_IDENTITY.skillVersion,
|
||||
toolCount: CATALOG_IDENTITY.counts.mcp_tools,
|
||||
catalogDigest: CATALOG_IDENTITY.digests.combined_sha256,
|
||||
catalogDigest: null,
|
||||
canonical: true,
|
||||
}),
|
||||
...overrides,
|
||||
});
|
||||
|
||||
@@ -177,6 +177,7 @@ describe("doctor installation identity", () => {
|
||||
version: "10",
|
||||
toolCount: null,
|
||||
catalogDigest: null,
|
||||
canonical: false,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
readLinuxDistribution,
|
||||
type LinuxDistribution,
|
||||
} from "./LinuxHopper.js";
|
||||
import { canonicalSkillNeedsInstall } from "./SetupSkill.js";
|
||||
import { CATALOG_IDENTITY } from "../catalogIdentity.js";
|
||||
import { PRODUCT_IDENTITY, SDK_IDENTITY } from "../identity.js";
|
||||
import {
|
||||
@@ -95,11 +96,13 @@ export interface DoctorHost {
|
||||
ilspyCmdVersion?(path: string): Promise<string | undefined>;
|
||||
}
|
||||
|
||||
/** Parsed identity committed by an installed REA skill. */
|
||||
/** Observed skill metadata and comparison with the packaged instruction bundle. */
|
||||
interface InstalledSkillIdentity {
|
||||
readonly version: string | null;
|
||||
readonly toolCount: number | null;
|
||||
readonly catalogDigest: string | null;
|
||||
/** Whether all managed instruction and reference files match this package. */
|
||||
readonly canonical: boolean;
|
||||
}
|
||||
|
||||
/** Structured result returned by the read-only doctor workflow. */
|
||||
@@ -139,6 +142,7 @@ interface DoctorIdentity {
|
||||
readonly skill: {
|
||||
readonly installed_version: string | null;
|
||||
readonly installed_tool_count: number | null;
|
||||
/** Legacy catalog digest observed in older bundles; current bundles omit it. */
|
||||
readonly installed_catalog_digest: string | null;
|
||||
readonly state: "aligned" | "stale" | "missing";
|
||||
readonly remediation: string | null;
|
||||
@@ -253,7 +257,7 @@ const skillIdentityAligned = (
|
||||
): boolean =>
|
||||
identity?.version === PRODUCT_IDENTITY.skillVersion &&
|
||||
identity.toolCount === CATALOG_IDENTITY.counts.mcp_tools &&
|
||||
identity.catalogDigest === CATALOG_IDENTITY.digests.combined_sha256;
|
||||
identity.canonical;
|
||||
|
||||
const skillIdentityCheck = (
|
||||
identity: InstalledSkillIdentity | undefined,
|
||||
@@ -507,6 +511,7 @@ const installedSkillIdentity = async (
|
||||
const content = await readInstalledSkill(home);
|
||||
const countText = /^\s{2}tool_count:\s*(\d+)\s*$/mu.exec(content)?.[1];
|
||||
return {
|
||||
canonical: !(await canonicalSkillNeedsInstall(home)),
|
||||
version: /^\s{2}version:\s*"([^"]+)"\s*$/mu.exec(content)?.[1] ?? null,
|
||||
toolCount:
|
||||
countText === undefined ? null : Number.parseInt(countText, 10),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
describeEvidenceBundleFailure,
|
||||
describeValidationFailure,
|
||||
parseEvidenceBundle,
|
||||
serializeEvidenceBundle,
|
||||
type EvidenceBundle,
|
||||
} from "../domain/evidenceBundle.js";
|
||||
import {
|
||||
@@ -9,7 +10,11 @@ import {
|
||||
} from "../domain/evidenceErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { parseProcessCapture } from "../domain/process/processCapture.js";
|
||||
import { readJsonFile, writeTextFile } from "./JsonFiles.js";
|
||||
import {
|
||||
bufferedJsonParts,
|
||||
canonicalJsonParts,
|
||||
} from "../domain/jsonSerialization.js";
|
||||
import { readJsonFile, writeTextParts } from "./JsonFiles.js";
|
||||
|
||||
type EvidenceReadFailure = EvidenceFileError | EvidenceIntegrityError;
|
||||
type EvidenceWriteFailure = EvidenceFileError | EvidenceIntegrityError;
|
||||
@@ -20,20 +25,31 @@ export const readEvidenceBundle = async (
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> => {
|
||||
const loaded = await readJsonFile(path);
|
||||
if (!loaded.ok) return loaded;
|
||||
let bundle: EvidenceBundle;
|
||||
try {
|
||||
const bundle = parseEvidenceBundle(loaded.value);
|
||||
for (const record of bundle.records) {
|
||||
if (record.predicate_type === "rea.process-capture")
|
||||
parseProcessCapture(record.normalized_result);
|
||||
}
|
||||
return ok(bundle);
|
||||
bundle = parseEvidenceBundle(loaded.value);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence bundle validation failed", {
|
||||
cause,
|
||||
userMessage: describeEvidenceBundleFailure(loaded.value, cause),
|
||||
}),
|
||||
);
|
||||
}
|
||||
for (const record of bundle.records) {
|
||||
if (record.predicate_type !== "rea.process-capture") continue;
|
||||
try {
|
||||
parseProcessCapture(record.normalized_result);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence bundle validation failed", {
|
||||
cause,
|
||||
userMessage: `Process capture record ${record.evidence_id} has an invalid normalized_result (${describeValidationFailure(cause)}). Recreate or re-export the bundle, then try again.`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
return ok(bundle);
|
||||
};
|
||||
|
||||
/** Atomically write deterministic evidence JSON at the caller-supplied path. */
|
||||
@@ -47,9 +63,9 @@ export const writeEvidenceBundle = async (
|
||||
EvidenceWriteFailure
|
||||
>
|
||||
> => {
|
||||
let encoded: string;
|
||||
let checked: EvidenceBundle;
|
||||
try {
|
||||
encoded = serializeEvidenceBundle(bundle);
|
||||
checked = parseEvidenceBundle(bundle);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence bundle validation failed", {
|
||||
@@ -57,5 +73,9 @@ export const writeEvidenceBundle = async (
|
||||
}),
|
||||
);
|
||||
}
|
||||
return writeTextFile(encoded, path, overwrite);
|
||||
return writeTextParts(
|
||||
bufferedJsonParts(canonicalJsonParts(checked)),
|
||||
path,
|
||||
overwrite,
|
||||
);
|
||||
};
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
import { link, lstat, mkdtemp, readFile, realpath, rm } from "node:fs/promises";
|
||||
import {
|
||||
link,
|
||||
lstat,
|
||||
mkdtemp,
|
||||
open,
|
||||
readFile,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
} from "node:fs/promises";
|
||||
import { basename, dirname, resolve } from "node:path";
|
||||
|
||||
import writeFileAtomic from "write-file-atomic";
|
||||
|
||||
import { EvidenceFileError } from "../domain/evidenceErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -10,10 +17,14 @@ import { err, ok, type Result } from "../domain/result.js";
|
||||
export const readJsonFile = async (
|
||||
path: string,
|
||||
): Promise<Result<unknown, EvidenceFileError>> => {
|
||||
const requestedPath = resolve(path);
|
||||
try {
|
||||
const canonicalPath = await realpath(resolve(path));
|
||||
const canonicalPath = await realpath(requestedPath);
|
||||
const stats = await lstat(canonicalPath);
|
||||
if (!stats.isFile()) return err(new EvidenceFileError("read", "not-file"));
|
||||
if (!stats.isFile())
|
||||
return err(
|
||||
new EvidenceFileError("read", "not-file", { path: requestedPath }),
|
||||
);
|
||||
const encoded = await readFile(canonicalPath);
|
||||
let decoded: unknown;
|
||||
try {
|
||||
@@ -23,11 +34,21 @@ export const readJsonFile = async (
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
return err(new EvidenceFileError("read", "invalid-json", { cause }));
|
||||
return err(
|
||||
new EvidenceFileError("read", "invalid-json", {
|
||||
cause,
|
||||
path: requestedPath,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return ok(decoded);
|
||||
} catch (cause: unknown) {
|
||||
return err(new EvidenceFileError("read", "io", { cause }));
|
||||
return err(
|
||||
new EvidenceFileError("read", missingOrIo(cause), {
|
||||
cause,
|
||||
path: requestedPath,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -38,10 +59,18 @@ export const writeTextFile = async (
|
||||
overwrite: boolean,
|
||||
): Promise<
|
||||
Result<{ readonly path: string; readonly bytes: number }, EvidenceFileError>
|
||||
> => writeTextParts([encoded], path, overwrite);
|
||||
|
||||
/** Publish complete streamed text atomically, keeping only one encoded part in memory. */
|
||||
export const writeTextParts = async (
|
||||
parts: Iterable<string>,
|
||||
path: string,
|
||||
overwrite: boolean,
|
||||
): Promise<
|
||||
Result<{ readonly path: string; readonly bytes: number }, EvidenceFileError>
|
||||
> => {
|
||||
const bytes = Buffer.byteLength(encoded, "utf8");
|
||||
const requestedPath = resolve(path);
|
||||
try {
|
||||
const requestedPath = resolve(path);
|
||||
const canonicalParent = await realpath(dirname(requestedPath));
|
||||
const destination = resolve(canonicalParent, basename(requestedPath));
|
||||
const existing = await lstat(destination).catch((cause: unknown) => {
|
||||
@@ -49,41 +78,65 @@ export const writeTextFile = async (
|
||||
throw cause;
|
||||
});
|
||||
if (existing !== undefined) {
|
||||
if (!overwrite) return err(new EvidenceFileError("write", "exists"));
|
||||
if (!overwrite)
|
||||
return err(
|
||||
new EvidenceFileError("write", "exists", { path: requestedPath }),
|
||||
);
|
||||
if (!existing.isFile() || existing.isSymbolicLink())
|
||||
return err(new EvidenceFileError("write", "not-file"));
|
||||
return err(
|
||||
new EvidenceFileError("write", "not-file", { path: requestedPath }),
|
||||
);
|
||||
}
|
||||
if (overwrite)
|
||||
await writeFileAtomic(destination, encoded, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
fsync: true,
|
||||
});
|
||||
else await publishNewFile(destination, encoded);
|
||||
const bytes = await publishFile(destination, parts, overwrite);
|
||||
return ok({ path: requestedPath, bytes });
|
||||
} catch (cause: unknown) {
|
||||
if (!overwrite && fileErrorCode(cause) === "EEXIST")
|
||||
return err(new EvidenceFileError("write", "exists", { cause }));
|
||||
return err(new EvidenceFileError("write", "io", { cause }));
|
||||
return err(
|
||||
new EvidenceFileError("write", "exists", {
|
||||
cause,
|
||||
path: requestedPath,
|
||||
}),
|
||||
);
|
||||
return err(
|
||||
new EvidenceFileError("write", missingOrIo(cause), {
|
||||
cause,
|
||||
path: requestedPath,
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const publishNewFile = async (
|
||||
/** A missing path or parent is a selection error, not a permission failure. */
|
||||
const missingOrIo = (cause: unknown): "missing" | "io" => {
|
||||
const code = fileErrorCode(cause);
|
||||
return code === "ENOENT" || code === "ENOTDIR" ? "missing" : "io";
|
||||
};
|
||||
|
||||
const publishFile = async (
|
||||
destination: string,
|
||||
encoded: string,
|
||||
): Promise<void> => {
|
||||
parts: Iterable<string>,
|
||||
overwrite: boolean,
|
||||
): Promise<number> => {
|
||||
const stagingDirectory = await mkdtemp(
|
||||
resolve(dirname(destination), ".rea-write-"),
|
||||
);
|
||||
try {
|
||||
const staged = resolve(stagingDirectory, "content");
|
||||
await writeFileAtomic(staged, encoded, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
fsync: true,
|
||||
});
|
||||
const file = await open(staged, "wx", 0o600);
|
||||
let bytes = 0;
|
||||
try {
|
||||
for (const part of parts) {
|
||||
await file.writeFile(part, { encoding: "utf8" });
|
||||
bytes += Buffer.byteLength(part, "utf8");
|
||||
}
|
||||
await file.sync();
|
||||
} finally {
|
||||
await file.close();
|
||||
}
|
||||
if (overwrite) await rename(staged, destination);
|
||||
// A hard link publishes complete bytes atomically and cannot replace an existing name.
|
||||
await link(staged, destination);
|
||||
else await link(staged, destination);
|
||||
return bytes;
|
||||
} finally {
|
||||
await rm(stagingDirectory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ import { dirname } from "node:path";
|
||||
import writeFileAtomic from "write-file-atomic";
|
||||
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js";
|
||||
import { MCP_STARTUP_POLICY } from "../mcpStartupPolicy.js";
|
||||
import { resolveClientConfigTransactionPath } from "./ClientConfigPath.js";
|
||||
import type {
|
||||
@@ -24,12 +25,7 @@ import type {
|
||||
} from "./SetupTypes.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
|
||||
const defaultCommand = (): readonly string[] => [
|
||||
"npx",
|
||||
"-y",
|
||||
PRODUCT_IDENTITY.registrationPackageSpecifier,
|
||||
"mcp",
|
||||
];
|
||||
const defaultCommand = (): readonly string[] => npxRegistrationCommand();
|
||||
|
||||
/** Back up, atomically update, and semantically read back one JSON MCP configuration. */
|
||||
export const configureJsonClient = (
|
||||
|
||||
@@ -3,6 +3,7 @@ import { homedir } from "node:os";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { npxRegistrationCommand } from "./ClientRegistrationIdentity.js";
|
||||
import {
|
||||
SUPPORTED_NODE_VERSION_PROSE,
|
||||
supportsNodeVersion,
|
||||
@@ -35,7 +36,7 @@ export const setupRegistrationCommand = (
|
||||
useNpmRunner: boolean = process.env.npm_command === "exec",
|
||||
): readonly string[] =>
|
||||
useNpmRunner
|
||||
? PRODUCT_IDENTITY.mcpCommand.split(" ")
|
||||
? npxRegistrationCommand(platform)
|
||||
: platform === "win32"
|
||||
? [
|
||||
process.execPath,
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import {
|
||||
REA_WORKFLOW_PROVIDER,
|
||||
workflowAnalysisProfile,
|
||||
} from "./InvestigationProviders.js";
|
||||
import type { AnalysisOperation } from "./AnalysisProvider.js";
|
||||
import type { AnalysisProfileCommitment } from "../domain/analysisProfile.js";
|
||||
import type { WorkflowSnapshotRecordInput } from "./binary/BinarySessionRecords.js";
|
||||
import {
|
||||
analysisProfileSchema,
|
||||
committedProviderSchema,
|
||||
} from "../domain/analysisProfile.js";
|
||||
import type { BinaryTarget } from "../domain/binaryTarget.js";
|
||||
import { createEvidence, type Evidence } from "../domain/evidence.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
|
||||
/** Build the shared Evidence record for a successful composed binary workflow. */
|
||||
export const createWorkflowEvidence = (input: {
|
||||
readonly target: BinaryTarget | undefined;
|
||||
readonly operation: AnalysisOperation;
|
||||
readonly parameters: Readonly<Record<string, JsonValue>>;
|
||||
readonly result: JsonValue;
|
||||
readonly upstreamProfile: AnalysisProfileCommitment | undefined;
|
||||
}): Evidence =>
|
||||
createEvidence(input.target, REA_WORKFLOW_PROVIDER, {
|
||||
operation: input.operation,
|
||||
parameters: input.parameters,
|
||||
result: input.result,
|
||||
...(input.upstreamProfile === undefined
|
||||
? {}
|
||||
: { analysisProfile: workflowAnalysisProfile(input.upstreamProfile) }),
|
||||
confidence: "derived",
|
||||
limitations: ["Derived by an REA composed workflow."],
|
||||
});
|
||||
|
||||
/** Convert workflow Evidence into its exact replay binding, when it has a profile. */
|
||||
export const workflowSnapshotRecord = (
|
||||
evidence: Evidence,
|
||||
operation: AnalysisOperation,
|
||||
): WorkflowSnapshotRecordInput | undefined => {
|
||||
if (!("analysis_profile" in evidence)) return undefined;
|
||||
return {
|
||||
operation,
|
||||
parameters: evidence.parameters,
|
||||
execution: {
|
||||
result: evidence.normalized_result,
|
||||
rawResult: evidence.raw_result,
|
||||
provider: committedProviderSchema.parse(evidence.provider),
|
||||
analysisProfile: analysisProfileSchema.parse(evidence.analysis_profile),
|
||||
limitations: evidence.limitations,
|
||||
locations: evidence.locations,
|
||||
subject:
|
||||
evidence.subject === null
|
||||
? null
|
||||
: {
|
||||
path: evidence.subject.local_path,
|
||||
sha256: evidence.subject.digest.sha256,
|
||||
format: evidence.subject.format,
|
||||
...(evidence.subject.architecture === null
|
||||
? {}
|
||||
: { architecture: evidence.subject.architecture }),
|
||||
},
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -7,6 +7,8 @@ import type { AnalysisProfileCommitment } from "../../domain/analysisProfile.js"
|
||||
import type { AnalysisProviderSelector } from "../../contracts/providerSelection.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Result } from "../../domain/result.js";
|
||||
import type { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import type { WorkflowSnapshotRecordInput } from "./BinarySessionRecords.js";
|
||||
|
||||
import type {
|
||||
AnalysisOperation,
|
||||
@@ -43,6 +45,10 @@ export interface BinarySessionPort
|
||||
analysisProfile(
|
||||
operation?: AnalysisOperation,
|
||||
): AnalysisProfileCommitment | undefined;
|
||||
allowsSnapshotReplay(operation: AnalysisOperation): boolean;
|
||||
recordWorkflowSnapshot(
|
||||
input: WorkflowSnapshotRecordInput,
|
||||
): Result<null, EvidenceIntegrityError>;
|
||||
openCompatibility(): Readonly<Record<string, JsonValue>>;
|
||||
onAvailabilityChanged?(listener: () => void | Promise<void>): () => void;
|
||||
onAnalysisSnapshotChanged?(listener: () => void | Promise<void>): () => void;
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js";
|
||||
import type { BinaryTarget } from "../../domain/binaryTarget.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import type { EvidenceBundle } from "../../domain/evidenceBundle.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { evidenceBundleForTarget } from "../../domain/evidenceBundle.js";
|
||||
import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
@@ -26,6 +27,15 @@ export interface ActiveAnalysisBinding {
|
||||
readonly profile: AnalysisProfileCommitment | null;
|
||||
}
|
||||
|
||||
/** Validated composed-workflow payload bound to the active target and profile. */
|
||||
export interface WorkflowSnapshotRecordInput {
|
||||
readonly operation: AnalysisOperation;
|
||||
readonly parameters: Readonly<Record<string, JsonValue>>;
|
||||
readonly execution: Parameters<
|
||||
AnalysisSnapshotCache["recordWorkflow"]
|
||||
>[0]["execution"];
|
||||
}
|
||||
|
||||
/** Binary snapshot owner and compatibility facade for composed investigation records. */
|
||||
export abstract class BinarySessionRecords {
|
||||
readonly #records: InvestigationRecords;
|
||||
@@ -59,10 +69,20 @@ export abstract class BinarySessionRecords {
|
||||
return this.#records.evidenceById(evidenceId);
|
||||
}
|
||||
|
||||
/** Borrow immutable investigation Evidence without copying a complete graph. */
|
||||
evidenceForAnalysis(evidenceId: string): Evidence | undefined {
|
||||
return this.#records.evidenceForAnalysis(evidenceId);
|
||||
}
|
||||
|
||||
exportEvidenceBundle(): EvidenceBundle {
|
||||
return this.#records.exportEvidenceBundle();
|
||||
}
|
||||
|
||||
/** Borrow a sealed bundle for complete serialization without cloning retained graphs. */
|
||||
evidenceBundleForSerialization(): EvidenceBundle {
|
||||
return this.#records.evidenceBundleForSerialization();
|
||||
}
|
||||
|
||||
importEvidenceBundle(
|
||||
bundle: unknown,
|
||||
): Result<number, EvidenceIntegrityError> {
|
||||
@@ -161,15 +181,9 @@ export abstract class BinarySessionRecords {
|
||||
}
|
||||
|
||||
/** Retain one derived workflow result alongside its provider cache entries. */
|
||||
recordWorkflowSnapshot(input: {
|
||||
readonly operation: string;
|
||||
readonly parameters: Readonly<
|
||||
Record<string, import("../../domain/jsonValue.js").JsonValue>
|
||||
>;
|
||||
readonly execution: Parameters<
|
||||
AnalysisSnapshotCache["recordWorkflow"]
|
||||
>[0]["execution"];
|
||||
}): Result<null, EvidenceIntegrityError> {
|
||||
recordWorkflowSnapshot(
|
||||
input: WorkflowSnapshotRecordInput,
|
||||
): Result<null, EvidenceIntegrityError> {
|
||||
const active = this.activeAnalysisBinding();
|
||||
if (active === undefined || active.profile === null)
|
||||
return err(
|
||||
|
||||
@@ -116,6 +116,36 @@ describe("evidence ledger recording", () => {
|
||||
ledger.clear();
|
||||
expect(ledger.export().records).toEqual([]);
|
||||
});
|
||||
|
||||
it("protects borrowed snapshots while preserving detached mutable public reads", () => {
|
||||
const evidence = createEvidence(TARGET, PROVIDER, {
|
||||
operation: "health",
|
||||
parameters: {},
|
||||
result: { nested: { value: "original" } },
|
||||
});
|
||||
const ledger = new EvidenceLedger();
|
||||
expect(ledger.record(evidence).ok).toBe(true);
|
||||
const borrowed = ledger.forAnalysis(evidence.evidence_id);
|
||||
if (borrowed === undefined) throw new Error("Missing recorded Evidence");
|
||||
expect(Object.isFrozen(borrowed)).toBe(true);
|
||||
expect(Object.isFrozen(borrowed.normalized_result)).toBe(true);
|
||||
expect(Reflect.set(borrowed, "operation", "forged")).toBe(false);
|
||||
expect(ledger.forAnalysis(evidence.evidence_id)).toBe(borrowed);
|
||||
expect(ledger.record(borrowed)).toEqual({ ok: true, value: "duplicate" });
|
||||
const detached = ledger.get(evidence.evidence_id);
|
||||
if (detached === undefined) throw new Error("Missing detached Evidence");
|
||||
expect(Reflect.set(detached, "operation", "forged")).toBe(true);
|
||||
expect(ledger.record(detached).ok).toBe(false);
|
||||
expect(ledger.forAnalysis(evidence.evidence_id)).toBe(borrowed);
|
||||
const snapshot = ledger.forSerialization();
|
||||
expect(snapshot).toEqual(ledger.export());
|
||||
expect(snapshot.records[0]).toBe(borrowed);
|
||||
expect(Object.isFrozen(snapshot.records)).toBe(true);
|
||||
ledger.clear();
|
||||
expect(ledger.forAnalysis(evidence.evidence_id)).toBeUndefined();
|
||||
expect(ledger.forSerialization().records).toEqual([]);
|
||||
expect(snapshot.records).toEqual([evidence]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("evidence ledger metadata imports", () => {
|
||||
|
||||
@@ -2,13 +2,19 @@ import canonicalize from "canonicalize";
|
||||
|
||||
import {
|
||||
createEvidenceBundle,
|
||||
createImmutableEvidenceBundle,
|
||||
describeEvidenceBundleFailure,
|
||||
parseEvidenceBundle,
|
||||
validateResidualUnknownAddition,
|
||||
type EvidenceBundle,
|
||||
} from "../../domain/evidenceBundle.js";
|
||||
import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import { UnknownRegistryError } from "../../domain/unknownRegistryError.js";
|
||||
import { parseEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import {
|
||||
immutableEvidence,
|
||||
parseEvidence,
|
||||
type Evidence,
|
||||
} from "../../domain/evidence.js";
|
||||
import {
|
||||
createResidualUnknown,
|
||||
updateResidualUnknown,
|
||||
@@ -41,7 +47,7 @@ export class EvidenceLedger {
|
||||
record(input: Evidence): RecordResult {
|
||||
let evidence: Evidence;
|
||||
try {
|
||||
evidence = parseEvidence(input);
|
||||
evidence = immutableEvidence(input);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence record validation failed", {
|
||||
@@ -51,6 +57,10 @@ export class EvidenceLedger {
|
||||
}
|
||||
const existing = this.#records.get(evidence.evidence_id);
|
||||
if (existing !== undefined) {
|
||||
// A borrowed record is already authenticated and deeply immutable. Its
|
||||
// identity cannot change between reads, so comparing document-sized
|
||||
// canonical strings again would add no integrity check.
|
||||
if (existing === evidence) return ok("duplicate");
|
||||
return recordsAgree(existing, evidence)
|
||||
? ok("duplicate")
|
||||
: err(new EvidenceIntegrityError("Conflicting evidence record"));
|
||||
@@ -68,6 +78,7 @@ export class EvidenceLedger {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence bundle validation failed", {
|
||||
cause,
|
||||
userMessage: describeEvidenceBundleFailure(input, cause),
|
||||
}),
|
||||
);
|
||||
}
|
||||
@@ -95,8 +106,9 @@ export class EvidenceLedger {
|
||||
additions.set(evidence.evidence_id, evidence);
|
||||
}
|
||||
for (const [id, evidence] of replacements)
|
||||
this.#records.set(id, evidence);
|
||||
for (const [id, evidence] of additions) this.#records.set(id, evidence);
|
||||
this.#records.set(id, immutableEvidence(evidence));
|
||||
for (const [id, evidence] of additions)
|
||||
this.#records.set(id, immutableEvidence(evidence));
|
||||
return ok({
|
||||
recordsAdded: additions.size,
|
||||
unknownsAdded: 0,
|
||||
@@ -163,6 +175,14 @@ export class EvidenceLedger {
|
||||
);
|
||||
}
|
||||
|
||||
/** Borrow a sealed canonical bundle for serialization without copying its records. */
|
||||
forSerialization(): EvidenceBundle {
|
||||
return createImmutableEvidenceBundle(
|
||||
[...this.#records.values()],
|
||||
[...this.#unknownRevisions.values()],
|
||||
);
|
||||
}
|
||||
|
||||
/** Check whether one immutable Evidence ID is present in this ledger. */
|
||||
has(evidenceId: string): boolean {
|
||||
return this.#records.has(evidenceId);
|
||||
@@ -174,6 +194,11 @@ export class EvidenceLedger {
|
||||
return evidence === undefined ? undefined : structuredClone(evidence);
|
||||
}
|
||||
|
||||
/** Borrow authenticated immutable Evidence for read-only analysis without a full clone. */
|
||||
forAnalysis(evidenceId: string): Evidence | undefined {
|
||||
return this.#records.get(evidenceId);
|
||||
}
|
||||
|
||||
/** Create one approved unknown and atomically append its mutation evidence. */
|
||||
recordUnknown(
|
||||
input: RecordUnknownInput,
|
||||
@@ -378,7 +403,8 @@ export class EvidenceLedger {
|
||||
} catch (cause: unknown) {
|
||||
return err(new UnknownRegistryError("integrity", { cause }));
|
||||
}
|
||||
for (const [id, evidence] of additions) this.#records.set(id, evidence);
|
||||
for (const [id, evidence] of additions)
|
||||
this.#records.set(id, immutableEvidence(evidence));
|
||||
if (unknown !== undefined && unknownKey !== undefined) {
|
||||
this.#unknownRevisions.set(unknownKey, unknown);
|
||||
this.#unknownHeads.set(unknown.unknown_id, unknown);
|
||||
@@ -405,7 +431,8 @@ export class EvidenceLedger {
|
||||
unknowns: ReadonlyMap<string, ResidualUnknown>,
|
||||
): void {
|
||||
this.#records.clear();
|
||||
for (const [id, evidence] of records) this.#records.set(id, evidence);
|
||||
for (const [id, evidence] of records)
|
||||
this.#records.set(id, immutableEvidence(evidence));
|
||||
this.#unknownRevisions.clear();
|
||||
this.#unknownHeads.clear();
|
||||
for (const [key, unknown] of unknowns) {
|
||||
|
||||
@@ -15,6 +15,10 @@ import type { Result } from "../../domain/result.js";
|
||||
export interface EvidenceReader {
|
||||
hasEvidence(evidenceId: string): boolean;
|
||||
evidenceById(evidenceId: string): Evidence | undefined;
|
||||
/** Optional immutable analysis read; older adapters can keep detached reads. */
|
||||
evidenceForAnalysis?(evidenceId: string): Evidence | undefined;
|
||||
/** Optional immutable bundle snapshot for complete serialization without graph copies. */
|
||||
evidenceBundleForSerialization?(): EvidenceBundle;
|
||||
exportEvidenceBundle(): EvidenceBundle;
|
||||
}
|
||||
|
||||
|
||||
@@ -51,11 +51,21 @@ export class InvestigationRecords
|
||||
return this.#evidence.get(evidenceId);
|
||||
}
|
||||
|
||||
/** Borrow an immutable authenticated record for a read-only application workflow. */
|
||||
evidenceForAnalysis(evidenceId: string): Evidence | undefined {
|
||||
return this.#evidence.forAnalysis(evidenceId);
|
||||
}
|
||||
|
||||
/** Export a detached deterministic bundle of Evidence and Unknown revisions. */
|
||||
exportEvidenceBundle(): EvidenceBundle {
|
||||
return this.#evidence.export();
|
||||
}
|
||||
|
||||
/** Borrow a sealed bundle snapshot for complete JSON serialization. */
|
||||
evidenceBundleForSerialization(): EvidenceBundle {
|
||||
return this.#evidence.forSerialization();
|
||||
}
|
||||
|
||||
/** Atomically merge a bundle and report changes for the owner's post-commit observers. */
|
||||
mergeEvidenceBundle(
|
||||
bundle: unknown,
|
||||
|
||||
@@ -8,6 +8,18 @@ import type {
|
||||
JavaScriptApplicationAnalysisResult,
|
||||
} from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import { JAVASCRIPT_APPLICATION_PROVIDER } from "../InvestigationProviders.js";
|
||||
import { freezeJsonSnapshot } from "../../domain/immutableJson.js";
|
||||
import { rememberOwnedApplicationGraphEvidence } from "./JavaScriptApplicationEvidenceGraph.js";
|
||||
|
||||
/** Transfer an already validated, application-owned result into immutable analysis Evidence. */
|
||||
export const createOwnedJavaScriptApplicationEvidence = (
|
||||
input: AnalyzeJavaScriptApplicationInput,
|
||||
result: JavaScriptApplicationAnalysisResult,
|
||||
): Evidence =>
|
||||
rememberOwnedApplicationGraphEvidence(
|
||||
createJavaScriptApplicationEvidence(input, freezeJsonSnapshot(result)),
|
||||
result,
|
||||
);
|
||||
|
||||
/** Create Evidence for one deterministic local JavaScript application graph. */
|
||||
export const createJavaScriptApplicationEvidence = (
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE } from "../../contracts/javascript/javascriptRuntimeReconciliationExample.js";
|
||||
import { parseEvidence } from "../../domain/evidence.js";
|
||||
import {
|
||||
analyzeJavaScriptApplicationInputSchema,
|
||||
javascriptApplicationAnalysisResultSchema,
|
||||
} from "../../domain/javascript/javascriptApplicationAnalysis.js";
|
||||
import {
|
||||
createJavaScriptApplicationEvidence,
|
||||
createOwnedJavaScriptApplicationEvidence,
|
||||
} from "./JavaScriptApplicationEvidence.js";
|
||||
import {
|
||||
parseApplicationGraphEvidence,
|
||||
rememberOwnedApplicationGraphEvidence,
|
||||
} from "./JavaScriptApplicationEvidenceGraph.js";
|
||||
|
||||
describe("owned application graph Evidence", () => {
|
||||
it("reuses the exact authenticated graph and preserves ordinary wire identity", () => {
|
||||
const result = javascriptApplicationAnalysisResultSchema.parse(
|
||||
JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.normalized_result,
|
||||
);
|
||||
const input = analyzeJavaScriptApplicationInputSchema.parse({
|
||||
input_path: result.input_path,
|
||||
...JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE.parameters,
|
||||
});
|
||||
const ordinary = createJavaScriptApplicationEvidence(input, result);
|
||||
const owned = createOwnedJavaScriptApplicationEvidence(input, result);
|
||||
expect(owned.evidence_id).toBe(ordinary.evidence_id);
|
||||
expect(JSON.stringify(owned)).toBe(JSON.stringify(ordinary));
|
||||
const parsed = parseApplicationGraphEvidence(owned);
|
||||
expect(parsed.evidence).toBe(owned);
|
||||
expect(parsed.graph).toBe(result.graph);
|
||||
expect(parsed.semanticGraph).toBe(result.semantic_graph);
|
||||
expect(Object.isFrozen(parsed.graph.nodes)).toBe(true);
|
||||
expect(parseApplicationGraphEvidence(owned)).toBe(parsed);
|
||||
const wire: unknown = JSON.parse(JSON.stringify(owned));
|
||||
expect(parseApplicationGraphEvidence(wire)).toEqual(parsed);
|
||||
expect(parseApplicationGraphEvidence(wire).graph).not.toBe(parsed.graph);
|
||||
expect(() =>
|
||||
rememberOwnedApplicationGraphEvidence(ordinary, result),
|
||||
).toThrow("exact validated immutable result");
|
||||
const tampered = { ...owned, normalized_result: {} };
|
||||
expect(() => parseEvidence(Object.freeze(tampered))).toThrow();
|
||||
expect(() => parseApplicationGraphEvidence(tampered)).toThrow();
|
||||
});
|
||||
});
|
||||
@@ -3,7 +3,11 @@ import {
|
||||
JAVASCRIPT_RUNTIME_RECONCILIATION_PROVIDER,
|
||||
MANAGED_WORKFLOW_PROVIDER,
|
||||
} from "../InvestigationProviders.js";
|
||||
import { parseEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import {
|
||||
isImmutableEvidence,
|
||||
parseEvidence,
|
||||
type Evidence,
|
||||
} from "../../domain/evidence.js";
|
||||
import {
|
||||
analyzeJavaScriptApplicationInputSchema,
|
||||
javascriptApplicationAnalysisResultSchema,
|
||||
@@ -26,11 +30,39 @@ export interface ApplicationGraphEvidenceSource {
|
||||
readonly semanticGraph: JavaScriptSemanticGraph | null;
|
||||
}
|
||||
|
||||
const ownedApplicationSources = new WeakMap<
|
||||
Evidence,
|
||||
ApplicationGraphEvidenceSource
|
||||
>();
|
||||
|
||||
/** Retain the producer-validated meaning of its exact immutable result snapshot. */
|
||||
export const rememberOwnedApplicationGraphEvidence = (
|
||||
evidence: Evidence,
|
||||
result: JavaScriptApplicationAnalysisResult,
|
||||
): Evidence => {
|
||||
if (
|
||||
!isImmutableEvidence(evidence) ||
|
||||
evidence.normalized_result !== result ||
|
||||
evidence.operation !== "analyze_javascript_application" ||
|
||||
!providerMatches(evidence, JAVASCRIPT_APPLICATION_PROVIDER)
|
||||
)
|
||||
throw new TypeError(
|
||||
"Owned application Evidence must bind its exact validated immutable result",
|
||||
);
|
||||
ownedApplicationSources.set(
|
||||
evidence,
|
||||
Object.freeze(staticApplicationSourceForResult(evidence, result)),
|
||||
);
|
||||
return evidence;
|
||||
};
|
||||
|
||||
/** Parse and authenticate one REA-produced JavaScript Application Graph Evidence. */
|
||||
export const parseApplicationGraphEvidence = (
|
||||
input: unknown,
|
||||
): ApplicationGraphEvidenceSource => {
|
||||
const evidence = parseEvidence(input);
|
||||
const owned = ownedApplicationSources.get(evidence);
|
||||
if (owned !== undefined) return owned;
|
||||
if (
|
||||
evidence.operation === "analyze_javascript_application" &&
|
||||
evidence.predicate_type === "rea.javascript-application-analysis" &&
|
||||
@@ -56,6 +88,16 @@ export const parseApplicationGraphEvidence = (
|
||||
|
||||
const staticApplicationSource = (
|
||||
evidence: Evidence,
|
||||
): ApplicationGraphEvidenceSource => {
|
||||
const result = javascriptApplicationAnalysisResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
return staticApplicationSourceForResult(evidence, result);
|
||||
};
|
||||
|
||||
const staticApplicationSourceForResult = (
|
||||
evidence: Evidence,
|
||||
result: JavaScriptApplicationAnalysisResult,
|
||||
): ApplicationGraphEvidenceSource => {
|
||||
if (
|
||||
evidence.authority !== "shipped-artifact" ||
|
||||
@@ -64,9 +106,6 @@ const staticApplicationSource = (
|
||||
throw new TypeError(
|
||||
"JavaScript application Evidence authority or confidence is invalid",
|
||||
);
|
||||
const result = javascriptApplicationAnalysisResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
if (evidence.predicate_type !== "rea.javascript-application-analysis")
|
||||
throw new TypeError(
|
||||
"JavaScript application Evidence predicate does not match its result shape",
|
||||
|
||||
@@ -16,7 +16,7 @@ import { projectInputIssues } from "../../domain/inputIssueProjection.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import type { ExecutionOptions } from "../AnalysisProvider.js";
|
||||
import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js";
|
||||
import { createOwnedJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js";
|
||||
import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js";
|
||||
import { JAVASCRIPT_APPLICATION_PROVIDER } from "../InvestigationProviders.js";
|
||||
|
||||
@@ -77,7 +77,7 @@ export const analyzeJavaScriptApplicationValidated = async (
|
||||
total: 1,
|
||||
message: "Creating and hashing application analysis Evidence",
|
||||
});
|
||||
const evidence = createJavaScriptApplicationEvidence(input, result);
|
||||
const evidence = createOwnedJavaScriptApplicationEvidence(input, result);
|
||||
await options.progress?.report({
|
||||
phase: "analyze_javascript_application",
|
||||
completed: 1,
|
||||
|
||||
@@ -362,6 +362,35 @@ describe("contextual JavaScript package path resolution", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("generic package entry heuristics", () => {
|
||||
it.each([
|
||||
['{"main":"actual.cjs","module":"bundler.mjs"}', "actual.cjs"],
|
||||
['{"module":"bundler.mjs"}', "bundler.mjs"],
|
||||
])(
|
||||
"preserves generic entry heuristics without a module kind for %s",
|
||||
(metadata, entry) => {
|
||||
const files = fileMap([
|
||||
file("app/consumer.js", "root"),
|
||||
file("app/node_modules/fixture/package.json", "root", metadata),
|
||||
file("app/node_modules/fixture/actual.cjs", "root"),
|
||||
file("app/node_modules/fixture/bundler.mjs", "root"),
|
||||
file("app/node_modules/fixture/index.js", "root"),
|
||||
]);
|
||||
expect(
|
||||
resolve({
|
||||
declaredPath: "fixture",
|
||||
sourcePath: "app/consumer.js",
|
||||
context: "module-specifier",
|
||||
files,
|
||||
}),
|
||||
).toMatchObject({
|
||||
resolution_status: "resolved",
|
||||
resolved_path: `app/node_modules/fixture/${entry}`,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("directory package entrypoint precedence", () => {
|
||||
it.each([
|
||||
["require", '{"main":"actual.cjs"}', "actual.cjs"],
|
||||
|
||||
@@ -461,10 +461,12 @@ const packageEntry = (
|
||||
? { ...exported, source: "exports" }
|
||||
: exported;
|
||||
}
|
||||
const preferred =
|
||||
moduleKind === "import"
|
||||
? [Reflect.get(value, "module"), Reflect.get(value, "main")]
|
||||
: [Reflect.get(value, "main"), Reflect.get(value, "module")];
|
||||
// Node imports and requires ignore the bundler-only module field. Keep
|
||||
// that fallback for callers without an explicit module-loading kind.
|
||||
const preferred = [
|
||||
Reflect.get(value, "main"),
|
||||
...(moduleKind === undefined ? [Reflect.get(value, "module")] : []),
|
||||
];
|
||||
const entry = preferred.find((candidate) => candidate !== undefined);
|
||||
if (entry === undefined) return { status: "missing" };
|
||||
const legacy = packagePathValue(entry);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
@@ -113,6 +114,12 @@ const parseCaptureEvidence = (input: unknown) => {
|
||||
try {
|
||||
evidence = parseEvidence(input);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof z.ZodError)
|
||||
throw new AnalysisInputError(
|
||||
"compare_process_captures",
|
||||
{ cause },
|
||||
projectInputIssues(cause.issues, input),
|
||||
);
|
||||
throw invalidCaptureEvidence();
|
||||
}
|
||||
if (
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { Transform, type TransformCallback } from "node:stream";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
/** Terminal exhaustion of the budget shared by all readers in one traversal. */
|
||||
export class ArtifactBudgetExceeded extends ArtifactReaderFailure {
|
||||
constructor(message: string) {
|
||||
super("limit", message);
|
||||
}
|
||||
}
|
||||
|
||||
/** One decoded-byte ceiling shared by an archive and all nested readers. */
|
||||
export class ArtifactDecodedBudget {
|
||||
#used = 0;
|
||||
#metadataUsed = 0;
|
||||
|
||||
constructor(
|
||||
private readonly maximum = 512 * 1024 * 1024,
|
||||
private readonly maximumMetadata = 128 * 1024 * 1024,
|
||||
) {}
|
||||
|
||||
/** Charge decoding work, including repeated reads, before exposing a chunk. */
|
||||
consume(bytes: number, path: string): void {
|
||||
if (
|
||||
!Number.isSafeInteger(bytes) ||
|
||||
bytes < 0 ||
|
||||
bytes > this.maximum - this.#used
|
||||
)
|
||||
throw new ArtifactBudgetExceeded(
|
||||
`Decoded archive budget of ${this.maximum} bytes exhausted at ${path}`,
|
||||
);
|
||||
this.#used += bytes;
|
||||
}
|
||||
|
||||
/** Charge retained path text, path-trie segments, and per-entry graph/index capacity. */
|
||||
consumeEntry(path: string): void {
|
||||
const segments = path.split("/").length;
|
||||
const units = 1024 + Buffer.byteLength(path) * 2 + segments * 256;
|
||||
if (units > this.maximumMetadata - this.#metadataUsed)
|
||||
throw new ArtifactBudgetExceeded(
|
||||
`Retained archive metadata budget of ${this.maximumMetadata} capacity units exhausted at ${path}`,
|
||||
);
|
||||
this.#metadataUsed += units;
|
||||
}
|
||||
}
|
||||
|
||||
/** Decoder-stage budget/cancellation guard, independent of member metadata. */
|
||||
export class ArtifactBudgetTransform extends Transform {
|
||||
constructor(
|
||||
private readonly budget: ArtifactDecodedBudget,
|
||||
private readonly path: string,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer,
|
||||
_encoding: BufferEncoding,
|
||||
done: TransformCallback,
|
||||
): void {
|
||||
if (this.signal?.aborted) {
|
||||
done(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`Archive decoding cancelled at ${this.path}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.budget.consume(chunk.length, this.path);
|
||||
} catch (cause: unknown) {
|
||||
done(cause instanceof Error ? cause : new Error(String(cause)));
|
||||
return;
|
||||
}
|
||||
done(null, chunk);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ export type HashResult = {
|
||||
readonly sha256: string;
|
||||
readonly bytes: number;
|
||||
readonly prefix: Buffer;
|
||||
/** Digest in the additionally requested algorithm, when one was requested. */
|
||||
readonly also?: string;
|
||||
};
|
||||
|
||||
/** Bounded classification evidence; this is not an executable format size limit. */
|
||||
@@ -27,8 +29,11 @@ export const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
export const hashReadable = async (
|
||||
stream: Readable,
|
||||
signal?: AbortSignal,
|
||||
alsoAlgorithm?: string,
|
||||
): Promise<HashResult> => {
|
||||
const hash = createHash("sha256");
|
||||
const also =
|
||||
alsoAlgorithm === undefined ? undefined : createHash(alsoAlgorithm);
|
||||
const prefixes: Buffer[] = [];
|
||||
let prefixBytes = 0;
|
||||
let bytes = 0;
|
||||
@@ -37,6 +42,7 @@ export const hashReadable = async (
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
hash.update(chunk);
|
||||
also?.update(chunk);
|
||||
if (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) {
|
||||
const selected = chunk.subarray(
|
||||
0,
|
||||
@@ -46,5 +52,10 @@ export const hashReadable = async (
|
||||
prefixBytes += selected.length;
|
||||
}
|
||||
}
|
||||
return { sha256: hash.digest("hex"), bytes, prefix: Buffer.concat(prefixes) };
|
||||
return {
|
||||
sha256: hash.digest("hex"),
|
||||
bytes,
|
||||
prefix: Buffer.concat(prefixes),
|
||||
...(also === undefined ? {} : { also: also.digest("hex") }),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -21,7 +21,10 @@ import {
|
||||
type ArtifactAnalysisOperation,
|
||||
} from "../contracts/artifactToolContracts.js";
|
||||
import type { BinaryTarget } from "../domain/binaryTarget.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
|
||||
import {
|
||||
AnalysisCapabilityUnavailableError,
|
||||
AnalysisInputError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { ArtifactOperationError } from "../domain/artifactOperationError.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
@@ -116,10 +119,15 @@ class ArtifactClient implements AnalysisClient {
|
||||
parameters.path !== "." &&
|
||||
parameters.path !== basename(this.target.path)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
"For an active plist, path must select that archive (omit path or use its basename)",
|
||||
);
|
||||
throw new AnalysisInputError(operation, undefined, [
|
||||
{
|
||||
path: ["path"],
|
||||
reason: "invalid_value",
|
||||
message:
|
||||
"For an active plist, path must select that archive (omit path or use its basename).",
|
||||
expected: basename(this.target.path),
|
||||
},
|
||||
]);
|
||||
const bundlePath = standalone
|
||||
? dirname(this.target.path)
|
||||
: this.target.sourcePath;
|
||||
@@ -352,6 +360,8 @@ const translateFailure = (
|
||||
cause.details,
|
||||
cause.message,
|
||||
);
|
||||
// Caller-selection failures are already typed; keep their correction details.
|
||||
if (cause instanceof AnalysisInputError) return cause;
|
||||
return new ArtifactOperationError(operation, "io");
|
||||
};
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ export const artifactCapabilities = (
|
||||
requiresRoot: false,
|
||||
}),
|
||||
limitations: Object.freeze([
|
||||
"DMG child inventory automatically uses a read-only native macOS mount; PKG remains root-hash-only.",
|
||||
"DMG child inventory automatically uses a read-only native macOS mount. PKG (xar) inventory expands gzip-cpio Scripts and Payload members; pbzx payloads are recorded without expansion.",
|
||||
"ASAR files discovered in filesystem-backed inventories are expanded without bulk extraction; other nested containers remain recorded only.",
|
||||
]),
|
||||
};
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import type { Readable } from "node:stream";
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
|
||||
import type { ArtifactDecodedBudget } from "./ArtifactDecodedBudget.js";
|
||||
import type {
|
||||
ArtifactChecksumObservation,
|
||||
ArtifactIntegrityFailureDetails,
|
||||
} from "../domain/artifactOperationError.js";
|
||||
export type { ArtifactChecksumObservation } from "../domain/artifactOperationError.js";
|
||||
|
||||
/** Archive-neutral entry metadata. Reader adapters never choose output paths. */
|
||||
export interface ArtifactEntry {
|
||||
@@ -15,6 +21,21 @@ export interface ArtifactEntry {
|
||||
readonly unpacked: boolean;
|
||||
readonly limitations: readonly string[];
|
||||
readonly adapterKey: string;
|
||||
/**
|
||||
* Container checksum of the member's bytes in an algorithm other than
|
||||
* SHA-256 (xar declares SHA-1 or MD5). The scan verifies it under the
|
||||
* caller's integrity policy.
|
||||
*/
|
||||
readonly declaredChecksum?: {
|
||||
readonly algorithm: "sha1" | "md5" | "sha512";
|
||||
readonly value: string;
|
||||
};
|
||||
/** The container records this member but does not hold its bytes, such as an unresolved hard link. */
|
||||
readonly contentUnavailable?: boolean;
|
||||
/** A consumed representation contradicted its declared integrity metadata. */
|
||||
readonly integrityMismatched?: boolean;
|
||||
/** Archive format of this member's own bytes, when the scan should expand it. */
|
||||
readonly nestedArchive?: "gzip-cpio";
|
||||
/** Filesystem identity captured during traversal, when supplied by an adapter. */
|
||||
readonly sourceIdentity?: {
|
||||
readonly device: number;
|
||||
@@ -24,9 +45,15 @@ export interface ArtifactEntry {
|
||||
|
||||
/** Read-only adapter over one directory, archive, or virtual container. */
|
||||
export interface ArtifactReader {
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "file";
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "pkg" | "file";
|
||||
/** Shared safety budget for compressed representations and nested archives. */
|
||||
readonly decodedBudget?: ArtifactDecodedBudget;
|
||||
entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry>;
|
||||
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable>;
|
||||
/** Checksums observed after the complete member stream has been consumed. */
|
||||
integrityObservations?(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[];
|
||||
provenance(): readonly ArtifactCommand[];
|
||||
close(): Promise<void>;
|
||||
}
|
||||
@@ -44,12 +71,7 @@ export class ArtifactReaderFailure extends Error {
|
||||
| "unavailable",
|
||||
message: string,
|
||||
options?: ErrorOptions,
|
||||
readonly details?: Readonly<{
|
||||
logicalPath: string;
|
||||
declaredSha256: string | null;
|
||||
calculatedSha256: string | null;
|
||||
unpacked: boolean;
|
||||
}>,
|
||||
readonly details?: ArtifactIntegrityFailureDetails,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = "ArtifactReaderFailure";
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { Readable, Transform } from "node:stream";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { expect, it } from "vitest";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import { artifactStreamPipeline } from "./ArtifactStreamPipeline.js";
|
||||
|
||||
it("preserves tagged failures from intermediate verification stages", async () => {
|
||||
const failure = new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"stored checksum disagrees",
|
||||
);
|
||||
const verifier = new Transform({
|
||||
transform(_chunk, _encoding, done) {
|
||||
done(failure);
|
||||
},
|
||||
});
|
||||
const output = artifactStreamPipeline(
|
||||
Readable.from([Buffer.from("encoded")]),
|
||||
[verifier],
|
||||
{ path: "member" },
|
||||
);
|
||||
await expect(buffer(output)).rejects.toBe(failure);
|
||||
});
|
||||
|
||||
it("closes the source when the consumer stops before exhausting member data", async () => {
|
||||
const source = Readable.from(
|
||||
(function* () {
|
||||
for (let index = 0; index < 1000; index++) yield Buffer.alloc(64 * 1024);
|
||||
})(),
|
||||
);
|
||||
const closed = new Promise<void>((resolve) =>
|
||||
source.once("close", () => resolve()),
|
||||
);
|
||||
const output = artifactStreamPipeline(source, [], { path: "member" });
|
||||
for await (const chunk of output) {
|
||||
expect(Buffer.isBuffer(chunk)).toBe(true);
|
||||
break;
|
||||
}
|
||||
await closed;
|
||||
expect(source.destroyed).toBe(true);
|
||||
});
|
||||
|
||||
it("observes abort before a consumer starts reading", async () => {
|
||||
const controller = new AbortController();
|
||||
const source = Readable.from([Buffer.from("data")]);
|
||||
const output = artifactStreamPipeline(source, [], {
|
||||
path: "member",
|
||||
signal: controller.signal,
|
||||
});
|
||||
controller.abort();
|
||||
await expect(buffer(output)).rejects.toMatchObject({ reason: "cancelled" });
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import { PassThrough, Readable, pipeline, type Transform } from "node:stream";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
/**
|
||||
* Connect a decoder chain as one lifecycle: failures propagate in both
|
||||
* directions, cancellation destroys every stage, and consumer early-return
|
||||
* cannot leave the archive source reading after its handle is closed.
|
||||
*/
|
||||
export const artifactStreamPipeline = (
|
||||
source: Readable,
|
||||
stages: readonly Transform[],
|
||||
context: {
|
||||
readonly path: string;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
},
|
||||
): Readable => {
|
||||
const output = new PassThrough();
|
||||
const aborted = () =>
|
||||
output.destroy(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`Archive read cancelled: ${context.path}`,
|
||||
),
|
||||
);
|
||||
pipeline([source, ...stages, output], () => {
|
||||
// The readable destination retains any failure for its consumer. Keep
|
||||
// cancellation active until buffered output has also been consumed.
|
||||
});
|
||||
context.signal?.addEventListener("abort", aborted, { once: true });
|
||||
if (context.signal?.aborted) aborted();
|
||||
const dispose = () => {
|
||||
context.signal?.removeEventListener("abort", aborted);
|
||||
output.destroy();
|
||||
};
|
||||
const consumer = Readable.from(
|
||||
(async function* () {
|
||||
try {
|
||||
for await (const chunk of output) yield chunk;
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
if (context.signal?.aborted)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`Archive read cancelled: ${context.path}`,
|
||||
{ cause },
|
||||
);
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Archive member could not be decoded: ${context.path}`,
|
||||
{ cause },
|
||||
);
|
||||
} finally {
|
||||
dispose();
|
||||
}
|
||||
})(),
|
||||
);
|
||||
// Also covers destroy() before the async generator is first advanced.
|
||||
consumer.once("close", dispose);
|
||||
return consumer;
|
||||
};
|
||||
@@ -0,0 +1,94 @@
|
||||
import { isAbsolute } from "node:path";
|
||||
import { normalizeArtifactPath } from "./ArtifactPaths.js";
|
||||
import type { ArtifactEntry, ArtifactReader } from "./ArtifactReader.js";
|
||||
import { AsarArtifactReader } from "./AsarArtifactReader.js";
|
||||
import { CpioArtifactReader } from "./CpioArtifactReader.js";
|
||||
|
||||
/** One reader entry plus its stable logical path and expansion eligibility. */
|
||||
export interface ArtifactTreeEntry {
|
||||
readonly reader: ArtifactReader;
|
||||
readonly entry: ArtifactEntry;
|
||||
readonly path: string;
|
||||
readonly container: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared inventory/extraction traversal. The visitor decides whether verified
|
||||
* content may expand; this boundary owns nested paths, reader creation and
|
||||
* cleanup so the two workflows cannot disagree about container topology.
|
||||
*/
|
||||
export const visitArtifactTree = async (
|
||||
root: ArtifactReader,
|
||||
visit: (item: ArtifactTreeEntry) => Promise<boolean>,
|
||||
options: {
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly integrity?: "fail" | "record-and-continue";
|
||||
readonly expandAsar?: boolean;
|
||||
} = {},
|
||||
): Promise<void> => {
|
||||
const stack = [
|
||||
{
|
||||
reader: root,
|
||||
prefix: "",
|
||||
iterator: root.entries(options.signal)[Symbol.asyncIterator](),
|
||||
owned: false,
|
||||
},
|
||||
];
|
||||
try {
|
||||
while (stack.length > 0) {
|
||||
const frame = stack.at(-1);
|
||||
if (frame === undefined) break;
|
||||
const next = await frame.iterator.next();
|
||||
if (next.done === true) {
|
||||
stack.pop();
|
||||
if (frame.owned) await frame.reader.close();
|
||||
continue;
|
||||
}
|
||||
const entry = next.value;
|
||||
// Charge the full retained namespace, including all enclosing prefixes,
|
||||
// independently from each reader's own raw metadata/index charge.
|
||||
const combined =
|
||||
frame.prefix === "" ? entry.path : `${frame.prefix}/${entry.path}`;
|
||||
root.decodedBudget?.consumeEntry(combined);
|
||||
const path = normalizeArtifactPath(combined);
|
||||
const cpio = entry.nestedArchive === "gzip-cpio";
|
||||
const asar =
|
||||
options.expandAsar !== false &&
|
||||
entry.kind === "file" &&
|
||||
path.toLowerCase().endsWith(".asar") &&
|
||||
isAbsolute(entry.adapterKey);
|
||||
const container =
|
||||
!entry.encrypted && entry.contentUnavailable !== true && (cpio || asar);
|
||||
const expand = await visit({
|
||||
reader: frame.reader,
|
||||
entry,
|
||||
path,
|
||||
container,
|
||||
});
|
||||
if (!container || !expand) continue;
|
||||
const parent = frame.reader;
|
||||
const nested = cpio
|
||||
? new CpioArtifactReader(
|
||||
(signal) => parent.open(entry, signal),
|
||||
options.integrity ?? "fail",
|
||||
parent.decodedBudget,
|
||||
)
|
||||
: new AsarArtifactReader(entry.adapterKey);
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: path,
|
||||
iterator: nested.entries(options.signal)[Symbol.asyncIterator](),
|
||||
owned: true,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await Promise.allSettled(
|
||||
stack
|
||||
.filter(({ owned }) => owned)
|
||||
.map(async ({ reader, iterator }) => {
|
||||
await iterator.return?.();
|
||||
await reader.close();
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,804 @@
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
ArtifactBudgetTransform,
|
||||
ArtifactDecodedBudget,
|
||||
} from "./ArtifactDecodedBudget.js";
|
||||
import { createGunzip } from "node:zlib";
|
||||
import { artifactStreamPipeline } from "./ArtifactStreamPipeline.js";
|
||||
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactChecksumObservation,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
|
||||
/** Longest member name accepted, including its terminating NUL. */
|
||||
const MAX_NAME_BYTES = 64 * 1024;
|
||||
/** Hard-link bytes are buffered to serve every linked path, within these bounds. */
|
||||
const MAX_LINK_BUFFER_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_LINK_BUFFER_TOTAL = 64 * 1024 * 1024;
|
||||
/** Longest symlink target read; PATH_MAX is 1024 on macOS and 4096 on Linux. */
|
||||
const MAX_SYMLINK_TARGET_BYTES = 4096;
|
||||
const TRAILER = "TRAILER!!!";
|
||||
const S_IFMT = 0o170000;
|
||||
const S_IFDIR = 0o040000;
|
||||
const S_IFREG = 0o100000;
|
||||
const S_IFLNK = 0o120000;
|
||||
|
||||
const UNRESOLVED_LINK =
|
||||
"Hard-link bytes are stored with another member of this archive and could not be associated with this path.";
|
||||
|
||||
/** Pull exactly the requested bytes from a stream, holding at most one chunk. */
|
||||
class ByteSource {
|
||||
readonly #iterator: AsyncIterator<unknown>;
|
||||
#buffer: Buffer = Buffer.alloc(0);
|
||||
|
||||
constructor(stream: Readable) {
|
||||
this.#iterator = stream[Symbol.asyncIterator]();
|
||||
}
|
||||
|
||||
async #fill(): Promise<boolean> {
|
||||
let next: IteratorResult<unknown>;
|
||||
try {
|
||||
next = await this.#iterator.next();
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio stream is not valid gzip data",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
if (next.done === true) return false;
|
||||
if (!Buffer.isBuffer(next.value))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio stream yielded non-binary data",
|
||||
);
|
||||
this.#buffer =
|
||||
this.#buffer.length === 0
|
||||
? next.value
|
||||
: Buffer.concat([this.#buffer, next.value]);
|
||||
return true;
|
||||
}
|
||||
|
||||
async read(length: number, label: string): Promise<Buffer> {
|
||||
while (this.#buffer.length < length)
|
||||
if (!(await this.#fill()))
|
||||
throw new ArtifactReaderFailure("format", `cpio ${label} is truncated`);
|
||||
const bytes = this.#buffer.subarray(0, length);
|
||||
this.#buffer = this.#buffer.subarray(length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Consume the NUL alignment bytes required by newc and crc records. */
|
||||
async padding(length: number, label: string): Promise<void> {
|
||||
if ((await this.read(length, label)).some((byte) => byte !== 0))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`cpio ${label} contains nonzero bytes`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Yield `length` bytes in chunks without buffering them together. */
|
||||
async *take(length: number, label: string): AsyncGenerator<Buffer> {
|
||||
let remaining = length;
|
||||
while (remaining > 0) {
|
||||
if (this.#buffer.length === 0 && !(await this.#fill()))
|
||||
throw new ArtifactReaderFailure("format", `cpio ${label} is truncated`);
|
||||
const size = Math.min(remaining, this.#buffer.length);
|
||||
const chunk = this.#buffer.subarray(0, size);
|
||||
this.#buffer = this.#buffer.subarray(size);
|
||||
remaining -= size;
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.#iterator.return?.();
|
||||
}
|
||||
|
||||
/** Consume legal zero padding through decoder EOF, including gzip trailer verification. */
|
||||
async finish(): Promise<void> {
|
||||
do {
|
||||
if (this.#buffer.some((byte) => byte !== 0))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio has non-padding bytes after its trailer",
|
||||
);
|
||||
this.#buffer = Buffer.alloc(0);
|
||||
} while (await this.#fill());
|
||||
}
|
||||
}
|
||||
|
||||
interface CpioHeader {
|
||||
readonly format: "odc" | "newc" | "crc";
|
||||
readonly mode: number;
|
||||
readonly fileSize: number;
|
||||
readonly nameSize: number;
|
||||
readonly links: number;
|
||||
/** Device and inode identifying hard links. */
|
||||
readonly identity: string;
|
||||
/** `c_check` of `070702` archives: the unsigned 32-bit sum of the data bytes. */
|
||||
readonly check: number;
|
||||
}
|
||||
|
||||
const field = (
|
||||
bytes: Buffer,
|
||||
start: number,
|
||||
width: number,
|
||||
radix: 8 | 16,
|
||||
): number => {
|
||||
const text = bytes.toString("latin1", start, start + width);
|
||||
const pattern = radix === 8 ? /^[0-7]+$/u : /^[0-9a-fA-F]+$/u;
|
||||
const value = pattern.test(text) ? Number.parseInt(text, radix) : Number.NaN;
|
||||
if (!Number.isSafeInteger(value))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio header has a malformed numeric field",
|
||||
);
|
||||
return value;
|
||||
};
|
||||
|
||||
/** odc (`070707`, octal) and newc/crc (`070701`/`070702`, hexadecimal) headers. */
|
||||
const readHeader = async (source: ByteSource): Promise<CpioHeader> => {
|
||||
const magic = (await source.read(6, "header")).toString("latin1");
|
||||
if (magic === "070707") {
|
||||
const rest = await source.read(70, "header");
|
||||
return {
|
||||
format: "odc",
|
||||
identity: `${field(rest, 0, 6, 8)}:${field(rest, 6, 6, 8)}`,
|
||||
mode: field(rest, 12, 6, 8),
|
||||
links: field(rest, 30, 6, 8),
|
||||
nameSize: field(rest, 53, 6, 8),
|
||||
fileSize: field(rest, 59, 11, 8),
|
||||
check: 0,
|
||||
};
|
||||
}
|
||||
if (magic === "070701" || magic === "070702") {
|
||||
const rest = await source.read(104, "header");
|
||||
return {
|
||||
format: magic === "070702" ? "crc" : "newc",
|
||||
identity: `${field(rest, 56, 8, 16)}:${field(rest, 64, 8, 16)}:${field(rest, 0, 8, 16)}`,
|
||||
mode: field(rest, 8, 8, 16),
|
||||
links: field(rest, 32, 8, 16),
|
||||
fileSize: field(rest, 48, 8, 16),
|
||||
nameSize: field(rest, 88, 8, 16),
|
||||
check: field(rest, 96, 8, 16),
|
||||
};
|
||||
}
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member has an unsupported header; only odc and newc archives are expanded",
|
||||
);
|
||||
};
|
||||
|
||||
const padding = (format: CpioHeader["format"], length: number): number =>
|
||||
format === "odc" ? 0 : (4 - (length % 4)) % 4;
|
||||
|
||||
/** Normalize a member name; reject absolute and traversing names. */
|
||||
const memberPath = (raw: string): string | undefined => {
|
||||
const segments = raw
|
||||
.split("/")
|
||||
.filter((segment) => segment !== "" && segment !== ".");
|
||||
if (raw.startsWith("/") || segments.includes(".."))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`cpio member escapes its archive: ${raw}`,
|
||||
);
|
||||
return segments.length === 0 ? undefined : segments.join("/");
|
||||
};
|
||||
|
||||
/** Project one cpio member as an archive-neutral entry. */
|
||||
const entryOf = (member: {
|
||||
readonly path: string;
|
||||
readonly kind: ArtifactEntry["kind"];
|
||||
readonly key: string;
|
||||
readonly header: CpioHeader;
|
||||
readonly limitations: readonly string[];
|
||||
readonly size?: number | null;
|
||||
readonly contentUnavailable?: boolean;
|
||||
readonly integrityMismatched?: boolean;
|
||||
}): ArtifactEntry => ({
|
||||
path: member.path,
|
||||
kind: member.kind,
|
||||
declaredSize:
|
||||
member.size !== undefined
|
||||
? member.size
|
||||
: member.kind === "file"
|
||||
? member.header.fileSize
|
||||
: null,
|
||||
compressedSize: null,
|
||||
executable: (member.header.mode & 0o111) !== 0,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: member.limitations,
|
||||
adapterKey: member.key,
|
||||
...(member.contentUnavailable === true ? { contentUnavailable: true } : {}),
|
||||
...(member.integrityMismatched === true ? { integrityMismatched: true } : {}),
|
||||
});
|
||||
|
||||
interface PendingLink {
|
||||
readonly path: string;
|
||||
readonly key: string;
|
||||
readonly header: CpioHeader;
|
||||
}
|
||||
|
||||
const drain = async (chunks: AsyncIterable<Buffer>): Promise<void> => {
|
||||
for await (const chunk of chunks) void chunk;
|
||||
};
|
||||
|
||||
const crcObservation = (
|
||||
declared: number,
|
||||
observed: number,
|
||||
): ArtifactChecksumObservation => ({
|
||||
representation: "decoded",
|
||||
algorithm: "cpio-byte-sum",
|
||||
declared: declared.toString(16).padStart(8, "0"),
|
||||
observed: observed.toString(16).padStart(8, "0"),
|
||||
});
|
||||
|
||||
/** Same sentence the scanner uses for a streamed regular-file CRC contradiction. */
|
||||
const crcMismatchDetail = (observation: ArtifactChecksumObservation): string =>
|
||||
`Declared ${observation.representation} ${observation.algorithm} ${observation.declared} disagrees with observed ${observation.observed}.`;
|
||||
|
||||
/** Human-readable limitation for a collected (or skipped) symlink target. */
|
||||
const symlinkTargetLimitation = (
|
||||
header: CpioHeader,
|
||||
target: Buffer | undefined,
|
||||
): string => {
|
||||
if (target === undefined)
|
||||
return `Symlink target of ${header.fileSize} bytes exceeds ${MAX_SYMLINK_TARGET_BYTES} bytes and was not read.`;
|
||||
try {
|
||||
return `Symlink target: ${new TextDecoder("utf-8", { fatal: true }).decode(target)}`;
|
||||
} catch {
|
||||
return "Symlink target is not valid UTF-8 and was not decoded; the archived bytes are preserved only as a byte count.";
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Sequential reader for a gzip-compressed cpio archive, such as an installer
|
||||
* package's Scripts or Payload. Members are decompressed once, in order: open
|
||||
* an entry before advancing to the next one. `070702` data is checked against
|
||||
* its CRC. Hard links whose bytes are stored on another member are served
|
||||
* from that member's bytes; device nodes, FIFOs, and sockets are skipped.
|
||||
*/
|
||||
export class CpioArtifactReader implements ArtifactReader {
|
||||
readonly format = "file" as const;
|
||||
#source: ByteSource | undefined;
|
||||
#signal: AbortSignal | undefined;
|
||||
#current:
|
||||
| {
|
||||
readonly key: string;
|
||||
readonly path: string;
|
||||
readonly header: CpioHeader;
|
||||
}
|
||||
| undefined;
|
||||
#consumed = false;
|
||||
readonly #links = new Map<string, Buffer>();
|
||||
readonly #pending = new Map<string, PendingLink[]>();
|
||||
readonly #aliases = new Map<string, Buffer>();
|
||||
readonly #integrity = new Map<
|
||||
string,
|
||||
readonly ArtifactChecksumObservation[]
|
||||
>();
|
||||
readonly #linkIntegrity = new Map<
|
||||
string,
|
||||
readonly ArtifactChecksumObservation[]
|
||||
>();
|
||||
#buffered = 0;
|
||||
|
||||
constructor(
|
||||
private readonly openCompressed: (
|
||||
signal?: AbortSignal,
|
||||
) => Promise<Readable>,
|
||||
/**
|
||||
* Integrity mode for member CRC failures. Every member representation is
|
||||
* checked before the next header is read. `fail` throws. Under
|
||||
* record-and-continue the disagreement is returned as checksum evidence
|
||||
* so the occurrence and later siblings are still inventoried.
|
||||
*/
|
||||
private readonly integrity: "fail" | "record-and-continue" = "fail",
|
||||
readonly decodedBudget: ArtifactDecodedBudget = new ArtifactDecodedBudget(),
|
||||
) {}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
this.#signal = signal;
|
||||
const compressed = await this.openCompressed(signal);
|
||||
const source = new ByteSource(
|
||||
artifactStreamPipeline(
|
||||
compressed,
|
||||
[
|
||||
createGunzip(),
|
||||
new ArtifactBudgetTransform(this.decodedBudget, "gzip-cpio", signal),
|
||||
],
|
||||
{ path: "gzip-cpio", signal },
|
||||
),
|
||||
);
|
||||
this.#source = source;
|
||||
for (let index = 0; ; index++) {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"cpio expansion was cancelled",
|
||||
);
|
||||
yield* this.#finishCurrent(source);
|
||||
const header = await readHeader(source);
|
||||
if (header.nameSize < 1 || header.nameSize > MAX_NAME_BYTES)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member name size is invalid",
|
||||
);
|
||||
const nameBytes = await source.read(header.nameSize, "member name");
|
||||
await source.padding(
|
||||
padding(header.format, 110 + header.nameSize),
|
||||
"name padding",
|
||||
);
|
||||
// cpio names require exactly one terminal NUL: a missing terminator or
|
||||
// an embedded NUL with trailing bytes is malformed. Decode without
|
||||
// replacement so distinct byte names cannot collapse to one path.
|
||||
if (
|
||||
nameBytes.length === 0 ||
|
||||
nameBytes[nameBytes.length - 1] !== 0 ||
|
||||
nameBytes.indexOf(0) !== nameBytes.length - 1
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member name is not NUL-terminated",
|
||||
);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = new TextDecoder("utf-8", { fatal: true }).decode(
|
||||
nameBytes.subarray(0, nameBytes.length - 1),
|
||||
);
|
||||
} catch {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member name is not valid UTF-8",
|
||||
);
|
||||
}
|
||||
this.decodedBudget.consumeEntry(raw);
|
||||
if (raw === TRAILER) {
|
||||
if (header.fileSize !== 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio trailer declares member data",
|
||||
);
|
||||
await drain(this.#verified(source, header, raw, undefined, "fail"));
|
||||
await source.finish();
|
||||
yield* this.#unresolvedLinks();
|
||||
return;
|
||||
}
|
||||
yield* this.#member(source, header, raw, `${index}:${raw}`);
|
||||
}
|
||||
}
|
||||
|
||||
async *#member(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
raw: string,
|
||||
key: string,
|
||||
): AsyncGenerator<ArtifactEntry> {
|
||||
const type = header.mode & S_IFMT;
|
||||
const path = memberPath(raw);
|
||||
if (type === S_IFDIR) {
|
||||
const consumed = await this.#consume(source, header, raw, false);
|
||||
if (path !== undefined)
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "directory",
|
||||
key,
|
||||
header,
|
||||
limitations:
|
||||
consumed.mismatch === undefined
|
||||
? []
|
||||
: [crcMismatchDetail(consumed.mismatch)],
|
||||
integrityMismatched: consumed.mismatch !== undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (type === S_IFLNK) {
|
||||
yield* this.#symlink(source, { header, raw, key }, path);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
path !== undefined &&
|
||||
type === S_IFREG &&
|
||||
header.links > 1 &&
|
||||
header.fileSize === 0
|
||||
) {
|
||||
// Zero-size headers carry no bytes, so a CRC archive must declare zero.
|
||||
// Under record-and-continue this is the occurrence's own forgotten
|
||||
// bytes: yield it unavailable so later siblings still inventory.
|
||||
const consumed = await this.#consume(source, header, raw, false);
|
||||
if (consumed.mismatch !== undefined) {
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: null,
|
||||
limitations: [crcMismatchDetail(consumed.mismatch)],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const stored = this.#links.get(header.identity);
|
||||
if (stored !== undefined) {
|
||||
yield this.#alias(path, key, header, stored);
|
||||
return;
|
||||
}
|
||||
// newc stores a hard link's bytes on its last member; wait for them.
|
||||
// An all-empty group is resolved to empty files at TRAILER!!!.
|
||||
const waiting = this.#pending.get(header.identity) ?? [];
|
||||
waiting.push({ path, key, header });
|
||||
this.#pending.set(header.identity, waiting);
|
||||
return;
|
||||
}
|
||||
this.#current = { key, path: raw, header };
|
||||
this.#consumed = false;
|
||||
if (path === undefined) return;
|
||||
if (type !== S_IFREG) {
|
||||
// FIFOs, device nodes, sockets and other types are not expanded; keep
|
||||
// an explicit unavailable occurrence instead of dropping the path.
|
||||
const consumed = await this.#consume(source, header, raw, false);
|
||||
this.#current = undefined;
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: null,
|
||||
limitations: [
|
||||
`Unsupported cpio member type ${type.toString(8)}; content not expanded.`,
|
||||
...(consumed.mismatch === undefined
|
||||
? []
|
||||
: [crcMismatchDetail(consumed.mismatch)]),
|
||||
],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
limitations: [],
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One symlink member. A hostile header can declare a multi-gigabyte target,
|
||||
* so oversized targets stream past. The CRC is resolved before the entry is
|
||||
* yielded; under record-and-continue a disagreement stays on the occurrence
|
||||
* so later siblings are still inventoried.
|
||||
*/
|
||||
async *#symlink(
|
||||
source: ByteSource,
|
||||
member: {
|
||||
readonly header: CpioHeader;
|
||||
readonly raw: string;
|
||||
readonly key: string;
|
||||
},
|
||||
path: string | undefined,
|
||||
): AsyncGenerator<ArtifactEntry> {
|
||||
const { header, raw, key } = member;
|
||||
const consumed = await this.#consume(
|
||||
source,
|
||||
header,
|
||||
raw,
|
||||
header.fileSize <= MAX_SYMLINK_TARGET_BYTES,
|
||||
);
|
||||
if (path === undefined) return;
|
||||
if (consumed.mismatch !== undefined) {
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "symlink",
|
||||
key,
|
||||
header,
|
||||
limitations: [
|
||||
crcMismatchDetail(consumed.mismatch),
|
||||
symlinkTargetLimitation(header, consumed.bytes),
|
||||
],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "symlink",
|
||||
key,
|
||||
header,
|
||||
limitations: [symlinkTargetLimitation(header, consumed.bytes)],
|
||||
});
|
||||
}
|
||||
|
||||
#alias(
|
||||
path: string,
|
||||
key: string,
|
||||
header: CpioHeader,
|
||||
bytes: Buffer,
|
||||
): ArtifactEntry {
|
||||
this.#aliases.set(key, bytes);
|
||||
const sourceIntegrity = this.#linkIntegrity.get(header.identity);
|
||||
if (sourceIntegrity !== undefined)
|
||||
this.#integrity.set(key, sourceIntegrity);
|
||||
return entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: bytes.length,
|
||||
limitations: [
|
||||
"Hard link: bytes are stored with another member of this archive.",
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
*#unresolvedLinks(): Generator<ArtifactEntry> {
|
||||
for (const links of this.#pending.values())
|
||||
for (const { path, key, header } of links) {
|
||||
// A complete zero-length group establishes empty content. An orphan
|
||||
// that claims more links than appear is missing its bytes. Pending
|
||||
// groups contain only zero-length members; do not rescan per alias.
|
||||
const complete = links.length >= header.links;
|
||||
if (header.fileSize === 0 && complete) {
|
||||
const empty = Buffer.alloc(0);
|
||||
this.#aliases.set(key, empty);
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: 0,
|
||||
limitations: [
|
||||
"Hard link: every member of this link group is empty.",
|
||||
],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: null,
|
||||
limitations: [UNRESOLVED_LINK],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
}
|
||||
this.#pending.clear();
|
||||
}
|
||||
|
||||
/** Finish the current member, then serve hard links that were waiting for it. */
|
||||
async *#finishCurrent(source: ByteSource): AsyncGenerator<ArtifactEntry> {
|
||||
const current = this.#current;
|
||||
if (current === undefined) return;
|
||||
this.#current = undefined;
|
||||
if (!this.#consumed) {
|
||||
if (this.#shouldKeep(current.header))
|
||||
this.#remember(
|
||||
current.header,
|
||||
await this.#collect(source, current.header, current.path),
|
||||
);
|
||||
else
|
||||
await drain(
|
||||
this.#verified(
|
||||
source,
|
||||
current.header,
|
||||
current.path,
|
||||
undefined,
|
||||
"fail",
|
||||
),
|
||||
);
|
||||
}
|
||||
const waiting = this.#pending.get(current.header.identity);
|
||||
const stored = this.#links.get(current.header.identity);
|
||||
if (waiting === undefined || stored === undefined) return;
|
||||
this.#pending.delete(current.header.identity);
|
||||
for (const { path, key, header } of waiting)
|
||||
yield this.#alias(path, key, header, stored);
|
||||
}
|
||||
|
||||
#shouldKeep(header: CpioHeader): boolean {
|
||||
return (
|
||||
(header.mode & S_IFMT) === S_IFREG &&
|
||||
header.links > 1 &&
|
||||
header.fileSize > 0 &&
|
||||
header.fileSize <= MAX_LINK_BUFFER_BYTES &&
|
||||
this.#buffered + header.fileSize <= MAX_LINK_BUFFER_TOTAL
|
||||
);
|
||||
}
|
||||
|
||||
#remember(
|
||||
header: CpioHeader,
|
||||
bytes: Buffer,
|
||||
observations: readonly ArtifactChecksumObservation[] = [],
|
||||
): void {
|
||||
if (this.#links.has(header.identity)) return;
|
||||
this.#links.set(header.identity, bytes);
|
||||
if (observations.length > 0)
|
||||
this.#linkIntegrity.set(header.identity, observations);
|
||||
this.#buffered += bytes.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Member bytes and padding. `sum` receives the unsigned 32-bit total used
|
||||
* by `070702`. Padding is consumed before the caller reports a mismatch, so
|
||||
* a recovered archive stays aligned on the next header.
|
||||
*/
|
||||
async *#payload(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
sum: { value: number },
|
||||
): AsyncGenerator<Buffer> {
|
||||
this.#cancelled();
|
||||
for await (const chunk of source.take(header.fileSize, "member data")) {
|
||||
this.#cancelled();
|
||||
if (header.format === "crc")
|
||||
for (const byte of chunk) sum.value = (sum.value + byte) >>> 0;
|
||||
yield chunk;
|
||||
}
|
||||
await source.padding(
|
||||
padding(header.format, header.fileSize),
|
||||
"data padding",
|
||||
);
|
||||
this.#cancelled();
|
||||
}
|
||||
|
||||
/**
|
||||
* One CRC check for every member kind. `fail` throws with the observation
|
||||
* in the message. record-and-continue returns the observation so callers
|
||||
* keep declared and observed values instead of catching a string.
|
||||
*/
|
||||
#crcMismatch(
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
sum: number,
|
||||
policy: "fail" | "record-and-continue" = this.integrity,
|
||||
): ArtifactChecksumObservation | undefined {
|
||||
if (header.format !== "crc" || sum === header.check) return undefined;
|
||||
const observation = crcObservation(header.check, sum);
|
||||
if (policy === "fail")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`cpio CRC disagrees with content: ${path} (${crcMismatchDetail(observation)})`,
|
||||
);
|
||||
return observation;
|
||||
}
|
||||
|
||||
/** Read one member fully. `collect` retains bytes for symlinks and hard links. */
|
||||
async #consume(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
collect: boolean,
|
||||
policy?: "fail" | "record-and-continue",
|
||||
): Promise<{
|
||||
readonly bytes: Buffer | undefined;
|
||||
readonly mismatch: ArtifactChecksumObservation | undefined;
|
||||
}> {
|
||||
const sum = { value: 0 };
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of this.#payload(source, header, sum))
|
||||
if (collect) chunks.push(chunk);
|
||||
return {
|
||||
bytes: collect ? Buffer.concat(chunks) : undefined,
|
||||
mismatch: this.#crcMismatch(header, path, sum.value, policy),
|
||||
};
|
||||
}
|
||||
|
||||
/** Member data and its padding, checked against a `070702` CRC. */
|
||||
async *#verified(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
record?: (observation: ArtifactChecksumObservation) => void,
|
||||
policy?: "fail" | "record-and-continue",
|
||||
): AsyncGenerator<Buffer> {
|
||||
const sum = { value: 0 };
|
||||
yield* this.#payload(source, header, sum);
|
||||
const mismatch = this.#crcMismatch(
|
||||
header,
|
||||
path,
|
||||
sum.value,
|
||||
policy ?? this.integrity,
|
||||
);
|
||||
if (mismatch !== undefined) record?.(mismatch);
|
||||
}
|
||||
|
||||
async #collect(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
): Promise<Buffer> {
|
||||
// Unopened linked bytes are not a recovered occurrence. A disagreement
|
||||
// still fails the archive instead of being remembered as verified content.
|
||||
const consumed = await this.#consume(source, header, path, true, "fail");
|
||||
return consumed.bytes ?? Buffer.alloc(0);
|
||||
}
|
||||
|
||||
#cancelled(): void {
|
||||
if (this.#signal?.aborted)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"cpio expansion was cancelled",
|
||||
);
|
||||
}
|
||||
|
||||
open(entry: ArtifactEntry): Promise<Readable> {
|
||||
const alias = this.#aliases.get(entry.adapterKey);
|
||||
if (alias !== undefined) {
|
||||
this.decodedBudget.consume(alias.length, entry.path);
|
||||
return Promise.resolve(Readable.from([alias]));
|
||||
}
|
||||
const source = this.#source;
|
||||
const current = this.#current;
|
||||
if (
|
||||
source === undefined ||
|
||||
current === undefined ||
|
||||
current.key !== entry.adapterKey ||
|
||||
this.#consumed
|
||||
)
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
entry.contentUnavailable === true
|
||||
? UNRESOLVED_LINK
|
||||
: "cpio members can be read only once, in archive order",
|
||||
),
|
||||
);
|
||||
this.#consumed = true;
|
||||
const chunks = this.#verified(
|
||||
source,
|
||||
current.header,
|
||||
current.path,
|
||||
this.integrity === "record-and-continue"
|
||||
? (observed) => this.#integrity.set(entry.adapterKey, [observed])
|
||||
: undefined,
|
||||
);
|
||||
if (!this.#shouldKeep(current.header))
|
||||
return Promise.resolve(Readable.from(chunks));
|
||||
// Keep a linked member's bytes for paths that share them.
|
||||
const kept: Buffer[] = [];
|
||||
const remember = (): void => {
|
||||
this.#remember(
|
||||
current.header,
|
||||
Buffer.concat(kept),
|
||||
this.#integrity.get(entry.adapterKey),
|
||||
);
|
||||
};
|
||||
return Promise.resolve(
|
||||
Readable.from(
|
||||
(async function* () {
|
||||
for await (const chunk of chunks) {
|
||||
kept.push(chunk);
|
||||
yield chunk;
|
||||
}
|
||||
remember();
|
||||
})(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
provenance(): readonly ArtifactCommand[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Completed member CRC evidence, including inherited hard-link source contradictions. */
|
||||
integrityObservations(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[] {
|
||||
return this.#integrity.get(entry.adapterKey) ?? [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.#source?.close();
|
||||
this.#source = undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/** xar archives and cpio payloads built field by field for installer-package tests. */
|
||||
import { createHash } from "node:crypto";
|
||||
import { deflateSync, gzipSync, inflateSync } from "node:zlib";
|
||||
|
||||
export interface XarFixtureMember {
|
||||
readonly name: string;
|
||||
readonly type?: "file" | "directory" | "symlink";
|
||||
readonly data?: Uint8Array;
|
||||
readonly encoding?: "raw" | "zlib" | "bzip2";
|
||||
readonly link?: string;
|
||||
readonly mode?: string;
|
||||
/** Replace the declared extracted checksum, to model tampering. */
|
||||
readonly extractedSha1?: string;
|
||||
/** Replace the declared stored checksum while retaining decodable content. */
|
||||
readonly archivedSha1?: string;
|
||||
/** Store these bytes instead of encoding `data`, to model corrupt members. */
|
||||
readonly archived?: Uint8Array;
|
||||
readonly children?: readonly XarFixtureMember[];
|
||||
}
|
||||
|
||||
const sha1 = (bytes: Uint8Array): string =>
|
||||
createHash("sha1").update(bytes).digest("hex");
|
||||
|
||||
const escape = (text: string): string =>
|
||||
text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
||||
|
||||
/** Build a xar archive with a SHA-1 TOC checksum and per-member checksums. */
|
||||
export const xarArchive = (
|
||||
members: readonly XarFixtureMember[],
|
||||
options: {
|
||||
readonly corruptTocChecksum?: boolean;
|
||||
/** Declared TOC checksum size; the real digest is 20 bytes. */
|
||||
readonly tocChecksumSize?: number;
|
||||
/** Override the fixed-header decoded length without altering the TOC. */
|
||||
readonly tocDecodedSize?: number;
|
||||
} = {},
|
||||
): Uint8Array => {
|
||||
const heap: Uint8Array[] = [new Uint8Array(20)];
|
||||
let heapLength = 20;
|
||||
let nextId = 1;
|
||||
const fileXml = (member: XarFixtureMember): string => {
|
||||
const id = nextId++;
|
||||
const type = member.type ?? "file";
|
||||
let data = "";
|
||||
if (type === "file" && member.data !== undefined) {
|
||||
const archived =
|
||||
member.archived ??
|
||||
(member.encoding === "zlib" ? deflateSync(member.data) : member.data);
|
||||
const offset = heapLength;
|
||||
heap.push(archived);
|
||||
heapLength += archived.length;
|
||||
const style =
|
||||
member.encoding === "zlib"
|
||||
? "application/x-gzip"
|
||||
: member.encoding === "bzip2"
|
||||
? "application/x-bzip2"
|
||||
: "application/octet-stream";
|
||||
data = `<data><archived-checksum style="sha1">${member.archivedSha1 ?? sha1(archived)}</archived-checksum><extracted-checksum style="sha1">${member.extractedSha1 ?? sha1(member.data)}</extracted-checksum><encoding style="${style}"/><size>${member.data.length}</size><offset>${offset}</offset><length>${archived.length}</length></data>`;
|
||||
}
|
||||
const link =
|
||||
type === "symlink"
|
||||
? `<link type="file">${escape(member.link ?? "")}</link>`
|
||||
: "";
|
||||
const children = (member.children ?? []).map(fileXml).join("");
|
||||
return `<file id="${id}"><name>${escape(member.name)}</name><type>${type}</type><mode>${member.mode ?? (type === "directory" ? "0755" : "0644")}</mode>${data}${link}${children}</file>`;
|
||||
};
|
||||
const files = members.map(fileXml).join("");
|
||||
const toc = Buffer.from(
|
||||
`<?xml version="1.0" encoding="UTF-8"?><xar><toc><checksum style="sha1"><size>${options.tocChecksumSize ?? 20}</size><offset>0</offset></checksum>${files}</toc></xar>`,
|
||||
);
|
||||
const compressed = deflateSync(toc);
|
||||
const checksum = createHash("sha1").update(compressed).digest();
|
||||
if (options.corruptTocChecksum === true)
|
||||
checksum[0] = (checksum[0] ?? 0) ^ 0xff;
|
||||
heap[0] = checksum;
|
||||
const header = Buffer.alloc(28);
|
||||
header.writeUInt32BE(0x78617221, 0);
|
||||
header.writeUInt16BE(28, 4);
|
||||
header.writeUInt16BE(1, 6);
|
||||
header.writeBigUInt64BE(BigInt(compressed.length), 8);
|
||||
header.writeBigUInt64BE(BigInt(options.tocDecodedSize ?? toc.length), 16);
|
||||
header.writeUInt32BE(1, 24);
|
||||
return Buffer.concat([header, compressed, ...heap]);
|
||||
};
|
||||
|
||||
export interface CpioFixtureMember {
|
||||
readonly name: string;
|
||||
readonly mode: number;
|
||||
readonly data?: Uint8Array | string;
|
||||
readonly ino?: number;
|
||||
readonly links?: number;
|
||||
/** Override a crc archive's c_check, to model corruption. */
|
||||
readonly check?: number;
|
||||
}
|
||||
|
||||
const octal = (value: number, width: number): string =>
|
||||
value.toString(8).padStart(width, "0");
|
||||
const hex = (value: number): string => value.toString(16).padStart(8, "0");
|
||||
|
||||
/** Build a gzip-compressed odc or newc cpio archive ending in its trailer. */
|
||||
export const gzipCpio = (
|
||||
members: readonly CpioFixtureMember[],
|
||||
format: "odc" | "newc" | "crc" = "odc",
|
||||
): Uint8Array => {
|
||||
const parts: Buffer[] = [];
|
||||
const pad = (length: number): void => {
|
||||
if (format !== "odc" && length % 4 !== 0)
|
||||
parts.push(Buffer.alloc(4 - (length % 4)));
|
||||
};
|
||||
for (const member of [...members, { name: "TRAILER!!!", mode: 0 }]) {
|
||||
const data =
|
||||
typeof member.data === "string"
|
||||
? Buffer.from(member.data)
|
||||
: Buffer.from(member.data ?? new Uint8Array());
|
||||
const name = Buffer.from(`${member.name}\0`);
|
||||
const ino = "ino" in member ? (member.ino ?? 1) : 1;
|
||||
const links = "links" in member ? (member.links ?? 1) : 1;
|
||||
const sum = [...data].reduce((total, byte) => (total + byte) >>> 0, 0);
|
||||
const check = "check" in member ? (member.check ?? sum) : sum;
|
||||
if (format === "odc")
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`070707${octal(0, 6)}${octal(ino, 6)}${octal(member.mode, 6)}${octal(0, 6)}${octal(0, 6)}${octal(links, 6)}${octal(0, 6)}${octal(0, 11)}${octal(name.length, 6)}${octal(data.length, 11)}`,
|
||||
),
|
||||
name,
|
||||
data,
|
||||
);
|
||||
else {
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`${format === "crc" ? "070702" : "070701"}${hex(ino)}${hex(member.mode)}${hex(0)}${hex(0)}${hex(links)}${hex(0)}${hex(data.length)}${hex(0)}${hex(0)}${hex(0)}${hex(0)}${hex(name.length)}${hex(format === "crc" ? check : 0)}`,
|
||||
),
|
||||
name,
|
||||
);
|
||||
pad(110 + name.length);
|
||||
parts.push(data);
|
||||
pad(data.length);
|
||||
}
|
||||
}
|
||||
return gzipSync(Buffer.concat(parts));
|
||||
};
|
||||
|
||||
export const MODE = {
|
||||
directory: 0o040755,
|
||||
file: 0o100644,
|
||||
executable: 0o100755,
|
||||
symlink: 0o120755,
|
||||
fifo: 0o010644,
|
||||
} as const;
|
||||
|
||||
/** Mutate a fixture's XML while keeping its compressed TOC checksum/lengths consistent. */
|
||||
export const rewriteXarToc = (
|
||||
archive: Uint8Array,
|
||||
rewrite: (xml: string) => string,
|
||||
): Uint8Array => {
|
||||
const input = Buffer.from(archive);
|
||||
const oldLength = Number(input.readBigUInt64BE(8));
|
||||
const decoded = inflateSync(input.subarray(28, 28 + oldLength));
|
||||
const xml = Buffer.from(rewrite(decoded.toString("utf8")));
|
||||
const compressed = deflateSync(xml);
|
||||
const header = Buffer.from(input.subarray(0, 28));
|
||||
header.writeBigUInt64BE(BigInt(compressed.length), 8);
|
||||
header.writeBigUInt64BE(BigInt(xml.length), 16);
|
||||
const heap = Buffer.from(input.subarray(28 + oldLength));
|
||||
createHash("sha1").update(compressed).digest().copy(heap, 0);
|
||||
return Buffer.concat([header, compressed, heap]);
|
||||
};
|
||||
@@ -0,0 +1,604 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { Readable } from "node:stream";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { ArtifactDecodedBudget } from "./ArtifactDecodedBudget.js";
|
||||
import { CpioArtifactReader } from "./CpioArtifactReader.js";
|
||||
import {
|
||||
MODE,
|
||||
gzipCpio,
|
||||
xarArchive,
|
||||
rewriteXarToc,
|
||||
type XarFixtureMember,
|
||||
} from "./InstallerPackage.fixture.js";
|
||||
import { XarArtifactReader } from "./XarArtifactReader.js";
|
||||
import { scanReader } from "./inventory/scanReader.js";
|
||||
import { visitArtifactTree } from "./ArtifactTraversal.js";
|
||||
|
||||
const withReader = async (
|
||||
bytes: Uint8Array,
|
||||
run: (reader: XarArtifactReader) => Promise<void>,
|
||||
decodedBudget?: ArtifactDecodedBudget,
|
||||
) => {
|
||||
const root = await createTestTempDirectory("rea-pkg-limits-");
|
||||
const path = join(root, "fixture.pkg");
|
||||
await writeFile(path, bytes);
|
||||
const reader = new XarArtifactReader(
|
||||
path,
|
||||
decodedBudget === undefined ? {} : { decodedBudget },
|
||||
);
|
||||
try {
|
||||
await run(reader);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
};
|
||||
|
||||
it("shares an inflation budget across separate zlib members", async () => {
|
||||
const bytes = xarArchive([
|
||||
{ name: "a", data: Buffer.alloc(40, 65), encoding: "zlib" },
|
||||
{ name: "b", data: Buffer.alloc(40, 66), encoding: "zlib" },
|
||||
]);
|
||||
await withReader(
|
||||
bytes,
|
||||
async (reader) => {
|
||||
let read = 0;
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) {
|
||||
await buffer(await reader.open(entry));
|
||||
read++;
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Decoded archive budget"),
|
||||
});
|
||||
expect(read).toBe(1);
|
||||
},
|
||||
new ArtifactDecodedBudget(64),
|
||||
);
|
||||
});
|
||||
|
||||
it("charges raw member reads to the same cumulative work budget", async () => {
|
||||
await withReader(
|
||||
xarArchive([
|
||||
{ name: "a", data: Buffer.alloc(40, 65) },
|
||||
{ name: "b", data: Buffer.alloc(40, 66) },
|
||||
]),
|
||||
async (reader) => {
|
||||
let read = 0;
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) {
|
||||
await buffer(await reader.open(entry));
|
||||
read++;
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Decoded archive budget"),
|
||||
});
|
||||
expect(read).toBe(1);
|
||||
},
|
||||
new ArtifactDecodedBudget(64),
|
||||
);
|
||||
});
|
||||
|
||||
it("validates unsafe extents even when a member is unsupported and would not be opened", async () => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", encoding: "bzip2", data: Buffer.from("opaque") }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<offset>20<\/offset>/u,
|
||||
`<offset>${Number.MAX_SAFE_INTEGER}</offset>`,
|
||||
),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("heap extent"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["", " ", "1e2", "0x10"])(
|
||||
"rejects nondecimal/empty numeric TOC data: %j",
|
||||
async (value) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("bytes") }]),
|
||||
(xml) =>
|
||||
xml.replace(/<length>\d+<\/length>/u, `<length>${value}</length>`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar TOC has an invalid data length",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects unsafe absolute heap ranges with a tagged format failure", async () => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "Payload", data: Buffer.from("gzip") }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<offset>20<\/offset>/u,
|
||||
`<offset>${Number.MAX_SAFE_INTEGER}</offset>`,
|
||||
),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("heap extent"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds retained full paths from deeply nested TOCs", async () => {
|
||||
let member: XarFixtureMember = { name: "leaf", data: Buffer.from("x") };
|
||||
for (let depth = 0; depth < 300; depth++)
|
||||
member = {
|
||||
name: "directory-segment",
|
||||
type: "directory",
|
||||
children: [member],
|
||||
};
|
||||
await withReader(xarArchive([member]), async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("path text"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("observes cancellation during internal symlink drains", async () => {
|
||||
const controller = new AbortController();
|
||||
const archive = gzipCpio([
|
||||
{
|
||||
name: "huge-link",
|
||||
mode: MODE.symlink,
|
||||
data: "x".repeat(4 * 1024 * 1024),
|
||||
},
|
||||
]);
|
||||
let decoded = 0;
|
||||
const budget = new (class extends ArtifactDecodedBudget {
|
||||
override consume(bytes: number, path: string): void {
|
||||
super.consume(bytes, path);
|
||||
decoded += bytes;
|
||||
if (decoded >= 64 * 1024) controller.abort();
|
||||
}
|
||||
})();
|
||||
const reader = new CpioArtifactReader(
|
||||
async () => Readable.from([archive]),
|
||||
"fail",
|
||||
budget,
|
||||
);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries(controller.signal)) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("verifies gzip integrity even when the cpio trailer has already been decoded", async () => {
|
||||
const bytes = Buffer.from(
|
||||
gzipCpio([{ name: "a", mode: MODE.file, data: "ok" }]),
|
||||
);
|
||||
bytes[bytes.length - 8] = (bytes[bytes.length - 8] ?? 0) ^ 1;
|
||||
const reader = new CpioArtifactReader(async () => Readable.from([bytes]));
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "format" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["0755junk", "-1", "", "08"])(
|
||||
"rejects partially parsed or malformed mode %j",
|
||||
async (mode) => {
|
||||
await withReader(
|
||||
xarArchive([{ name: "a", mode, data: Buffer.from("x") }]),
|
||||
async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar TOC has an invalid mode",
|
||||
});
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves CRC contradictions for special nodes and reaches later siblings", async () => {
|
||||
const bytes = gzipCpio(
|
||||
[
|
||||
{ name: "fifo", mode: MODE.fifo, data: "abc", check: 1 },
|
||||
{ name: "good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
);
|
||||
const reader = new CpioArtifactReader(
|
||||
async () => Readable.from([bytes]),
|
||||
"record-and-continue",
|
||||
);
|
||||
try {
|
||||
const paths: string[] = [];
|
||||
for await (const entry of reader.entries()) {
|
||||
paths.push(entry.path);
|
||||
if (entry.path === "fifo")
|
||||
expect(entry.limitations).toContain(
|
||||
"Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000126.",
|
||||
);
|
||||
if (entry.path === "good")
|
||||
expect((await buffer(await reader.open(entry))).toString()).toBe("ok");
|
||||
}
|
||||
expect(paths).toEqual(["fifo", "good"]);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose a partial inventory when a previous TOC load failed", async () => {
|
||||
await withReader(
|
||||
xarArchive([{ name: "a", mode: "broken", data: Buffer.from("x") }]),
|
||||
async (reader) => {
|
||||
const scan = async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
};
|
||||
await expect(scan()).rejects.toMatchObject({ reason: "format" });
|
||||
await expect(scan()).rejects.toMatchObject({ reason: "format" });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a TOC whose element count would build an unbounded DOM", async () => {
|
||||
const noise = "<n/>".repeat(100_001);
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
(xml) => xml.replace("<toc>", `<toc>${noise}`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: "xar TOC exceeds 100000 XML nodes",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a TOC checksum when the fixed header declares none", async () => {
|
||||
const bytes = Buffer.from(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
);
|
||||
bytes.writeUInt32BE(0, 24);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("declares no TOC checksum"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["", "nothex", "a".repeat(63)])(
|
||||
"rejects malformed checksum text %j before applying integrity policy",
|
||||
async (digest) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<extracted-checksum style="sha1">[^<]*<\/extracted-checksum>/u,
|
||||
`<extracted-checksum style="sha256">${digest}</extracted-checksum>`,
|
||||
),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar extracted-checksum has malformed sha256 digest text",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a CRC trailer whose declared checksum is not empty-data zero", async () => {
|
||||
const raw = gunzipSync(gzipCpio([], "crc"));
|
||||
raw.write("00000001", 102, "ascii");
|
||||
const reader = new CpioArtifactReader(async () =>
|
||||
Readable.from([gzipSync(raw)]),
|
||||
);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
message:
|
||||
"cpio CRC disagrees with content: TRAILER!!! (Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000000.)",
|
||||
});
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds retained metadata independently of small decoded byte counts", async () => {
|
||||
const members = Array.from({ length: 100 }, (_, index) => ({
|
||||
name: `empty-${index}`,
|
||||
mode: MODE.file,
|
||||
}));
|
||||
const reader = new CpioArtifactReader(
|
||||
async () => Readable.from([gzipCpio(members)]),
|
||||
"fail",
|
||||
new ArtifactDecodedBudget(1024 * 1024, 32 * 1024),
|
||||
);
|
||||
let seen = 0;
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) {
|
||||
void entry;
|
||||
seen++;
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Retained archive metadata budget"),
|
||||
});
|
||||
expect(seen).toBeGreaterThan(0);
|
||||
expect(seen).toBeLessThan(members.length);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("recovers directory CRC contradictions before yielding and reaches regular siblings", async () => {
|
||||
const reader = new CpioArtifactReader(
|
||||
async () =>
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[
|
||||
{ name: "dir", mode: MODE.directory, check: 1 },
|
||||
{ name: "dir/good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
]),
|
||||
"record-and-continue",
|
||||
);
|
||||
try {
|
||||
const seen: string[] = [];
|
||||
for await (const entry of reader.entries()) {
|
||||
seen.push(entry.path);
|
||||
if (entry.kind === "directory")
|
||||
expect(entry.limitations).toContain(
|
||||
"Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000000.",
|
||||
);
|
||||
else
|
||||
expect((await buffer(await reader.open(entry))).toString()).toBe("ok");
|
||||
}
|
||||
expect(seen).toEqual(["dir", "dir/good"]);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["directory", "symlink"])(
|
||||
"rejects data declarations attached to non-file type %s",
|
||||
async (type) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("data") }]),
|
||||
(xml) => xml.replace("<type>file</type>", `<type>${type}</type>`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar non-file member declares data: a",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("charges long enclosing prefixes for every retained nested entry", async () => {
|
||||
const files = Array.from({ length: 100 }, (_, index) => ({
|
||||
name: `empty-${index}`,
|
||||
mode: MODE.file,
|
||||
}));
|
||||
let member: XarFixtureMember = { name: "Payload", data: gzipCpio(files) };
|
||||
for (let index = 0; index < 18; index++)
|
||||
member = { name: "d".repeat(200), type: "directory", children: [member] };
|
||||
await withReader(
|
||||
xarArchive([member]),
|
||||
async (reader) => {
|
||||
let retained = 0;
|
||||
await expect(
|
||||
visitArtifactTree(reader, async ({ container }) => {
|
||||
if (!container) retained++;
|
||||
return container;
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Retained archive metadata budget"),
|
||||
});
|
||||
expect(retained).toBeLessThan(files.length + 18);
|
||||
},
|
||||
new ArtifactDecodedBudget(1024 * 1024, 512 * 1024),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["<!--n-->", "<?n value?>", "<![CDATA[n]]>"])(
|
||||
"bounds DOM nodes produced by %s before parsing",
|
||||
async (node) => {
|
||||
const bytes = rewriteXarToc(xarArchive([]), (xml) =>
|
||||
xml.replace("<toc>", `<toc>${node.repeat(100_001)}`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("XML nodes"),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["data", "extracted-checksum", "offset", "encoding", "type"])(
|
||||
"rejects duplicate singleton <%s> declarations",
|
||||
async (tag) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
(xml) => {
|
||||
const pattern = new RegExp(
|
||||
`<${tag}(?:\\s[^>]*|)>(?:[^]*?)</${tag}>|<${tag}[^>]*/>`,
|
||||
);
|
||||
return xml.replace(pattern, (element) => element + element);
|
||||
},
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining(`duplicate <${tag}>`),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects unsupported xar versions before reading the TOC", async () => {
|
||||
const bytes = Buffer.from(xarArchive([]));
|
||||
bytes.writeUInt16BE(2, 6);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("version 2"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("ends record-and-continue traversal when its shared decoded budget is exhausted", async () => {
|
||||
const bytes = xarArchive(
|
||||
Array.from({ length: 10 }, (_, index) => ({
|
||||
name: `file-${index}`,
|
||||
data: Buffer.alloc(40, index),
|
||||
})),
|
||||
);
|
||||
await withReader(
|
||||
bytes,
|
||||
async (reader) => {
|
||||
const opened: string[] = [];
|
||||
const observed = {
|
||||
format: reader.format,
|
||||
provenance: reader.provenance.bind(reader),
|
||||
entries: reader.entries.bind(reader),
|
||||
close: reader.close.bind(reader),
|
||||
decodedBudget: reader.decodedBudget,
|
||||
open: async (...args: Parameters<typeof reader.open>) => {
|
||||
opened.push(args[0].path);
|
||||
return reader.open(...args);
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
scanReader(observed, undefined, { mode: "record-and-continue" }),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Decoded archive budget"),
|
||||
});
|
||||
expect(opened).toEqual(["file-0", "file-1"]);
|
||||
},
|
||||
new ArtifactDecodedBudget(64),
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts productbuild repeated identical names but rejects contradictory names", async () => {
|
||||
const original = xarArchive([{ name: "a", data: Buffer.from("ok") }]);
|
||||
for (const name of ["a", "different"]) {
|
||||
const bytes = rewriteXarToc(original, (xml) =>
|
||||
xml.replace("<name>a</name>", `<name>a</name><name>${name}</name>`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
if (name === "a")
|
||||
expect((await scanReader(reader)).occurrences).toHaveLength(1);
|
||||
else
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("conflicting <name>"),
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("counts a comment as one node even when its text contains markup openers", async () => {
|
||||
const bytes = rewriteXarToc(xarArchive([]), (xml) =>
|
||||
xml.replace("<toc>", `<toc><!--${"<".repeat(100_001)}-->`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
expect((await scanReader(reader)).occurrences).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
it("resolves a large complete empty hard-link group without per-alias rescans", async () => {
|
||||
const count = 20_000;
|
||||
const bytes = gzipCpio(
|
||||
Array.from({ length: count }, (_, index) => ({
|
||||
name: `link-${index}`,
|
||||
mode: MODE.file,
|
||||
data: "",
|
||||
ino: 1,
|
||||
links: count,
|
||||
})),
|
||||
"newc",
|
||||
);
|
||||
const reader = new CpioArtifactReader(async () => Readable.from([bytes]));
|
||||
try {
|
||||
let retained = 0;
|
||||
for await (const entry of reader.entries()) {
|
||||
expect(entry.declaredSize).toBe(0);
|
||||
expect(entry.contentUnavailable).not.toBe(true);
|
||||
retained++;
|
||||
}
|
||||
expect(retained).toBe(count);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,444 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ArtifactEntry } from "./ArtifactReader.js";
|
||||
import { CpioArtifactReader } from "./CpioArtifactReader.js";
|
||||
import { MODE, gzipCpio, xarArchive } from "./InstallerPackage.fixture.js";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { XarArtifactReader } from "./XarArtifactReader.js";
|
||||
|
||||
const writePackage = async (bytes: Uint8Array): Promise<string> => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-reader-");
|
||||
const path = join(directory, "fixture.pkg");
|
||||
await writeFile(path, bytes);
|
||||
return path;
|
||||
};
|
||||
|
||||
const collect = async (reader: {
|
||||
entries(): AsyncIterable<ArtifactEntry>;
|
||||
}): Promise<ArtifactEntry[]> => {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
return entries;
|
||||
};
|
||||
|
||||
describe("xar installer package reader", () => {
|
||||
it("lists nested members and marks gzip-cpio scripts and payloads", async () => {
|
||||
const scripts = gzipCpio([
|
||||
{ name: "./postinstall", mode: MODE.executable, data: "#!/bin/sh\n" },
|
||||
]);
|
||||
const reader = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{
|
||||
name: "Distribution",
|
||||
data: Buffer.from("<installer-gui-script/>"),
|
||||
encoding: "zlib",
|
||||
},
|
||||
{
|
||||
name: "App.pkg",
|
||||
type: "directory",
|
||||
children: [
|
||||
{
|
||||
name: "PackageInfo",
|
||||
data: Buffer.from("<pkg-info/>"),
|
||||
encoding: "zlib",
|
||||
},
|
||||
{ name: "Scripts", data: scripts },
|
||||
{ name: "Payload", data: Buffer.from("pbzx\0\0\0\0\x01\0\0\0") },
|
||||
{ name: "Current", type: "symlink", link: "App.pkg" },
|
||||
],
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const entries = await collect(reader);
|
||||
expect(entries.map(({ path, kind }) => `${kind}:${path}`)).toEqual([
|
||||
"file:Distribution",
|
||||
"directory:App.pkg",
|
||||
"file:App.pkg/PackageInfo",
|
||||
"file:App.pkg/Scripts",
|
||||
"file:App.pkg/Payload",
|
||||
"symlink:App.pkg/Current",
|
||||
]);
|
||||
const scriptsEntry = entries.find(
|
||||
({ path }) => path === "App.pkg/Scripts",
|
||||
);
|
||||
expect(scriptsEntry?.nestedArchive).toBe("gzip-cpio");
|
||||
expect(
|
||||
entries.find(({ path }) => path === "App.pkg/Payload")?.limitations,
|
||||
).toEqual([expect.stringContaining("pbzx")]);
|
||||
const distribution = entries[0];
|
||||
if (distribution === undefined) throw new Error("missing entry");
|
||||
expect((await buffer(await reader.open(distribution))).toString()).toBe(
|
||||
"<installer-gui-script/>",
|
||||
);
|
||||
if (scriptsEntry === undefined) throw new Error("missing scripts");
|
||||
const nested = new CpioArtifactReader(() => reader.open(scriptsEntry));
|
||||
try {
|
||||
const members: string[] = [];
|
||||
for await (const entry of nested.entries()) {
|
||||
members.push(entry.path);
|
||||
expect((await buffer(await nested.open(entry))).toString()).toBe(
|
||||
"#!/bin/sh\n",
|
||||
);
|
||||
expect(entry.executable).toBe(true);
|
||||
}
|
||||
expect(members).toEqual(["postinstall"]);
|
||||
} finally {
|
||||
await nested.close();
|
||||
}
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects tampered members, TOCs and unsupported encodings", async () => {
|
||||
const tampered = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{
|
||||
name: "Bom",
|
||||
data: Buffer.from("bom"),
|
||||
extractedSha1: "0".repeat(40),
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const [entry] = await collect(tampered);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
await expect(buffer(await tampered.open(entry))).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
} finally {
|
||||
await tampered.close();
|
||||
}
|
||||
const corrupt = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([{ name: "Bom", data: Buffer.from("bom") }], {
|
||||
corruptTocChecksum: true,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await expect(collect(corrupt)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
await corrupt.close();
|
||||
const bzip = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{ name: "Bom", data: Buffer.from("bom"), encoding: "bzip2" },
|
||||
]),
|
||||
),
|
||||
);
|
||||
const [entry] = await collect(bzip);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
expect(entry.contentUnavailable).toBe(true);
|
||||
await expect(bzip.open(entry)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
await bzip.close();
|
||||
const notXar = new XarArtifactReader(
|
||||
await writePackage(Buffer.from("not an archive at all")),
|
||||
);
|
||||
await expect(collect(notXar)).rejects.toMatchObject({ reason: "format" });
|
||||
await notXar.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("gzip cpio reader", () => {
|
||||
for (const format of ["odc", "newc"] as const)
|
||||
it(`reads ${format} members in order and skips unopened data`, async () => {
|
||||
const archive = gzipCpio(
|
||||
[
|
||||
{ name: ".", mode: MODE.directory },
|
||||
{ name: "./usr", mode: MODE.directory },
|
||||
{
|
||||
name: "./usr/bin/tool",
|
||||
mode: MODE.executable,
|
||||
data: "skipped bytes",
|
||||
},
|
||||
{ name: "./usr/lib/link", mode: MODE.symlink, data: "../bin/tool" },
|
||||
{ name: "./usr/fifo", mode: MODE.fifo },
|
||||
{ name: "./usr/share/readme.txt", mode: MODE.file, data: "read me" },
|
||||
],
|
||||
format,
|
||||
);
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(Readable.from([archive])),
|
||||
);
|
||||
const seen: string[] = [];
|
||||
try {
|
||||
for await (const entry of reader.entries()) {
|
||||
seen.push(`${entry.kind}:${entry.path}`);
|
||||
if (entry.path === "usr/share/readme.txt") {
|
||||
expect((await buffer(await reader.open(entry))).toString()).toBe(
|
||||
"read me",
|
||||
);
|
||||
await expect(reader.open(entry)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
}
|
||||
if (entry.kind === "symlink")
|
||||
expect(entry.limitations).toEqual(["Symlink target: ../bin/tool"]);
|
||||
}
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
expect(seen).toEqual([
|
||||
"directory:usr",
|
||||
"file:usr/bin/tool",
|
||||
"symlink:usr/lib/link",
|
||||
"file:usr/fifo",
|
||||
"file:usr/share/readme.txt",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects traversal and truncated archives", async () => {
|
||||
for (const name of ["../escape", "/absolute"]) {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([gzipCpio([{ name, mode: MODE.file, data: "x" }])]),
|
||||
),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({ reason: "path" });
|
||||
await reader.close();
|
||||
}
|
||||
const whole = gzipCpio([{ name: "a", mode: MODE.file, data: "abc" }]);
|
||||
const { gunzipSync, gzipSync } = await import("node:zlib");
|
||||
const truncated = gzipSync(gunzipSync(whole).subarray(0, 60));
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(Readable.from([truncated])),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({ reason: "format" });
|
||||
await reader.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("installer package reader hardening", () => {
|
||||
it("refuses a TOC checksum whose declared size is not one digest", async () => {
|
||||
const reader = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([{ name: "Bom", data: Buffer.from("bom") }], {
|
||||
tocChecksumSize: 0x7fffffff,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar TOC declares a 2147483647-byte sha1 checksum",
|
||||
});
|
||||
await reader.close();
|
||||
});
|
||||
|
||||
it("reports corrupt zlib members as malformed and keeps cancellation tagged", async () => {
|
||||
const reader = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{
|
||||
name: "PackageInfo",
|
||||
data: Buffer.from("<pkg-info/>"),
|
||||
encoding: "zlib",
|
||||
archived: Buffer.from("not zlib at all"),
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const [entry] = await collect(reader);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
});
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(reader.open(entry, controller.signal)).rejects.toMatchObject(
|
||||
{
|
||||
reason: "cancelled",
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("cpio symlink targets", () => {
|
||||
it("streams past an oversized symlink target and keeps reading", async () => {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[
|
||||
{ name: "./huge", mode: MODE.symlink, data: "x".repeat(4097) },
|
||||
{ name: "./after", mode: MODE.file, data: "after" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const entries = await collect(reader);
|
||||
expect(
|
||||
entries.map(({ kind, path, limitations }) => [kind, path, limitations]),
|
||||
).toEqual([
|
||||
[
|
||||
"symlink",
|
||||
"huge",
|
||||
["Symlink target of 4097 bytes exceeds 4096 bytes and was not read."],
|
||||
],
|
||||
["file", "after", []],
|
||||
]);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("cpio hard links and CRC archives", () => {
|
||||
const readAll = async (archive: Uint8Array) => {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(Readable.from([archive])),
|
||||
);
|
||||
const files: Record<string, string | undefined> = {};
|
||||
try {
|
||||
for await (const entry of reader.entries())
|
||||
if (entry.kind === "file")
|
||||
files[entry.path] =
|
||||
entry.contentUnavailable === true
|
||||
? undefined
|
||||
: (await buffer(await reader.open(entry))).toString();
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
return files;
|
||||
};
|
||||
|
||||
it("serves newc hard links stored on the last member to every path", async () => {
|
||||
expect(
|
||||
await readAll(
|
||||
gzipCpio(
|
||||
[
|
||||
{ name: "./first", mode: MODE.file, ino: 7, links: 2 },
|
||||
{
|
||||
name: "./second",
|
||||
mode: MODE.file,
|
||||
ino: 7,
|
||||
links: 2,
|
||||
data: "shared",
|
||||
},
|
||||
{
|
||||
name: "./third",
|
||||
mode: MODE.file,
|
||||
ino: 9,
|
||||
links: 2,
|
||||
data: "kept",
|
||||
},
|
||||
{ name: "./fourth", mode: MODE.file, ino: 9, links: 2 },
|
||||
],
|
||||
"newc",
|
||||
),
|
||||
),
|
||||
).toEqual({
|
||||
first: "shared",
|
||||
second: "shared",
|
||||
third: "kept",
|
||||
fourth: "kept",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a hard link whose bytes never appear as unavailable", async () => {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[{ name: "./orphan", mode: MODE.file, ino: 4, links: 2 }],
|
||||
"newc",
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
const entries = await collect(reader);
|
||||
expect(entries).toEqual([
|
||||
expect.objectContaining({
|
||||
path: "orphan",
|
||||
contentUnavailable: true,
|
||||
declaredSize: null,
|
||||
}),
|
||||
]);
|
||||
const orphan = entries[0];
|
||||
if (orphan === undefined) throw new Error("missing entry");
|
||||
await expect(reader.open(orphan)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
await reader.close();
|
||||
});
|
||||
|
||||
it("verifies 070702 member checksums whether or not members are opened", async () => {
|
||||
expect(
|
||||
await readAll(
|
||||
gzipCpio([{ name: "./ok", mode: MODE.file, data: "abc" }], "crc"),
|
||||
),
|
||||
).toEqual({ ok: "abc" });
|
||||
await expect(
|
||||
readAll(
|
||||
gzipCpio(
|
||||
[{ name: "./bad", mode: MODE.file, data: "abc", check: 1 }],
|
||||
"crc",
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ reason: "integrity" });
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[{ name: "./bad", mode: MODE.file, data: "abc", check: 1 }],
|
||||
"crc",
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
await reader.close();
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["newc", "crc"] as const)(
|
||||
"rejects nonzero %s alignment under either integrity policy",
|
||||
async (format) => {
|
||||
for (const policy of ["fail", "record-and-continue"] as const) {
|
||||
for (const [offset, label] of [
|
||||
[113, "name padding"],
|
||||
[117, "data padding"],
|
||||
] as const) {
|
||||
// 110-byte header, a three-byte NUL-terminated name, then one data byte.
|
||||
const decoded = gunzipSync(
|
||||
gzipCpio([{ name: "ab", mode: MODE.file, data: "x" }], format),
|
||||
);
|
||||
decoded[offset] = 1;
|
||||
const reader = new CpioArtifactReader(
|
||||
() => Promise.resolve(Readable.from([gzipSync(decoded)])),
|
||||
policy,
|
||||
);
|
||||
try {
|
||||
await expect(collect(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: `cpio ${label} contains nonzero bytes`,
|
||||
});
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,875 @@
|
||||
import { createHash, type Hash } from "node:crypto";
|
||||
import { open, type FileHandle } from "node:fs/promises";
|
||||
import { Readable, Transform, type TransformCallback } from "node:stream";
|
||||
import { createInflate, inflateSync } from "node:zlib";
|
||||
|
||||
import { DOMParser, type Element, type Node } from "@xmldom/xmldom";
|
||||
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
ArtifactDecodedBudget,
|
||||
ArtifactBudgetTransform,
|
||||
} from "./ArtifactDecodedBudget.js";
|
||||
import { artifactStreamPipeline } from "./ArtifactStreamPipeline.js";
|
||||
import {
|
||||
assertXarTocNodes,
|
||||
xarInteger as integer,
|
||||
xarHeapPosition,
|
||||
XarPathBudget,
|
||||
xarMode,
|
||||
} from "./XarValidation.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactChecksumObservation,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
|
||||
const XAR_MAGIC = 0x78617221;
|
||||
/** A TOC describes members only; real installer TOCs are kilobytes. */
|
||||
const MAX_TOC_BYTES = 64 * 1024 * 1024;
|
||||
const READ_CHUNK_BYTES = 1024 * 1024;
|
||||
|
||||
const CHECKSUM_ALGORITHMS: ReadonlyMap<string, string> = new Map([
|
||||
["sha1", "sha1"],
|
||||
["md5", "md5"],
|
||||
["sha256", "sha256"],
|
||||
["sha512", "sha512"],
|
||||
]);
|
||||
const DIGEST_BYTES: ReadonlyMap<string, number> = new Map([
|
||||
["sha1", 20],
|
||||
["md5", 16],
|
||||
["sha256", 32],
|
||||
["sha512", 64],
|
||||
]);
|
||||
|
||||
const cancelled = (signal?: AbortSignal): void => {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure("cancelled", "PKG traversal was cancelled");
|
||||
};
|
||||
|
||||
/** One TOC `<data>` element: where a member's archived bytes live in the heap. */
|
||||
interface XarData {
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
readonly size: number;
|
||||
readonly encoding: string;
|
||||
readonly extractedChecksum:
|
||||
| { readonly algorithm: string; readonly value: string }
|
||||
| undefined;
|
||||
readonly unsupportedChecksum:
|
||||
| { readonly style: string; readonly value: string }
|
||||
| undefined;
|
||||
readonly archivedChecksum:
|
||||
| { readonly algorithm: string; readonly value: string }
|
||||
| undefined;
|
||||
readonly unsupportedArchivedChecksum:
|
||||
| { readonly style: string; readonly value: string }
|
||||
| undefined;
|
||||
}
|
||||
|
||||
interface XarMember {
|
||||
readonly path: string;
|
||||
readonly kind: "file" | "directory" | "symlink";
|
||||
/** Raw TOC type when it is not file/directory/symlink (FIFO, device, ...). */
|
||||
readonly unsupportedType?: string;
|
||||
readonly mode: number | null;
|
||||
readonly data: XarData | undefined;
|
||||
readonly link: string | undefined;
|
||||
}
|
||||
|
||||
const isElement = (node: Node): node is Element => node.nodeType === 1;
|
||||
|
||||
/** Direct child elements with one tag name, in document order. */
|
||||
const childElements = (element: Element, name: string): Element[] =>
|
||||
Array.from(element.childNodes).filter(
|
||||
(child): child is Element => isElement(child) && child.tagName === name,
|
||||
);
|
||||
|
||||
const childElement = (element: Element, name: string): Element | undefined => {
|
||||
const children = childElements(element, name);
|
||||
if (children.length > 1)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC has duplicate <${name}> declarations in <${element.tagName}>`,
|
||||
);
|
||||
return children[0];
|
||||
};
|
||||
|
||||
// productbuild repeats an identical <name> when flattening a component PKG.
|
||||
// Preserve that observed producer form while rejecting contradictory names.
|
||||
const memberName = (file: Element): string | undefined => {
|
||||
const names = childElements(file, "name").map(
|
||||
(name) => name.textContent ?? "",
|
||||
);
|
||||
if (names.some((name) => name !== names[0]))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar TOC has conflicting <name> declarations",
|
||||
);
|
||||
return names[0];
|
||||
};
|
||||
|
||||
const textOf = (element: Element, name: string): string | undefined =>
|
||||
childElement(element, name)?.textContent ?? undefined;
|
||||
|
||||
const parseChecksum = (
|
||||
parent: Element,
|
||||
tag: string,
|
||||
):
|
||||
| { readonly algorithm: string; readonly value: string }
|
||||
| { readonly unsupported: string; readonly value: string }
|
||||
| undefined => {
|
||||
const element = childElement(parent, tag);
|
||||
if (element === undefined) return undefined;
|
||||
const style = element.getAttribute("style")?.toLowerCase() ?? "";
|
||||
const value = (element.textContent ?? "").trim().toLowerCase();
|
||||
if (CHECKSUM_ALGORITHMS.get(style) === undefined)
|
||||
return { unsupported: style || "(missing style)", value };
|
||||
const expected = DIGEST_BYTES.get(style);
|
||||
if (
|
||||
expected === undefined ||
|
||||
value.length !== expected * 2 ||
|
||||
!/^[0-9a-f]+$/u.test(value)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar ${tag} has malformed ${style} digest text`,
|
||||
);
|
||||
return { algorithm: style, value };
|
||||
};
|
||||
|
||||
const parseData = (file: Element): XarData | undefined => {
|
||||
const data = childElement(file, "data");
|
||||
if (data === undefined) return undefined;
|
||||
const extracted = parseChecksum(data, "extracted-checksum");
|
||||
const archived = parseChecksum(data, "archived-checksum");
|
||||
return {
|
||||
offset: integer(textOf(data, "offset"), "data offset"),
|
||||
length: integer(textOf(data, "length"), "data length"),
|
||||
size: integer(textOf(data, "size"), "data size"),
|
||||
encoding:
|
||||
childElement(data, "encoding")?.getAttribute("style") ??
|
||||
"application/octet-stream",
|
||||
extractedChecksum:
|
||||
extracted === undefined || "unsupported" in extracted
|
||||
? undefined
|
||||
: { algorithm: extracted.algorithm, value: extracted.value },
|
||||
unsupportedChecksum:
|
||||
extracted !== undefined && "unsupported" in extracted
|
||||
? { style: extracted.unsupported, value: extracted.value }
|
||||
: undefined,
|
||||
archivedChecksum:
|
||||
archived === undefined || "unsupported" in archived
|
||||
? undefined
|
||||
: { algorithm: archived.algorithm, value: archived.value },
|
||||
unsupportedArchivedChecksum:
|
||||
archived !== undefined && "unsupported" in archived
|
||||
? { style: archived.unsupported, value: archived.value }
|
||||
: undefined,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Raw members must describe their stored bytes exactly; otherwise the source
|
||||
* range would extend into adjacent heap entries.
|
||||
*/
|
||||
const assertRawSize = (member: XarMember, data: XarData | undefined): void => {
|
||||
if (
|
||||
data !== undefined &&
|
||||
data.encoding === "application/octet-stream" &&
|
||||
data.size !== data.length
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${member.path} declares size ${data.size} with length ${data.length}`,
|
||||
);
|
||||
};
|
||||
|
||||
/** Entry limitations from the TOC record and nested-archive classification. */
|
||||
const memberLimitations = (
|
||||
member: XarMember,
|
||||
unsupportedChecksum: XarData["unsupportedChecksum"],
|
||||
unsupportedArchived: XarData["unsupportedArchivedChecksum"],
|
||||
classified: { readonly limitations: readonly string[] },
|
||||
): string[] => [
|
||||
...(member.kind === "symlink" && member.link !== undefined
|
||||
? [`Symlink target recorded in the TOC: ${member.link}`]
|
||||
: []),
|
||||
...(unsupportedChecksum === undefined
|
||||
? []
|
||||
: [
|
||||
`Member declares unsupported checksum ${unsupportedChecksum.style} ${unsupportedChecksum.value}; integrity not verified.`,
|
||||
]),
|
||||
...(unsupportedArchived === undefined
|
||||
? []
|
||||
: [
|
||||
`Member declares unsupported archived checksum ${unsupportedArchived.style} ${unsupportedArchived.value}; stored-byte integrity not verified.`,
|
||||
]),
|
||||
...(member.data !== undefined &&
|
||||
member.data.encoding !== "application/octet-stream" &&
|
||||
member.data.encoding !== "application/x-gzip"
|
||||
? [
|
||||
`Unsupported xar encoding ${member.data.encoding}; content is unavailable and was not expanded.`,
|
||||
]
|
||||
: []),
|
||||
...classified.limitations,
|
||||
];
|
||||
|
||||
/** File, directory, symlink, or an explicitly unsupported TOC type. */
|
||||
const classifyMemberType = (
|
||||
type: string | undefined,
|
||||
): { readonly kind: XarMember["kind"]; readonly unsupportedType?: string } => {
|
||||
if (type === "directory" || type === "symlink") return { kind: type };
|
||||
if (type === undefined || type === "file") return { kind: "file" };
|
||||
return { kind: "file", unsupportedType: type };
|
||||
};
|
||||
|
||||
/** Walk nested `<file>` elements in document order. */
|
||||
const collectMembers = (
|
||||
toc: Element,
|
||||
budget: ArtifactDecodedBudget,
|
||||
): XarMember[] => {
|
||||
const members: XarMember[] = [];
|
||||
const paths = new XarPathBudget();
|
||||
const pending: Array<{ readonly element: Element; readonly parent: string }> =
|
||||
childElements(toc, "file")
|
||||
.map((element) => ({ element, parent: "" }))
|
||||
.reverse();
|
||||
while (pending.length > 0) {
|
||||
const next = pending.pop();
|
||||
if (next === undefined) break;
|
||||
const name = memberName(next.element);
|
||||
if (
|
||||
name === undefined ||
|
||||
name === "" ||
|
||||
name.includes("/") ||
|
||||
name === "." ||
|
||||
name === ".."
|
||||
)
|
||||
throw new ArtifactReaderFailure("path", "xar member has an unsafe name");
|
||||
const path = paths.join(next.parent, name);
|
||||
budget.consumeEntry(path);
|
||||
const type = textOf(next.element, "type")?.trim();
|
||||
const mode = xarMode(textOf(next.element, "mode"));
|
||||
const { kind, unsupportedType } = classifyMemberType(type);
|
||||
if (kind !== "file" && childElement(next.element, "data") !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar non-file member declares data: ${path}`,
|
||||
);
|
||||
members.push({
|
||||
path,
|
||||
kind,
|
||||
...(unsupportedType === undefined ? {} : { unsupportedType }),
|
||||
mode,
|
||||
data: parseData(next.element),
|
||||
link: kind === "symlink" ? textOf(next.element, "link") : undefined,
|
||||
});
|
||||
const children = childElements(next.element, "file");
|
||||
for (let index = children.length - 1; index >= 0; index--) {
|
||||
const element = children[index];
|
||||
if (element !== undefined) pending.push({ element, parent: path });
|
||||
}
|
||||
}
|
||||
return members;
|
||||
};
|
||||
|
||||
/** Expose a member's extracted checksum so a caller can apply its own policy. */
|
||||
const declaredDigest = (
|
||||
checksum: { readonly algorithm: string; readonly value: string } | undefined,
|
||||
): Pick<ArtifactEntry, "declaredSha256" | "declaredChecksum"> => {
|
||||
if (checksum === undefined) return { declaredSha256: null };
|
||||
if (checksum.algorithm === "sha256")
|
||||
return { declaredSha256: checksum.value };
|
||||
const algorithm = checksum.algorithm;
|
||||
return algorithm === "sha1" || algorithm === "md5" || algorithm === "sha512"
|
||||
? {
|
||||
declaredSha256: null,
|
||||
declaredChecksum: { algorithm, value: checksum.value },
|
||||
}
|
||||
: { declaredSha256: null };
|
||||
};
|
||||
|
||||
/**
|
||||
* Absolute decoded-byte ceiling for one gzip member. The declared size is
|
||||
* attacker-controlled, so it cannot be the only inflation bound.
|
||||
*/
|
||||
const MAX_DECODED_MEMBER_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
/** Bound decoded output to the TOC-declared extracted size. */
|
||||
class SizeBound extends Transform {
|
||||
#seen = 0;
|
||||
#cap: number;
|
||||
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
private readonly expected: number,
|
||||
private readonly budget: ArtifactDecodedBudget,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
super();
|
||||
this.#cap = Math.min(expected, MAX_DECODED_MEMBER_BYTES);
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer,
|
||||
_encoding: BufferEncoding,
|
||||
done: TransformCallback,
|
||||
): void {
|
||||
if (this.signal?.aborted) {
|
||||
done(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`xar member decoding cancelled: ${this.path}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.budget.consume(chunk.length, this.path);
|
||||
} catch (cause: unknown) {
|
||||
done(cause instanceof Error ? cause : new Error(String(cause)));
|
||||
return;
|
||||
}
|
||||
this.#seen += chunk.length;
|
||||
if (this.#seen > this.#cap) {
|
||||
done(
|
||||
new ArtifactReaderFailure(
|
||||
"limit",
|
||||
this.expected > MAX_DECODED_MEMBER_BYTES
|
||||
? `xar member ${this.path} decodes beyond the ${MAX_DECODED_MEMBER_BYTES}-byte member ceiling`
|
||||
: `xar member ${this.path} decodes beyond its declared ${this.expected} bytes`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
done(null, chunk);
|
||||
}
|
||||
|
||||
override _flush(done: TransformCallback): void {
|
||||
done(
|
||||
this.#seen === this.expected
|
||||
? null
|
||||
: new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${this.path} decoded ${this.#seen} bytes, expected ${this.expected}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Verify a member's extracted checksum as its decoded bytes stream past. */
|
||||
class ChecksumVerifier extends Transform {
|
||||
readonly #hash: Hash;
|
||||
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
private readonly expected: {
|
||||
readonly algorithm: string;
|
||||
readonly value: string;
|
||||
},
|
||||
private readonly record?: (
|
||||
observation: ArtifactChecksumObservation,
|
||||
) => void,
|
||||
) {
|
||||
super();
|
||||
this.#hash = createHash(expected.algorithm);
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer,
|
||||
_encoding: BufferEncoding,
|
||||
done: TransformCallback,
|
||||
): void {
|
||||
this.#hash.update(chunk);
|
||||
done(null, chunk);
|
||||
}
|
||||
|
||||
override _flush(done: TransformCallback): void {
|
||||
const observed = this.#hash.digest("hex");
|
||||
if (this.record !== undefined) {
|
||||
this.record({
|
||||
representation: "stored",
|
||||
algorithm: this.expected.algorithm,
|
||||
declared: this.expected.value,
|
||||
observed,
|
||||
});
|
||||
done();
|
||||
return;
|
||||
}
|
||||
done(
|
||||
observed === this.expected.value
|
||||
? null
|
||||
: new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`xar ${this.expected.algorithm} checksum disagrees with content: ${this.path} (declared ${this.expected.value}, observed ${observed})`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** First bytes that identify an installer archive member's own format. */
|
||||
export type XarNestedArchive = "gzip-cpio";
|
||||
|
||||
/**
|
||||
* Read-only reader for xar archives, the container of flat installer
|
||||
* packages. Members are decoded and checksum-verified on demand. Gzip cpio
|
||||
* `Scripts` and `Payload` members are marked for nested expansion; pbzx
|
||||
* (LZMA) payloads are recorded as files with a limitation.
|
||||
*/
|
||||
export class XarArtifactReader implements ArtifactReader {
|
||||
readonly format = "pkg" as const;
|
||||
#handle: FileHandle | undefined;
|
||||
#heap = 0;
|
||||
#size = 0;
|
||||
readonly decodedBudget: ArtifactDecodedBudget;
|
||||
#members: readonly XarMember[] = [];
|
||||
readonly #byPath = new Map<string, XarMember>();
|
||||
#loading: Promise<void> | undefined;
|
||||
#closed = false;
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #integrity = new Map<string, ArtifactChecksumObservation[]>();
|
||||
|
||||
readonly #verifyChecksums: boolean;
|
||||
|
||||
/**
|
||||
* @param options.verifyChecksums Fail `open()` streams whose bytes disagree
|
||||
* with the member's extracted checksum (default). Inventory passes false and
|
||||
* verifies the declared checksum itself under the caller's integrity policy.
|
||||
* Archived checksums are always computed. When inventory owns the policy,
|
||||
* integrityObservations exposes their declared/observed values after reading.
|
||||
*/
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
options: {
|
||||
readonly verifyChecksums?: boolean;
|
||||
readonly decodedBudget?: ArtifactDecodedBudget;
|
||||
} = {},
|
||||
) {
|
||||
this.#verifyChecksums = options.verifyChecksums ?? true;
|
||||
this.decodedBudget = options.decodedBudget ?? new ArtifactDecodedBudget();
|
||||
}
|
||||
|
||||
async #load(signal?: AbortSignal): Promise<void> {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure("unavailable", "xar archive is closed");
|
||||
this.#loading ??= this.#readToc(signal);
|
||||
await this.#loading;
|
||||
}
|
||||
|
||||
/** Load/index once; a rejected parse remains rejected, never a partial inventory. */
|
||||
async #readToc(signal?: AbortSignal): Promise<void> {
|
||||
cancelled(signal);
|
||||
const handle = await open(this.path, "r");
|
||||
this.#handle = handle;
|
||||
this.#size = (await handle.stat()).size;
|
||||
const header = Buffer.alloc(28);
|
||||
const { bytesRead } = await handle.read(header, 0, 28, 0);
|
||||
if (bytesRead < 28 || header.readUInt32BE(0) !== XAR_MAGIC)
|
||||
throw new ArtifactReaderFailure("format", "File is not a xar archive");
|
||||
const version = header.readUInt16BE(6);
|
||||
if (version !== 1)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Unsupported xar header version ${version}; expected 1`,
|
||||
);
|
||||
const headerSize = header.readUInt16BE(4);
|
||||
const tocCompressed = header.readBigUInt64BE(8);
|
||||
const tocSize = header.readBigUInt64BE(16);
|
||||
// Fixed-header checksum algorithm at offset 24: 0 is none, 1 is SHA-1,
|
||||
// 2 is MD5. A declared algorithm requires a matching <checksum> element.
|
||||
const headerChecksumAlg = header.readUInt32BE(24);
|
||||
if (headerSize < 28)
|
||||
throw new ArtifactReaderFailure("format", "xar header is too short");
|
||||
if (
|
||||
tocCompressed > BigInt(MAX_TOC_BYTES) ||
|
||||
tocSize > BigInt(MAX_TOC_BYTES)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`xar TOC exceeds the ${MAX_TOC_BYTES}-byte limit`,
|
||||
);
|
||||
const compressed = Buffer.alloc(Number(tocCompressed));
|
||||
const toc = await handle.read(compressed, 0, compressed.length, headerSize);
|
||||
if (toc.bytesRead < compressed.length)
|
||||
throw new ArtifactReaderFailure("format", "xar TOC is truncated");
|
||||
this.#heap = headerSize + compressed.length;
|
||||
let xml: string;
|
||||
try {
|
||||
const inflated = inflateSync(compressed, {
|
||||
maxOutputLength: MAX_TOC_BYTES,
|
||||
});
|
||||
if (BigInt(inflated.length) !== tocSize)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC decoded ${inflated.length} bytes, expected ${tocSize}`,
|
||||
);
|
||||
try {
|
||||
xml = new TextDecoder("utf-8", { fatal: true }).decode(inflated);
|
||||
} catch (cause: unknown) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar TOC is not valid UTF-8",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
assertXarTocNodes(xml);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
throw new ArtifactReaderFailure("format", "xar TOC is not zlib data", {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
let document;
|
||||
try {
|
||||
document = new DOMParser({
|
||||
onError: (level, message) => {
|
||||
if (level !== "warning") throw new Error(message);
|
||||
},
|
||||
}).parseFromString(xml, "text/xml");
|
||||
} catch (cause: unknown) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar TOC is not well-formed XML",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
const root = document.documentElement;
|
||||
const tocElement = root === null ? undefined : childElement(root, "toc");
|
||||
if (tocElement === undefined)
|
||||
throw new ArtifactReaderFailure("format", "xar TOC has no <toc> element");
|
||||
await this.#verifyToc(tocElement, compressed, headerChecksumAlg);
|
||||
this.#members = collectMembers(tocElement, this.decodedBudget);
|
||||
for (const member of this.#members) {
|
||||
if (member.data !== undefined)
|
||||
xarHeapPosition(
|
||||
this.#heap,
|
||||
member.data.offset,
|
||||
member.data.length,
|
||||
this.#size,
|
||||
);
|
||||
if (this.#byPath.has(member.path))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Duplicate xar member path: ${member.path}`,
|
||||
);
|
||||
this.#byPath.set(member.path, member);
|
||||
}
|
||||
}
|
||||
|
||||
/** The TOC checksum in the heap covers the compressed TOC bytes. */
|
||||
async #verifyToc(
|
||||
toc: Element,
|
||||
compressed: Buffer,
|
||||
headerAlg: number,
|
||||
): Promise<void> {
|
||||
// Fixed-header checksum IDs follow libarchive's XAR reader: 0 is none,
|
||||
// 1 is SHA-1, 2 is MD5, 3 is SHA-256, 4 is SHA-512.
|
||||
const expectedStyle =
|
||||
headerAlg === 0
|
||||
? null
|
||||
: headerAlg === 1
|
||||
? "sha1"
|
||||
: headerAlg === 2
|
||||
? "md5"
|
||||
: headerAlg === 3
|
||||
? "sha256"
|
||||
: headerAlg === 4
|
||||
? "sha512"
|
||||
: undefined;
|
||||
if (expectedStyle === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar header declares unsupported TOC checksum algorithm ${headerAlg}`,
|
||||
);
|
||||
const checksum = childElement(toc, "checksum");
|
||||
if (expectedStyle === null && checksum !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar header declares no TOC checksum but the TOC supplies one",
|
||||
);
|
||||
if (checksum === undefined) {
|
||||
if (headerAlg !== 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar header declares a TOC checksum but the TOC has no <checksum> element",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const style = checksum.getAttribute("style")?.toLowerCase() ?? "";
|
||||
if (expectedStyle !== null && style !== expectedStyle)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC checksum style ${style || "(missing)"} does not match header algorithm ${expectedStyle ?? "none"}`,
|
||||
);
|
||||
const algorithm = CHECKSUM_ALGORITHMS.get(style);
|
||||
if (algorithm === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC declares unsupported checksum style ${style || "(missing)"}`,
|
||||
);
|
||||
const size = integer(textOf(checksum, "size"), "checksum size");
|
||||
// The stored checksum is exactly one digest; never allocate a declared size.
|
||||
if (size !== DIGEST_BYTES.get(style))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC declares a ${size}-byte ${style} checksum`,
|
||||
);
|
||||
const stored = await this.#heapBytes(
|
||||
integer(textOf(checksum, "offset"), "checksum offset"),
|
||||
size,
|
||||
);
|
||||
if (!createHash(algorithm).update(compressed).digest().equals(stored))
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"xar TOC checksum disagrees with the TOC",
|
||||
);
|
||||
}
|
||||
|
||||
async #heapBytes(offset: number, length: number): Promise<Buffer> {
|
||||
const handle = this.#handle;
|
||||
if (handle === undefined)
|
||||
throw new ArtifactReaderFailure("unavailable", "xar archive is closed");
|
||||
const position = xarHeapPosition(this.#heap, offset, length, this.#size);
|
||||
const bytes = Buffer.alloc(length);
|
||||
let done = 0;
|
||||
while (done < length) {
|
||||
const { bytesRead } = await handle.read(
|
||||
bytes,
|
||||
done,
|
||||
length - done,
|
||||
position + done,
|
||||
);
|
||||
if (bytesRead === 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar member extends beyond the archive",
|
||||
);
|
||||
done += bytesRead;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Classify a raw-stored installer member by its leading bytes. */
|
||||
async #nestedArchive(member: XarMember): Promise<{
|
||||
readonly nested: XarNestedArchive | undefined;
|
||||
readonly limitations: readonly string[];
|
||||
}> {
|
||||
const name = member.path.split("/").at(-1);
|
||||
const data = member.data;
|
||||
if (
|
||||
(name !== "Payload" && name !== "Scripts") ||
|
||||
data === undefined ||
|
||||
data.length < 4
|
||||
)
|
||||
return { nested: undefined, limitations: [] };
|
||||
if (data.encoding !== "application/octet-stream")
|
||||
return {
|
||||
nested: undefined,
|
||||
limitations: [
|
||||
`${name} is stored with ${data.encoding}; its archive is not expanded.`,
|
||||
],
|
||||
};
|
||||
const head = await this.#heapBytes(data.offset, 4);
|
||||
if (head[0] === 0x1f && head[1] === 0x8b)
|
||||
return { nested: "gzip-cpio", limitations: [] };
|
||||
if (head.toString("latin1") === "pbzx")
|
||||
return {
|
||||
nested: undefined,
|
||||
limitations: [
|
||||
`${name} is a pbzx (LZMA) archive; its members are not expanded. Expand it with pkgutil --expand-full to inspect them.`,
|
||||
],
|
||||
};
|
||||
return {
|
||||
nested: undefined,
|
||||
limitations: [
|
||||
`${name} has an unrecognized archive format and is not expanded.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
await this.#load(signal);
|
||||
for (const member of this.#members) {
|
||||
cancelled(signal);
|
||||
yield await this.#memberEntry(member);
|
||||
}
|
||||
}
|
||||
|
||||
/** Project one TOC member as an archive-neutral entry. */
|
||||
async #memberEntry(member: XarMember): Promise<ArtifactEntry> {
|
||||
if (member.unsupportedType !== undefined)
|
||||
return {
|
||||
path: member.path,
|
||||
kind: "file",
|
||||
declaredSize: null,
|
||||
compressedSize: null,
|
||||
executable: false,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: [
|
||||
`Unsupported xar member type ${member.unsupportedType}; content not expanded.`,
|
||||
],
|
||||
adapterKey: member.path,
|
||||
contentUnavailable: true,
|
||||
};
|
||||
const data = member.data;
|
||||
assertRawSize(member, data);
|
||||
const unsupportedChecksum = data?.unsupportedChecksum;
|
||||
const unsupportedArchived = data?.unsupportedArchivedChecksum;
|
||||
const unsupportedEncoding =
|
||||
data !== undefined &&
|
||||
data.encoding !== "application/octet-stream" &&
|
||||
data.encoding !== "application/x-gzip";
|
||||
const classified =
|
||||
member.kind === "file" && !unsupportedEncoding
|
||||
? await this.#nestedArchive(member)
|
||||
: {
|
||||
nested: undefined as XarNestedArchive | undefined,
|
||||
limitations: [] as readonly string[],
|
||||
};
|
||||
return {
|
||||
path: member.path,
|
||||
kind: member.kind,
|
||||
declaredSize: member.data?.size ?? (member.kind === "file" ? 0 : null),
|
||||
compressedSize: member.data?.length ?? null,
|
||||
executable: member.mode !== null && (member.mode & 0o111) !== 0,
|
||||
encrypted: false,
|
||||
byteOffset:
|
||||
member.data?.encoding === "application/octet-stream"
|
||||
? this.#heap + member.data.offset
|
||||
: null,
|
||||
...declaredDigest(member.data?.extractedChecksum),
|
||||
unpacked: false,
|
||||
limitations: memberLimitations(
|
||||
member,
|
||||
unsupportedChecksum,
|
||||
unsupportedArchived,
|
||||
classified,
|
||||
),
|
||||
adapterKey: member.path,
|
||||
...(unsupportedEncoding ? { contentUnavailable: true } : {}),
|
||||
...(classified.nested === undefined
|
||||
? {}
|
||||
: { nestedArchive: classified.nested }),
|
||||
};
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
cancelled(signal);
|
||||
this.#integrity.delete(entry.adapterKey);
|
||||
await this.#load(signal);
|
||||
const member = this.#byPath.get(entry.adapterKey);
|
||||
if (member === undefined || member.kind !== "file")
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`xar member is not a file: ${entry.path}`,
|
||||
);
|
||||
if (member.unsupportedType !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`xar member ${entry.path} has unsupported type ${member.unsupportedType}`,
|
||||
);
|
||||
const data = member.data;
|
||||
if (data === undefined) return Readable.from([]);
|
||||
if (entry.contentUnavailable === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`xar member ${entry.path} is not expandable`,
|
||||
);
|
||||
// Raw members must describe their stored bytes exactly.
|
||||
if (
|
||||
data.encoding === "application/octet-stream" &&
|
||||
data.size !== data.length
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${entry.path} declares size ${data.size} with length ${data.length}`,
|
||||
);
|
||||
// Validate before creating any streams: otherwise an early throw would
|
||||
// orphan a flowing source that keeps reading after close().
|
||||
if (
|
||||
data.encoding !== "application/octet-stream" &&
|
||||
data.encoding !== "application/x-gzip"
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${entry.path} uses unsupported encoding ${data.encoding}`,
|
||||
);
|
||||
// Archived checksums cover the stored stream, which inventory's scanner
|
||||
// never observes: it hashes decoded bytes. Always verify them here; the
|
||||
// scanner's per-entry recovery still honors record-and-continue.
|
||||
const archived = data.archivedChecksum;
|
||||
const archivedVerifier =
|
||||
archived === undefined
|
||||
? undefined
|
||||
: new ChecksumVerifier(
|
||||
`${entry.path} (archived)`,
|
||||
archived,
|
||||
this.#verifyChecksums
|
||||
? undefined
|
||||
: (observation) => {
|
||||
this.#integrity.set(entry.adapterKey, [observation]);
|
||||
},
|
||||
);
|
||||
const stages: Transform[] = [
|
||||
new ArtifactBudgetTransform(this.decodedBudget, entry.path, signal),
|
||||
];
|
||||
if (archivedVerifier !== undefined) stages.push(archivedVerifier);
|
||||
if (data.encoding === "application/x-gzip")
|
||||
stages.push(
|
||||
createInflate(),
|
||||
new SizeBound(entry.path, data.size, this.decodedBudget, signal),
|
||||
);
|
||||
const checksum = data.extractedChecksum;
|
||||
if (checksum !== undefined && this.#verifyChecksums)
|
||||
stages.push(new ChecksumVerifier(entry.path, checksum));
|
||||
const stream = artifactStreamPipeline(
|
||||
Readable.from(this.#chunks(data.offset, data.length, signal)),
|
||||
stages,
|
||||
{ path: entry.path, signal },
|
||||
);
|
||||
this.#streams.add(stream);
|
||||
stream.once("close", () => this.#streams.delete(stream));
|
||||
return stream;
|
||||
}
|
||||
|
||||
async *#chunks(
|
||||
offset: number,
|
||||
length: number,
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<Buffer> {
|
||||
for (let done = 0; done < length;) {
|
||||
cancelled(signal);
|
||||
const size = Math.min(READ_CHUNK_BYTES, length - done);
|
||||
yield await this.#heapBytes(offset + done, size);
|
||||
done += size;
|
||||
}
|
||||
}
|
||||
|
||||
provenance(): readonly ArtifactCommand[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Stored-byte checksums computed while consuming the member stream. */
|
||||
integrityObservations(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[] {
|
||||
return this.#integrity.get(entry.adapterKey) ?? [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
for (const stream of this.#streams) stream.destroy();
|
||||
await this.#loading?.catch(() => undefined);
|
||||
const handle = this.#handle;
|
||||
this.#handle = undefined;
|
||||
this.#members = [];
|
||||
this.#byPath.clear();
|
||||
this.#integrity.clear();
|
||||
await handle?.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
/**
|
||||
* Node ceiling for one TOC. The byte cap still allows millions of tiny
|
||||
* elements, comments, and processing instructions. The DOM is built before
|
||||
* member collection can charge the metadata budget.
|
||||
*/
|
||||
const MAX_TOC_NODES = 100_000;
|
||||
|
||||
/** Bound DOM-producing markup, including comments, CDATA and processing instructions. */
|
||||
export const assertXarTocNodes = (xml: string): void => {
|
||||
let nodes = 0;
|
||||
let index = xml.indexOf("<");
|
||||
while (index !== -1) {
|
||||
if (xml[index + 1] !== "/") {
|
||||
nodes += 1;
|
||||
if (nodes > MAX_TOC_NODES)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`xar TOC exceeds ${MAX_TOC_NODES} XML nodes`,
|
||||
);
|
||||
}
|
||||
const terminator = xml.startsWith("<!--", index)
|
||||
? "-->"
|
||||
: xml.startsWith("<![CDATA[", index)
|
||||
? "]]>"
|
||||
: xml.startsWith("<?", index)
|
||||
? "?>"
|
||||
: ">";
|
||||
const end = xml.indexOf(terminator, index + 1);
|
||||
if (end === -1) return; // The XML parser reports malformed framing.
|
||||
index = xml.indexOf("<", end + terminator.length);
|
||||
}
|
||||
};
|
||||
|
||||
/** Required XAR integer syntax: nonempty unsigned decimal, exactly representable. */
|
||||
export const xarInteger = (
|
||||
value: string | undefined,
|
||||
label: string,
|
||||
): number => {
|
||||
const text = value?.trim() ?? "";
|
||||
const parsed = /^\d+$/u.test(text) ? Number(text) : Number.NaN;
|
||||
if (!Number.isSafeInteger(parsed))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC has an invalid ${label}`,
|
||||
);
|
||||
return parsed;
|
||||
};
|
||||
|
||||
/** Optional mode metadata is accepted only as a complete unsigned octal field. */
|
||||
export const xarMode = (value: string | undefined): number | null => {
|
||||
if (value === undefined) return null;
|
||||
const text = value.trim();
|
||||
const mode = /^[0-7]+$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN;
|
||||
if (!Number.isSafeInteger(mode))
|
||||
throw new ArtifactReaderFailure("format", "xar TOC has an invalid mode");
|
||||
return mode;
|
||||
};
|
||||
|
||||
/** Validate the entire absolute heap range before allocating or requesting it. */
|
||||
export const xarHeapPosition = (
|
||||
heap: number,
|
||||
offset: number,
|
||||
length: number,
|
||||
size: number,
|
||||
): number => {
|
||||
const position = heap + offset;
|
||||
const end = position + length;
|
||||
if (
|
||||
![position, end, offset, length].every(Number.isSafeInteger) ||
|
||||
offset < 0 ||
|
||||
length < 0 ||
|
||||
end > size
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar member has an invalid or out-of-file heap extent",
|
||||
);
|
||||
return position;
|
||||
};
|
||||
|
||||
/** Bound retained path text before constructing paths from deeply nested TOCs. */
|
||||
export class XarPathBudget {
|
||||
#used = 0;
|
||||
private readonly maximumPathBytes = 4096;
|
||||
private readonly maximumTotalBytes = 16 * 1024 * 1024;
|
||||
|
||||
/** Form the next path only when individual and cumulative budgets permit it. */
|
||||
join(parent: string, name: string): string {
|
||||
const bytes =
|
||||
Buffer.byteLength(parent) +
|
||||
Buffer.byteLength(name) +
|
||||
(parent.length > 0 ? 1 : 0);
|
||||
if (
|
||||
bytes > this.maximumPathBytes ||
|
||||
bytes > this.maximumTotalBytes - this.#used
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`xar TOC path text exceeds its ${this.maximumPathBytes}-byte path or ${this.maximumTotalBytes}-byte cumulative budget`,
|
||||
);
|
||||
this.#used += bytes;
|
||||
return parent === "" ? name : `${parent}/${name}`;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,14 @@
|
||||
import { mkdir, symlink, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { collectInterfaceBuilderResourceKeys } from "./AppleAssetCatalogAnalysis.js";
|
||||
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import {
|
||||
analyzeAppleAssetCatalogs,
|
||||
collectInterfaceBuilderResourceKeys,
|
||||
} from "./AppleAssetCatalogAnalysis.js";
|
||||
|
||||
describe("Apple asset catalog application workflow", () => {
|
||||
it("joins only explicit resource-key fields with archive provenance", () => {
|
||||
@@ -40,4 +48,70 @@ describe("Apple asset catalog application workflow", () => {
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports an app without compiled catalogs as an empty observed inventory", async () => {
|
||||
const bundle = await createTestTempDirectory("rea-asset-catalog-absent-");
|
||||
await mkdir(join(bundle, "Contents", "Resources"), { recursive: true });
|
||||
await writeFile(join(bundle, "Contents", "Resources", "icon.icns"), "");
|
||||
const result = await analyzeAppleAssetCatalogs({
|
||||
bundlePath: bundle,
|
||||
targetSha256: "a".repeat(64),
|
||||
runAssetUtil: () => {
|
||||
throw new Error("assetutil must not run without a catalog");
|
||||
},
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
catalogs: [],
|
||||
total_records: 0,
|
||||
records: [],
|
||||
resource_key_matches: [],
|
||||
next_offset: null,
|
||||
truncated: false,
|
||||
});
|
||||
expect(result.limitations).toContain(
|
||||
"No compiled Assets.car catalog was found among the app bundle's regular files (symlinks are not followed); any resource keys are reported as unmatched.",
|
||||
);
|
||||
});
|
||||
|
||||
it("does not report absence when the only catalog entry is a symlink", async () => {
|
||||
const bundle = await createTestTempDirectory("rea-asset-catalog-link-");
|
||||
const resources = join(bundle, "Contents", "Resources");
|
||||
await mkdir(resources, { recursive: true });
|
||||
await writeFile(join(bundle, "outside.car"), "catalog");
|
||||
await symlink("../../outside.car", join(resources, "Assets.car"));
|
||||
const rejected = await analyzeAppleAssetCatalogs({
|
||||
bundlePath: bundle,
|
||||
targetSha256: "a".repeat(64),
|
||||
runAssetUtil: () => Promise.resolve("[]"),
|
||||
}).catch((cause: unknown) => cause);
|
||||
expect(rejected).toBeInstanceOf(ArtifactReaderFailure);
|
||||
expect(rejected).toMatchObject({
|
||||
reason: "path",
|
||||
message: expect.stringContaining(
|
||||
"Contents/Resources/Assets.car (symlink)",
|
||||
),
|
||||
});
|
||||
});
|
||||
|
||||
it("discloses skipped catalog entries beside inspected catalogs", async () => {
|
||||
const bundle = await createTestTempDirectory("rea-asset-catalog-mixed-");
|
||||
const resources = join(bundle, "Contents", "Resources");
|
||||
await mkdir(join(resources, "Nested"), { recursive: true });
|
||||
await writeFile(join(resources, "Assets.car"), "catalog");
|
||||
await symlink("../Assets.car", join(resources, "Nested", "Assets.car"));
|
||||
const result = await analyzeAppleAssetCatalogs({
|
||||
bundlePath: bundle,
|
||||
targetSha256: "a".repeat(64),
|
||||
runAssetUtil: () => Promise.resolve("[]"),
|
||||
});
|
||||
expect(result.catalogs.map(({ path }) => path)).toEqual([
|
||||
"Contents/Resources/Assets.car",
|
||||
]);
|
||||
expect(result.limitations).toContainEqual(
|
||||
expect.stringContaining("Contents/Resources/Nested/Assets.car (symlink)"),
|
||||
);
|
||||
expect(result.limitations).not.toContainEqual(
|
||||
expect.stringContaining("No compiled Assets.car catalog"),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -48,10 +48,15 @@ export const analyzeAppleAssetCatalogs = async (input: {
|
||||
sha256: string;
|
||||
records: ReturnType<typeof parseAppleAssetCatalogRecords>;
|
||||
}[] = [];
|
||||
const uninspected: string[] = [];
|
||||
let totalMetadataBytes = 0;
|
||||
try {
|
||||
for await (const entry of reader.entries(input.signal)) {
|
||||
if (entry.kind !== "file" || !entry.path.endsWith("Assets.car")) continue;
|
||||
if (!entry.path.endsWith("Assets.car")) continue;
|
||||
if (entry.kind !== "file") {
|
||||
uninspected.push(`${entry.path} (${entry.kind})`);
|
||||
continue;
|
||||
}
|
||||
const inspected = await inspectAssetCatalogEntry({
|
||||
reader,
|
||||
entry,
|
||||
@@ -69,10 +74,12 @@ export const analyzeAppleAssetCatalogs = async (input: {
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
if (catalogs.length === 0)
|
||||
// Without any readable catalog, unmatched resource keys would overstate an
|
||||
// absence that a skipped catalog entry may contradict.
|
||||
if (catalogs.length === 0 && uninspected.length > 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"No compiled Assets.car catalogs were found in the app bundle",
|
||||
"path",
|
||||
`Asset catalog entries are not regular files and were not inspected: ${uninspected.join(", ")}`,
|
||||
);
|
||||
const interfaceBuilder = await analyzeInterfaceBuilderBundle({
|
||||
bundlePath: input.bundlePath,
|
||||
@@ -93,6 +100,18 @@ export const analyzeAppleAssetCatalogs = async (input: {
|
||||
...result,
|
||||
limitations: [
|
||||
...result.limitations,
|
||||
// An app without a compiled catalog is an observed empty inventory, not
|
||||
// a missing host tool.
|
||||
...(catalogs.length === 0
|
||||
? [
|
||||
"No compiled Assets.car catalog was found among the app bundle's regular files (symlinks are not followed); any resource keys are reported as unmatched.",
|
||||
]
|
||||
: []),
|
||||
...(uninspected.length === 0
|
||||
? []
|
||||
: [
|
||||
`Asset catalog entries that are not regular files were not inspected, so resource keys reported as unmatched may resolve there: ${uninspected.join(", ")}`,
|
||||
]),
|
||||
"Exact resource-key joins use only recognized image or asset key fields on decoded Interface Builder resource objects; other archive resource references are not inferred.",
|
||||
...(interfaceBuilder.graph.truncated
|
||||
? [
|
||||
|
||||
@@ -125,6 +125,16 @@ describe("inert keyed archive decoding", () => {
|
||||
offset: 0,
|
||||
limit: 1,
|
||||
}),
|
||||
).toThrow("does not exist");
|
||||
).toThrow(
|
||||
expect.objectContaining({
|
||||
issues: [
|
||||
expect.objectContaining({
|
||||
path: ["root"],
|
||||
message: expect.stringContaining("does not exist"),
|
||||
expected: ["root", "other"],
|
||||
}),
|
||||
],
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,6 +2,7 @@ import { createHash } from "node:crypto";
|
||||
import { TextDecoder } from "node:util";
|
||||
import { parseBinary } from "plist";
|
||||
import { z } from "zod";
|
||||
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { projectPlistValue } from "../../domain/apple/plistValue.js";
|
||||
import {
|
||||
@@ -25,6 +26,10 @@ export const decodeKeyedArchiveBytes = (
|
||||
) => {
|
||||
if (bytes.length > MAX_BYTES)
|
||||
throw new RangeError("Keyed archive exceeds 64 MiB");
|
||||
if (bytes.subarray(0, 10).toString("ascii") === "NIBArchive")
|
||||
throw new TypeError(
|
||||
"Selected file is a compiled NIBArchive, not a Foundation plist archive; decode it with decode_interface_builder",
|
||||
);
|
||||
const binary = bytes.subarray(0, 8).toString("ascii") === "bplist00";
|
||||
const parsed = binary
|
||||
? { value: parseBinary(bytes), omittedPrototypeKeys: 0 }
|
||||
@@ -83,18 +88,20 @@ export const inspectBundleKeyedArchive = async (input: {
|
||||
.split("/")
|
||||
.some((part) => part === ".." || part === "." || part.length === 0)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Archive path must be a canonical relative bundle path",
|
||||
throw archivePathError(
|
||||
"invalid_format",
|
||||
selected.path === "."
|
||||
? "Select a keyed archive path relative to the active app bundle."
|
||||
: `Archive path must be a canonical relative bundle path without empty, '.', or '..' segments: ${selected.path}`,
|
||||
);
|
||||
const reader = new DirectoryArtifactReader(input.bundlePath);
|
||||
try {
|
||||
for await (const entry of reader.entries(input.signal)) {
|
||||
if (entry.path !== selected.path) continue;
|
||||
if (entry.kind !== "file")
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Archive must be a regular file, not a symlink or directory",
|
||||
throw archivePathError(
|
||||
"invalid_value",
|
||||
`Archive path selects a ${entry.kind}, not a regular file: ${selected.path}`,
|
||||
);
|
||||
if ((entry.declaredSize ?? 0) > MAX_BYTES)
|
||||
throw new ArtifactReaderFailure(
|
||||
@@ -132,18 +139,28 @@ export const inspectBundleKeyedArchive = async (input: {
|
||||
...decodeKeyedArchiveBytes(bytes, selected),
|
||||
});
|
||||
} catch (cause) {
|
||||
if (cause instanceof AnalysisInputError) throw cause;
|
||||
throw new ArtifactReaderFailure(
|
||||
cause instanceof RangeError ? "limit" : "format",
|
||||
"Cannot decode selected Foundation keyed archive",
|
||||
`Cannot decode selected Foundation keyed archive: ${cause instanceof Error ? cause.message : String(cause)}`,
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
}
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Archive not found in bundle: ${selected.path}`,
|
||||
throw archivePathError(
|
||||
"invalid_value",
|
||||
`No regular file exists at ${selected.path} in the active app bundle.`,
|
||||
);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
};
|
||||
|
||||
/** A caller-selected archive path, not the artifact, failed its constraint. */
|
||||
const archivePathError = (
|
||||
reason: "invalid_format" | "invalid_value",
|
||||
message: string,
|
||||
) =>
|
||||
new AnalysisInputError("inspect_keyed_archive", undefined, [
|
||||
{ path: ["path"], reason, message },
|
||||
]);
|
||||
|
||||
@@ -2,10 +2,9 @@ import { lstat, realpath } from "node:fs/promises";
|
||||
|
||||
import { canonicalDigest } from "../../domain/comparisonSemantics.js";
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import {
|
||||
ArtifactPathRegistry,
|
||||
normalizeArtifactPath,
|
||||
} from "../ArtifactPaths.js";
|
||||
import { XarArtifactReader } from "../XarArtifactReader.js";
|
||||
import { ArtifactPathRegistry } from "../ArtifactPaths.js";
|
||||
import { visitArtifactTree } from "../ArtifactTraversal.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
@@ -44,6 +43,7 @@ export const extractArtifact = async (
|
||||
const selectedOccurrences = snapshot.occurrences.filter(
|
||||
(occurrence) =>
|
||||
(occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") &&
|
||||
occurrence.hash_status !== "unavailable" &&
|
||||
occurrence.logical_path !== ".",
|
||||
);
|
||||
const selectedIds = new Set(
|
||||
@@ -63,6 +63,11 @@ export const extractArtifact = async (
|
||||
manifest: snapshot.manifest,
|
||||
occurrences,
|
||||
nodes,
|
||||
unavailablePaths: new Set(
|
||||
snapshot.occurrences
|
||||
.filter(({ hash_status }) => hash_status === "unavailable")
|
||||
.map(({ logical_path }) => logical_path),
|
||||
),
|
||||
};
|
||||
const selected = selectedOccurrences.map((occurrence) => {
|
||||
if (
|
||||
@@ -128,35 +133,52 @@ const materializeSelection = async ({
|
||||
try {
|
||||
const materialized: SelectedOccurrence[] = [];
|
||||
const registry = new ArtifactPathRegistry();
|
||||
for await (const entry of reader.entries(signal)) {
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
registry.add(path, entry.kind);
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await reader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
stream,
|
||||
selectedItem.node.sha256,
|
||||
signal,
|
||||
);
|
||||
extracted.push({
|
||||
artifact_id: selectedItem.node.artifact_id,
|
||||
relative_path: written.relativePath,
|
||||
sha256: written.sha256,
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
materialized.push(selectedItem);
|
||||
}
|
||||
await visitArtifactTree(
|
||||
reader,
|
||||
async ({ reader: currentReader, entry, path, container: expandable }) => {
|
||||
// Traverse expanded PKG Payload/Scripts like inventory does, so nested
|
||||
// members are materialized instead of silently skipped. An expanded
|
||||
// member acts as a container: register it as a directory and leave its
|
||||
// archive bytes to its children, since a file and a directory cannot
|
||||
// share one output path.
|
||||
registry.add(path, expandable ? "directory" : entry.kind);
|
||||
if (expandable) {
|
||||
return true;
|
||||
}
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (
|
||||
entry.contentUnavailable === true &&
|
||||
inventory.unavailablePaths.has(path)
|
||||
)
|
||||
return false;
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await currentReader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
stream,
|
||||
selectedItem.node.sha256,
|
||||
signal,
|
||||
);
|
||||
extracted.push({
|
||||
artifact_id: selectedItem.node.artifact_id,
|
||||
relative_path: written.relativePath,
|
||||
sha256: written.sha256,
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
materialized.push(selectedItem);
|
||||
return false;
|
||||
},
|
||||
{ signal, expandAsar: false },
|
||||
);
|
||||
await reader.close();
|
||||
readerClosed = true;
|
||||
extracted.sort((left, right) =>
|
||||
@@ -207,7 +229,11 @@ const createExtractionResult = (
|
||||
cleanup: { attempted: false, verified: true, residual_paths: [] },
|
||||
provenance: [],
|
||||
limitations: [
|
||||
"All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file.",
|
||||
"All available selected regular files were materialized; expanded PKG Payload/Scripts archives are represented by their extracted children.",
|
||||
...[...inventory.unavailablePaths].map(
|
||||
(path) =>
|
||||
`Unavailable member ${path} was not extracted; see its inventory occurrence for the reason.`,
|
||||
),
|
||||
],
|
||||
});
|
||||
};
|
||||
@@ -216,6 +242,7 @@ interface LoadedInventory {
|
||||
readonly manifest: ArtifactGraphManifest;
|
||||
readonly occurrences: ReadonlyMap<string, ArtifactOccurrence>;
|
||||
readonly nodes: ReadonlyMap<string, ArtifactNode>;
|
||||
readonly unavailablePaths: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
const collectOccurrences = (
|
||||
@@ -256,6 +283,7 @@ const createReader = async (
|
||||
)
|
||||
return new ZipArtifactReader(path, format);
|
||||
if (format === "mach-o") return new MachOSliceArtifactReader(path);
|
||||
if (format === "pkg") return new XarArtifactReader(path);
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`Artifact format has no extraction reader: ${format}`,
|
||||
|
||||
@@ -50,7 +50,12 @@ export const createOccurrence = (
|
||||
compressed_size: entry.compressedSize,
|
||||
executable: entry.executable,
|
||||
encrypted: entry.encrypted,
|
||||
hash_status: entry.encrypted ? "unavailable" : "not-hashed",
|
||||
hash_status:
|
||||
entry.integrityMismatched === true
|
||||
? "mismatched"
|
||||
: entry.encrypted || entry.contentUnavailable === true
|
||||
? "unavailable"
|
||||
: "not-hashed",
|
||||
source_location:
|
||||
entry.byteOffset === null || entry.declaredSize === null
|
||||
? null
|
||||
@@ -104,6 +109,14 @@ export const materializeDirectoryNodes = (
|
||||
format: "directory",
|
||||
executable: false,
|
||||
contentState: "virtual",
|
||||
...(directory.hash_status === "mismatched"
|
||||
? {
|
||||
limitations: [
|
||||
...directory.limitations,
|
||||
"Directory structure identity is derived; its declared member integrity metadata disagreed.",
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
const existing = nodes.get(node.artifact_id);
|
||||
nodes.set(
|
||||
@@ -111,7 +124,8 @@ export const materializeDirectoryNodes = (
|
||||
existing?.kind === "framework" ? existing : node,
|
||||
);
|
||||
directory.artifact_id = node.artifact_id;
|
||||
directory.hash_status = "verified";
|
||||
if (directory.hash_status !== "mismatched")
|
||||
directory.hash_status = "verified";
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user