Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Kaoru0822
|
7aee9900da | ||
|
Kaoru0822andClaude Opus 5.5
|
e1def2bf1a |
@@ -25,7 +25,6 @@ node scripts/release-notes.mjs inventory \
|
||||
--manifest /tmp/rea-VERSION.contributions.json
|
||||
node scripts/release-notes.mjs record \
|
||||
--manifest /tmp/rea-VERSION.contributions.json \
|
||||
--summary \
|
||||
--output /tmp/rea-VERSION.record.md
|
||||
```
|
||||
|
||||
@@ -57,31 +56,25 @@ version/date/link heading. Use single-line Markdown list items under:
|
||||
PR/commit references required by the existing checkpoint validator.
|
||||
|
||||
Group related work by analyst workflow or user impact. Explain the resulting
|
||||
behavior in short, concrete entries. Highlights summarize the release; avoid
|
||||
repeating their full descriptions in Changes. Internal cleanup, tests, CI, and
|
||||
routine documentation generally belong only in the provenance file. Read ambiguous PRs/diffs before making
|
||||
behavior; internal cleanup, tests, CI, and routine documentation generally
|
||||
belong only in the contribution record. Read ambiguous PRs/diffs before making
|
||||
claims. Distinguish simulated-provider checks from real-provider verification.
|
||||
|
||||
Credit each represented contribution once in the public notes with its link
|
||||
and `Thanks @...` for its verified human author, known co-authors, and confirmed
|
||||
closing-issue reporters. Grouping preserves all those credits in one line.
|
||||
When Highlights, detailed entries or migrations reference the same work,
|
||||
put its thanks in one of those entries, not all of them. Include issue links.
|
||||
Direct-commit bullets carry the commit link and known human credit.
|
||||
Each bullet covering a PR carries its link and `Thanks @...` for its verified
|
||||
human author, known co-authors, and confirmed closing-issue reporters. Grouping
|
||||
must preserve **all** those credits in the same bullet. Include the issue
|
||||
links. Direct-commit bullets carry the commit link and known human credit.
|
||||
Exclude GitHub bots and known agent accounts. REA's observed Anthropic and
|
||||
Cursor co-author trailers resolve to `@claude` and `@cursoragent` with GitHub
|
||||
type `User`; the credit policy excludes them while retaining their metadata.
|
||||
At the REA maintainer's request, omit `@morluto` self-thanks while preserving
|
||||
authorship in the inventory. Other human contributors remain eligible.
|
||||
Do not copy OpenClaw's maintainer-specific exclusions.
|
||||
Use explicit GitHub links for cross-repository references to avoid ambiguous
|
||||
`#NNN` numbers. Unknown handles stay unknown; they are not fabricated credit.
|
||||
|
||||
Append the generated `record --summary` provenance link verbatim at the end
|
||||
of this release. The linked inventory accounts for every discovered in-range
|
||||
PR and direct contribution, including internal work omitted from the prose.
|
||||
Do not paste the full `record` output into public notes: it duplicates thanks
|
||||
and obscures the user-facing changes. Full records remain available for review.
|
||||
Do not hand-edit generated credits to make validation pass. Refresh the inventory
|
||||
Append the generated `### Complete contribution record` verbatim as the last
|
||||
section of this release. It accounts for every discovered in-range PR and
|
||||
direct contribution, including internal work omitted from the prose. Do not
|
||||
hand-edit generated credits to make validation pass. Refresh the inventory
|
||||
when product history changes; a prose-only correction can reuse the snapshot.
|
||||
|
||||
## Finalize with Release Please
|
||||
|
||||
@@ -135,16 +135,6 @@ session to a selected path. The analysis remains complete in the ledger; avoid
|
||||
repeating it merely to request the same oversized response. Complete CLI JSON
|
||||
output also streams without a single MCP frame.
|
||||
|
||||
When the retained parent ID is unknown, use `get_evidence_bundle` with
|
||||
`detail: "summary"` and exact filters for operation, target SHA-256,
|
||||
analysis-profile digest or canonical procedure address. Select its returned
|
||||
Evidence ID for the next view instead of requesting the complete bundle.
|
||||
`retention: "complete-record"` means the whole record is stored; native
|
||||
value-flow truncation and unknown metadata remain separate. For an exported
|
||||
portable bundle, use `rea inspect-evidence-bundle` with JSON containing `path`,
|
||||
`detail: "summary"` and the same `filters`. Reading that file still requires
|
||||
validating its full contents within the local memory budget.
|
||||
|
||||
A client can also truncate a successful tool result without a REA transport
|
||||
error. Treat that preview as incomplete; use the retained parent ID for smaller
|
||||
views or export when the full record is needed. Do not parse the preview as
|
||||
|
||||
+31
-43
@@ -115,8 +115,14 @@ jobs:
|
||||
- run: npm ci
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rea-test-runtime
|
||||
path: .
|
||||
name: rea-dist
|
||||
path: dist
|
||||
- name: Generate test artifacts from the downloaded runtime
|
||||
run: |
|
||||
node scripts/generate-mcp-tool-catalog.mjs
|
||||
node scripts/generate-skill-metadata.mjs
|
||||
node scripts/generate-product-catalog.mjs
|
||||
node scripts/generate-completion-ledger.mjs
|
||||
- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -140,17 +146,11 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run test:prepare
|
||||
- run: npm run build:cached
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rea-test-runtime
|
||||
path: |
|
||||
dist/
|
||||
skills/
|
||||
.cache/mcp-tool-catalog.json
|
||||
docs/public/product-catalog.json
|
||||
docs/verification/managed-conformance-*.json
|
||||
include-hidden-files: true
|
||||
name: rea-dist
|
||||
path: dist
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
@@ -274,10 +274,7 @@ jobs:
|
||||
if: needs.changes.outputs.runtime == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
# The four platform jobs use independent hosted runners; keep each
|
||||
# runner's existing heap and threadpool limits while avoiding a queued
|
||||
# second platform wave.
|
||||
max-parallel: 4
|
||||
max-parallel: 2
|
||||
matrix: ${{ fromJSON(needs.changes.outputs.matrix) }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 20
|
||||
@@ -323,34 +320,25 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Verify Apple artifact formats and metadata
|
||||
run: |
|
||||
npm run verify:keyed-archive
|
||||
npm run verify:asset-catalog
|
||||
npm run verify:macos-bundle
|
||||
npm run verify:legacy-simulator
|
||||
npm run verify:apple-dispatch
|
||||
npm run verify:interface-builder
|
||||
- name: Run native Apple and process boundary regressions
|
||||
run: >-
|
||||
npm run test:focused --
|
||||
tests/boundary/filesystem/installScript.test.ts
|
||||
tests/boundary/filesystem/nibNativeHierarchy.test.ts
|
||||
tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
tests/boundary/cli/nativePlistCli.test.ts
|
||||
tests/boundary/cli/nativePlistPrototypeKeyCli.test.ts
|
||||
tests/boundary/cli/nativePlistSelectionFailureCli.test.ts
|
||||
tests/boundary/cli/nativeDemangleCli.test.ts
|
||||
tests/boundary/mcp/nativeDemangleMcp.test.ts
|
||||
tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts
|
||||
tests/boundary/cli/nativeSignatureCli.test.ts
|
||||
tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts
|
||||
src/process/capture/FilesystemSnapshot.test.ts
|
||||
tests/boundary/process/darwinProcessRunTokenReader.test.ts
|
||||
tests/boundary/process/directAnalysisSignals.test.ts
|
||||
tests/boundary/process/processCaptureLifecycle.test.ts
|
||||
tests/boundary/process/processCli.test.ts
|
||||
tests/boundary/mcp/sessionMcp.test.ts
|
||||
- run: npm run verify:keyed-archive
|
||||
- run: npm run verify:asset-catalog
|
||||
- run: npm run verify:macos-bundle
|
||||
- name: Legacy simulator platform verification
|
||||
run: npm run verify:legacy-simulator
|
||||
- name: Native installer and compiled NIB hierarchy regressions
|
||||
run: npm run test:focused -- tests/boundary/filesystem/installScript.test.ts tests/boundary/filesystem/nibNativeHierarchy.test.ts tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
- name: Real binary metadata and Interface Builder integration
|
||||
run: npm run verify:apple-dispatch && npm run verify:interface-builder
|
||||
- name: Native plist CLI selection regressions
|
||||
run: npm run test:focused -- tests/boundary/cli/nativePlistCli.test.ts tests/boundary/cli/nativePlistPrototypeKeyCli.test.ts tests/boundary/cli/nativePlistSelectionFailureCli.test.ts
|
||||
- name: Native Swift demangling CLI and MCP regressions
|
||||
run: npm run test:focused -- tests/boundary/cli/nativeDemangleCli.test.ts tests/boundary/mcp/nativeDemangleMcp.test.ts
|
||||
- name: Native signature target binding and CLI/MCP recovery regressions
|
||||
run: npm run test:focused -- tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts tests/boundary/cli/nativeSignatureCli.test.ts tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts
|
||||
- name: Process filesystem snapshot descriptor regressions
|
||||
run: npm run test:focused -- src/process/capture/FilesystemSnapshot.test.ts
|
||||
- name: Native Darwin ownership and process cleanup regressions
|
||||
run: npm run test:focused -- tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/directAnalysisSignals.test.ts tests/boundary/process/processCaptureLifecycle.test.ts tests/boundary/process/processCli.test.ts tests/boundary/mcp/sessionMcp.test.ts
|
||||
|
||||
reconstruction-readiness:
|
||||
name: Reconstruction readiness
|
||||
|
||||
@@ -30,9 +30,5 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
|
||||
python3 -m pip install -r website/requirements.txt
|
||||
- name: Set up Node.js for website script tests
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Check shared HTML, links, assets and contributor workflows
|
||||
run: python3 scripts/website.py check
|
||||
|
||||
@@ -59,15 +59,8 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
|
||||
python3 -m pip install -r website/requirements.txt
|
||||
- name: Set up Node.js for website script tests
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Check shared HTML, links, assets and contributor workflows
|
||||
run: python3 scripts/website.py check
|
||||
# An out-of-date translation only warns in checks; publish it with its notice.
|
||||
- name: Refresh translation notices
|
||||
run: python3 scripts/website.py sync
|
||||
- name: Record the published commit
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
|
||||
+1
-2
@@ -22,8 +22,7 @@
|
||||
],
|
||||
"max-params": ["warn", 4],
|
||||
"no-var": "warn",
|
||||
"no-var-requires": "error",
|
||||
"typescript/no-import-type-side-effects": "error"
|
||||
"no-var-requires": "error"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
{
|
||||
".": "6.4.0"
|
||||
".": "6.3.0"
|
||||
}
|
||||
|
||||
@@ -1,69 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## [6.4.0](https://github.com/morluto/rea/compare/rea-agents-6.3.0...rea-agents-6.4.0) (2026-10-11)
|
||||
|
||||
### Highlights
|
||||
|
||||
- Android: added device inspection, split-APK acquisition, logs, screenshots, file transfers and explicit app lifecycle actions through ADB, plus manifest and localized-string decoding with Apktool. ([#1552](https://github.com/morluto/rea/pull/1552), [#1556](https://github.com/morluto/rea/pull/1556)) Thanks @3QL.
|
||||
- JEB: browse project units and decompile selected items through a caller-started local JEB MCP server. ([#1536](https://github.com/morluto/rea/pull/1536)) Thanks @3QL.
|
||||
- Large analyses: discover retained Evidence without loading every payload, and inspect JavaScript graphs, binary layouts and function dossiers through summaries and selected views. ([#1682](https://github.com/morluto/rea/pull/1682), [#1568](https://github.com/morluto/rea/pull/1568), [#1570](https://github.com/morluto/rea/pull/1570), [#1345](https://github.com/morluto/rea/pull/1345), [#1507](https://github.com/morluto/rea/pull/1507), [#1247](https://github.com/morluto/rea/issues/1247)) Thanks @jhuang-tw, @kaoru0822-kitauji, @N0zoM1z0.
|
||||
- JavaScript: isolate source analysis in an owned process with configurable heap and timeout limits; retain completed facts and explicit unknowns when analysis runs out of resources. ([#1651](https://github.com/morluto/rea/pull/1651), [#1563](https://github.com/morluto/rea/issues/1563), [#1565](https://github.com/morluto/rea/issues/1565)) Thanks @N0zoM1z0, @peter-research.
|
||||
- Ghidra: configure import language/compiler specifications and seed functions, code and labels before auto-analysis. ([#1667](https://github.com/morluto/rea/pull/1667)) Thanks @sqmagellan.
|
||||
- RPC discovery: recognize supported tRPC calls, HTTP-link URLs and named GraphQL operations in JavaScript applications and web bundles. ([#1670](https://github.com/morluto/rea/pull/1670)) Thanks @pompiliodonatoapp.
|
||||
|
||||
### Changes
|
||||
|
||||
- Added Ghidra support for Windows PE DLLs and bounded MIPS ELF analysis, including carved MIPS inputs with incomplete section tables. Windows P0 remains read-only. ([#1303](https://github.com/morluto/rea/pull/1303), [#1362](https://github.com/morluto/rea/pull/1362), [#1629](https://github.com/morluto/rea/pull/1629), [#1290](https://github.com/morluto/rea/issues/1290), [#718](https://github.com/morluto/rea/issues/718), [#1619](https://github.com/morluto/rea/issues/1619)) Thanks @bqw371, @jaelliot, @kvalls, @N0zoM1z0.
|
||||
- Enabled `set_address_name` and `set_addresses_names` for Ghidra sessions with mutation authority, including namespace-aware data labels and per-address batch results. ([#1635](https://github.com/morluto/rea/pull/1635), [#1634](https://github.com/morluto/rea/issues/1634)) Thanks @giri256.
|
||||
- Recovered additional signed AArch64 relative jump-table patterns. ([#1592](https://github.com/morluto/rea/pull/1592)) Thanks @RishTheFish21.
|
||||
- Added PE resource inspection and read-only NativeAOT metadata parsing for supported x64 PE32+/.NET 8 RTR 9.1 inputs. The NativeAOT parser’s Windows-host Ghidra workflow remains unverified on a real Windows host. ([#1482](https://github.com/morluto/rea/pull/1482), [76d8f68](https://github.com/morluto/rea/commit/76d8f68cb918dfb43869f2b649a0b6c0c1cf266d), [#1254](https://github.com/morluto/rea/issues/1254)) Thanks @N0zoM1z0, @thangldw.
|
||||
- Added Pi, Hermes and Qwen Code setup, including Hermes active-profile support. ([#1376](https://github.com/morluto/rea/pull/1376), [#1403](https://github.com/morluto/rea/pull/1403), [#1415](https://github.com/morluto/rea/pull/1415), [#1409](https://github.com/morluto/rea/issues/1409)) Thanks @rasatpetabit, @yiliang114, @zaydiscold.
|
||||
- Added `REA_MCP_INPUT_SCHEMA_PROFILE=compact` for size-capped MCP clients such as Kimi/Moonshot, while retaining canonical runtime validation. ([#1514](https://github.com/morluto/rea/pull/1514), [#1580](https://github.com/morluto/rea/pull/1580), [#1484](https://github.com/morluto/rea/issues/1484), [#1581](https://github.com/morluto/rea/issues/1581)) Thanks @haythemsellami, @N0zoM1z0.
|
||||
- Added live process-capture progress and optional `finalization_ms` to capture shutdown output before timeout escalation; cancellation remains immediate. ([#1483](https://github.com/morluto/rea/pull/1483), [#1583](https://github.com/morluto/rea/pull/1583), [#1252](https://github.com/morluto/rea/issues/1252), [#1253](https://github.com/morluto/rea/issues/1253), [modelcontextprotocol/typescript-sdk#2997](https://github.com/modelcontextprotocol/typescript-sdk/issues/2997)) Thanks @N0zoM1z0, @nikolay-dementiev, @sathariels.
|
||||
- Improved Codex onboarding and recovery from observed client, PATH and provider blockers. ([#1613](https://github.com/morluto/rea/pull/1613)) Thanks @N0zoM1z0.
|
||||
- Reduced redirected JSON and terminal output size, and deferred heavy imports until their commands need them. ([#1413](https://github.com/morluto/rea/pull/1413), [#1450](https://github.com/morluto/rea/pull/1450), [#1439](https://github.com/morluto/rea/pull/1439), [21dbcf7](https://github.com/morluto/rea/commit/21dbcf7a1458219989ad291e44ce7d017378b727)) Thanks @kaoru0822-kitauji.
|
||||
|
||||
### Fixes
|
||||
|
||||
- Electron analysis: follow renamed and default-imported APIs and optional calls, ignore overwritten identities and media preload attributes, and resolve packaged `app.asar` pages and unpacked companions. ([#1391](https://github.com/morluto/rea/pull/1391), [#1393](https://github.com/morluto/rea/pull/1393), [#1400](https://github.com/morluto/rea/pull/1400), [#1664](https://github.com/morluto/rea/pull/1664), [#1673](https://github.com/morluto/rea/pull/1673)) Thanks @kaoru0822-kitauji, @marnixpluim, @N0zoM1z0, @suninrain086.
|
||||
- JavaScript data flow: preserve returned/yielded reference uncertainty, invalidate property-stored aliases correctly, and retain bundled-module runtime scope. ([#1437](https://github.com/morluto/rea/pull/1437), [#1445](https://github.com/morluto/rea/pull/1445), [#1471](https://github.com/morluto/rea/pull/1471), [#1577](https://github.com/morluto/rea/pull/1577), [#1428](https://github.com/morluto/rea/issues/1428), [#1442](https://github.com/morluto/rea/issues/1442), [#1449](https://github.com/morluto/rea/issues/1449), [#1576](https://github.com/morluto/rea/issues/1576)) Thanks @CassieFox23, @dakjdakd, @N0zoM1z0.
|
||||
- JavaScript packages: resolve exact exports subpaths, preserve trailing-slash directory requests and exact CommonJS keys, and respect shadowed `require` bindings. ([#1424](https://github.com/morluto/rea/pull/1424), [#1615](https://github.com/morluto/rea/pull/1615), [5485483](https://github.com/morluto/rea/commit/5485483f7da2f6cb66441bb784aaf2a1a5f8731a), [8b2f851](https://github.com/morluto/rea/commit/8b2f8514c50e469c3add8ce9ba7ebebef888b14f), [f998fcb](https://github.com/morluto/rea/commit/f998fcb1cca9994c3589fb70d6e0127fc215f76f), [1e51c73](https://github.com/morluto/rea/commit/1e51c732bbf08191688d6526bf58ab2650b601e3), [#1365](https://github.com/morluto/rea/issues/1365)) Thanks @akshitkhokhani, @AWUQAQ, @immanuwell, @rudycelekli.
|
||||
- Large JavaScript inputs: handle oversized inline source maps, deep returns and reassignment loops; avoid argument expansion limits and yield during large analysis passes. ([#1386](https://github.com/morluto/rea/pull/1386), [#1368](https://github.com/morluto/rea/pull/1368), [#1390](https://github.com/morluto/rea/pull/1390), [#1539](https://github.com/morluto/rea/pull/1539), [#1525](https://github.com/morluto/rea/pull/1525), [#1542](https://github.com/morluto/rea/pull/1542), [#1545](https://github.com/morluto/rea/pull/1545), [#1557](https://github.com/morluto/rea/pull/1557), [#1380](https://github.com/morluto/rea/issues/1380), [#1401](https://github.com/morluto/rea/issues/1401), [#1538](https://github.com/morluto/rea/issues/1538), [#1462](https://github.com/morluto/rea/issues/1462)) Thanks @cold-atom, @KadirJeager, @kaoru0822-kitauji, @marnixpluim, @N0zoM1z0, @rudycelekli.
|
||||
- JavaScript receivers: expand escaped methods once, index returned-object targets, and avoid replaying earlier writes or duplicating retained graph allocations. ([#1517](https://github.com/morluto/rea/pull/1517), [#1537](https://github.com/morluto/rea/pull/1537), [#1444](https://github.com/morluto/rea/pull/1444), [#1601](https://github.com/morluto/rea/pull/1601), [#1495](https://github.com/morluto/rea/issues/1495)) Thanks @CassieFox23, @geekwolf, @kaoru0822-kitauji, @N0zoM1z0, @zeynelacikgoez.
|
||||
- Browser captures: retain WebSocket URLs, empty connections and frame order; compare only common known payload coverage and preserve incomplete WebMCP replay coverage. ([db4e360](https://github.com/morluto/rea/commit/db4e3608465486466591b594cf77037ac150c988), [aa257ea](https://github.com/morluto/rea/commit/aa257ea4b16f2d2a35c12df63359d650b48b1154), [e52c0a7](https://github.com/morluto/rea/commit/e52c0a72c59c740315e89149881a27ce4d89a11c))
|
||||
- Browser scenarios: seed storage in the correct partition, normalize loopback attach identities, retain declared-secret redaction and require verified lineage before Windows Electron signals. ([#1604](https://github.com/morluto/rea/pull/1604), [#1590](https://github.com/morluto/rea/pull/1590), [#1588](https://github.com/morluto/rea/pull/1588), [8b4c3de](https://github.com/morluto/rea/commit/8b4c3de3b3a3b580adec266f94f29ccedf1da228), [#1497](https://github.com/morluto/rea/issues/1497), [#1591](https://github.com/morluto/rea/issues/1591), [#1589](https://github.com/morluto/rea/issues/1589)) Thanks @Adolanium, @akushonkamen, @dakjdakd.
|
||||
- Added Termux Android browser-scenario support. ([#1655](https://github.com/morluto/rea/pull/1655)) Thanks @tyloryang.
|
||||
- Web inputs: accept BOM-prefixed HAR, preserve the first duplicate DOM attribute and resolve SVG/XLink URLs in HTML and XHTML. ([#1509](https://github.com/morluto/rea/pull/1509), [#1516](https://github.com/morluto/rea/pull/1516), [#1602](https://github.com/morluto/rea/pull/1602), [#1683](https://github.com/morluto/rea/pull/1683), [#1496](https://github.com/morluto/rea/issues/1496), [#1586](https://github.com/morluto/rea/issues/1586)) Thanks @CassieFox23, @kaoru0822-kitauji, @sathariels, @taljeri.
|
||||
- Screenshot comparisons: validate PNG chunks, distinguish corruption from unsupported encodings and bound comparison memory. ([4e4b15b](https://github.com/morluto/rea/commit/4e4b15b8250b29e176d4191c9610a3931b693474), [b3e977e](https://github.com/morluto/rea/commit/b3e977e6d073ab1783cece8581279f87fc360411)) Thanks @rudycelekli.
|
||||
- Archives: preserve case-distinct names, report collisions at their diverging segment, detect nested formats from bytes and reject Windows alternate-stream extraction paths. ([#1480](https://github.com/morluto/rea/pull/1480), [#1505](https://github.com/morluto/rea/pull/1505), [#1553](https://github.com/morluto/rea/pull/1553), [573598c](https://github.com/morluto/rea/commit/573598c6324698cc48ceafd24ed3864670e08d00), [#1470](https://github.com/morluto/rea/issues/1470), [#1491](https://github.com/morluto/rea/issues/1491)) Thanks @CassieFox23, @kaoru0822-kitauji.
|
||||
- Cleanup: retain failed readers, ASAR handles, output trees and provider workers for retry without discarding completed output or partial observations. ([ffe0574](https://github.com/morluto/rea/commit/ffe0574e17b9d973860371083cb1a351be912131), [de0d341](https://github.com/morluto/rea/commit/de0d341524dc68250a45b41eb7e49fcecc1e32a9), [3f8c51e](https://github.com/morluto/rea/commit/3f8c51e9b506d88fc41aded7b1a22b9a09037c26), [d1235fa](https://github.com/morluto/rea/commit/d1235fadc901e1fb9ab3149d53063306c8545414), [026126e](https://github.com/morluto/rea/commit/026126eafe741e5a8b857433587fd615cdd44c19))
|
||||
- Providers: canonicalize IDA workspace roots before identity checks, respect IDA cancellation, serialize JEB acquisition and correct Ghidra no-return/imported-symbol recovery. ([#1481](https://github.com/morluto/rea/pull/1481), [#1419](https://github.com/morluto/rea/pull/1419), [5f30ea6](https://github.com/morluto/rea/commit/5f30ea6ff2dd89aab7af6da6e8dfa5e508ed459b), [f88c990](https://github.com/morluto/rea/commit/f88c99021b88ce67e513025901f57478d8107d0c), [49e6f00](https://github.com/morluto/rea/commit/49e6f00ed08b225abf9ddb7c762553607fdab115)) Thanks @AS9929, @JonahSimon, @Y0UW1SH.
|
||||
- Binary metadata: respect PE/.NET section extents, bound binary-plist expansion, scope keyed-archive traversal and decode app `Info.plist` files without `plutil`. ([#1678](https://github.com/morluto/rea/pull/1678), [#1515](https://github.com/morluto/rea/pull/1515), [#1378](https://github.com/morluto/rea/pull/1378), [#1476](https://github.com/morluto/rea/pull/1476), [975dff2](https://github.com/morluto/rea/commit/975dff2e25b14b634e14dd5e0db1537d0ca9815b), [76296c5](https://github.com/morluto/rea/commit/76296c5a3868ba3b54be6ed3896719283d6de5f8)) Thanks @jayhemnani9910, @kaoru0822-kitauji, @rudycelekli.
|
||||
- Windows runtimes: preserve drive-root paths when creating private provider directories. ([ee46323](https://github.com/morluto/rea/commit/ee46323b493994902b04824a7dc8aae2a8117903))
|
||||
- Compatibility: load Windows native controls under Node-API hosts with renamed executables, preserve dylib directory requirements, support Wakaru 1.x and reject directory-valued Hopper launchers. ([#1531](https://github.com/morluto/rea/pull/1531), [#1623](https://github.com/morluto/rea/pull/1623), [#1540](https://github.com/morluto/rea/pull/1540), [#1426](https://github.com/morluto/rea/pull/1426)) Thanks @evenafterallisaid, @Fazmin, @pionxzh, @xeropresence.
|
||||
- Reference imports: normalize internal symlink lookup consistently, preserve exact paths and exclusion provenance, report root access failures and reject FIFO replacements without blocking. ([#1535](https://github.com/morluto/rea/pull/1535), [2ca3ad2](https://github.com/morluto/rea/commit/2ca3ad238608adfca80a7ff1aaadea6e59a58225), [6636045](https://github.com/morluto/rea/commit/6636045c9c5b472bba51afee9d434394861e93a4), [3967cfa](https://github.com/morluto/rea/commit/3967cfa43e786c65a57e89c265343983d19ef9d0), [f1c58cc](https://github.com/morluto/rea/commit/f1c58cc53fbd83b968e11b3e2eccebca51299c54)) Thanks @anujsingh-cse.
|
||||
- Setup: honor active configuration and skill roots, preserve configured MCP environments, validate Hermes/OMP profiles before writing, recognize Omarchy and retain recovery backups. ([#1504](https://github.com/morluto/rea/pull/1504), [#1607](https://github.com/morluto/rea/pull/1607), [#1528](https://github.com/morluto/rea/pull/1528), [#1562](https://github.com/morluto/rea/pull/1562), [#1662](https://github.com/morluto/rea/pull/1662), [a011485](https://github.com/morluto/rea/commit/a01148553e2a37c72ae78405418ab648876a96a6), [#1606](https://github.com/morluto/rea/issues/1606), [#1521](https://github.com/morluto/rea/issues/1521), [#1561](https://github.com/morluto/rea/issues/1561)) Thanks @Adkid-Zephyr, @CassieFox23, @LuisTaffelli, @N0zoM1z0, @peter-research.
|
||||
- Process capture: retain descendant lineage after launcher exit, preserve identity uncertainty before signaling and release terminals after observation failures. ([80896a1](https://github.com/morluto/rea/commit/80896a1d64b216ef893907249cd4252f86215d49), [c916850](https://github.com/morluto/rea/commit/c9168508527302ac57bc86385ccf0d824697b5dc)) Thanks @Y0UW1SH.
|
||||
- CLI/MCP errors: accept JSON files for web comparisons, preserve native option help and union correction details, remove invalid aliases and validate doctor against its discovered output contract. ([#1397](https://github.com/morluto/rea/pull/1397), [#1395](https://github.com/morluto/rea/pull/1395), [#1425](https://github.com/morluto/rea/pull/1425), [#1461](https://github.com/morluto/rea/pull/1461), [8b4c96e](https://github.com/morluto/rea/commit/8b4c96e6c6b3ed6806e4cd72b556b5db6114304a), [#1421](https://github.com/morluto/rea/issues/1421), [#1454](https://github.com/morluto/rea/issues/1454)) Thanks @Fazmin, @kaoru0822-kitauji, @N0zoM1z0.
|
||||
|
||||
### ⚠ BREAKING CHANGES
|
||||
|
||||
- WebSocket results: use `network.websocket_connections` instead of `network.websocket_events`. Connections contain `request_id`, `url` and `events`; allow `payload_bytes: null` when size is unknown. No legacy alias remains. ([db4e360](https://github.com/morluto/rea/commit/db4e3608465486466591b594cf77037ac150c988))
|
||||
- Direct resource APIs: pass a caller-owned `ArtifactResourceScope` to inventory/extraction, JavaScript analysis, reference imports and web exports; retain it through cleanup/shutdown. For `readJavaScriptArtifactFiles`, scope is argument three and abort signal is argument four. Keep each root inventory source’s single cleanup owner. Use `OwnedFileHandle` owners, injected ASAR `fd`/`read` handles instead of `createReadStream`, VCS `Result` values, and `WebScriptExportError`’s options object. CLI/MCP inputs for the scoped operations are unchanged. ([ffe0574](https://github.com/morluto/rea/commit/ffe0574e17b9d973860371083cb1a351be912131), [53e7fad](https://github.com/morluto/rea/commit/53e7fadbd078422ba3d55b5499a70d237c5f4908), [684199b](https://github.com/morluto/rea/commit/684199ba0be7822266d57ddb8a40e95f087fdc42), [be75ed5](https://github.com/morluto/rea/commit/be75ed5e4b4bfb146af497f8fa5d172a1ea19462), [be6272f](https://github.com/morluto/rea/commit/be6272f26115bd8e94f2ab0df4cf092004587a18), [e0e49e9](https://github.com/morluto/rea/commit/e0e49e9dd53d9d1f1d2330b23f7f880dc1c81774), [95a77b6](https://github.com/morluto/rea/commit/95a77b6e25420b3879bd0e32bdbb14d452b38776))
|
||||
- Saved reference graphs: exclusions require their matching `pattern` and distinguish `project-ignored` from `default-ignored`. Re-import graphs missing that provenance. ([6585293](https://github.com/morluto/rea/commit/6585293c562853b95c3a70df02c12ad8cc17afc4))
|
||||
- Terminal captures: keep initial and resized buffers within 1,000,000 cells using `columns × (rows + scrollback)`; oversized scenarios are rejected before launch. ([f9ea9a9](https://github.com/morluto/rea/commit/f9ea9a9268c854d31048da9f8a50cfa37703fe5a))
|
||||
- Optional agent-verification reports: accept schema version 4, nullable `expectedFirstTool` and `targetClarificationPassed`. Production CLI/MCP contracts are unchanged. ([#1613](https://github.com/morluto/rea/pull/1613))
|
||||
- Capture comparisons: recapture both sides with matching `literal-replacements-v1` semantics when custom normalization patterns are used. Legacy unversioned pattern captures remain readable but cannot be compared with new pattern captures; captures without patterns are unaffected. ([#1456](https://github.com/morluto/rea/pull/1456), [2b6f14f](https://github.com/morluto/rea/commit/2b6f14f528297cf12c776c960af0859f5cfbefbf)) Thanks @Madhavan006, @N0zoM1z0.
|
||||
- Artifact graph edges: read child artifact IDs and paths from the occurrence selected by `occurrence_id`, not edge fields `child_artifact_id`/`logical_path`. Earlier inventory Evidence no longer validates against the new commitment. ([#1493](https://github.com/morluto/rea/pull/1493)) Thanks @kaoru0822-kitauji.
|
||||
- Semantic graph schema: nodes and relations no longer include `identifier_strategy`, and relation IDs no longer hash it. Earlier Evidence no longer validates against this schema. ([#1474](https://github.com/morluto/rea/pull/1474)) Thanks @kaoru0822-kitauji.
|
||||
- Semantic identities: select IDs from the parent graph; read exact event names from `event_name`, endpoints from `endpoint` and literal values from `properties.value`. New analyses use digest-based IDs for long/reserved event and endpoint names and `value-sha256` literal role keys, with shorter labels; string literal labels now read `string literal`. Previously stored graphs retain their original IDs and remain valid workflow inputs. ([#1427](https://github.com/morluto/rea/pull/1427), [#1423](https://github.com/morluto/rea/pull/1423), [#1407](https://github.com/morluto/rea/pull/1407)) Thanks @N0zoM1z0.
|
||||
|
||||
### Contributors
|
||||
|
||||
- Thanks also to @BaaRai, @commit-regrets, @coygeek, @Franky100-pig, @inii-sayaa, @jopemachine, @JosephDemarest, @keemsisi, @Totopo27 for contributions included in this release.
|
||||
|
||||
Full contribution record: [336 commits and 163 merged PRs](https://github.com/morluto/rea/blob/rea-agents-6.4.0/docs/releases/6.4.0.contributions.json).
|
||||
|
||||
## [6.3.0](https://github.com/morluto/rea/compare/rea-agents-6.2.0...rea-agents-6.3.0) (2026-10-09)
|
||||
|
||||
|
||||
|
||||
+183
-388
@@ -2004,20 +2004,9 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
);
|
||||
JumpTable.LoadTable[] loadTables = jumpTable.getLoadTables();
|
||||
RecoveredAArch64RelativeTable recoveredAArch64 =
|
||||
recoverAArch64RelativeTable(jumpTable);
|
||||
if (recoveredAArch64 != null && loadTables.length > 0) {
|
||||
// A decoded table only adds evidence when Ghidra's own load table
|
||||
// describes the same bytes; any disagreement keeps Ghidra's view.
|
||||
boolean agrees = false;
|
||||
for (JumpTable.LoadTable loadTable : loadTables) {
|
||||
if (loadTable.getAddress().equals(recoveredAArch64.tableAddress) &&
|
||||
loadTable.getSize() == recoveredAArch64.entrySize &&
|
||||
loadTable.getNum() == recoveredAArch64.mappings.size()) {
|
||||
agrees = true;
|
||||
}
|
||||
}
|
||||
if (!agrees) recoveredAArch64 = null;
|
||||
}
|
||||
loadTables.length == 0
|
||||
? recoverAArch64RelativeTable(jumpTable)
|
||||
: null;
|
||||
List<Address> referencedDataAddresses = loadTables.length == 0
|
||||
? dispatchPathDataReferences(jumpTable)
|
||||
: List.of();
|
||||
@@ -2051,7 +2040,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
source.add("evidence", evidence);
|
||||
dataSources.add(source);
|
||||
}
|
||||
if (recoveredAArch64 != null && loadTables.length == 0) {
|
||||
if (recoveredAArch64 != null) {
|
||||
JsonObject source = new JsonObject();
|
||||
source.addProperty(
|
||||
"address",
|
||||
@@ -2066,16 +2055,9 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
inferenceEvidence(
|
||||
"jump-table",
|
||||
"ghidra-instruction-and-memory",
|
||||
"Decoded " + recoveredAArch64.mappings.size() + " " +
|
||||
(recoveredAArch64.signedEntries ? "signed" : "unsigned") +
|
||||
"Decoded " + recoveredAArch64.mappings.size() +
|
||||
" entries of size " + recoveredAArch64.entrySize + " bytes from " + canonicalAddress(recoveredAArch64.tableAddress) +
|
||||
"; the AArch64 ADRP/ADD table, indexed load, ADR base, ADD" +
|
||||
(recoveredAArch64.offsetShift == 0 ? "" : " LSL #" + recoveredAArch64.offsetShift) +
|
||||
"/BR chain uses a selector bounded by CMP #" + recoveredAArch64.compareImmediate +
|
||||
" and " + recoveredAArch64.guardMnemonic + " on one straight-line path" +
|
||||
(recoveredAArch64.nullTarget == null
|
||||
? "."
|
||||
: "; zero entries are routed by CBZ to " + canonicalAddress(recoveredAArch64.nullTarget) + ".")
|
||||
"; the AArch64 LDRB-or-LDRH/ADR/ADD/BR chain uses a selector guarded by an unsigned upper-bound branch."
|
||||
)
|
||||
);
|
||||
source.add("evidence", evidence);
|
||||
@@ -2114,33 +2096,6 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
result.add("data_sources", dataSources);
|
||||
Address[] targets = jumpTable.getCases();
|
||||
TypedJumpTableCases typedCases = typedJumpTableCases(jumpTable, caseTokens);
|
||||
// Table indices identify the register bounded by CMP. Typed labels may
|
||||
// instead identify a selector before normalization (for example, SUB
|
||||
// index, selector, #10). Do not merge those distinct value identities.
|
||||
if (recoveredAArch64 != null && recoveredAArch64.priorSelectorDefinition != null) {
|
||||
typedCases.limitations.add(
|
||||
"REA decoded the relative table, but the compared selector has an earlier register definition at " +
|
||||
canonicalAddress(recoveredAArch64.priorSelectorDefinition) + "; its table indices are not proven to use the typed switch selector's case values. Ghidra's typed labels are retained and additional case labels remain unknown."
|
||||
);
|
||||
recoveredAArch64 = null;
|
||||
}
|
||||
if (recoveredAArch64 != null) {
|
||||
for (Map.Entry<Long, Address> typed : typedCases.cases.entrySet()) {
|
||||
long value = typed.getKey();
|
||||
if (value < 0 || value >= recoveredAArch64.mappings.size()) {
|
||||
typedCases.limitations.add("Typed case labels and decoded table indices do not use a verified common selector; Ghidra's typed labels are retained.");
|
||||
recoveredAArch64 = null;
|
||||
break;
|
||||
}
|
||||
RecoveredAArch64Mapping decoded = recoveredAArch64.mappings.get((int) value);
|
||||
boolean divertedZero = decoded.nullEntry && typed.getValue().equals(recoveredAArch64.unreachableZeroTarget);
|
||||
if (!typed.getValue().equals(decoded.targetAddress) && !divertedZero) {
|
||||
typedCases.limitations.add("Typed case evidence and the decoded relative table disagree on a destination; Ghidra's typed labels are retained.");
|
||||
recoveredAArch64 = null;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
JsonArray mappings = new JsonArray();
|
||||
JsonArray defaultTargets = new JsonArray();
|
||||
for (Address target : typedCases.defaults) {
|
||||
@@ -2148,30 +2103,27 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
}
|
||||
Set<Address> coveredTargets = new HashSet<>(typedCases.cases.values());
|
||||
coveredTargets.addAll(typedCases.defaults);
|
||||
if (recoveredAArch64 != null && recoveredAArch64.unreachableZeroTarget != null) {
|
||||
Address unreachable = recoveredAArch64.unreachableZeroTarget;
|
||||
coveredTargets.add(unreachable);
|
||||
for (RecoveredAArch64Mapping recovered : recoveredAArch64.mappings) {
|
||||
if (recovered.nullEntry && unreachable.equals(typedCases.cases.get((long) recovered.tableIndex))) {
|
||||
typedCases.cases.remove((long) recovered.tableIndex);
|
||||
}
|
||||
}
|
||||
typedCases.limitations.add(
|
||||
"Ghidra recovered destination " + canonicalAddress(unreachable) +
|
||||
" as the zero-offset decode of the relative table, but the CBZ before the dispatch diverts zero entries to " +
|
||||
canonicalAddress(recoveredAArch64.nullTarget) + ", so the indirect branch never reaches it for an in-range case."
|
||||
);
|
||||
}
|
||||
if (recoveredAArch64 != null) {
|
||||
for (int index = 0; index < recoveredAArch64.mappings.size(); index += 1) {
|
||||
monitor.checkCancelled();
|
||||
RecoveredAArch64Mapping recovered = recoveredAArch64.mappings.get(index);
|
||||
if (
|
||||
typedCases.unknownTargets.contains(recovered.targetAddress) ||
|
||||
typedCases.ambiguousValues.contains((long) recovered.tableIndex)
|
||||
typedCases.ambiguousValues.contains((long) recovered.caseValue)
|
||||
) continue;
|
||||
Address typedTarget = typedCases.cases.get((long) recovered.caseValue);
|
||||
if (typedTarget != null && !typedTarget.equals(recovered.targetAddress)) {
|
||||
typedCases.cases.remove((long) recovered.caseValue);
|
||||
typedCases.ambiguousValues.add((long) recovered.caseValue);
|
||||
typedCases.unknownTargets.add(typedTarget);
|
||||
typedCases.unknownTargets.add(recovered.targetAddress);
|
||||
typedCases.limitations.add(
|
||||
"Typed case evidence and the verified relative table disagree on a destination; both destinations remain unknown for that case."
|
||||
);
|
||||
continue;
|
||||
}
|
||||
JsonObject mapping = new JsonObject();
|
||||
mapping.addProperty("case_value", recovered.tableIndex);
|
||||
mapping.addProperty("case_value", recovered.caseValue);
|
||||
String targetAddress = canonicalAddress(recovered.targetAddress);
|
||||
mapping.addProperty("target_address", targetAddress);
|
||||
mapping.addProperty("confidence", "high");
|
||||
@@ -2180,18 +2132,14 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
inferenceEvidence(
|
||||
"jump-table",
|
||||
"ghidra-instruction-and-memory",
|
||||
"Case value " + recovered.tableIndex + (recoveredAArch64.entrySize == 1 ? " indexes byte " : " indexes halfword at byte offset ") +
|
||||
(recovered.tableIndex * recoveredAArch64.entrySize) + " at " + canonicalAddress(recoveredAArch64.tableAddress) +
|
||||
(recovered.nullEntry
|
||||
? "; the entry is zero, so the CBZ before the dispatch branches to " + targetAddress + "."
|
||||
: "; the loaded " + (recoveredAArch64.signedEntries ? "signed" : "unsigned") +
|
||||
" entry " + recovered.encodedEntry + " selects a " +
|
||||
(recoveredAArch64.offsetShift == 2 ? "four-byte instruction" : "byte") +
|
||||
" offset from " + canonicalAddress(recoveredAArch64.branchBase) + ", reaching " + targetAddress + ".")
|
||||
"Case value " + recovered.caseValue + (recoveredAArch64.entrySize == 1 ? " indexes byte " : " indexes halfword at byte offset ") +
|
||||
(recovered.caseValue * recoveredAArch64.entrySize) + " at " + canonicalAddress(recoveredAArch64.tableAddress) +
|
||||
"; the loaded unsigned entry selects a four-byte instruction offset from " +
|
||||
canonicalAddress(recoveredAArch64.branchBase) + ", reaching " + targetAddress + "."
|
||||
)
|
||||
);
|
||||
mapping.add("evidence", evidence);
|
||||
if (!typedCases.cases.containsKey((long) recovered.tableIndex)) {
|
||||
if (!typedCases.cases.containsKey((long) recovered.caseValue)) {
|
||||
mappings.add(mapping);
|
||||
}
|
||||
coveredTargets.add(recovered.targetAddress);
|
||||
@@ -2265,7 +2213,7 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
);
|
||||
}
|
||||
if (recoveredAArch64 == null && loadTables.length == 0 && currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:")) {
|
||||
limitations.add("REA did not verify a supported signed or unsigned byte/halfword relative dispatch chain with exact unsigned bounds and register definitions; unsupported encodings and ambiguous bounds remain unknown.");
|
||||
limitations.add("REA did not verify a supported byte/halfword relative dispatch chain with exact unsigned bounds and register definitions; unsupported encodings and ambiguous bounds remain unknown.");
|
||||
}
|
||||
if (targets.length == 0) {
|
||||
limitations.add(
|
||||
@@ -2395,21 +2343,12 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
}
|
||||
|
||||
private static final class RecoveredAArch64Mapping {
|
||||
private final int tableIndex;
|
||||
private final int caseValue;
|
||||
private final Address targetAddress;
|
||||
private final long encodedEntry;
|
||||
private final boolean nullEntry;
|
||||
|
||||
private RecoveredAArch64Mapping(
|
||||
int tableIndex,
|
||||
Address targetAddress,
|
||||
long encodedEntry,
|
||||
boolean nullEntry
|
||||
) {
|
||||
this.tableIndex = tableIndex;
|
||||
private RecoveredAArch64Mapping(int caseValue, Address targetAddress) {
|
||||
this.caseValue = caseValue;
|
||||
this.targetAddress = targetAddress;
|
||||
this.encodedEntry = encodedEntry;
|
||||
this.nullEntry = nullEntry;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2417,70 +2356,24 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
private final Address tableAddress;
|
||||
private final Address branchBase;
|
||||
private final int entrySize;
|
||||
private final Address priorSelectorDefinition;
|
||||
private final boolean signedEntries;
|
||||
private final int offsetShift;
|
||||
private final String guardMnemonic;
|
||||
private final long compareImmediate;
|
||||
private final Address nullTarget;
|
||||
private final Address unreachableZeroTarget;
|
||||
private final List<RecoveredAArch64Mapping> mappings;
|
||||
|
||||
private RecoveredAArch64RelativeTable(
|
||||
Address tableAddress,
|
||||
Address branchBase,
|
||||
int entrySize,
|
||||
Address priorSelectorDefinition,
|
||||
boolean signedEntries,
|
||||
int offsetShift,
|
||||
String guardMnemonic,
|
||||
long compareImmediate,
|
||||
Address nullTarget,
|
||||
Address unreachableZeroTarget,
|
||||
List<RecoveredAArch64Mapping> mappings
|
||||
) {
|
||||
this.tableAddress = tableAddress;
|
||||
this.branchBase = branchBase;
|
||||
this.entrySize = entrySize;
|
||||
this.priorSelectorDefinition = priorSelectorDefinition;
|
||||
this.signedEntries = signedEntries;
|
||||
this.offsetShift = offsetShift;
|
||||
this.guardMnemonic = guardMnemonic;
|
||||
this.compareImmediate = compareImmediate;
|
||||
this.nullTarget = nullTarget;
|
||||
this.unreachableZeroTarget = unreachableZeroTarget;
|
||||
this.mappings = mappings;
|
||||
}
|
||||
}
|
||||
|
||||
private static final Pattern AARCH64_DISPATCH_ADD = Pattern.compile(
|
||||
"(?i)^([wx]\\d+),\\s*([wx]\\d+),\\s*([wx]\\d+)(?:,\\s*LSL\\s*#(?:0x)?([0-9a-f]+))?$"
|
||||
);
|
||||
private static final Pattern AARCH64_TABLE_LOAD = Pattern.compile(
|
||||
"(?i)^([wx]\\d+),\\s*\\[([wx]\\d+),\\s*([wx]\\d+)(?:,\\s*(LSL|UXTW)\\s*(?:#(?:0x)?([0-9a-f]+))?)?\\s*\\]$"
|
||||
);
|
||||
private static final Pattern AARCH64_REGISTER_AND_ADDRESS = Pattern.compile(
|
||||
"(?i)^([wx]\\d+),\\s*(0x[0-9a-f]+)$"
|
||||
);
|
||||
private static final Pattern AARCH64_ADD_IMMEDIATE = Pattern.compile(
|
||||
"(?i)^([wx]\\d+),\\s*([wx]\\d+),\\s*#?(0x[0-9a-f]+|[0-9]+)$"
|
||||
);
|
||||
private static final Pattern AARCH64_COMPARE_IMMEDIATE = Pattern.compile(
|
||||
"(?i)^([wx]\\d+),\\s*#?(0x[0-9a-f]+|[0-9]+)$"
|
||||
);
|
||||
private static final Pattern AARCH64_MOVE_REGISTER = Pattern.compile(
|
||||
"(?i)^([wx]\\d+),\\s*([wx]\\d+)$"
|
||||
);
|
||||
private static final Set<String> AARCH64_FLAG_PRESERVING =
|
||||
Set.of("MOV", "ADRP", "ADR", "ADD");
|
||||
|
||||
/**
|
||||
* Recover only directly verified AArch64 relative branch tables:
|
||||
* ADRP/ADD table base, LDRB/LDRH/LDRSB/LDRSH entry load with an exact
|
||||
* index scale, ADR branch base, optional CBZ on the loaded entry, and
|
||||
* ADD (byte or LSL #2 scale)/BR, with a selector bounded by CMP #imm and an
|
||||
* unsigned B.HI or B.CS/B.HS guard on one straight-line path. Any other
|
||||
* instruction sequence keeps its case labels unknown.
|
||||
* Recover only the directly verified AArch64 byte/halfword-indexed relative branch
|
||||
* table form. Other instruction sequences keep their case labels unknown.
|
||||
*/
|
||||
private RecoveredAArch64RelativeTable recoverAArch64RelativeTable(
|
||||
JumpTable jumpTable
|
||||
@@ -2502,265 +2395,167 @@ public final class ReaGhidraBridge extends HeadlessScript {
|
||||
if (instruction.getAddress().equals(jumpTable.getSwitchAddress())) break;
|
||||
}
|
||||
int branchIndex = instructions.size() - 1;
|
||||
if (branchIndex < 1 || !mnemonic(instructions.get(branchIndex)).equals("BR")) {
|
||||
if (branchIndex < 1 ||
|
||||
!instructions.get(branchIndex).getMnemonicString().equalsIgnoreCase("BR")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
Address[] recoveredTargets = jumpTable.getCases();
|
||||
Set<String> recoveredTargetSet = new HashSet<>();
|
||||
for (Address target : jumpTable.getCases()) recoveredTargetSet.add(canonicalAddress(target));
|
||||
if (recoveredTargetSet.isEmpty()) return null;
|
||||
return matchAArch64RelativeDispatch(
|
||||
instructions,
|
||||
branchIndex,
|
||||
recoveredTargetSet,
|
||||
dispatchPathDataReferences(jumpTable)
|
||||
);
|
||||
}
|
||||
for (Address target : recoveredTargets) recoveredTargetSet.add(canonicalAddress(target));
|
||||
List<Address> dataAddresses = dispatchPathDataReferences(jumpTable);
|
||||
|
||||
private RecoveredAArch64RelativeTable matchAArch64RelativeDispatch(
|
||||
List<Instruction> instructions,
|
||||
int branchIndex,
|
||||
Set<String> recoveredTargetSet,
|
||||
List<Address> dataAddresses
|
||||
) throws Exception {
|
||||
String branchRegister = normalizedRegister(operands(instructions.get(branchIndex)));
|
||||
for (int loadIndex = 0; loadIndex < branchIndex; loadIndex += 1) {
|
||||
Instruction load = instructions.get(loadIndex);
|
||||
int entrySize = load.getMnemonicString().equalsIgnoreCase("LDRB") ? 1 : load.getMnemonicString().equalsIgnoreCase("LDRH") ? 2 : 0;
|
||||
if (entrySize == 0 || branchIndex != loadIndex + 2 || loadIndex < 3 ||
|
||||
!instructions.get(loadIndex - 1).getMnemonicString().equalsIgnoreCase("ADR") ||
|
||||
!instructions.get(loadIndex - 2).getMnemonicString().equalsIgnoreCase("ADD") ||
|
||||
!instructions.get(loadIndex - 3).getMnemonicString().equalsIgnoreCase("ADRP")) continue;
|
||||
String loadOperands = operands(load);
|
||||
java.util.regex.Matcher loadMatcher = java.util.regex.Pattern
|
||||
.compile(entrySize == 1 ? "(?i)^[wx]\\d+,\\s*\\[([wx]\\d+),\\s*([wx]\\d+)\\]$" : "(?i)^[wx]\\d+,\\s*\\[([wx]\\d+),\\s*([wx]\\d+),\\s*LSL\\s*#(?:0x)?1\\]$")
|
||||
.matcher(loadOperands);
|
||||
if (!loadMatcher.matches()) continue;
|
||||
String loadedRegister = normalizedRegister(loadOperands.substring(0, loadOperands.indexOf(',')));
|
||||
String tableBaseRegister = normalizedRegister(loadMatcher.group(1));
|
||||
String indexRegister = normalizedRegister(loadMatcher.group(2));
|
||||
|
||||
// ADD Xd, Xn, Xm{, LSL #s}; BR Xd. LSL applies to Xm only.
|
||||
Instruction dispatchAdd = instructions.get(branchIndex - 1);
|
||||
if (!mnemonic(dispatchAdd).equals("ADD")) return null;
|
||||
java.util.regex.Matcher addMatcher = AARCH64_DISPATCH_ADD.matcher(operands(dispatchAdd));
|
||||
if (!addMatcher.matches() || !addMatcher.group(1).toLowerCase(Locale.ROOT).startsWith("x")) return null;
|
||||
if (!normalizedRegister(addMatcher.group(1)).equals(branchRegister)) return null;
|
||||
int offsetShift = addMatcher.group(4) == null ? 0 : Integer.parseInt(addMatcher.group(4), 16);
|
||||
if (offsetShift != 0 && offsetShift != 2) return null;
|
||||
String firstSource = normalizedRegister(addMatcher.group(2));
|
||||
String secondSource = normalizedRegister(addMatcher.group(3));
|
||||
if (firstSource.equals(secondSource)) return null;
|
||||
|
||||
// The entry register is the source whose latest definition is the
|
||||
// table load; the other source must be defined by ADR.
|
||||
int loadIndex = -1;
|
||||
String entryRegister = null;
|
||||
String baseRegister = null;
|
||||
for (String candidate : offsetShift == 0 ? List.of(secondSource, firstSource) : List.of(secondSource)) {
|
||||
int writer = latestWriter(instructions, branchIndex - 1, candidate);
|
||||
if (writer >= 0 && Set.of("LDRB", "LDRH", "LDRSB", "LDRSH").contains(mnemonic(instructions.get(writer)))) {
|
||||
loadIndex = writer;
|
||||
entryRegister = candidate;
|
||||
baseRegister = candidate.equals(firstSource) ? secondSource : firstSource;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (loadIndex < 0) return null;
|
||||
|
||||
Instruction load = instructions.get(loadIndex);
|
||||
String loadMnemonic = mnemonic(load);
|
||||
int entrySize = loadMnemonic.endsWith("B") ? 1 : 2;
|
||||
boolean signedEntries = loadMnemonic.startsWith("LDRS");
|
||||
java.util.regex.Matcher loadMatcher = AARCH64_TABLE_LOAD.matcher(operands(load));
|
||||
if (!loadMatcher.matches()) return null;
|
||||
// A signed load into a W register sign-extends only to 32 bits; the
|
||||
// 64-bit dispatch ADD would then read a zero-extended value.
|
||||
if (signedEntries && !loadMatcher.group(1).toLowerCase(Locale.ROOT).startsWith("x")) return null;
|
||||
String tableBaseRegister = normalizedRegister(loadMatcher.group(2));
|
||||
String indexRegister = normalizedRegister(loadMatcher.group(3));
|
||||
boolean wordIndex = loadMatcher.group(3).toLowerCase(Locale.ROOT).startsWith("w");
|
||||
String extend = loadMatcher.group(4) == null ? null : loadMatcher.group(4).toUpperCase(Locale.ROOT);
|
||||
int indexShift = loadMatcher.group(5) == null ? 0 : Integer.parseInt(loadMatcher.group(5), 16);
|
||||
if (wordIndex ? !"UXTW".equals(extend) : "UXTW".equals(extend)) return null;
|
||||
if (indexShift != (entrySize == 1 ? 0 : 1)) return null;
|
||||
|
||||
// Only ADR and one CBZ on the entry may sit between load and ADD.
|
||||
Address nullTarget = null;
|
||||
for (int index = loadIndex + 1; index < branchIndex - 1; index += 1) {
|
||||
Instruction instruction = instructions.get(index);
|
||||
String name = mnemonic(instruction);
|
||||
if (name.equals("ADR")) continue;
|
||||
if (name.equals("CBZ") && nullTarget == null) {
|
||||
String[] parts = operands(instruction).split(",");
|
||||
Address[] flows = instruction.getFlows();
|
||||
if (parts.length != 2 || flows.length != 1 ||
|
||||
!normalizedRegister(parts[0]).equals(entryRegister)) return null;
|
||||
nullTarget = flows[0];
|
||||
continue;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
int baseIndex = latestWriter(instructions, branchIndex - 1, baseRegister);
|
||||
if (baseIndex < 0 || !mnemonic(instructions.get(baseIndex)).equals("ADR")) return null;
|
||||
java.util.regex.Matcher baseMatcher = AARCH64_REGISTER_AND_ADDRESS.matcher(operands(instructions.get(baseIndex)));
|
||||
if (!baseMatcher.matches()) return null;
|
||||
Address branchBase = addressFromOperand(baseMatcher.group(2));
|
||||
|
||||
int tableAddIndex = latestWriter(instructions, loadIndex, tableBaseRegister);
|
||||
if (tableAddIndex < 0 || !mnemonic(instructions.get(tableAddIndex)).equals("ADD")) return null;
|
||||
java.util.regex.Matcher tableAddMatcher = AARCH64_ADD_IMMEDIATE.matcher(operands(instructions.get(tableAddIndex)));
|
||||
if (!tableAddMatcher.matches() ||
|
||||
!normalizedRegister(tableAddMatcher.group(1)).equals(tableBaseRegister) ||
|
||||
!normalizedRegister(tableAddMatcher.group(2)).equals(tableBaseRegister)) return null;
|
||||
int tablePageIndex = latestWriter(instructions, tableAddIndex, tableBaseRegister);
|
||||
if (tablePageIndex < 0 || !mnemonic(instructions.get(tablePageIndex)).equals("ADRP")) return null;
|
||||
java.util.regex.Matcher pageMatcher = AARCH64_REGISTER_AND_ADDRESS.matcher(operands(instructions.get(tablePageIndex)));
|
||||
if (!pageMatcher.matches() || !normalizedRegister(pageMatcher.group(1)).equals(tableBaseRegister)) return null;
|
||||
Address tableAddress = addressFromOperand(pageMatcher.group(2)).add(parseInteger(tableAddMatcher.group(3)));
|
||||
if (!currentProgram.getMemory().contains(tableAddress) ||
|
||||
(!dataAddresses.isEmpty() && !dataAddresses.contains(tableAddress))) return null;
|
||||
|
||||
// Unsigned guard: CMP Rs, #imm, then B.HI (imm + 1 entries) or
|
||||
// B.CS/B.HS (imm entries), with only flag-preserving moves and
|
||||
// address arithmetic between the compare and the load.
|
||||
int guardIndex = -1;
|
||||
for (int index = loadIndex - 1; index >= 0; index -= 1) {
|
||||
String name = mnemonic(instructions.get(index));
|
||||
if (name.startsWith("B.")) {
|
||||
guardIndex = index;
|
||||
break;
|
||||
}
|
||||
if (!AARCH64_FLAG_PRESERVING.contains(name)) return null;
|
||||
}
|
||||
if (guardIndex < 1) return null;
|
||||
Instruction guard = instructions.get(guardIndex);
|
||||
String guardMnemonic = mnemonic(guard);
|
||||
if (!Set.of("B.HI", "B.CS", "B.HS").contains(guardMnemonic)) return null;
|
||||
Address[] guardFlows = guard.getFlows();
|
||||
if (guardFlows.length != 1) return null;
|
||||
Address defaultTarget = guardFlows[0];
|
||||
int compareIndex = -1;
|
||||
for (int index = guardIndex - 1; index >= 0; index -= 1) {
|
||||
String name = mnemonic(instructions.get(index));
|
||||
if (name.equals("CMP")) {
|
||||
compareIndex = index;
|
||||
break;
|
||||
}
|
||||
if (!AARCH64_FLAG_PRESERVING.contains(name)) return null;
|
||||
}
|
||||
if (compareIndex < 0) return null;
|
||||
java.util.regex.Matcher compareMatcher = AARCH64_COMPARE_IMMEDIATE.matcher(operands(instructions.get(compareIndex)));
|
||||
if (!compareMatcher.matches()) return null;
|
||||
String comparedRegister = normalizedRegister(compareMatcher.group(1));
|
||||
boolean comparedFullWidth = compareMatcher.group(1).toLowerCase(Locale.ROOT).startsWith("x");
|
||||
long compareImmediate = parseInteger(compareMatcher.group(2));
|
||||
long entryCountLong = guardMnemonic.equals("B.HI") ? compareImmediate + 1 : compareImmediate;
|
||||
if (entryCountLong <= 0 || entryCountLong > 4096) return null;
|
||||
int entryCount = (int) entryCountLong;
|
||||
|
||||
// The loaded index must carry the compared value: either the compared
|
||||
// register itself, unchanged since CMP, or a MOV copy made after CMP.
|
||||
// A W compare bounds only the low 32 bits, so an X index needs a
|
||||
// zero-extending W copy; a W index is zero-extended by UXTW.
|
||||
int indexWriter = latestWriter(instructions, loadIndex, indexRegister);
|
||||
boolean bounded = false;
|
||||
if (indexWriter < compareIndex) {
|
||||
bounded = indexRegister.equals(comparedRegister) && (wordIndex || comparedFullWidth);
|
||||
}
|
||||
else if (mnemonic(instructions.get(indexWriter)).equals("MOV")) {
|
||||
java.util.regex.Matcher moveMatcher = AARCH64_MOVE_REGISTER.matcher(operands(instructions.get(indexWriter)));
|
||||
if (moveMatcher.matches() &&
|
||||
normalizedRegister(moveMatcher.group(2)).equals(comparedRegister) &&
|
||||
latestWriter(instructions, indexWriter, comparedRegister) < compareIndex) {
|
||||
boolean wordCopy = moveMatcher.group(1).toLowerCase(Locale.ROOT).startsWith("w");
|
||||
bounded = wordCopy || comparedFullWidth;
|
||||
}
|
||||
}
|
||||
if (!bounded) return null;
|
||||
|
||||
// Every fact above holds only on one straight-line fall-through path
|
||||
// with no other entry into the window.
|
||||
int windowStart = Math.min(Math.min(tablePageIndex, baseIndex), compareIndex);
|
||||
for (int index = windowStart + 1; index <= branchIndex; index += 1) {
|
||||
monitor.checkCancelled();
|
||||
Instruction previous = instructions.get(index - 1);
|
||||
Instruction current = instructions.get(index);
|
||||
Address fallThrough = previous.getFallThrough();
|
||||
if (fallThrough == null || !fallThrough.equals(current.getAddress())) return null;
|
||||
ReferenceIterator incoming = currentProgram.getReferenceManager().getReferencesTo(current.getAddress());
|
||||
while (incoming.hasNext()) {
|
||||
if (incoming.next().getReferenceType().isFlow()) return null;
|
||||
}
|
||||
}
|
||||
|
||||
List<RecoveredAArch64Mapping> mappings = new ArrayList<>();
|
||||
Set<String> mappedTargets = new HashSet<>();
|
||||
for (int tableIndex = 0; tableIndex < entryCount; tableIndex += 1) {
|
||||
monitor.checkCancelled();
|
||||
Address entry = tableAddress.add((long) tableIndex * entrySize);
|
||||
if (!currentProgram.getMemory().contains(entry)) return null;
|
||||
long encoded;
|
||||
if (entrySize == 1) {
|
||||
byte value = currentProgram.getMemory().getByte(entry);
|
||||
encoded = signedEntries ? value : value & 0xffL;
|
||||
}
|
||||
else {
|
||||
short value = currentProgram.getMemory().getShort(entry);
|
||||
encoded = signedEntries ? value : value & 0xffffL;
|
||||
}
|
||||
if (encoded == 0 && nullTarget != null) {
|
||||
mappings.add(new RecoveredAArch64Mapping(tableIndex, nullTarget, encoded, true));
|
||||
continue;
|
||||
}
|
||||
Address target;
|
||||
try {
|
||||
target = branchBase.addNoWrap(encoded << offsetShift);
|
||||
}
|
||||
catch (ghidra.program.model.address.AddressOverflowException overflow) {
|
||||
return null;
|
||||
}
|
||||
String canonicalTarget = canonicalAddress(target);
|
||||
if (!recoveredTargetSet.contains(canonicalTarget)) return null;
|
||||
mappedTargets.add(canonicalTarget);
|
||||
mappings.add(new RecoveredAArch64Mapping(tableIndex, target, encoded, false));
|
||||
}
|
||||
if (mappedTargets.contains(canonicalAddress(defaultTarget))) return null;
|
||||
// Ghidra's recovered destinations must be fully accounted for by the
|
||||
// decoded entries, the guard default, and the CBZ null path.
|
||||
Set<String> accountedTargets = new HashSet<>(mappedTargets);
|
||||
accountedTargets.add(canonicalAddress(defaultTarget));
|
||||
if (nullTarget != null) accountedTargets.add(canonicalAddress(nullTarget));
|
||||
// Ghidra does not model the CBZ, so it may decode a zero entry as a
|
||||
// branch to the base itself. Only a zero entry encodes offset 0, and
|
||||
// the CBZ diverts every zero entry, so the BR never reaches the base.
|
||||
Address unreachableZeroTarget = null;
|
||||
boolean divertedZeroEntry = mappings.stream().anyMatch(mapping -> mapping.nullEntry);
|
||||
if (divertedZeroEntry &&
|
||||
recoveredTargetSet.contains(canonicalAddress(branchBase)) &&
|
||||
!mappedTargets.contains(canonicalAddress(branchBase)) &&
|
||||
!branchBase.equals(defaultTarget) &&
|
||||
!branchBase.equals(nullTarget)) {
|
||||
unreachableZeroTarget = branchBase;
|
||||
accountedTargets.add(canonicalAddress(branchBase));
|
||||
}
|
||||
if (!accountedTargets.containsAll(recoveredTargetSet)) return null;
|
||||
int selectorWriter = latestWriter(instructions, compareIndex, comparedRegister);
|
||||
return new RecoveredAArch64RelativeTable(
|
||||
tableAddress,
|
||||
branchBase,
|
||||
entrySize,
|
||||
selectorWriter < 0 ? null : instructions.get(selectorWriter).getAddress(),
|
||||
signedEntries,
|
||||
offsetShift,
|
||||
guardMnemonic,
|
||||
compareImmediate,
|
||||
nullTarget,
|
||||
unreachableZeroTarget,
|
||||
List.copyOf(mappings)
|
||||
);
|
||||
}
|
||||
|
||||
private static String mnemonic(Instruction instruction) {
|
||||
return instruction.getMnemonicString().toUpperCase(Locale.ROOT);
|
||||
}
|
||||
|
||||
/** Index of the last instruction before {@code before} that writes {@code register}, or -1. */
|
||||
private static int latestWriter(List<Instruction> instructions, int before, String register) {
|
||||
for (int index = before - 1; index >= 0; index -= 1) {
|
||||
for (Object result : instructions.get(index).getResultObjects()) {
|
||||
if (result instanceof Register written &&
|
||||
normalizedRegister(written.getBaseRegister().getName()).equals(register)) {
|
||||
return index;
|
||||
Address resolvedTableAddress = null;
|
||||
for (int index = loadIndex - 3; index < loadIndex - 1; index += 1) {
|
||||
Instruction instruction = instructions.get(index);
|
||||
String mnemonic = instruction.getMnemonicString().toUpperCase(Locale.ROOT);
|
||||
String text = operands(instruction);
|
||||
if (mnemonic.equals("ADRP") &&
|
||||
text.matches("(?i)^[wx]\\d+,\\s*0x[0-9a-f]+$")) {
|
||||
String[] parts = text.split(",");
|
||||
if (normalizedRegister(parts[0]).equals(tableBaseRegister)) {
|
||||
resolvedTableAddress = addressFromOperand(parts[1]);
|
||||
}
|
||||
}
|
||||
else if (mnemonic.equals("ADD") && resolvedTableAddress != null &&
|
||||
text.matches("(?i)^[wx]\\d+,\\s*[wx]\\d+,\\s*#?(?:0x[0-9a-f]+|[0-9]+)$")) {
|
||||
String[] parts = text.split(",");
|
||||
if (normalizedRegister(parts[0]).equals(tableBaseRegister) &&
|
||||
normalizedRegister(parts[1]).equals(tableBaseRegister)) {
|
||||
resolvedTableAddress = resolvedTableAddress.add(parseInteger(parts[2].trim().replaceFirst("^#", "")));
|
||||
}
|
||||
}
|
||||
}
|
||||
if (resolvedTableAddress == null || !currentProgram.getMemory().contains(resolvedTableAddress) ||
|
||||
(!dataAddresses.isEmpty() && !dataAddresses.contains(resolvedTableAddress))) continue;
|
||||
Address tableAddress = resolvedTableAddress;
|
||||
|
||||
Address branchBase = null;
|
||||
boolean addressChain = false;
|
||||
for (int index = loadIndex - 1; index < branchIndex; index += 1) {
|
||||
Instruction instruction = instructions.get(index);
|
||||
String mnemonic = instruction.getMnemonicString().toUpperCase(Locale.ROOT);
|
||||
String text = operands(instruction);
|
||||
if (index < loadIndex && mnemonic.equals("ADR") && text.matches("(?i)^[wx]\\d+,\\s*0x[0-9a-f]+$")) {
|
||||
int comma = text.indexOf(',');
|
||||
String destination = normalizedRegister(text.substring(0, comma));
|
||||
if (!destination.equals(normalizedRegister(operands(instructions.get(branchIndex))))) continue;
|
||||
branchBase = addressFromOperand(text.substring(comma + 1));
|
||||
}
|
||||
if (index > loadIndex && mnemonic.equals("ADD") && branchBase != null &&
|
||||
text.matches("(?i)^[wx]\\d+,\\s*[wx]\\d+,\\s*[wx]\\d+,\\s*LSL\\s*#(?:0x)?2$")) {
|
||||
String[] parts = text.split(",");
|
||||
String destination = normalizedRegister(parts[0]);
|
||||
String base = normalizedRegister(parts[1]);
|
||||
String offset = normalizedRegister(parts[2]);
|
||||
if (destination.equals(base) &&
|
||||
base.equals(normalizedRegister(operands(instructions.get(branchIndex)))) &&
|
||||
offset.equals(loadedRegister)) {
|
||||
addressChain = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!addressChain || branchBase == null) continue;
|
||||
|
||||
int entryCount = -1;
|
||||
Address defaultTarget = null;
|
||||
for (int compareIndex = 0; compareIndex < loadIndex; compareIndex += 1) {
|
||||
Instruction compare = instructions.get(compareIndex);
|
||||
if (!compare.getMnemonicString().equalsIgnoreCase("CMP")) continue;
|
||||
java.util.regex.Matcher compareMatcher = java.util.regex.Pattern
|
||||
.compile("(?i)^([wx]\\d+),\\s*#?(0x[0-9a-f]+|[0-9]+)$")
|
||||
.matcher(operands(compare));
|
||||
if (!compareMatcher.matches()) continue;
|
||||
String comparedRegister = normalizedRegister(compareMatcher.group(1));
|
||||
long upperBound = parseInteger(compareMatcher.group(2));
|
||||
if (upperBound < 0 || upperBound >= 4096) continue;
|
||||
for (int guardIndex = compareIndex + 1; guardIndex < loadIndex; guardIndex += 1) {
|
||||
Instruction guard = instructions.get(guardIndex);
|
||||
String mnemonic = guard.getMnemonicString().toUpperCase(Locale.ROOT);
|
||||
if (!mnemonic.equals("B.HI")) continue;
|
||||
Address[] flows = guard.getFlows();
|
||||
if (flows.length != 1 || !recoveredTargetSet.contains(canonicalAddress(flows[0]))) continue;
|
||||
boolean supportedSequence = true;
|
||||
for (int sequenceIndex = compareIndex + 1; sequenceIndex < loadIndex; sequenceIndex += 1) {
|
||||
String sequenceMnemonic = instructions.get(sequenceIndex)
|
||||
.getMnemonicString().toUpperCase(Locale.ROOT);
|
||||
if (!Set.of("B.HI", "MOV", "ADRP", "ADD", "ADR").contains(sequenceMnemonic)) {
|
||||
supportedSequence = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Set<String> selectorAliases = new HashSet<>();
|
||||
selectorAliases.add(comparedRegister);
|
||||
Set<String> boundedIndexes = new HashSet<>();
|
||||
if (compareMatcher.group(1).toLowerCase(Locale.ROOT).startsWith("x")) boundedIndexes.add(comparedRegister);
|
||||
for (int sequenceIndex = compareIndex + 1; sequenceIndex < loadIndex; sequenceIndex++) {
|
||||
Instruction instruction = instructions.get(sequenceIndex);
|
||||
String[] parts = operands(instruction).split(",");
|
||||
if (parts.length == 0) continue;
|
||||
if (instruction.getMnemonicString().equalsIgnoreCase("MOV")) {
|
||||
String destination = normalizedRegister(parts[0]);
|
||||
String source = parts.length == 2 ? normalizedRegister(parts[1]) : "";
|
||||
boolean aliasesSelector = selectorAliases.contains(source);
|
||||
boolean fullWidthBounded = boundedIndexes.contains(source) ||
|
||||
(parts[0].trim().toLowerCase(Locale.ROOT).startsWith("w") && aliasesSelector);
|
||||
if (aliasesSelector) selectorAliases.add(destination);
|
||||
else selectorAliases.remove(destination);
|
||||
if (fullWidthBounded) boundedIndexes.add(destination);
|
||||
else boundedIndexes.remove(destination);
|
||||
} else if (Set.of("ADRP", "ADD", "ADR").contains(instruction.getMnemonicString().toUpperCase(Locale.ROOT))) {
|
||||
selectorAliases.remove(normalizedRegister(parts[0]));
|
||||
boundedIndexes.remove(normalizedRegister(parts[0]));
|
||||
}
|
||||
}
|
||||
boolean selectorFeedsTable = boundedIndexes.contains(indexRegister);
|
||||
if (compareIndex < loadIndex - 3 && selectorFeedsTable && supportedSequence) {
|
||||
entryCount = (int) upperBound + 1;
|
||||
defaultTarget = flows[0];
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (entryCount > 0) break;
|
||||
}
|
||||
if (entryCount <= 0 || entryCount > 4096 || defaultTarget == null) continue;
|
||||
|
||||
List<RecoveredAArch64Mapping> mappings = new ArrayList<>();
|
||||
Set<String> mappedTargets = new HashSet<>();
|
||||
boolean valid = true;
|
||||
for (int caseValue = 0; caseValue < entryCount; caseValue += 1) {
|
||||
Address entry = tableAddress.add((long) caseValue * entrySize);
|
||||
int encodedOffset = entrySize == 1 ? currentProgram.getMemory().getByte(entry) & 0xff : currentProgram.getMemory().getShort(entry) & 0xffff;
|
||||
Address target = branchBase.add((long) encodedOffset * 4L);
|
||||
String canonicalTarget = canonicalAddress(target);
|
||||
if (!recoveredTargetSet.contains(canonicalTarget)) {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
mappedTargets.add(canonicalTarget);
|
||||
mappings.add(new RecoveredAArch64Mapping(caseValue, target));
|
||||
}
|
||||
if (!valid || mappedTargets.contains(canonicalAddress(defaultTarget))) continue;
|
||||
Set<String> accountedTargets = new HashSet<>(mappedTargets);
|
||||
accountedTargets.add(canonicalAddress(defaultTarget));
|
||||
if (!accountedTargets.equals(recoveredTargetSet)) continue;
|
||||
return new RecoveredAArch64RelativeTable(tableAddress, branchBase, entrySize, List.copyOf(mappings));
|
||||
}
|
||||
return -1;
|
||||
return null;
|
||||
}
|
||||
|
||||
private static String operands(Instruction instruction) {
|
||||
|
||||
@@ -24,29 +24,6 @@ REA can attach to a user-owned Chrome-family browser through the Chrome DevTools
|
||||
|
||||
Target discovery returns the complete in-scope target array in one result.
|
||||
|
||||
## Screenshot comparison memory budget
|
||||
|
||||
`compare_web_screenshots` admits both PNGs against a 256 MiB estimated
|
||||
working-memory budget before inflating either image. This is a processing
|
||||
buffer budget for this operation, not a PNG dimension limit, CDP transport
|
||||
limit, or process RSS guarantee. Header-valid dimensions that exceed the
|
||||
budget return `resource_constraint` with `details.resource: "memory"` and the
|
||||
estimated and maximum working-set bytes. Invalid PNG structure remains
|
||||
malformed input; encodings outside supported pixel-comparison coverage remain
|
||||
unsupported-target errors.
|
||||
|
||||
The conservative estimate includes the two decoder-owned decoded input
|
||||
buffers, the current image's concatenated IDAT buffer, Node zlib's default
|
||||
16 KiB output chunk backing rounded up for retained chunks, the concatenated
|
||||
inflate result while its output chunks are retained, one possible extra output
|
||||
chunk before zlib checks `maxOutputLength`, unfiltered scanline storage,
|
||||
RGB-to-RGBA expansion, and the first retained RGBA image while the second is
|
||||
decoded. RGBA output aliases its unfiltered buffer and is counted once.
|
||||
Admission uses checked integer arithmetic before image-sized allocation.
|
||||
Caller-owned JSON/base64 strings, input parsing and validation, transport
|
||||
buffers, garbage-collector behavior, and other runtime memory are outside this
|
||||
estimate; the budget does not promise a bound on total process memory.
|
||||
|
||||
Electron `file://` pages use a separate provider and target boundary; see [electron-observation.md](electron-observation.md).
|
||||
|
||||
## Provider authorities
|
||||
|
||||
+33
-13
@@ -68,15 +68,23 @@ selects the npm release, and persistent MCP registrations are pinned to the
|
||||
version that performed setup. Installing newer instructions does not update a
|
||||
running server or its registration.
|
||||
|
||||
The npm release checked on **2026-10-11** was **6.4.0**, published from
|
||||
[`a56f7bb6`](https://github.com/morluto/rea/commit/a56f7bb668ab23fec8d33f7af353ac7ed2fe1895).
|
||||
The actual registry package accepted dry-run setup plans for all 17 client IDs
|
||||
listed below, including Qwen Code, Pi and Hermes. OpenCode's plan selected
|
||||
`OPENCODE_CONFIG_DIR` even when that directory did not yet exist. These checks
|
||||
used an isolated home on macOS with Node 24; they verify planning and argument
|
||||
admission, without writing client configuration or establishing live integration
|
||||
with every client. This resolves the npm 6.3.0 setup gaps tracked in
|
||||
[#1650](https://github.com/morluto/rea/issues/1650).
|
||||
The npm release checked on **2026-10-11** was **6.3.0**, published from
|
||||
[`74cf9e14`](https://github.com/morluto/rea/commit/74cf9e1401fcd89482fe39c27cc898f5858e9b72),
|
||||
with bundled skill version 34. It does not include automatic Qwen Code, Pi, or
|
||||
Hermes registration: `setup --client qwen_code`, `setup --client pi`, and
|
||||
`setup --client hermes` reject those IDs before writing configuration. It also
|
||||
does not honor `OPENCODE_CONFIG_DIR` when selecting OpenCode's configuration
|
||||
directory. These workflows below require a source build or a later release
|
||||
containing the relevant changes; `@latest` alone cannot select unpublished
|
||||
changes.
|
||||
|
||||
The earlier release checked on **2026-10-07** was **5.0.0** (133 MCP tools), published
|
||||
from the fixed checkpoint
|
||||
[`b33236ec`](https://github.com/morluto/rea/releases/tag/rea-agents-5.0.0).
|
||||
The public CLI, MCP catalog and target-free session, and isolated update from
|
||||
4.1.0 to 5.0.0 were verified through npm. The artifact includes Windows native
|
||||
controls, Android/JADX and firmware tools, Ghidra function annotations, and
|
||||
retained application-Evidence references.
|
||||
|
||||
Main's catalog describes the current code. A source build or a subsequent
|
||||
release containing changes after this checkpoint is required for newer
|
||||
@@ -164,6 +172,9 @@ schema constraints; REA still validates complete canonical inputs. Full-profile
|
||||
Windows remain unverified; see [native client verification](testing.md).
|
||||
|
||||
For [Pi](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/mcp.md),
|
||||
automatic setup requires a build containing the integration; npm 6.3.0 does not
|
||||
include it. See [released package and main](#released-package-and-main).
|
||||
In a supported build,
|
||||
setup writes a `type: "stdio"` entry under `mcpServers.rea` in the user-level
|
||||
`~/.pi/agent/mcp.json` and installs the bundled skill under `~/.agents/skills`.
|
||||
Use `rea setup --client pi --dry-run --json` to inspect the plan, then
|
||||
@@ -190,7 +201,10 @@ but unavailable in a project. An extension that registers `/mcp` can replace
|
||||
Pi's built-in MCP support and use its own configuration instead. Run `/reload`
|
||||
in an existing Pi session after changing servers outside the session.
|
||||
|
||||
For Qwen Code, setup registers `rea` in the `mcpServers` table of the
|
||||
For Qwen Code, automatic setup requires a build containing the integration;
|
||||
npm 6.3.0 does not include it. See
|
||||
[released package and main](#released-package-and-main). In a supported build,
|
||||
setup registers `rea` in the `mcpServers` table of the
|
||||
user-level `~/.qwen/settings.json`, or `$QWEN_HOME/settings.json` when configured.
|
||||
Setup and doctor follow Qwen Code's tilde and working-directory-relative home
|
||||
overrides. Qwen Code also discovers personal skills in `~/.agents/skills`,
|
||||
@@ -222,7 +236,10 @@ registration. Doctor treats an `enabled: false` entry as active when
|
||||
`enabledServers` lists `rea`, as OMP does, unless `disabledServers` also lists
|
||||
it. Run setup under each profile that should load REA.
|
||||
|
||||
For Hermes, setup writes a `mcp_servers.rea` entry to `config.yaml`, preserving
|
||||
For Hermes, automatic setup also requires a build containing the integration;
|
||||
npm 6.3.0 does not include it. See
|
||||
[released package and main](#released-package-and-main).
|
||||
In a supported build, setup writes a `mcp_servers.rea` entry to `config.yaml`, preserving
|
||||
comments and unrelated settings. It resolves that file from `HERMES_HOME`, else
|
||||
from the platform default Hermes itself uses — `%LOCALAPPDATA%\hermes` on
|
||||
Windows, `~/.hermes` elsewhere — including any `HERMES_DATA_DIR_SUFFIX`. Hermes
|
||||
@@ -276,7 +293,10 @@ for its native configuration contract.
|
||||
|
||||
### Other client settings
|
||||
|
||||
For OpenCode, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2
|
||||
For OpenCode, npm 6.3.0 does not honor `OPENCODE_CONFIG_DIR`; use a source build
|
||||
or a later release containing the override support when selecting a custom
|
||||
configuration directory. See [released package and main](#released-package-and-main).
|
||||
In a supported build, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2
|
||||
both load. If the configuration already uses OpenCode V2's native
|
||||
`mcp.servers` table, setup registers REA there instead and replaces any earlier
|
||||
`mcp.rea` entry from REA. Setup honors `OPENCODE_CONFIG_DIR` and
|
||||
@@ -860,7 +880,7 @@ For a client that requires manual configuration, use:
|
||||
"mcpServers": {
|
||||
"rea": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "rea-agents@6.4.0", "mcp"]
|
||||
"args": ["-y", "rea-agents@6.3.0", "mcp"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -60,12 +60,6 @@ manifest and graph commitments, JavaScript Application Graph, static
|
||||
Electron summary, reconstruction statistics, and explicit limitations. It does
|
||||
not require a live Hopper, Ghidra, browser, or Electron process.
|
||||
|
||||
The application graph selects the shallowest `package.json` outside
|
||||
`node_modules` as its package root, breaking equal-depth ties by Unicode
|
||||
code-point order. Dependency manifests remain package observations but do not
|
||||
declare Electron main or renderer entries. If only dependency manifests are
|
||||
present, the inventory artifact remains the graph root.
|
||||
|
||||
## ASAR integrity
|
||||
|
||||
ASAR inventory checks Electron integrity metadata for embedded archive entries
|
||||
|
||||
@@ -308,23 +308,6 @@ that reference for a summary, one section/module, or a stable page, or call
|
||||
bundle exports stream canonical JSON into an atomically published file. A broad
|
||||
follow-up or `get_evidence_bundle` can also exceed the response budget; exporting
|
||||
preserves the complete session without sending it through a single MCP frame.
|
||||
To discover existing records without their large payloads, call
|
||||
`get_evidence_bundle` with `detail: "summary"`. Its `result.records` contain
|
||||
Evidence IDs, artifact/provider identities, analysis-profile digests and native
|
||||
procedure metadata. Optional `filters` match exact `evidence_id`, `operation`,
|
||||
`target_sha256`, `analysis_profile_digest` or `procedure_address`. Use a discovered
|
||||
ID with `inspect_analysis_view`; neither discovery nor retained selection starts
|
||||
a provider. `retention: "complete-record"` describes stored Evidence, while
|
||||
`native_dossier.value_flow` independently reports truncation, unavailable or
|
||||
unknown metadata. Missing omission counts remain null. Summary output carries
|
||||
every matching record; actual transport admission still applies. Omitted
|
||||
`detail` preserves complete canonical bundle delivery; filters require summary.
|
||||
|
||||
For a portable exported bundle, the CLI shares this projection:
|
||||
`rea inspect-evidence-bundle '{"path":"/tmp/evidence.json","detail":"summary","filters":{"operation":"analyze_function"}}' --format json`.
|
||||
The CLI validates and materializes that local bundle under its existing memory
|
||||
budget; summary delivery does not make arbitrary large files cheap to import.
|
||||
|
||||
Cancelling an export stops further serialization and removes its staging file.
|
||||
The destination changes only when a complete export is atomically published.
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+1
-5
@@ -21,11 +21,7 @@ Use [the rea-changelog-update skill](https://github.com/morluto/rea/blob/main/.a
|
||||
to turn the bot draft into grouped Highlights, Changes, and Fixes with inline
|
||||
`Thanks @...` credit. `npm run release:notes -- inventory` collects the full
|
||||
selected Git range and paginated GitHub PR/author/closing-issue metadata;
|
||||
`record --summary` produces a compact provenance link for public notes, while
|
||||
`record` produces the full record for review. Credit a contribution once even
|
||||
when it appears in highlights, details and migrations; omit maintainer
|
||||
`@morluto` self-thanks. Keep complete authorship metadata in the inventory
|
||||
instead of duplicating a credit ledger in the release body. `check` validates credits
|
||||
`record` produces the complete contribution record, `check` validates credits
|
||||
and optional PR-body equality, and `render` writes the verified release section.
|
||||
Run a subcommand without its required options to see its usage. GitHub reads
|
||||
require an authenticated `gh`; record/check/render work offline from the saved
|
||||
|
||||
+17
-38
@@ -23,11 +23,6 @@ cases. Share setup without hiding inputs or expected evidence. For successful
|
||||
`.ok` assertion adds no coverage. Protocol fixtures must reject unmodeled
|
||||
commands and preserve actual producer reply shapes.
|
||||
|
||||
Stateless source-analysis matrices may share a file-owned stdio MCP connection
|
||||
and retain representative compiled CLI parity checks. Every distinct source
|
||||
scenario still runs through the public workflow; session-lifecycle scenarios
|
||||
need their own sessions and cleanup assertions.
|
||||
|
||||
Report unavailable authority as a named skip. Await owned cleanup before a
|
||||
verifier reports success. Native process inspectors are released at Vitest
|
||||
worker teardown; fork termination does not run Node exit hooks, and per-file
|
||||
@@ -455,15 +450,15 @@ installation (`application.java.min` through `application.java.max`). Current
|
||||
12.1 releases require JDK 21 or newer and set no maximum. The lanes below still
|
||||
prove behavior on the verified Ghidra 12.1.4 and JDK 21 build.
|
||||
|
||||
| Ghidra lane | Supported runner/target | Additional local tools |
|
||||
| ------------------------------------------ | ----------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
|
||||
| `npm run verify:ghidra` | Linux x64/arm64 ELF or macOS x64/arm64 Mach-O | Host C compiler, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:seeds` | POSIX host; source-owned 32-bit x86 raw fixture; function order and BSS labels | Ghidra 12.1.4 and full JDK 21 |
|
||||
| `npm run verify:ghidra:swift` | macOS x64/arm64 Mach-O | Host Swift compiler, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:switch` | Linux x64 ELF; GCC/Clang optimized and stripped switch fixtures | GCC, Clang, GNU nm/objdump/strip, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:aarch64-jump-table` | Any supported Ghidra host; AArch64 ELF; signed/unsigned byte/halfword tables; host ARM64 Mach-O | Clang with AArch64 target support, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:cross-format` | Any supported Ghidra host; also analyzes AArch64 ELF, x86-64 PE, and x86-64 Mach-O | `clang`, LLD, and `lld-link` in addition to host-lane prerequisites |
|
||||
| `npm run verify:ghidra:windows` | Controlled Windows x64 with native x86-64 PE; `-- --x86` selects native x86 PE | Ghidra 12.1.4, full JDK 21, and the matching native artifact |
|
||||
| Ghidra lane | Supported runner/target | Additional local tools |
|
||||
| ------------------------------------------ | ---------------------------------------------------------------------------------- | ------------------------------------------------------------------- |
|
||||
| `npm run verify:ghidra` | Linux x64/arm64 ELF or macOS x64/arm64 Mach-O | Host C compiler, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:seeds` | POSIX host; source-owned 32-bit x86 raw fixture; function order and BSS labels | Ghidra 12.1.4 and full JDK 21 |
|
||||
| `npm run verify:ghidra:swift` | macOS x64/arm64 Mach-O | Host Swift compiler, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:switch` | Linux x64 ELF; GCC/Clang optimized and stripped switch fixtures | GCC, Clang, GNU nm/objdump/strip, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:aarch64-jump-table` | Any supported Ghidra host; AArch64 ELF; byte/halfword tables; host ARM64 Mach-O | Clang with AArch64 target support, Ghidra 12.1.4, and full JDK 21 |
|
||||
| `npm run verify:ghidra:cross-format` | Any supported Ghidra host; also analyzes AArch64 ELF, x86-64 PE, and x86-64 Mach-O | `clang`, LLD, and `lld-link` in addition to host-lane prerequisites |
|
||||
| `npm run verify:ghidra:windows` | Controlled Windows x64 with native x86-64 PE; `-- --x86` selects native x86 PE | Ghidra 12.1.4, full JDK 21, and the matching native artifact |
|
||||
|
||||
The optional Swift lane compiles a source-owned fixture and checks modern and
|
||||
provider-demangled procedure names through real stdio MCP, with and without the
|
||||
@@ -828,24 +823,11 @@ CI exercises the pinned Node.js runtime on native hosted runners:
|
||||
|
||||
Package lanes assert the actual Node platform/architecture before verification
|
||||
and record those values with the Node version. The four-host package matrix
|
||||
runs up to four jobs concurrently on independent hosted runners with explicit
|
||||
timeouts and per-runner Node heap/thread limits; Windows retains its curated
|
||||
capability lane. Package checks cover installation, CLI/MCP discovery,
|
||||
target-free analysis, configuration backups/recovery, Evidence and owned
|
||||
lifecycle; Inspector checks execute source-owned loopback targets and special
|
||||
filename cases.
|
||||
|
||||
The Apple artifact CLI/MCP lane uses one `macos-15` job to avoid adding demand
|
||||
for scarce macOS runner capacity. It runs the artifact-format and metadata
|
||||
verifiers, followed by one consolidated focused invocation containing all
|
||||
existing native and process-boundary regression paths.
|
||||
|
||||
The Linux source-test shards download a shared runtime and generated test
|
||||
metadata snapshot produced by the build job with `npm run test:prepare`. The
|
||||
snapshot includes `dist/`, generated skills, the MCP tool catalog, product
|
||||
catalog, and managed-conformance metadata. Each shard still installs the locked
|
||||
dependencies and runs its assigned coverage-enabled tests; it no longer
|
||||
regenerates the same metadata independently.
|
||||
runs at most two jobs concurrently with explicit timeouts and Node heap/thread
|
||||
limits; Windows retains its curated capability lane. Package checks cover
|
||||
installation, CLI/MCP discovery, target-free
|
||||
analysis, configuration backups/recovery, Evidence and owned lifecycle; Inspector
|
||||
checks execute source-owned loopback targets and special filename cases.
|
||||
|
||||
These native baseline checks complement the Linux source-test shards and the
|
||||
separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,
|
||||
@@ -948,12 +930,9 @@ provider evidence.
|
||||
Vitest projects use up to two workers, bounded by host parallelism.
|
||||
`process-boundary` runs later with serial files because process-tree sampling
|
||||
shares host resources. Acceptance and process-global cases use isolated forks
|
||||
without serial scheduling. Pure domain/service, composition, conformance,
|
||||
evaluation, and MCP boundary projects share module graphs; their tests avoid
|
||||
process-global state and keep resource cleanup test-scoped. Each MCP session
|
||||
still owns its resources. Adapter, non-MCP boundary, acceptance, and
|
||||
process-global projects retain per-file isolation. See `vitest.config.ts` for
|
||||
current settings.
|
||||
without serial scheduling. Pure domain/service and MCP boundary projects share
|
||||
immutable module graphs; each MCP session still owns its resources. Other
|
||||
projects retain per-file isolation. See `vitest.config.ts` for current settings.
|
||||
|
||||
Build/documentation writers hold checkout-local output locks. Tests do not
|
||||
hold a broad command lock; `check:pr` finishes tests before document validation.
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
#include "authority.hpp"
|
||||
|
||||
#include <delayimp.h>
|
||||
|
||||
namespace rea {
|
||||
|
||||
static const napi_type_tag RESOURCE_TAG = {0x45fcbf0b49e84fe3ULL, 0x9e3e5074130de843ULL};
|
||||
@@ -123,13 +121,3 @@ static napi_value initialize(napi_env env, napi_value exports) {
|
||||
} // namespace rea
|
||||
|
||||
NAPI_MODULE(NODE_GYP_MODULE_NAME, rea::initialize)
|
||||
|
||||
// Node-API symbols are delay-imported from "node.exe". Resolve them against the
|
||||
// host executable so Node.js, Bun, and other Node-API hosts load the addon
|
||||
// regardless of their executable file name.
|
||||
static FARPROC WINAPI reaHostHook(unsigned event, PDelayLoadInfo info) {
|
||||
if (event != dliNotePreLoadLibrary || _stricmp(info->szDll, "node.exe") != 0) return nullptr;
|
||||
return reinterpret_cast<FARPROC>(GetModuleHandleW(nullptr));
|
||||
}
|
||||
|
||||
decltype(__pfnDliNotifyHook2) __pfnDliNotifyHook2 = reaHostHook;
|
||||
|
||||
@@ -219,10 +219,9 @@ std::unique_ptr<Runtime> createRuntime(const std::wstring& parentPath, const std
|
||||
require(BCryptGenRandom(nullptr, random.data(), random.size(), BCRYPT_USE_SYSTEM_PREFERRED_RNG) == 0,
|
||||
"Generate runtime identity failed", parentPath, ERROR_GEN_FAILURE);
|
||||
auto result = std::make_unique<Runtime>();
|
||||
const auto name = prefix + hex(random.data(), random.size());
|
||||
result->path = parent->path + (parent->path.back() == L'\\' ? L"" : L"\\") + name;
|
||||
result->path = parent->path + L"\\" + prefix + hex(random.data(), random.size());
|
||||
Security security;
|
||||
result->directory = createRelative(parent->get(), name,
|
||||
result->directory = createRelative(parent->get(), prefix + hex(random.data(), random.size()),
|
||||
DELETE | FILE_LIST_DIRECTORY | FILE_READ_ATTRIBUTES | READ_CONTROL,
|
||||
true, security, result->path);
|
||||
try {
|
||||
|
||||
Generated
+2
-3
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "rea-agents",
|
||||
"version": "6.4.0",
|
||||
"version": "6.3.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "rea-agents",
|
||||
"version": "6.4.0",
|
||||
"version": "6.3.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@babel/parser": "8.0.4",
|
||||
@@ -39,7 +39,6 @@
|
||||
"plist": "5.0.0",
|
||||
"semver": "7.8.5",
|
||||
"smol-toml": "1.9.0",
|
||||
"stream-chain": "4.2.6",
|
||||
"stream-json": "3.7.0",
|
||||
"write-file-atomic": "7.0.0",
|
||||
"ws": "8.21.0",
|
||||
|
||||
+1
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "rea-agents",
|
||||
"mcpName": "io.github.morluto/rea",
|
||||
"version": "6.4.0",
|
||||
"version": "6.3.0",
|
||||
"description": "Reverse engineer anything from your terminal or agent with one CLI and MCP server.",
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
@@ -292,7 +292,6 @@
|
||||
"plist": "5.0.0",
|
||||
"semver": "7.8.5",
|
||||
"smol-toml": "1.9.0",
|
||||
"stream-chain": "4.2.6",
|
||||
"stream-json": "3.7.0",
|
||||
"write-file-atomic": "7.0.0",
|
||||
"ws": "8.21.0",
|
||||
|
||||
@@ -84,8 +84,7 @@ if (!(await exists(join(headers, "node_api.h")))) {
|
||||
}
|
||||
const artifact = "rea-windows-x64.node";
|
||||
// MSVC short-import libraries do not produce a correct Node import table with
|
||||
// GNU ld. Generate a GNU delay-import library for the exact stable Node-API
|
||||
// symbols; addon.cc binds it to the host executable, whatever its file name.
|
||||
// GNU ld. Generate a GNU import library for the exact stable Node-API symbols.
|
||||
const sources = ["addon.cc", "filesystem.cc", "process.cc"];
|
||||
const sourceText = (
|
||||
await Promise.all(
|
||||
@@ -107,7 +106,7 @@ await writeFile(
|
||||
definition,
|
||||
`LIBRARY node.exe\nEXPORTS\n${symbols.join("\n")}\n`,
|
||||
);
|
||||
run("x86_64-w64-mingw32-dlltool", ["-d", definition, "-y", library]);
|
||||
run("x86_64-w64-mingw32-dlltool", ["-d", definition, "-l", library]);
|
||||
run(compiler, [
|
||||
"-std=c++17",
|
||||
"-O2",
|
||||
|
||||
@@ -222,7 +222,6 @@ export const SCOPES = {
|
||||
website: [
|
||||
"website/**",
|
||||
"scripts/*website*.py",
|
||||
"scripts/test-website-scripts.mjs",
|
||||
".github/workflows/website-check.yml",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
/* Keep the configured launcher identity alive while its resource-limited
|
||||
* Python child runs. Both remain in REA's owned process group. */
|
||||
#define _POSIX_C_SOURCE 200809L
|
||||
#include <errno.h>
|
||||
#include <signal.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <sys/resource.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
char *python = getenv("REA_VERIFY_LIMITED_PYTHON");
|
||||
char *limit = getenv("REA_VERIFY_LIMIT_OPTION");
|
||||
if (python == NULL || python[0] != '/' || limit == NULL) {
|
||||
fputs("limited Python fixture requires an absolute Python and limit\n", stderr);
|
||||
return 127;
|
||||
}
|
||||
struct rlimit core = {0, 0};
|
||||
if (setrlimit(RLIMIT_CORE, &core) != 0) {
|
||||
perror("setrlimit");
|
||||
return 127;
|
||||
}
|
||||
char **arguments = calloc((size_t)argc + 4, sizeof(char *));
|
||||
if (arguments == NULL) return 127;
|
||||
arguments[0] = "/usr/bin/prlimit";
|
||||
arguments[1] = limit;
|
||||
arguments[2] = "--";
|
||||
arguments[3] = python;
|
||||
for (int index = 1; index < argc; index++) arguments[index + 3] = argv[index];
|
||||
pid_t child = fork();
|
||||
if (child == 0) {
|
||||
execv(arguments[0], arguments);
|
||||
perror("execv");
|
||||
_exit(127);
|
||||
}
|
||||
free(arguments);
|
||||
if (child < 0) {
|
||||
perror("fork");
|
||||
return 127;
|
||||
}
|
||||
int status;
|
||||
while (waitpid(child, &status, 0) < 0) {
|
||||
if (errno == EINTR) continue;
|
||||
perror("waitpid");
|
||||
return 127;
|
||||
}
|
||||
if (WIFEXITED(status)) return WEXITSTATUS(status);
|
||||
if (WIFSIGNALED(status)) {
|
||||
int signum = WTERMSIG(status);
|
||||
signal(signum, SIG_DFL);
|
||||
sigset_t signals;
|
||||
sigemptyset(&signals);
|
||||
sigaddset(&signals, signum);
|
||||
sigprocmask(SIG_UNBLOCK, &signals, NULL);
|
||||
raise(signum);
|
||||
return 128 + signum;
|
||||
}
|
||||
return 127;
|
||||
}
|
||||
@@ -28,11 +28,6 @@ import {
|
||||
isolatedFixtureUrl,
|
||||
} from "./browser-runtime-world-e2e.mjs";
|
||||
|
||||
// Arming, progress delivery, isolated-world proof and the page action cross
|
||||
// separate transports. Leave time for their round trips on shared CI runners;
|
||||
// coverage can include a late action even after request collection has ended.
|
||||
const executionObservationMs = 2_000;
|
||||
|
||||
/** Exercise native browser runtime attribution through public CLI and stdio MCP, including arming. */
|
||||
export async function verifyBrowserRuntime(
|
||||
executable,
|
||||
@@ -215,7 +210,7 @@ async function executionMcp(
|
||||
const response = await client.callTool(
|
||||
{
|
||||
name: "observe_web_execution",
|
||||
arguments: { ...input, observation_ms: executionObservationMs },
|
||||
arguments: { ...input, observation_ms: 300 },
|
||||
},
|
||||
{
|
||||
timeout: 40_000,
|
||||
@@ -264,7 +259,7 @@ async function executionCli(entrypoint, input, env, action, isolatedContext) {
|
||||
input.cdp_endpoint,
|
||||
input.target_id,
|
||||
"--observation-ms",
|
||||
String(executionObservationMs),
|
||||
"300",
|
||||
"--json",
|
||||
],
|
||||
{ env, stdio: ["ignore", "pipe", "pipe"] },
|
||||
@@ -387,14 +382,7 @@ function assertExecution(evidence, site, requireBlock = true) {
|
||||
const request = result.requests.find((item) =>
|
||||
item.url.endsWith("/evidence?marker=chosen"),
|
||||
);
|
||||
assert.ok(
|
||||
request,
|
||||
`The selected request initiator must be retained: ${JSON.stringify({
|
||||
window: result.window,
|
||||
requests: result.requests.map((item) => item.url),
|
||||
excluded_requests: result.excluded_requests,
|
||||
})}`,
|
||||
);
|
||||
assert.ok(request, "The selected request initiator must be retained");
|
||||
assert.ok(
|
||||
request.callsites.some(
|
||||
(site) =>
|
||||
|
||||
@@ -176,9 +176,6 @@ const startStorageSite = async () => {
|
||||
origin: `http://127.0.0.1:${port}`,
|
||||
close: async () => {
|
||||
server.close();
|
||||
// Chrome can leave speculative TCP connections without an HTTP request;
|
||||
// server.close() alone waits indefinitely for those owned sockets.
|
||||
server.closeAllConnections();
|
||||
await once(server, "close");
|
||||
},
|
||||
};
|
||||
|
||||
@@ -101,13 +101,10 @@ export async function readReleaseHistory(run, base, target) {
|
||||
// REA history maps Anthropic and Cursor's agent trailers to ordinary User actors.
|
||||
// Keep those source identities in the inventory while excluding automation credit.
|
||||
const AGENT_ACCOUNTS = new Set(["claude", "cursoragent"]);
|
||||
// REA's maintainer requested external-contributor thanks without self-credit.
|
||||
const MAINTAINER_ACCOUNTS = new Set(["morluto"]);
|
||||
const human = (value) =>
|
||||
value?.__typename === "User" &&
|
||||
!value.login.endsWith("[bot]") &&
|
||||
!AGENT_ACCOUNTS.has(value.login.toLowerCase()) &&
|
||||
!MAINTAINER_ACCOUNTS.has(value.login.toLowerCase())
|
||||
!AGENT_ACCOUNTS.has(value.login.toLowerCase())
|
||||
? value.login
|
||||
: undefined;
|
||||
const unique = (values) =>
|
||||
@@ -166,12 +163,6 @@ export function contributionRecord(raw) {
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
/** Keep the full authorship snapshot available without duplicating public thanks. */
|
||||
export function contributionSummary(raw) {
|
||||
const inventory = releaseInventorySchema.parse(raw);
|
||||
return `Full contribution record: [${inventory.commits.length} commits and ${inventory.pullRequests.length} merged PR${inventory.pullRequests.length === 1 ? "" : "s"}](https://github.com/${inventory.repository}/blob/rea-agents-${inventory.version}/docs/releases/${inventory.version}.contributions.json).\n`;
|
||||
}
|
||||
|
||||
export function selectedReleaseNotes(text, version) {
|
||||
const headings = [...text.matchAll(/^## \[([^\]]+)\].*$/gmu)];
|
||||
if (headings[0]?.[1] !== version)
|
||||
@@ -245,8 +236,7 @@ function requiredCredits(inventory, bullet, references) {
|
||||
if (matches[0].parents.length > 1) continue;
|
||||
for (const identity of matches[0].authors) {
|
||||
const handle = human(identity.user);
|
||||
if (handle)
|
||||
required.push({ handle, context: `commit ${matches[0].sha}` });
|
||||
if (handle) required.push({ handle, context: `commit ${match[2]}` });
|
||||
}
|
||||
}
|
||||
return required;
|
||||
@@ -274,8 +264,6 @@ function verifyEditorialCredits(inventory, prose) {
|
||||
]).map((handle) => handle.toLowerCase()),
|
||||
);
|
||||
const errors = [];
|
||||
const requiredByContext = new Map();
|
||||
const thankedByContext = new Map();
|
||||
for (const bullet of proseBullets(prose)) {
|
||||
const references = bulletPrs(bullet, inventory.repository);
|
||||
const start = bullet.indexOf("Thanks ");
|
||||
@@ -294,21 +282,13 @@ function verifyEditorialCredits(inventory, prose) {
|
||||
bullet,
|
||||
references,
|
||||
)) {
|
||||
const required = requiredByContext.get(context) ?? new Set();
|
||||
required.add(handle);
|
||||
requiredByContext.set(context, required);
|
||||
const credited = thankedByContext.get(context) ?? new Set();
|
||||
for (const login of thanked) credited.add(login);
|
||||
thankedByContext.set(context, credited);
|
||||
if (!thanked.has(handle.toLowerCase()))
|
||||
errors.push(`Missing Thanks @${handle} in bullet for ${context}`);
|
||||
}
|
||||
for (const handle of thanked)
|
||||
if (!eligible.has(handle))
|
||||
errors.push(`Unverified human credit @${handle}`);
|
||||
}
|
||||
for (const [context, required] of requiredByContext)
|
||||
for (const handle of required)
|
||||
if (!thankedByContext.get(context)?.has(handle.toLowerCase()))
|
||||
errors.push(`Missing Thanks @${handle} for ${context}`);
|
||||
return errors;
|
||||
}
|
||||
|
||||
@@ -325,15 +305,8 @@ export function verifyReleaseNotes(raw, text, body) {
|
||||
"Release notes exceed GitHub's body budget; shorten prose without dropping credit",
|
||||
);
|
||||
const record = contributionRecord(inventory).trim();
|
||||
const summary = contributionSummary(inventory).trim();
|
||||
const start = notes.endsWith(summary)
|
||||
? notes.length - summary.length
|
||||
: notes.indexOf("### Complete contribution record");
|
||||
if (
|
||||
start < 0 ||
|
||||
(notes.slice(start).trim() !== record &&
|
||||
notes.slice(start).trim() !== summary)
|
||||
)
|
||||
const start = notes.indexOf("### Complete contribution record");
|
||||
if (start < 0 || notes.slice(start).trim() !== record)
|
||||
throw new Error(
|
||||
"Contribution record differs from inventory; regenerate it with release-notes record",
|
||||
);
|
||||
|
||||
@@ -6,7 +6,6 @@ import { collectGitHubCredits } from "./lib/release-notes-github.mjs";
|
||||
import { verifyNotesPublication } from "./lib/release-notes-publication.mjs";
|
||||
import {
|
||||
contributionRecord,
|
||||
contributionSummary,
|
||||
readReleaseHistory,
|
||||
releaseInventorySchema,
|
||||
selectedReleaseNotes,
|
||||
@@ -74,7 +73,6 @@ async function main() {
|
||||
"pr-body": { type: "string" },
|
||||
source: { type: "string" },
|
||||
output: { type: "string" },
|
||||
summary: { type: "boolean", default: false },
|
||||
},
|
||||
});
|
||||
const [command] = positionals;
|
||||
@@ -94,7 +92,7 @@ async function main() {
|
||||
!values.manifest
|
||||
)
|
||||
throw new Error(
|
||||
"Usage: release-notes inventory|record|check|render --manifest FILE [--repository OWNER/REPO --base TAG --target SHA --version VERSION] [--notes FILE --pr-body FILE --source SHA --output FILE] [--summary for record]",
|
||||
"Usage: release-notes inventory|record|check|render --manifest FILE [--repository OWNER/REPO --base TAG --target SHA --version VERSION] [--notes FILE --pr-body FILE --source SHA --output FILE]",
|
||||
);
|
||||
if (command === "inventory") {
|
||||
await createInventory(values);
|
||||
@@ -105,14 +103,11 @@ async function main() {
|
||||
);
|
||||
await verifyInventoryHistory(run, inventory, values.source);
|
||||
if (command === "record") {
|
||||
const record = values.summary
|
||||
? contributionSummary(inventory)
|
||||
: contributionRecord(inventory);
|
||||
if (values.output)
|
||||
await writeFile(values.output, record, {
|
||||
await writeFile(values.output, contributionRecord(inventory), {
|
||||
flag: "wx",
|
||||
});
|
||||
else process.stdout.write(record);
|
||||
else process.stdout.write(contributionRecord(inventory));
|
||||
return;
|
||||
}
|
||||
const notes = await readFile(values.notes, "utf8");
|
||||
|
||||
@@ -1,481 +0,0 @@
|
||||
// Run the website's browser scripts against a small document model, so missing
|
||||
// page-owned messages are checked without a browser.
|
||||
import assert from "node:assert/strict";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { test } from "node:test";
|
||||
import { setImmediate as nextTurn } from "node:timers/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const ASSETS = new URL("../website/public/assets/", import.meta.url);
|
||||
|
||||
class Element {
|
||||
constructor(tag, attributes = {}, children = [], text = "") {
|
||||
this.tagName = tag.toUpperCase();
|
||||
this.attributes = new Map(Object.entries(attributes));
|
||||
this.children = [];
|
||||
this.parent = null;
|
||||
this.listeners = new Map();
|
||||
this.ownText = text;
|
||||
this.hidden = "hidden" in attributes;
|
||||
this.disabled = "disabled" in attributes;
|
||||
this.checked = "checked" in attributes;
|
||||
this.value = attributes.value ?? "";
|
||||
for (const child of children) this.append(child);
|
||||
}
|
||||
|
||||
append(child) {
|
||||
child.parent = this;
|
||||
this.children.push(child);
|
||||
}
|
||||
|
||||
get textContent() {
|
||||
return (
|
||||
this.ownText + this.children.map((child) => child.textContent).join("")
|
||||
);
|
||||
}
|
||||
|
||||
set textContent(value) {
|
||||
this.ownText = String(value);
|
||||
this.children = [];
|
||||
}
|
||||
|
||||
get id() {
|
||||
return this.getAttribute("id") ?? "";
|
||||
}
|
||||
|
||||
get lang() {
|
||||
return this.getAttribute("lang") ?? "";
|
||||
}
|
||||
|
||||
get dataset() {
|
||||
const values = {};
|
||||
for (const [name, value] of this.attributes) {
|
||||
if (!name.startsWith("data-")) continue;
|
||||
const key = name
|
||||
.slice(5)
|
||||
.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
||||
values[key] = value;
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
getAttribute(name) {
|
||||
return this.attributes.has(name) ? this.attributes.get(name) : null;
|
||||
}
|
||||
|
||||
setAttribute(name, value) {
|
||||
this.attributes.set(name, String(value));
|
||||
}
|
||||
|
||||
removeAttribute(name) {
|
||||
this.attributes.delete(name);
|
||||
}
|
||||
|
||||
addEventListener(type, listener) {
|
||||
this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]);
|
||||
}
|
||||
|
||||
dispatch(type, event = {}) {
|
||||
for (const listener of this.listeners.get(type) ?? []) {
|
||||
listener({ preventDefault() {}, repeat: false, ...event });
|
||||
}
|
||||
}
|
||||
|
||||
click() {
|
||||
this.dispatch("click");
|
||||
}
|
||||
|
||||
focus() {}
|
||||
|
||||
descendants() {
|
||||
return this.children.flatMap((child) => [child, ...child.descendants()]);
|
||||
}
|
||||
|
||||
matches(selector) {
|
||||
return selector
|
||||
.split(",")
|
||||
.some((part) => matchesComplex(this, part.trim()));
|
||||
}
|
||||
|
||||
closest(selector) {
|
||||
for (let node = this; node !== null; node = node.parent) {
|
||||
if (node.matches(selector)) return node;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
querySelectorAll(selector) {
|
||||
return this.descendants().filter((node) => node.matches(selector));
|
||||
}
|
||||
|
||||
querySelector(selector) {
|
||||
return this.querySelectorAll(selector)[0] ?? null;
|
||||
}
|
||||
|
||||
getContext() {
|
||||
return this.context ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
// Supports the selector forms the website scripts use: tag, #id, .class and
|
||||
// [attribute], [attribute="value"] or [attribute^="value"], joined by spaces.
|
||||
function matchesCompound(node, compound) {
|
||||
const pattern =
|
||||
/^([a-z0-9]*)((?:#[\w-]+|\.[\w-]+|\[[\w-]+(?:\^?="[^"]*")?\])*)$/i;
|
||||
const parsed = pattern.exec(compound);
|
||||
if (parsed === null) throw new Error(`Unsupported selector: ${compound}`);
|
||||
const [, tag, rest] = parsed;
|
||||
if (tag && node.tagName !== tag.toUpperCase()) return false;
|
||||
for (const [, id, className, name, operator, value] of rest.matchAll(
|
||||
/#([\w-]+)|\.([\w-]+)|\[([\w-]+)(?:(\^?=)"([^"]*)")?\]/g,
|
||||
)) {
|
||||
if (id !== undefined) {
|
||||
if (node.id !== id) return false;
|
||||
continue;
|
||||
}
|
||||
if (className !== undefined) {
|
||||
const classes = (node.getAttribute("class") ?? "").split(/\s+/);
|
||||
if (!classes.includes(className)) return false;
|
||||
continue;
|
||||
}
|
||||
const actual = node.getAttribute(name);
|
||||
if (actual === null) return false;
|
||||
if (operator === "=" && actual !== value) return false;
|
||||
if (operator === "^=" && !actual.startsWith(value)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function matchesComplex(node, selector) {
|
||||
const parts = selector.split(/\s+/);
|
||||
if (!matchesCompound(node, parts.at(-1))) return false;
|
||||
let ancestor = node.parent;
|
||||
for (const part of parts.slice(0, -1).reverse()) {
|
||||
while (ancestor !== null && !matchesCompound(ancestor, part))
|
||||
ancestor = ancestor.parent;
|
||||
if (ancestor === null) return false;
|
||||
ancestor = ancestor.parent;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const h = (tag, attributes, ...children) => {
|
||||
const text = children.filter((child) => typeof child === "string").join("");
|
||||
return new Element(
|
||||
tag,
|
||||
attributes ?? {},
|
||||
children.filter((child) => child instanceof Element),
|
||||
text,
|
||||
);
|
||||
};
|
||||
|
||||
// Install a fresh document and browser globals, then run one script.
|
||||
async function run(
|
||||
script,
|
||||
body,
|
||||
{ lang = "en", clipboard = "resolve", stars = 1234 } = {},
|
||||
) {
|
||||
const root = h("html", { lang }, h("body", {}, ...body));
|
||||
const timers = [];
|
||||
const written = [];
|
||||
const canvasText = [];
|
||||
for (const canvas of root.querySelectorAll("canvas")) {
|
||||
canvas.context = new Proxy(
|
||||
{ fillText: (text) => canvasText.push(text) },
|
||||
{ get: (target, name) => target[name] ?? (() => {}), set: () => true },
|
||||
);
|
||||
}
|
||||
const document = {
|
||||
documentElement: root,
|
||||
hidden: false,
|
||||
getElementById: (id) =>
|
||||
root.descendants().find((node) => node.id === id) ?? null,
|
||||
querySelector: (selector) => root.querySelector(selector),
|
||||
querySelectorAll: (selector) => root.querySelectorAll(selector),
|
||||
createRange: () => ({ selectNodeContents() {} }),
|
||||
addEventListener() {},
|
||||
};
|
||||
const window = {
|
||||
scrollY: 0,
|
||||
innerHeight: 900,
|
||||
location: { hash: "" },
|
||||
addEventListener() {},
|
||||
getSelection: () => ({ removeAllRanges() {}, addRange() {} }),
|
||||
matchMedia: () => ({ matches: false, addEventListener() {} }),
|
||||
requestAnimationFrame: () => 1,
|
||||
scrollTo() {},
|
||||
setTimeout: (callback) => timers.push(callback),
|
||||
};
|
||||
Object.assign(globalThis, {
|
||||
document,
|
||||
window,
|
||||
requestAnimationFrame: () => 1,
|
||||
cancelAnimationFrame() {},
|
||||
Path2D: class {},
|
||||
fetch: async () => ({
|
||||
ok: true,
|
||||
json: async () => ({ stargazers_count: stars }),
|
||||
}),
|
||||
});
|
||||
Object.defineProperty(globalThis, "navigator", {
|
||||
configurable: true,
|
||||
value: {
|
||||
clipboard: {
|
||||
writeText: async (text) => {
|
||||
if (clipboard === "reject") throw new Error("denied");
|
||||
written.push(text);
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await import(
|
||||
`${pathToFileURL(new URL(script, ASSETS).pathname)}?load=${randomUUID()}`
|
||||
);
|
||||
await nextTurn();
|
||||
return { root, timers, written, canvasText };
|
||||
}
|
||||
|
||||
const copyBlock = () => [
|
||||
h("button", { "data-copy": "command" }, "Copy"),
|
||||
h("pre", {}, h("code", { id: "command" }, "npx rea-agents setup")),
|
||||
];
|
||||
const copyStatus = (messages = {}) =>
|
||||
h("p", {
|
||||
id: "copy-status",
|
||||
...Object.fromEntries(
|
||||
Object.entries(messages).map(([key, value]) => [`data-${key}`, value]),
|
||||
),
|
||||
});
|
||||
const englishCopyMessages = {
|
||||
copied: "Copied",
|
||||
selected: "Selected",
|
||||
"status-copied": "Code copied to clipboard.",
|
||||
"status-selected": "Code selected. Use your keyboard to copy it.",
|
||||
};
|
||||
|
||||
test("copy buttons report and restore localized labels", async () => {
|
||||
const { root, timers, written } = await run("main.js", [
|
||||
...copyBlock(),
|
||||
copyStatus(englishCopyMessages),
|
||||
]);
|
||||
const button = root.querySelector("[data-copy]");
|
||||
button.click();
|
||||
await nextTurn();
|
||||
assert.deepEqual(written, ["npx rea-agents setup"]);
|
||||
assert.equal(button.textContent, "Copied");
|
||||
assert.equal(
|
||||
root.querySelector("#copy-status").textContent,
|
||||
"Code copied to clipboard.",
|
||||
);
|
||||
timers.forEach((callback) => callback());
|
||||
assert.equal(button.textContent, "Copy");
|
||||
});
|
||||
|
||||
test("copy buttons keep their label when messages are missing", async () => {
|
||||
for (const [body, clipboard] of [
|
||||
[[...copyBlock(), copyStatus()], "resolve"],
|
||||
[[...copyBlock(), copyStatus()], "reject"],
|
||||
[
|
||||
[...copyBlock(), copyStatus({ copied: "", "status-copied": "" })],
|
||||
"resolve",
|
||||
],
|
||||
[copyBlock(), "resolve"],
|
||||
]) {
|
||||
const { root, timers, written } = await run("main.js", body, { clipboard });
|
||||
const button = root.querySelector("[data-copy]");
|
||||
button.click();
|
||||
await nextTurn();
|
||||
assert.equal(written.length, clipboard === "resolve" ? 1 : 0);
|
||||
assert.equal(button.textContent, "Copy");
|
||||
assert.equal(root.querySelector("#copy-status")?.textContent ?? "", "");
|
||||
timers.forEach((callback) => callback());
|
||||
assert.equal(button.textContent, "Copy");
|
||||
}
|
||||
});
|
||||
|
||||
const githubLink = (labels = {}) =>
|
||||
h(
|
||||
"a",
|
||||
{
|
||||
class: "nav-github",
|
||||
"aria-label": "REA on GitHub",
|
||||
title: "REA on GitHub",
|
||||
...labels,
|
||||
},
|
||||
h("span", { class: "nav-github-count", hidden: "" }),
|
||||
);
|
||||
|
||||
test("the star label follows the page language and plural rules", async () => {
|
||||
const labels = {
|
||||
"data-label-one": "REA on GitHub ({count} star)",
|
||||
"data-label-other": "REA on GitHub ({count} stars)",
|
||||
};
|
||||
for (const [stars, lang, expected] of [
|
||||
[1, "en", "REA on GitHub (1 star)"],
|
||||
[1234, "en", "REA on GitHub (1,234 stars)"],
|
||||
[1234, "de", "REA on GitHub (1.234 stars)"],
|
||||
]) {
|
||||
const { root } = await run("main.js", [githubLink(labels)], {
|
||||
stars,
|
||||
lang,
|
||||
});
|
||||
await nextTurn();
|
||||
assert.equal(
|
||||
root.querySelector(".nav-github").getAttribute("aria-label"),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
// Languages without a "one" category use the "other" label.
|
||||
const { root } = await run(
|
||||
"main.js",
|
||||
[githubLink({ "data-label-other": "{count}" })],
|
||||
{ stars: 1, lang: "ja" },
|
||||
);
|
||||
await nextTurn();
|
||||
assert.equal(
|
||||
root.querySelector(".nav-github").getAttribute("aria-label"),
|
||||
"1",
|
||||
);
|
||||
});
|
||||
|
||||
test("the star label stays unchanged without a localized label", async () => {
|
||||
const { root } = await run("main.js", [githubLink()], { stars: 1 });
|
||||
await nextTurn();
|
||||
const link = root.querySelector(".nav-github");
|
||||
assert.equal(link.getAttribute("aria-label"), "REA on GitHub");
|
||||
assert.equal(link.getAttribute("title"), "REA on GitHub");
|
||||
assert.equal(link.querySelector(".nav-github-count").hidden, true);
|
||||
});
|
||||
|
||||
const percentDemo = (messages) =>
|
||||
h(
|
||||
"figure",
|
||||
{ "data-percent-demo": "", ...messages },
|
||||
h(
|
||||
"div",
|
||||
{ class: "percent-operation", hidden: "" },
|
||||
h(
|
||||
"button",
|
||||
{ "data-percent-operation": "add", "aria-pressed": "true" },
|
||||
"200 + 10%",
|
||||
),
|
||||
h(
|
||||
"button",
|
||||
{ "data-percent-operation": "multiply", "aria-pressed": "false" },
|
||||
"200 × 10%",
|
||||
),
|
||||
),
|
||||
h("span", { "data-percent-expression": "" }, "200 +"),
|
||||
h("strong", { "data-percent-display": "" }, "20"),
|
||||
h(
|
||||
"div",
|
||||
{ class: "percent-controls", hidden: "" },
|
||||
h("button", { "data-percent-key": "percent" }, "%"),
|
||||
h("button", { "data-percent-key": "equals", disabled: "" }, "="),
|
||||
h("button", { "data-percent-key": "reset" }, "Reset"),
|
||||
),
|
||||
h(
|
||||
"p",
|
||||
{ "data-percent-status": "" },
|
||||
"10% of 200 = 20. Then 200 + 20 = 220.",
|
||||
),
|
||||
);
|
||||
|
||||
test("the percent demo uses page messages and keeps text when one is missing", async () => {
|
||||
const { root } = await run("home.js", [
|
||||
percentDemo({
|
||||
"data-msg-input": "Press % to see what happens to 10.",
|
||||
"data-msg-result": "200 {operator} {percent} = {result}.",
|
||||
}),
|
||||
]);
|
||||
const status = root.querySelector("[data-percent-status]");
|
||||
const key = (name) =>
|
||||
root.querySelector(`[data-percent-key="${name}"]`).click();
|
||||
assert.equal(status.textContent, "Press % to see what happens to 10.");
|
||||
key("percent");
|
||||
assert.equal(status.textContent, "Press % to see what happens to 10.");
|
||||
key("equals");
|
||||
assert.equal(status.textContent, "200 + 20 = 220.");
|
||||
|
||||
const bare = await run("home.js", [percentDemo({})]);
|
||||
const bareStatus = bare.root.querySelector("[data-percent-status]");
|
||||
for (const name of ["percent", "equals", "reset"]) {
|
||||
bare.root.querySelector(`[data-percent-key="${name}"]`).click();
|
||||
assert.equal(
|
||||
bareStatus.textContent,
|
||||
"10% of 200 = 20. Then 200 + 20 = 220.",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const dinoDemo = (messages) =>
|
||||
h(
|
||||
"figure",
|
||||
{ "data-dino-demo": "", ...messages },
|
||||
h("svg", { class: "dino-fallback" }),
|
||||
h("canvas", { hidden: "" }),
|
||||
h(
|
||||
"div",
|
||||
{ class: "dino-controls", hidden: "" },
|
||||
h("button", { "data-dino-start": "" }, "Play"),
|
||||
h("button", { "data-dino-jump": "" }, "Jump ↑"),
|
||||
h("button", { "data-dino-reset": "" }, "Restart"),
|
||||
h("output", { "data-dino-speed": "" }, "6.0"),
|
||||
h("input", { "data-dino-slider": "", value: "6" }),
|
||||
h("input", { "data-dino-auto": "", checked: "" }),
|
||||
),
|
||||
h(
|
||||
"p",
|
||||
{ "data-dino-status": "" },
|
||||
"Original rule: start at 6, then increase toward 13.",
|
||||
),
|
||||
);
|
||||
|
||||
test("the dinosaur game never shows a missing message", async () => {
|
||||
const { root, canvasText } = await run("dino-demo.js", [
|
||||
dinoDemo({}),
|
||||
h("button", { "data-speed-check": "", hidden: "" }, "Replay"),
|
||||
h("p", { "data-speed-check-status": "" }, "Not replayed yet."),
|
||||
]);
|
||||
const start = root.querySelector("[data-dino-start]");
|
||||
const status = root.querySelector("[data-dino-status]");
|
||||
const slider = root.querySelector("[data-dino-slider]");
|
||||
start.click();
|
||||
start.click();
|
||||
slider.value = "9.5";
|
||||
slider.dispatch("input");
|
||||
root.querySelector("[data-dino-reset]").click();
|
||||
assert.equal(start.textContent, "Play");
|
||||
assert.equal(
|
||||
status.textContent,
|
||||
"Original rule: start at 6, then increase toward 13.",
|
||||
);
|
||||
assert.deepEqual(canvasText, []);
|
||||
root.querySelector("[data-speed-check]").click();
|
||||
assert.equal(
|
||||
root.querySelector("[data-speed-check-status]").textContent,
|
||||
"Not replayed yet.",
|
||||
);
|
||||
});
|
||||
|
||||
test("the dinosaur game fills placeholders from page messages", async () => {
|
||||
const { root, canvasText } = await run("dino-demo.js", [
|
||||
dinoDemo({
|
||||
"data-msg-pause": "Pause",
|
||||
"data-msg-fixed-running":
|
||||
"Fixed speed: {speed}. Space or Jump clears a cactus.",
|
||||
"data-msg-distance": "DISTANCE {distance}",
|
||||
}),
|
||||
]);
|
||||
const start = root.querySelector("[data-dino-start]");
|
||||
const slider = root.querySelector("[data-dino-slider]");
|
||||
start.click();
|
||||
assert.equal(start.textContent, "Pause");
|
||||
slider.value = "9.5";
|
||||
slider.dispatch("input");
|
||||
assert.equal(
|
||||
root.querySelector("[data-dino-status]").textContent,
|
||||
"Fixed speed: 9.5. Space or Jump clears a cactus.",
|
||||
);
|
||||
assert.ok(canvasText.includes("DISTANCE 0000"));
|
||||
});
|
||||
+2
-212
@@ -2,10 +2,7 @@
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
from html import escape
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
@@ -140,7 +137,7 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
def test_shared_edits_preserve_body_and_catch_drift_before_rewriting(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates", "locales"):
|
||||
for name in ("public", "templates"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
page = root / "website/public/blog/touhou-reconstruction/index.html"
|
||||
original = page.read_text()
|
||||
@@ -162,7 +159,7 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
def test_new_page_escapes_metadata_stays_a_draft_and_joins_sitemap_when_ready(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates", "locales"):
|
||||
for name in ("public", "templates"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
site = root / "website/public"
|
||||
page = WEBSITE.new_page(root, "blog/new-example", 'A <test> & "result"', 'Check "quotes" & <markup>.')
|
||||
@@ -245,212 +242,5 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
worker.join()
|
||||
|
||||
|
||||
|
||||
def site_copy(directory):
|
||||
"""Copy the authored website inputs without existing translated pages."""
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates", "locales"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
for code in WEBSITE.LANGUAGES:
|
||||
if code != "en":
|
||||
shutil.rmtree(root / "website/public" / code, ignore_errors=True)
|
||||
return root
|
||||
|
||||
|
||||
def zh_hant(key):
|
||||
"""Return one zh-Hant UI string, so assertions follow the locale file."""
|
||||
strings = json.loads((ROOT / "website/locales/zh-hant.json").read_text())["strings"]
|
||||
return escape(strings[key])
|
||||
|
||||
|
||||
def quietly(function, *arguments):
|
||||
"""Run a command helper without its progress output."""
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return function(*arguments)
|
||||
|
||||
|
||||
def mark_reviewed(path):
|
||||
"""Apply the reviewer's change: reviewed status and date, without noindex.
|
||||
|
||||
A reviewed translation has its own search title, so give the test page one.
|
||||
"""
|
||||
source = re.sub(r"<title>(.*?)</title>", r"<title>\1 (zh-Hant)</title>", path.read_text(), count=1)
|
||||
source = source.replace('status="draft"', 'status="reviewed" reviewed="2026-10-11"', 1)
|
||||
path.write_text(source.replace(' <meta name="robots" content="noindex" />\n', "", 1))
|
||||
|
||||
|
||||
def translation_errors(root):
|
||||
"""Return the verifier's translation errors and warnings for a site copy."""
|
||||
return VERIFY.check_translations(root, VERIFY.parse_pages(root / "website/public"))
|
||||
|
||||
|
||||
class WebsiteTranslationChecks(unittest.TestCase):
|
||||
"""Create translations through the contributor command and check what readers get."""
|
||||
|
||||
def test_translate_creates_an_unlinked_noindex_draft(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
site = root / "website/public"
|
||||
english = (site / "get-started/index.html").read_text()
|
||||
page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
|
||||
source = page.read_text()
|
||||
self.assertEqual(page, site / "zh-hant/get-started/index.html")
|
||||
self.assertIn('<html lang="zh-Hant">', source)
|
||||
self.assertIn('<meta name="robots" content="noindex" />', source)
|
||||
self.assertEqual(
|
||||
WEBSITE.translation_record(source),
|
||||
{"source-sha256": WEBSITE.source_digest(english), "status": "draft"},
|
||||
)
|
||||
self.assertIn('href="../../assets/styles.css"', source)
|
||||
self.assertIn('<a class="brand" href="../../"', source)
|
||||
self.assertIn(
|
||||
'<a href="../../zh-hant/get-started/" class="nav-start" aria-current="page">'
|
||||
f'{zh_hant("navGetStarted")}</a>',
|
||||
source,
|
||||
)
|
||||
self.assertIn(
|
||||
f'<a href="../../showcase/" hreflang="en">{zh_hant("navShowcases")}'
|
||||
'<span class="nav-badge" lang="en">EN</span></a>',
|
||||
source,
|
||||
)
|
||||
self.assertIn('<li><a href="../../get-started/" hreflang="en" lang="en">English</a></li>', source)
|
||||
self.assertIn('<meta property="og:locale" content="zh_TW" />', source)
|
||||
self.assertNotIn('rel="alternate"', source)
|
||||
self.assertIn(WEBSITE.NOTICE_START + WEBSITE.NOTICE_END, source)
|
||||
self.assertEqual(english, (site / "get-started/index.html").read_text())
|
||||
errors, warnings = translation_errors(root)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertLessEqual({file for file, _ in warnings}, {"website/locales/zh-hant.json"})
|
||||
self.assertEqual(VERIFY.check_site(site)[3], [])
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
PREPARE.prepare_sitemap(site)
|
||||
self.assertNotIn("zh-hant", (site / "sitemap.xml").read_text())
|
||||
for language, route, message in (
|
||||
("zh-hant", "get-started", "already exists"),
|
||||
("en", "faq", "Choose one of"),
|
||||
("ja", "missing", "No English page"),
|
||||
("ja", "zh-hant/get-started", "without a language directory"),
|
||||
("ja", "../outside", "directory route"),
|
||||
):
|
||||
with self.subTest(language=language, route=route):
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
quietly(WEBSITE.new_translation, root, language, route)
|
||||
|
||||
def test_reviewed_translations_join_menus_hreflang_and_the_sitemap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
site = root / "website/public"
|
||||
mark_reviewed(quietly(WEBSITE.new_translation, root, "zh-hant", "get-started"))
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
english = (site / "get-started/index.html").read_text()
|
||||
for tag, url in (
|
||||
("en", "https://rea.tools/get-started/"),
|
||||
("zh-Hant", "https://rea.tools/zh-hant/get-started/"),
|
||||
("x-default", "https://rea.tools/get-started/"),
|
||||
):
|
||||
self.assertIn(f'<link rel="alternate" hreflang="{tag}" href="{url}" />', english)
|
||||
self.assertIn('<meta property="og:locale:alternate" content="zh_TW" />', english)
|
||||
self.assertIn(
|
||||
'<li><a href="../zh-hant/get-started/" hreflang="zh-Hant" lang="zh-Hant">繁體中文</a></li>', english
|
||||
)
|
||||
self.assertNotIn("language-menu", (site / "faq/index.html").read_text())
|
||||
translated = (site / "zh-hant/get-started/index.html").read_text()
|
||||
self.assertIn('<meta property="og:locale:alternate" content="en_US" />', translated)
|
||||
|
||||
# A reviewed home page becomes the menu's fallback for untranslated pages.
|
||||
mark_reviewed(quietly(WEBSITE.new_translation, root, "zh-hant", "/"))
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
self.assertIn(
|
||||
'<li><a href="../zh-hant/" hreflang="zh-Hant" lang="zh-Hant">繁體中文</a></li>',
|
||||
(site / "faq/index.html").read_text(),
|
||||
)
|
||||
translated = (site / "zh-hant/get-started/index.html").read_text()
|
||||
self.assertIn('<a class="brand" href="../../zh-hant/"', translated)
|
||||
home = (site / "zh-hant/index.html").read_text()
|
||||
self.assertIn('href="get-started/"', home)
|
||||
self.assertIn('href="../showcase/"', home)
|
||||
self.assertIn('href="../assets/styles.css"', home)
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
PREPARE.prepare_sitemap(site)
|
||||
sitemap = (site / "sitemap.xml").read_text()
|
||||
self.assertIn("https://rea.tools/zh-hant/get-started/", sitemap)
|
||||
self.assertIn("https://rea.tools/zh-hant/", sitemap)
|
||||
self.assertEqual(VERIFY.check_site(site)[3], [])
|
||||
self.assertEqual(translation_errors(root)[0], [])
|
||||
|
||||
def test_english_commands_sync_and_text_changes_show_a_notice(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
site = root / "website/public"
|
||||
english_path = site / "get-started/index.html"
|
||||
page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
|
||||
english = english_path.read_text()
|
||||
identifier = WEBSITE.synced_code_ids(english)[0]
|
||||
block = WEBSITE.code_block(identifier)
|
||||
|
||||
# A command-only change is copied and keeps the translation current.
|
||||
command = block.search(english).group(2) + " --test-flag"
|
||||
english_path.write_text(block.sub(lambda match: match.group(1) + command + match.group(3), english))
|
||||
with self.assertRaisesRegex(ValueError, "Shared HTML is stale: zh-hant/get-started/index.html"):
|
||||
quietly(WEBSITE.sync_pages, root, True)
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
self.assertEqual(block.search(page.read_text()).group(2), command)
|
||||
self.assertNotIn('class="translation-notice"', page.read_text())
|
||||
self.assertEqual(translation_errors(root), ([], []))
|
||||
|
||||
# A text change marks the translation out of date.
|
||||
unchanged = english_path.read_text()
|
||||
changed = unchanged.replace(" </main>", " <p>A new English paragraph.</p>\n </main>", 1)
|
||||
english_path.write_text(changed)
|
||||
self.assertEqual(quietly(WEBSITE.sync_pages, root, True), [page])
|
||||
errors, warnings = translation_errors(root)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertTrue(any("has changed since this translation" in message for _, message in warnings))
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
link = f'<a href="../../get-started/" hreflang="en">{zh_hant("staleNoticeLink")}</a>'
|
||||
self.assertIn(
|
||||
f'<p class="translation-notice" role="note">{zh_hant("staleNotice").replace("{link}", link)}</p>',
|
||||
page.read_text(),
|
||||
)
|
||||
report = quietly(WEBSITE.translation_report, root)
|
||||
self.assertIn(f' after updating the translation, record source-sha256="{WEBSITE.source_digest(changed)}"', report)
|
||||
|
||||
english_path.write_text(unchanged)
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
self.assertNotIn('class="translation-notice"', page.read_text())
|
||||
|
||||
page.write_text(block.sub(lambda match: match.group(1) + "edited" + match.group(3), page.read_text()))
|
||||
self.assertTrue(any(f"{identifier} must match" in error for error in translation_errors(root)[0]))
|
||||
|
||||
def test_translations_keep_structure_and_cjk_spacing(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
|
||||
original = page.read_text()
|
||||
first, second = WEBSITE.copy_targets(original)[:2]
|
||||
|
||||
def added(paragraph):
|
||||
return (" </main>", f" {paragraph}\n </main>")
|
||||
|
||||
cases = (
|
||||
(*added("<p>完成後\n 重新啟動。</p>"), "renders as a gap"),
|
||||
('<main id="main"', '<main id="moved"', "element IDs differ"),
|
||||
(f'data-copy="{first}"', f'data-copy="{second}"', "copy buttons differ"),
|
||||
(' <meta name="robots" content="noindex" />\n', "", "draft translation needs"),
|
||||
('status="draft"', 'status="final"', "status must be draft or reviewed"),
|
||||
)
|
||||
for old, new, message in cases:
|
||||
with self.subTest(message=message):
|
||||
self.assertIn(old, original)
|
||||
page.write_text(original.replace(old, new, 1))
|
||||
self.assertTrue(any(message in error for error in translation_errors(root)[0]))
|
||||
page.write_text(original.replace(*added("<p>重新啟動 REA 用戶端。</p>"), 1))
|
||||
self.assertEqual(translation_errors(root)[0], [])
|
||||
page.write_text(original.replace(*added("<p><code>extra-command</code></p>"), 1))
|
||||
errors, warnings = translation_errors(root)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertTrue(any("inline code differs" in message for _, message in warnings))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -15,10 +15,6 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
|
||||
import {
|
||||
OwnedCommandFailure,
|
||||
runOwnedCommand,
|
||||
} from "../dist/process/OwnedCommand.js";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
@@ -60,13 +56,6 @@ const root = await PrivateRuntimeRoot.create({
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
|
||||
);
|
||||
const limitedPython = join(root.path, "limited-python");
|
||||
const limitedEnvironment = (option) => ({
|
||||
...environment,
|
||||
REA_PWNTOOLS_PYTHON: limitedPython,
|
||||
REA_VERIFY_LIMITED_PYTHON: python,
|
||||
REA_VERIFY_LIMIT_OPTION: option,
|
||||
});
|
||||
const client = new Client({ name: "binary-layout-verifier", version: "1" });
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
@@ -84,46 +73,6 @@ let cases = 0;
|
||||
let bootstrapCases = 0;
|
||||
const failures = [];
|
||||
try {
|
||||
await execute(
|
||||
"gcc",
|
||||
[
|
||||
"-std=c11",
|
||||
"-Wall",
|
||||
"-Wextra",
|
||||
"-Werror",
|
||||
fileURLToPath(
|
||||
new URL("./fixtures/binary-layout-limited-python.c", import.meta.url),
|
||||
),
|
||||
"-o",
|
||||
limitedPython,
|
||||
],
|
||||
{ timeout: 30_000, maxBuffer: 1024 * 1024 },
|
||||
);
|
||||
// A live limited child must retain its configured launcher's identity so
|
||||
// timeout cleanup can still verify and terminate the entire owned group.
|
||||
await assert.rejects(
|
||||
runOwnedCommand(
|
||||
{
|
||||
command: limitedPython,
|
||||
arguments: [
|
||||
"-I",
|
||||
"-c",
|
||||
"import time; print('limited child ready', flush=True); time.sleep(60)",
|
||||
],
|
||||
cwd: root.path,
|
||||
runId: `${run.run_id}-limited-python-timeout`,
|
||||
hostEnvironment: limitedEnvironment("--as=2147483648"),
|
||||
},
|
||||
{ timeoutMs: 2000, diagnosticBytes: 1024 },
|
||||
),
|
||||
(error) => {
|
||||
assert.ok(error instanceof OwnedCommandFailure);
|
||||
assert.equal(error.reason, "timeout");
|
||||
assert.equal(error.cleanupFailure, null);
|
||||
assert.ok(error.snapshot.stdout.text.includes("limited child ready"));
|
||||
return true;
|
||||
},
|
||||
);
|
||||
const bootstrapRoot = join(root.path, "bootstrap-boundary");
|
||||
await mkdir(bootstrapRoot);
|
||||
const bootstrapPath = join(bootstrapRoot, "layout.py");
|
||||
@@ -847,12 +796,19 @@ try {
|
||||
assert.deepEqual(await readFile(extendedPath), extendedObject);
|
||||
cases++;
|
||||
}
|
||||
for (const [option, expected] of [
|
||||
["--as=2147483648", '"address_space_bytes": 2147483648'],
|
||||
["--cpu=20", '"cpu_seconds": 20'],
|
||||
["--fsize=33554432", '"file_size_bytes": 33554432'],
|
||||
for (const [name, option, expected] of [
|
||||
["address-space", "--as=2147483648", '"address_space_bytes": 2147483648'],
|
||||
["cpu", "--cpu=20", '"cpu_seconds": 20'],
|
||||
["file-size", "--fsize=33554432", '"file_size_bytes": 33554432'],
|
||||
]) {
|
||||
const selectedEnvironment = limitedEnvironment(option);
|
||||
const wrapper = join(root.path, `python-limit-${name}`);
|
||||
const quotedPython = "'" + python.replaceAll("'", "'\"'\"'") + "'";
|
||||
await writeFile(
|
||||
wrapper,
|
||||
`#!/bin/sh\nexec /usr/bin/prlimit ${option} -- ${quotedPython} "$@"\n`,
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
const limitedEnvironment = { ...environment, REA_PWNTOOLS_PYTHON: wrapper };
|
||||
const limitedClient = new Client({
|
||||
name: "limited-layout-verifier",
|
||||
version: "1",
|
||||
@@ -860,7 +816,7 @@ try {
|
||||
const limitedTransport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: selectedEnvironment,
|
||||
env: limitedEnvironment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
try {
|
||||
@@ -870,7 +826,7 @@ try {
|
||||
mode,
|
||||
join(root.path, "protected"),
|
||||
undefined,
|
||||
selectedEnvironment,
|
||||
limitedEnvironment,
|
||||
limitedClient,
|
||||
);
|
||||
assert.ok(value.limitations.some((item) => item.includes(expected)));
|
||||
@@ -906,29 +862,17 @@ try {
|
||||
sectionHeavy.writeUInt16LE(sectionCount, 60);
|
||||
const sectionHeavyPath = join(root.path, "section-heavy.o");
|
||||
await writeFile(sectionHeavyPath, sectionHeavy);
|
||||
// pwntools materializes each PT_LOAD zero-filled tail during inspection.
|
||||
// One allocation larger than the whole address-space budget fails promptly;
|
||||
// a large section count instead races CPU work against the command deadline.
|
||||
const memoryHeavy = Buffer.from(await readFile(join(root.path, "protected")));
|
||||
const memoryPhOffset = Number(memoryHeavy.readBigUInt64LE(32));
|
||||
const memoryPhSize = memoryHeavy.readUInt16LE(54);
|
||||
const memoryPhCount = memoryHeavy.readUInt16LE(56);
|
||||
const lastLoadIndex = Array.from(
|
||||
{ length: memoryPhCount },
|
||||
(_, index) => index,
|
||||
).findLast(
|
||||
(index) =>
|
||||
memoryHeavy.readUInt32LE(memoryPhOffset + index * memoryPhSize) === 1,
|
||||
const memoryWrapper = join(root.path, "python-memory-constraint");
|
||||
const quotedPython = "'" + python.replaceAll("'", "'\"'\"'") + "'";
|
||||
await writeFile(
|
||||
memoryWrapper,
|
||||
`#!/bin/sh\nexec /usr/bin/prlimit --as=100663296 -- ${quotedPython} "$@"\n`,
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
assert.notEqual(lastLoadIndex, undefined);
|
||||
const memorySizeOffset = memoryPhOffset + lastLoadIndex * memoryPhSize + 40;
|
||||
memoryHeavy.writeBigUInt64LE(
|
||||
memoryHeavy.readBigUInt64LE(memorySizeOffset) + 128n * 1024n * 1024n,
|
||||
memorySizeOffset,
|
||||
);
|
||||
const memoryHeavyPath = join(root.path, "memory-zero-tail");
|
||||
await writeFile(memoryHeavyPath, memoryHeavy);
|
||||
const memoryEnvironment = limitedEnvironment("--as=100663296");
|
||||
const memoryEnvironment = {
|
||||
...environment,
|
||||
REA_PWNTOOLS_PYTHON: memoryWrapper,
|
||||
};
|
||||
const memoryClient = new Client({
|
||||
name: "memory-layout-verifier",
|
||||
version: "1",
|
||||
@@ -944,7 +888,7 @@ try {
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const error = await inspect(
|
||||
mode,
|
||||
memoryHeavyPath,
|
||||
sectionHeavyPath,
|
||||
"resource_constraint",
|
||||
memoryEnvironment,
|
||||
memoryClient,
|
||||
@@ -959,7 +903,7 @@ try {
|
||||
assert.ok(error.remediation.action.includes("memory"));
|
||||
cases++;
|
||||
}
|
||||
assert.deepEqual(await readFile(memoryHeavyPath), memoryHeavy);
|
||||
assert.deepEqual(await readFile(sectionHeavyPath), sectionHeavy);
|
||||
} finally {
|
||||
try {
|
||||
await memoryClient.close();
|
||||
@@ -980,7 +924,13 @@ try {
|
||||
cpuHeavy.writeBigUInt64LE(BigInt(cpuSectionCount), sectionOffset + 32);
|
||||
const cpuHeavyPath = join(root.path, "cpu-section-heavy.o");
|
||||
await writeFile(cpuHeavyPath, cpuHeavy);
|
||||
const cpuEnvironment = limitedEnvironment("--cpu=1:");
|
||||
const cpuWrapper = join(root.path, "python-cpu-constraint");
|
||||
await writeFile(
|
||||
cpuWrapper,
|
||||
`#!/bin/sh\nexec /usr/bin/prlimit --cpu=1: --core=0: -- ${quotedPython} "$@"\n`,
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
const cpuEnvironment = { ...environment, REA_PWNTOOLS_PYTHON: cpuWrapper };
|
||||
const cpuClient = new Client({ name: "cpu-layout-verifier", version: "1" });
|
||||
const cpuTransport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
@@ -1013,7 +963,13 @@ try {
|
||||
}
|
||||
}
|
||||
for (const limit of [1024, 1]) {
|
||||
const fileEnvironment = limitedEnvironment(`--fsize=${limit}:`);
|
||||
const wrapper = join(root.path, `python-file-size-${limit}`);
|
||||
await writeFile(
|
||||
wrapper,
|
||||
`#!/bin/sh\nexec /usr/bin/prlimit --fsize=${limit}: --core=0: -- ${quotedPython} "$@"\n`,
|
||||
{ mode: 0o700 },
|
||||
);
|
||||
const fileEnvironment = { ...environment, REA_PWNTOOLS_PYTHON: wrapper };
|
||||
const fileClient = new Client({
|
||||
name: "file-size-layout-verifier",
|
||||
version: "1",
|
||||
|
||||
+97
-145
@@ -38,147 +38,104 @@ let update;
|
||||
let repositorySkill;
|
||||
let report;
|
||||
|
||||
const runStage = async (name, action) => {
|
||||
const started = performance.now();
|
||||
process.stderr.write(`[verify:package] ${name}: start\n`);
|
||||
const result = await action();
|
||||
process.stderr.write(
|
||||
`[verify:package] ${name}: complete (${Math.round(performance.now() - started)} ms)\n`,
|
||||
);
|
||||
return result;
|
||||
};
|
||||
|
||||
try {
|
||||
repositorySkill = await runStage("repository skill", () =>
|
||||
verifyRepositorySkill({ root, workspace }),
|
||||
);
|
||||
({ tarball } = await runStage("package archive", () =>
|
||||
verifyPackagePack({ root, workspace }),
|
||||
));
|
||||
const environmentData = await runStage("package environment", () =>
|
||||
verifyPackageEnvironment({
|
||||
root,
|
||||
workspace,
|
||||
evidenceRoot,
|
||||
referenceRoot,
|
||||
}),
|
||||
);
|
||||
const { cli, packageRunnerCli } = await runStage("package install", () =>
|
||||
verifyPackageInstall({
|
||||
tarball,
|
||||
prefix: environmentData.prefix,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
update = await runStage("package update", () =>
|
||||
verifyPackageUpdate({
|
||||
prefix: environmentData.prefix,
|
||||
packageRoot: join(
|
||||
environmentData.prefix,
|
||||
"lib",
|
||||
"node_modules",
|
||||
PRODUCT_IDENTITY.packageName,
|
||||
),
|
||||
tarball,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
mcpStartup = await runStage("MCP startup", () =>
|
||||
measureMcpStartup({
|
||||
command: cli,
|
||||
args: ["mcp"],
|
||||
environment: {
|
||||
...environmentData.environment,
|
||||
},
|
||||
policy: MCP_STARTUP_POLICY,
|
||||
}),
|
||||
);
|
||||
mcpModuleLoading = await runStage("MCP module loading", () =>
|
||||
profileMcpModuleLoading({
|
||||
command: cli,
|
||||
args: ["mcp"],
|
||||
environment: {
|
||||
...environmentData.environment,
|
||||
},
|
||||
policy: MCP_STARTUP_POLICY,
|
||||
packageName: PRODUCT_IDENTITY.packageName,
|
||||
}),
|
||||
);
|
||||
const { supportedSetupHost, hopperSetupSupported } = await runStage(
|
||||
"package discovery",
|
||||
() =>
|
||||
verifyPackageDiscovery({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
const { artifactArchive } = await runStage("artifact and Electron CLI", () =>
|
||||
verifyPackageArtifactAndElectron({
|
||||
cli,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
await runStage("managed analysis CLI", () =>
|
||||
verifyManaged({
|
||||
cli,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
await runStage("unknown provider CLI", () =>
|
||||
verifyUnknownProvider({
|
||||
repositorySkill = await verifyRepositorySkill({ root, workspace });
|
||||
({ tarball } = await verifyPackagePack({ root, workspace }));
|
||||
const environmentData = await verifyPackageEnvironment({
|
||||
root,
|
||||
workspace,
|
||||
evidenceRoot,
|
||||
referenceRoot,
|
||||
});
|
||||
const { cli, packageRunnerCli } = await verifyPackageInstall({
|
||||
tarball,
|
||||
prefix: environmentData.prefix,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
update = await verifyPackageUpdate({
|
||||
prefix: environmentData.prefix,
|
||||
packageRoot: join(
|
||||
environmentData.prefix,
|
||||
"lib",
|
||||
"node_modules",
|
||||
PRODUCT_IDENTITY.packageName,
|
||||
),
|
||||
tarball,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
mcpStartup = await measureMcpStartup({
|
||||
command: cli,
|
||||
args: ["mcp"],
|
||||
environment: {
|
||||
...environmentData.environment,
|
||||
},
|
||||
policy: MCP_STARTUP_POLICY,
|
||||
});
|
||||
mcpModuleLoading = await profileMcpModuleLoading({
|
||||
command: cli,
|
||||
args: ["mcp"],
|
||||
environment: {
|
||||
...environmentData.environment,
|
||||
},
|
||||
policy: MCP_STARTUP_POLICY,
|
||||
packageName: PRODUCT_IDENTITY.packageName,
|
||||
});
|
||||
const { supportedSetupHost, hopperSetupSupported } =
|
||||
await verifyPackageDiscovery({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
await runStage("platform CLI", () =>
|
||||
verifyPackagePlatform({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
await runStage("capabilities and search CLI", () =>
|
||||
verifyPackageCapabilitiesAndSearch({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
await runStage("evidence CLI", () =>
|
||||
verifyPackageEvidence({
|
||||
cli,
|
||||
evidenceRoot,
|
||||
referenceRoot,
|
||||
environment: environmentData.environment,
|
||||
}),
|
||||
);
|
||||
await runStage("setup and uninstall CLI", () =>
|
||||
verifyPackageSetup({
|
||||
cli,
|
||||
packageRunnerCli,
|
||||
environment: environmentData.environment,
|
||||
home: environmentData.home,
|
||||
npxLog: environmentData.npxLog,
|
||||
claudeConfig: environmentData.claudeConfig,
|
||||
codexConfig: environmentData.codexConfig,
|
||||
cursorConfig: environmentData.cursorConfig,
|
||||
codexTarget: environmentData.codexTarget,
|
||||
cursorTarget: environmentData.cursorTarget,
|
||||
supportedSetupHost,
|
||||
hopperSetupSupported,
|
||||
root,
|
||||
}),
|
||||
);
|
||||
await runStage("MCP tools and evidence", () =>
|
||||
verifyPackageMcp({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
evidenceRoot,
|
||||
artifactArchive,
|
||||
}),
|
||||
);
|
||||
});
|
||||
const { artifactArchive } = await verifyPackageArtifactAndElectron({
|
||||
cli,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
await verifyManaged({
|
||||
cli,
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
await verifyUnknownProvider({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
await verifyPackagePlatform({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
await verifyPackageCapabilitiesAndSearch({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
await verifyPackageEvidence({
|
||||
cli,
|
||||
evidenceRoot,
|
||||
referenceRoot,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
await verifyPackageSetup({
|
||||
cli,
|
||||
packageRunnerCli,
|
||||
environment: environmentData.environment,
|
||||
home: environmentData.home,
|
||||
npxLog: environmentData.npxLog,
|
||||
claudeConfig: environmentData.claudeConfig,
|
||||
codexConfig: environmentData.codexConfig,
|
||||
cursorConfig: environmentData.cursorConfig,
|
||||
codexTarget: environmentData.codexTarget,
|
||||
cursorTarget: environmentData.cursorTarget,
|
||||
supportedSetupHost,
|
||||
hopperSetupSupported,
|
||||
root,
|
||||
});
|
||||
await verifyPackageMcp({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
evidenceRoot,
|
||||
artifactArchive,
|
||||
});
|
||||
report = {
|
||||
cli: true,
|
||||
analysisCli: true,
|
||||
@@ -216,13 +173,8 @@ try {
|
||||
mcpModuleLoading,
|
||||
};
|
||||
} finally {
|
||||
await runStage("workspace cleanup", () =>
|
||||
rm(workspace, { recursive: true, force: true }),
|
||||
);
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
}
|
||||
const verifierLineage = await runStage("verifier lineage", () =>
|
||||
completeVerifierRun(verifierRun),
|
||||
);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ verifier_run: verifierLineage, ...report })}\n`,
|
||||
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
|
||||
);
|
||||
|
||||
@@ -277,13 +277,11 @@ try {
|
||||
throw new Error("Scenario attachment terminated its external browser");
|
||||
|
||||
const profilesBefore = await scenarioProfiles();
|
||||
process.stderr.write("Browser verifier: scenario environment\n");
|
||||
const scenarioEnvironment = await verifyScenarioEnvironment(
|
||||
endpoint,
|
||||
target,
|
||||
site.origin,
|
||||
);
|
||||
process.stderr.write("Browser verifier: scenario storage\n");
|
||||
const scenarioStorage = await verifyScenarioStorage({
|
||||
executable,
|
||||
endpoint,
|
||||
@@ -341,9 +339,7 @@ try {
|
||||
`Browser scenario included unselected structured value: ${unselectedValue}`,
|
||||
);
|
||||
|
||||
process.stderr.write("Browser verifier: large screenshot\n");
|
||||
const largeScreenshot = await verifyLargeScreenshotE2e(endpoint, site.origin);
|
||||
process.stderr.write("Browser verifier: popup coverage\n");
|
||||
const popupEvents = await verifyPopupEventCoverage(executable);
|
||||
const networkEvidence = await verifyBrowserNetworkEvidence(
|
||||
executable,
|
||||
|
||||
@@ -674,113 +674,6 @@ export async function verifyRelativeSwitch(client, procedures, entrySize) {
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Oracle for relative-switch-signed.S: signed halfword byte offsets from a base
|
||||
* inside the case bodies, a CBZ route for the zero entry, and a B.CS bound.
|
||||
*/
|
||||
const SIGNED_RELATIVE_SWITCH_CASES = [
|
||||
{ value: 0, immediate: 200 },
|
||||
{ value: 1, immediate: 201 },
|
||||
{ value: 2, immediate: 202 },
|
||||
{ value: 3, immediate: 250 },
|
||||
{ value: 4, immediate: 204 },
|
||||
];
|
||||
|
||||
export async function verifyRelativeSignedSwitch(client, selectorBias = 0) {
|
||||
const names = await inventoryCall(client, "list_names", {
|
||||
document: null,
|
||||
address: null,
|
||||
});
|
||||
const procedure = requireProcedure(names, "rea_relative_switch_signed");
|
||||
const dossier = await functionCall(client, "analyze_function", {
|
||||
procedure: procedure.address,
|
||||
});
|
||||
const table = dossier.native_api?.jump_tables.find((item) =>
|
||||
item.data_sources.some(
|
||||
(source) =>
|
||||
source.entry_size_bytes === 2 &&
|
||||
source.entry_count === SIGNED_RELATIVE_SWITCH_CASES.length,
|
||||
),
|
||||
);
|
||||
if (table === undefined)
|
||||
throw new Error(
|
||||
`Exact signed halfword relative table unavailable: ${JSON.stringify(dossier.native_api)}`,
|
||||
);
|
||||
const expectedValues = SIGNED_RELATIVE_SWITCH_CASES.map(
|
||||
({ value }) => value + selectorBias,
|
||||
);
|
||||
if (
|
||||
table.mappings.some(
|
||||
(mapping) =>
|
||||
mapping.case_value !== null &&
|
||||
!expectedValues.includes(mapping.case_value),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
`Signed relative switch emitted an unsupported case label: ${JSON.stringify(table)}`,
|
||||
);
|
||||
if (table.mappings.length !== SIGNED_RELATIVE_SWITCH_CASES.length)
|
||||
throw new Error(
|
||||
`Signed relative switch has unexpected mappings: ${JSON.stringify(table)}`,
|
||||
);
|
||||
if (selectorBias !== 0) {
|
||||
if (
|
||||
!table.limitations.some((limitation) =>
|
||||
limitation.includes("table indices are not proven"),
|
||||
) ||
|
||||
!table.mappings.some((mapping) => mapping.case_value === null) ||
|
||||
table.mappings.some((mapping) => mapping.case_value === selectorBias + 3)
|
||||
)
|
||||
throw new Error(
|
||||
`Normalized selector must retain unknown case evidence: ${JSON.stringify(table)}`,
|
||||
);
|
||||
}
|
||||
for (const { value: tableIndex, immediate } of SIGNED_RELATIVE_SWITCH_CASES) {
|
||||
const value = tableIndex + selectorBias;
|
||||
// Without a proven relationship between the machine index and the typed
|
||||
// switch selector, the CBZ-only case must stay unknown rather than being
|
||||
// assigned the normalized index or Ghidra's unreachable base destination.
|
||||
if (selectorBias !== 0 && tableIndex === 3) continue;
|
||||
const mapping = table.mappings.find((item) => item.case_value === value);
|
||||
if (
|
||||
mapping === undefined ||
|
||||
mapping.confidence !== "high" ||
|
||||
(selectorBias !== 0 &&
|
||||
!mapping.evidence.some(
|
||||
(evidence) => evidence.source === "ghidra-clang-case-token",
|
||||
))
|
||||
)
|
||||
throw new Error(
|
||||
`Signed relative switch case ${value} was not verified: ${JSON.stringify(table)}`,
|
||||
);
|
||||
const instruction = await functionCall(
|
||||
client,
|
||||
"inspect_native_instruction",
|
||||
{ address: mapping.target_address },
|
||||
);
|
||||
if (
|
||||
instruction.status !== "decoded" ||
|
||||
!instruction.operands.some((operand) =>
|
||||
operand.components.some(
|
||||
(component) =>
|
||||
component.kind === "immediate" &&
|
||||
component.value === `0x${immediate.toString(16)}`,
|
||||
),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
`Signed relative switch target does not implement source case ${value}: ${JSON.stringify(instruction)}`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
entry_size: 2,
|
||||
signed: true,
|
||||
selector_bias: selectorBias,
|
||||
dispatch_address: table.dispatch_address,
|
||||
mappings: table.mappings.length,
|
||||
};
|
||||
}
|
||||
|
||||
export async function verifyNativeValueTrace(client, procedures, target) {
|
||||
const entry = requireProcedure(procedures, "rea_ghidra_inventory_entry");
|
||||
const names = await inventoryCall(client, "list_names", {
|
||||
|
||||
@@ -38,7 +38,6 @@ import {
|
||||
verifyJumpTableDenseSwitch,
|
||||
verifyNativeTypeLayout,
|
||||
verifyRelativeSwitch,
|
||||
verifyRelativeSignedSwitch,
|
||||
verifyNativeValueTrace,
|
||||
} from "./verify-real-ghidra-function.mjs";
|
||||
const exec = promisify(execFile);
|
||||
@@ -174,36 +173,6 @@ try {
|
||||
]);
|
||||
}
|
||||
}
|
||||
const signedRelativeSource = fileURLToPath(
|
||||
new URL(
|
||||
"../tests/conformance/ghidra/relative-switch-signed.S",
|
||||
import.meta.url,
|
||||
),
|
||||
);
|
||||
for (const selectorBias of [0, 2, 10]) {
|
||||
for (const format of [
|
||||
"elf",
|
||||
...(expectedNativeTarget.architecture === "arm64" ? ["mach-o"] : []),
|
||||
]) {
|
||||
const targetPath = join(
|
||||
fixtureRoot,
|
||||
`relative-signed-${selectorBias}-${format}.o`,
|
||||
);
|
||||
await exec(clang, [
|
||||
...(format === "elf" ? ["--target=aarch64-linux-gnu"] : []),
|
||||
`-DREA_SELECTOR_BIAS=${selectorBias}`,
|
||||
"-c",
|
||||
signedRelativeSource,
|
||||
"-o",
|
||||
targetPath,
|
||||
]);
|
||||
crossTargets.push([
|
||||
targetPath,
|
||||
`relative-signed-${selectorBias}`,
|
||||
{ format, architecture: "arm64" },
|
||||
]);
|
||||
}
|
||||
}
|
||||
|
||||
await exec(clang, [
|
||||
"--target=aarch64-linux-gnu",
|
||||
@@ -553,30 +522,25 @@ async function verifyTarget(targetPath, variant, expectedTarget = null) {
|
||||
client,
|
||||
})
|
||||
: null;
|
||||
const probes = variant.startsWith("relative-signed-")
|
||||
? await verifyRelativeSignedSwitch(
|
||||
const probes = variant.startsWith("relative-")
|
||||
? await verifyRelativeSwitch(
|
||||
client,
|
||||
Number(variant.split("-").at(-1)),
|
||||
procedures,
|
||||
Number(variant.slice(-1)),
|
||||
)
|
||||
: variant.startsWith("relative-")
|
||||
? await verifyRelativeSwitch(
|
||||
client,
|
||||
procedures,
|
||||
Number(variant.slice(-1)),
|
||||
)
|
||||
: variant === "type-layout"
|
||||
? await verifyNativeTypeLayout(client, names)
|
||||
: variant === "jump-table"
|
||||
? await verifyJumpTableDenseSwitch(client, procedures)
|
||||
: variant === "custom" || variant === "aarch64-jump-table"
|
||||
? null
|
||||
: await verifyInventoryOperations({
|
||||
client,
|
||||
variant,
|
||||
procedures,
|
||||
names,
|
||||
strings,
|
||||
});
|
||||
: variant === "type-layout"
|
||||
? await verifyNativeTypeLayout(client, names)
|
||||
: variant === "jump-table"
|
||||
? await verifyJumpTableDenseSwitch(client, procedures)
|
||||
: variant === "custom" || variant === "aarch64-jump-table"
|
||||
? null
|
||||
: await verifyInventoryOperations({
|
||||
client,
|
||||
variant,
|
||||
procedures,
|
||||
names,
|
||||
strings,
|
||||
});
|
||||
const nativeValues =
|
||||
variant === "debug"
|
||||
? await verifyNativeValueTrace(client, procedures, parsedTarget.value)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { access, writeFile } from "node:fs/promises";
|
||||
import { access, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
@@ -24,6 +25,8 @@ const electronPath = join(fixtureRoot, "electron");
|
||||
const environment = {
|
||||
...process.env,
|
||||
};
|
||||
const temporaryRoot = await mkdtemp(join(tmpdir(), "rea-readiness-"));
|
||||
|
||||
await access(nativePath);
|
||||
const nativeEvidence = await runCli(["inspect-artifact", nativePath, "--json"]);
|
||||
const javascriptCliEvidence = await runCli([
|
||||
@@ -132,6 +135,7 @@ try {
|
||||
);
|
||||
} finally {
|
||||
await client.close();
|
||||
await rm(temporaryRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function runCli(arguments_) {
|
||||
|
||||
+8
-221
@@ -1,8 +1,6 @@
|
||||
"""Check website references, search metadata and the sole production publisher."""
|
||||
|
||||
from collections import Counter
|
||||
from html.parser import HTMLParser
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
@@ -12,21 +10,7 @@ from urllib.parse import unquote, urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
from zipfile import BadZipFile, ZipFile
|
||||
|
||||
from website import (
|
||||
LANGUAGES,
|
||||
NOTES_FIXTURES,
|
||||
SITE_ORIGIN,
|
||||
SiteState,
|
||||
authored_markdown,
|
||||
code_block,
|
||||
copy_targets,
|
||||
page_language,
|
||||
page_url,
|
||||
source_digest,
|
||||
sync_pages,
|
||||
synced_code_ids,
|
||||
translation_record,
|
||||
)
|
||||
from website import NOTES_FIXTURES, SITE_ORIGIN, authored_markdown, page_url, sync_pages
|
||||
|
||||
LEGACY_ORIGIN = "morluto.github.io"
|
||||
SITE_TEXT_URL = re.compile(
|
||||
@@ -183,9 +167,8 @@ def check_search_metadata(site, pages):
|
||||
title = page.titles[0].strip() if len(page.titles) == 1 else ""
|
||||
if not title:
|
||||
errors.append(f"{label}: needs exactly one nonempty head title")
|
||||
tag = LANGUAGES[page_language(site, path)[0]].tag
|
||||
if page.lang != tag or page.h1_count != 1:
|
||||
errors.append(f"{label}: needs lang={tag} and exactly one h1")
|
||||
if page.lang != "en" or page.h1_count != 1:
|
||||
errors.append(f"{label}: needs lang=en and exactly one h1")
|
||||
if label in NOTES_FIXTURES and not page.noindex():
|
||||
errors.append(
|
||||
f"{label}: example application must have a head noindex directive"
|
||||
@@ -297,21 +280,15 @@ def check_search_metadata(site, pages):
|
||||
return len(indexable), errors
|
||||
|
||||
|
||||
def parse_pages(site):
|
||||
"""Parse every HTML page once for reference, metadata and translation checks."""
|
||||
def check_site(site):
|
||||
"""Check assets, HTML fragment destinations and SVG syntax."""
|
||||
pages = {}
|
||||
errors = []
|
||||
references = 0
|
||||
for path in sorted(site.rglob("*.html")):
|
||||
parser = HtmlReferences()
|
||||
parser.feed(path.read_text(encoding="utf-8"))
|
||||
pages[path] = parser
|
||||
return pages
|
||||
|
||||
|
||||
def check_site(site):
|
||||
"""Check assets, HTML fragment destinations and SVG syntax."""
|
||||
pages = parse_pages(site)
|
||||
errors = []
|
||||
references = 0
|
||||
if site / "index.html" not in pages:
|
||||
errors.append("website/public/index.html is missing")
|
||||
|
||||
@@ -369,190 +346,6 @@ def check_site(site):
|
||||
return len(pages), references, indexable, errors
|
||||
|
||||
|
||||
# Han, kana and CJK punctuation; a space between two of them renders as a gap.
|
||||
CJK_GAP = re.compile(
|
||||
r"([、-〿-ヿ㐀-䶿一-鿿豈-！-])"
|
||||
r"\s+([、-〿-ヿ㐀-䶿一-鿿豈-！-])"
|
||||
)
|
||||
CODE_TAGS = {"code", "kbd", "pre", "script", "style"}
|
||||
|
||||
|
||||
class TranslationText(HTMLParser):
|
||||
"""Collect prose, inline code and message attributes for comparing translations."""
|
||||
|
||||
def __init__(self, synced):
|
||||
super().__init__()
|
||||
self.synced = set(synced)
|
||||
self.stack = []
|
||||
self.prose = []
|
||||
self.inline_code = Counter()
|
||||
self.messages = Counter()
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attributes = dict(attrs)
|
||||
self.messages.update(name for name in attributes if name.startswith("data-msg-"))
|
||||
if tag in CODE_TAGS:
|
||||
self.stack.append([tag, attributes.get("id"), ""])
|
||||
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
attributes = dict(attrs)
|
||||
self.messages.update(name for name in attributes if name.startswith("data-msg-"))
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.stack and self.stack[-1][0] == tag:
|
||||
name, identifier, text = self.stack.pop()
|
||||
if name in ("code", "kbd") and identifier not in self.synced:
|
||||
self.inline_code[" ".join(text.split())] += 1
|
||||
if self.stack:
|
||||
self.stack[-1][2] += text
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.stack:
|
||||
self.stack[-1][2] += data
|
||||
else:
|
||||
self.prose.append(data)
|
||||
|
||||
|
||||
def translation_warning(path, root, message):
|
||||
"""Return a warning tied to a repository file."""
|
||||
return (path.relative_to(root).as_posix(), message)
|
||||
|
||||
|
||||
def check_translations(root, pages):
|
||||
"""Check translation records, copied code, structure parity and CJK spacing.
|
||||
|
||||
A changed English source is a warning: its translation keeps working, shows a
|
||||
notice after sync and is updated later. Errors cover what sync or the
|
||||
translator controls directly.
|
||||
"""
|
||||
site = root / "website/public"
|
||||
errors = []
|
||||
warnings = []
|
||||
try:
|
||||
state = SiteState(root)
|
||||
except ValueError as error:
|
||||
return [str(error)], []
|
||||
for path, page in pages.items():
|
||||
language, english = page_language(site, path)
|
||||
if language == "en":
|
||||
continue
|
||||
label = path.relative_to(site).as_posix()
|
||||
source = path.read_text(encoding="utf-8")
|
||||
record = translation_record(source)
|
||||
english_path = site / english
|
||||
if not english_path.is_file():
|
||||
errors.append(f"{label}: translates {english}, which does not exist")
|
||||
continue
|
||||
english_source = english_path.read_text(encoding="utf-8")
|
||||
if record is None:
|
||||
errors.append(f"{label}: needs one <!-- website:translation source-sha256=... status=... --> record")
|
||||
continue
|
||||
status = record.get("status")
|
||||
if status not in ("draft", "reviewed"):
|
||||
errors.append(f"{label}: translation status must be draft or reviewed")
|
||||
if status == "draft" and not page.noindex():
|
||||
errors.append(f"{label}: a draft translation needs <meta name=\"robots\" content=\"noindex\" />")
|
||||
if status == "reviewed":
|
||||
if page.noindex():
|
||||
errors.append(f"{label}: remove noindex from a reviewed translation")
|
||||
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", record.get("reviewed", "")):
|
||||
errors.append(f"{label}: a reviewed translation needs reviewed=\"YYYY-MM-DD\"")
|
||||
fresh = record.get("source-sha256") == source_digest(english_source)
|
||||
if not fresh:
|
||||
warnings.append(translation_warning(
|
||||
path, root,
|
||||
f"The English page {english} has changed since this translation. Update the translation, "
|
||||
"then set source-sha256 to the new value from python3 scripts/website.py translations.",
|
||||
))
|
||||
|
||||
synced = synced_code_ids(english_source)
|
||||
for identifier in synced:
|
||||
expected = code_block(identifier).search(english_source).group(2)
|
||||
found = code_block(identifier).findall(source)
|
||||
if len(found) != 1 or found[0][1] != expected:
|
||||
errors.append(f"{label}: code block {identifier} must match the English page; run sync")
|
||||
|
||||
english_page = pages[english_path]
|
||||
differences = []
|
||||
if page.ids != english_page.ids:
|
||||
differences.append(
|
||||
f"element IDs differ (missing {sorted(english_page.ids - page.ids)}, extra {sorted(page.ids - english_page.ids)})"
|
||||
)
|
||||
if copy_targets(source) != copy_targets(english_source):
|
||||
differences.append("copy buttons differ from the English page")
|
||||
translated = TranslationText(synced)
|
||||
translated.feed(source)
|
||||
original = TranslationText(synced)
|
||||
original.feed(english_source)
|
||||
if translated.messages != original.messages:
|
||||
differences.append("data-msg-* message attributes differ from the English page")
|
||||
for difference in differences:
|
||||
if fresh:
|
||||
errors.append(f"{label}: {difference}")
|
||||
else:
|
||||
warnings.append(translation_warning(path, root, difference))
|
||||
if fresh and translated.inline_code != original.inline_code:
|
||||
changed = sorted((translated.inline_code - original.inline_code) + (original.inline_code - translated.inline_code))
|
||||
warnings.append(translation_warning(
|
||||
path, root, f"inline code differs from the English page: {', '.join(repr(text) for text in changed[:5])}",
|
||||
))
|
||||
|
||||
if language in ("ja", "zh-hans", "zh-hant"):
|
||||
for text in translated.prose:
|
||||
for match in CJK_GAP.finditer(text):
|
||||
errors.append(
|
||||
f"{label}: whitespace between {match.group(1)!r} and {match.group(2)!r} renders as a gap; "
|
||||
"keep CJK text on one line without spaces"
|
||||
)
|
||||
|
||||
for reference in page.references:
|
||||
url = urlsplit(reference)
|
||||
if url.scheme or url.netloc or not url.path or reference.startswith("/"):
|
||||
continue
|
||||
target = (path.parent / unquote(url.path)).resolve()
|
||||
if target.is_dir():
|
||||
target /= "index.html"
|
||||
if not target.is_relative_to(site.resolve()):
|
||||
continue
|
||||
target_language, target_english = page_language(site.resolve(), target)
|
||||
if (
|
||||
target_language == "en"
|
||||
and target_english != english
|
||||
and state.translations.get((language, target_english)) == "reviewed"
|
||||
):
|
||||
warnings.append(translation_warning(
|
||||
path, root, f"links to the English {target_english}; link to its reviewed translation instead",
|
||||
))
|
||||
|
||||
english_strings = state.locales["en"]["strings"]
|
||||
for code in LANGUAGES:
|
||||
if code == "en" or not any(key[0] == code for key in state.translations):
|
||||
continue
|
||||
locale = state.locales[code]
|
||||
same = sorted(
|
||||
key for key, value in locale["strings"].items()
|
||||
if value == english_strings[key] and key not in locale.get("sameAsEnglish", [])
|
||||
)
|
||||
if same:
|
||||
warnings.append((f"website/locales/{code}.json", f"UI strings still in English: {', '.join(same)}"))
|
||||
return errors, warnings
|
||||
|
||||
|
||||
def report_warnings(warnings):
|
||||
"""Print warnings, as GitHub annotations and a job summary when running in Actions."""
|
||||
for file, message in warnings:
|
||||
print(f"Warning: {file}: {message}")
|
||||
if os.environ.get("GITHUB_ACTIONS") != "true" or not warnings:
|
||||
return
|
||||
for file, message in warnings:
|
||||
print(f"::warning file={file}::{message}")
|
||||
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if summary:
|
||||
with open(summary, "a", encoding="utf-8") as output:
|
||||
output.write("### Website warnings\n\n")
|
||||
output.writelines(f"- `{file}`: {message}\n" for file, message in warnings)
|
||||
|
||||
|
||||
def check_publisher(root):
|
||||
"""Detect a second Pages action that could replace the website."""
|
||||
expected = root / ".github/workflows/website-pages.yml"
|
||||
@@ -611,16 +404,10 @@ def main():
|
||||
errors.extend(check_example_archive(root / "website/public"))
|
||||
errors.extend(check_publisher(root))
|
||||
errors.extend(check_repository_links(root))
|
||||
translation_errors, warnings = check_translations(root, parse_pages(root / "website/public"))
|
||||
errors.extend(translation_errors)
|
||||
try:
|
||||
for path in sync_pages(root, check=True):
|
||||
warnings.append(translation_warning(
|
||||
path, root, "the out-of-date notice needs refreshing; run python3 scripts/website.py sync",
|
||||
))
|
||||
sync_pages(root, check=True)
|
||||
except (OSError, ValueError, KeyError) as error:
|
||||
errors.append(str(error))
|
||||
report_warnings(warnings)
|
||||
if errors:
|
||||
print("Website checks failed:\n" + "\n".join(errors), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
@@ -6,7 +6,6 @@ import { createHash } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import {
|
||||
access,
|
||||
copyFile,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
@@ -17,7 +16,7 @@ import {
|
||||
open,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, parse, resolve } from "node:path";
|
||||
import { join, resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { verifyWindowsProtocolInput } from "./lib/windows-protocol-input.mjs";
|
||||
|
||||
@@ -33,9 +32,6 @@ const native = requireWindowsNativeAuthority();
|
||||
const { WindowsPrivateRuntime, windowsPrivateRuntime } = await import(
|
||||
pathToFileURL(join(packageRoot, "dist/windows/WindowsPrivateRuntime.js"))
|
||||
);
|
||||
const { ghidraSessionRoot } = await import(
|
||||
pathToFileURL(join(packageRoot, "dist/ghidra/GhidraSessionRoot.js"))
|
||||
);
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-native-conformance-"));
|
||||
const environment = Object.entries(process.env)
|
||||
.filter(([, value]) => value !== undefined)
|
||||
@@ -81,31 +77,6 @@ const wait = async (child) => {
|
||||
};
|
||||
const report = { ok: false, artifact: "verified", controls: {}, processes: {} };
|
||||
try {
|
||||
// Exercise the loader in a process whose image is not named node.exe. The
|
||||
// ordinary Windows lane otherwise cannot detect eager Node-API imports.
|
||||
const renamedHost = join(workspace, "rea-renamed-node.exe");
|
||||
const renamedHostFixture = join(workspace, "renamed-host.mjs");
|
||||
const loaderUrl = pathToFileURL(
|
||||
join(packageRoot, "dist/windows/WindowsNativeLoader.js"),
|
||||
).href;
|
||||
await copyFile(process.execPath, renamedHost);
|
||||
await writeFile(
|
||||
renamedHostFixture,
|
||||
`import {basename} from 'node:path';
|
||||
import {requireWindowsNativeAuthority} from ${JSON.stringify(loaderUrl)};
|
||||
const {inspection}=requireWindowsNativeAuthority();
|
||||
process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVersion:inspection.abiVersion}));\n`,
|
||||
);
|
||||
const renamedHostResult = execFileSync(renamedHost, [renamedHostFixture], {
|
||||
encoding: "utf8",
|
||||
timeout: 20_000,
|
||||
windowsHide: true,
|
||||
});
|
||||
assert.deepEqual(JSON.parse(renamedHostResult), {
|
||||
executable: "rea-renamed-node.exe",
|
||||
abiVersion: 1,
|
||||
});
|
||||
report.controls.renamedHostNodeApi = true;
|
||||
const sourceDirectory = join(workspace, "source");
|
||||
await mkdir(sourceDirectory);
|
||||
const source = join(sourceDirectory, "target.bin");
|
||||
@@ -139,47 +110,6 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
|
||||
])
|
||||
assert.throws(() => native.call("open", [rejected]));
|
||||
report.controls.ordinaryDriveSeparatorsAndRequestedIdentity = true;
|
||||
const driveRoot = parse(workspace).root;
|
||||
const runtimeParent = ghidraSessionRoot({
|
||||
base: join(driveRoot, ".Tmp", "scratch"),
|
||||
});
|
||||
assert.equal(runtimeParent, driveRoot);
|
||||
const driveRootRuntime = native.call("runtime_create", [
|
||||
runtimeParent,
|
||||
"rea-drive-root-",
|
||||
]);
|
||||
try {
|
||||
assert.equal(driveRootRuntime.path.startsWith(`${driveRoot}\\`), false);
|
||||
assert.equal(dirname(driveRootRuntime.path), driveRoot);
|
||||
const snapshot = await native.call("runtime_snapshot", [
|
||||
driveRootRuntime.handle,
|
||||
source,
|
||||
"probe.bin",
|
||||
]);
|
||||
const sourceBytes = await readFile(source);
|
||||
assert.equal(
|
||||
snapshot.sha256,
|
||||
createHash("sha256").update(sourceBytes).digest("hex"),
|
||||
);
|
||||
assert.equal(snapshot.source.requestedPath, source);
|
||||
const readback = native.call("runtime_open", [
|
||||
driveRootRuntime.handle,
|
||||
"probe.bin",
|
||||
]);
|
||||
try {
|
||||
assert.equal(readback.size, sourceBytes.byteLength);
|
||||
assert.deepEqual(
|
||||
native.call("read", [readback.handle, 0, 65536]),
|
||||
sourceBytes.subarray(0, 65536),
|
||||
);
|
||||
} finally {
|
||||
native.call("close", [readback.handle]);
|
||||
}
|
||||
} finally {
|
||||
native.call("runtime_close", [driveRootRuntime.handle]);
|
||||
}
|
||||
assert.equal(await exists(driveRootRuntime.path), false);
|
||||
report.controls.driveRootRuntimeParent = true;
|
||||
const snapshotOwner = WindowsPrivateRuntime.create(
|
||||
workspace.replace(/\\/u, "/"),
|
||||
"single-flight-",
|
||||
|
||||
+28
-442
@@ -1,53 +1,26 @@
|
||||
"""Maintain shared HTML and run the static website's contributor commands."""
|
||||
|
||||
import argparse
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from hashlib import sha256
|
||||
from html import escape
|
||||
from html.parser import HTMLParser
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import posixpath
|
||||
import re
|
||||
from string import Template
|
||||
import subprocess
|
||||
import sys
|
||||
from textwrap import indent
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
|
||||
SITE_ORIGIN = "https://rea.tools"
|
||||
NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
|
||||
# Each label is a key in website/locales/<language>.json.
|
||||
NAVIGATION = (
|
||||
("navShowcases", "showcase/"),
|
||||
("navBlog", "blog/"),
|
||||
("navGetStarted", "get-started/"),
|
||||
("navGuides", "guides/"),
|
||||
("navFaq", "faq/"),
|
||||
)
|
||||
Language = namedtuple("Language", "tag name og_locale")
|
||||
# English stays at the root; every other language lives under its directory.
|
||||
# The order is the language menu's order.
|
||||
LANGUAGES = {
|
||||
"en": Language("en", "English", "en_US"),
|
||||
"zh-hans": Language("zh-Hans", "简体中文", "zh_CN"),
|
||||
"zh-hant": Language("zh-Hant", "繁體中文", "zh_TW"),
|
||||
"ja": Language("ja", "日本語", "ja_JP"),
|
||||
"ko": Language("ko", "한국어", "ko_KR"),
|
||||
"de": Language("de", "Deutsch", "de_DE"),
|
||||
"fr": Language("fr", "Français", "fr_FR"),
|
||||
}
|
||||
TRANSLATION_STATUSES = ("draft", "reviewed")
|
||||
TRANSLATION_RECORD = re.compile(r"<!-- website:translation((?:\s+[\w-]+=\"[^\"]*\")*)\s*-->")
|
||||
SHARED_REGION = re.compile(r"<!-- website:([\w-]+):start -->.*?<!-- website:\1:end -->", re.S)
|
||||
NOTICE_START = " <!-- website:translation-notice:start -->\n"
|
||||
NOTICE_END = " <!-- website:translation-notice:end -->"
|
||||
GLOBE_PATH = (
|
||||
"M12 3a9 9 0 1 0 0 18a9 9 0 1 0 0-18Z M3.6 9h16.8 M3.6 15h16.8 "
|
||||
"M12 3c2.4 2.5 3.6 5.5 3.6 9s-1.2 6.5-3.6 9c-2.4-2.5-3.6-5.5-3.6-9s1.2-6.5 3.6-9Z"
|
||||
("Showcases", "showcase/"),
|
||||
("Blog", "blog/"),
|
||||
("Get started", "get-started/"),
|
||||
("Guides", "guides/"),
|
||||
("FAQ", "faq/"),
|
||||
)
|
||||
|
||||
|
||||
@@ -102,14 +75,10 @@ class PageMetadata(HTMLParser):
|
||||
self.titles[-1] += data
|
||||
|
||||
|
||||
def file_route(relative):
|
||||
"""Map a site-relative HTML file to its public route."""
|
||||
return relative.removesuffix("index.html") if PurePosixPath(relative).name == "index.html" else relative
|
||||
|
||||
|
||||
def page_route(site, path):
|
||||
"""Map an HTML file to its public route without a separate page registry."""
|
||||
return file_route(path.relative_to(site).as_posix())
|
||||
route = path.relative_to(site).as_posix()
|
||||
return route.removesuffix("index.html") if path.name == "index.html" else route
|
||||
|
||||
|
||||
def page_url(site, path):
|
||||
@@ -117,227 +86,30 @@ def page_url(site, path):
|
||||
return f"{SITE_ORIGIN}/{quote(page_route(site, path), safe='/')}"
|
||||
|
||||
|
||||
def page_language(site, path):
|
||||
"""Return a page's language directory and the site-relative file of its English source."""
|
||||
relative = path.relative_to(site).as_posix()
|
||||
first, separator, rest = relative.partition("/")
|
||||
if separator and first in LANGUAGES and first != "en":
|
||||
return first, rest
|
||||
return "en", relative
|
||||
|
||||
|
||||
def load_locales(root):
|
||||
"""Read the UI strings for every language and require the same keys as English."""
|
||||
directory = root / "website/locales"
|
||||
locales = {}
|
||||
for code in LANGUAGES:
|
||||
path = directory / f"{code}.json"
|
||||
try:
|
||||
locale = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
raise ValueError(f"website/locales/{code}.json: cannot read UI strings: {error}") from error
|
||||
locales[code] = locale
|
||||
expected = set(locales["en"]["strings"])
|
||||
for code, locale in locales.items():
|
||||
label = f"website/locales/{code}.json"
|
||||
strings = locale.get("strings")
|
||||
if not isinstance(strings, dict) or set(strings) != expected:
|
||||
missing = sorted(expected - set(strings or {}))
|
||||
extra = sorted(set(strings or {}) - expected)
|
||||
raise ValueError(f"{label}: strings must match en.json (missing {missing}, extra {extra})")
|
||||
if any(not isinstance(value, str) or not value.strip() for value in strings.values()):
|
||||
raise ValueError(f"{label}: every UI string must be nonempty text")
|
||||
stars = locale.get("githubStars")
|
||||
if not isinstance(stars, dict) or "other" not in stars or not set(stars) <= {"zero", "one", "two", "few", "many", "other"}:
|
||||
raise ValueError(f"{label}: githubStars needs an 'other' label and CLDR plural categories only")
|
||||
if any("{count}" not in value for value in stars.values()):
|
||||
raise ValueError(f"{label}: every githubStars label needs a {{count}} placeholder")
|
||||
if "{link}" not in strings["staleNotice"]:
|
||||
raise ValueError(f"{label}: staleNotice needs a {{link}} placeholder")
|
||||
same = locale.get("sameAsEnglish", [])
|
||||
if not isinstance(same, list) or not set(same) <= expected:
|
||||
raise ValueError(f"{label}: sameAsEnglish must list UI string keys")
|
||||
return locales
|
||||
|
||||
|
||||
def translation_record(source):
|
||||
"""Return the attributes of a translation's provenance comment, or None."""
|
||||
records = TRANSLATION_RECORD.findall(source)
|
||||
if len(records) != 1:
|
||||
return None
|
||||
return dict(re.findall(r'([\w-]+)="([^"]*)"', records[0]))
|
||||
|
||||
|
||||
def copy_targets(source):
|
||||
"""Return the IDs referenced by copy buttons, in page order."""
|
||||
return list(dict.fromkeys(re.findall(r'\bdata-copy="([^"]+)"', source)))
|
||||
|
||||
|
||||
def code_block(identifier):
|
||||
"""Match a code element by ID, capturing its opening tag, contents and end tag."""
|
||||
return re.compile(rf'(<code\b[^>]*\bid="{re.escape(identifier)}"[^>]*>)(.*?)(</code>)', re.S)
|
||||
|
||||
|
||||
def synced_code_ids(source):
|
||||
"""Return the copy targets that are code blocks; translations take them from English."""
|
||||
return [identifier for identifier in copy_targets(source) if code_block(identifier).search(source)]
|
||||
|
||||
|
||||
def source_digest(source):
|
||||
"""Hash an English page's translatable content.
|
||||
|
||||
Shared regions and copied code blocks are left out, so template edits and
|
||||
command-only changes do not mark translations out of date.
|
||||
"""
|
||||
content = SHARED_REGION.sub("", source)
|
||||
for identifier in synced_code_ids(source):
|
||||
content = code_block(identifier).sub(r"\1\3", content)
|
||||
return sha256(" ".join(content.split()).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def sync_code_blocks(english, source, label):
|
||||
"""Copy every synced code block's contents from the English source, matched by ID."""
|
||||
for identifier in synced_code_ids(english):
|
||||
contents = code_block(identifier).search(english).group(2)
|
||||
pattern = code_block(identifier)
|
||||
if len(pattern.findall(source)) != 1:
|
||||
raise ValueError(f"{label}: needs one <code id=\"{identifier}\"> block copied from the English page")
|
||||
source = pattern.sub(lambda match: match.group(1) + contents + match.group(3), source)
|
||||
return source
|
||||
|
||||
|
||||
class SiteState:
|
||||
"""Read the locales and every translation's status once for a sync or check."""
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
self.site = root / "website/public"
|
||||
self.locales = load_locales(root)
|
||||
self.translations = {}
|
||||
for path in sorted(self.site.rglob("*.html")):
|
||||
language, english = page_language(self.site, path)
|
||||
if language != "en":
|
||||
record = translation_record(path.read_text(encoding="utf-8")) or {}
|
||||
self.translations[(language, english)] = record.get("status")
|
||||
|
||||
def linkable(self, language, english, from_draft=False):
|
||||
"""Report whether other pages may link to this language's version of a page."""
|
||||
if language == "en":
|
||||
return (self.site / english).is_file()
|
||||
status = self.translations.get((language, english))
|
||||
return status == "reviewed" or (from_draft and status == "draft")
|
||||
|
||||
def alternates(self, english):
|
||||
"""Return the languages whose versions of a page join its hreflang cluster."""
|
||||
return [code for code in LANGUAGES if code == "en" or self.translations.get((code, english)) == "reviewed"]
|
||||
|
||||
|
||||
def language_url(language, english):
|
||||
"""Return the canonical URL of one language's version of a page."""
|
||||
prefix = "" if language == "en" else f"{language}/"
|
||||
return f"{SITE_ORIGIN}/{quote(prefix + file_route(english), safe='/')}"
|
||||
|
||||
|
||||
def shared_blocks(root, path, source, state=None):
|
||||
def shared_blocks(root, path, source):
|
||||
"""Render shared regions while keeping the page's body and metadata authored."""
|
||||
state = state or SiteState(root)
|
||||
site = state.site
|
||||
site = root / "website/public"
|
||||
metadata = PageMetadata()
|
||||
metadata.feed(source)
|
||||
for name, values in (("title", metadata.titles), ("description", metadata.descriptions)):
|
||||
if len(values) != 1 or not values[0].strip():
|
||||
raise ValueError(f"{path.relative_to(site)}: needs one nonempty head {name}")
|
||||
language, english = page_language(site, path)
|
||||
strings = state.locales[language]["strings"]
|
||||
depth = len(path.relative_to(site).parent.parts)
|
||||
base = "../" * depth or "./"
|
||||
route = file_route(english)
|
||||
draft = language != "en" and state.translations.get((language, english)) == "draft"
|
||||
|
||||
def href(target_language, target):
|
||||
prefix = "" if target_language == "en" else f"{target_language}/"
|
||||
return base + file_route(prefix + target)
|
||||
|
||||
def localized(target):
|
||||
"""Prefer this language's version of a target page, else the English page."""
|
||||
if language != "en" and state.linkable(language, target, from_draft=draft):
|
||||
return language, href(language, target)
|
||||
return "en", href("en", target)
|
||||
|
||||
route = page_route(site, path)
|
||||
links = []
|
||||
for key, destination in NAVIGATION:
|
||||
target_language, url = localized(destination + "index.html")
|
||||
for label, destination in NAVIGATION:
|
||||
attributes = ' class="nav-start"' if destination == "get-started/" else ""
|
||||
if route.startswith(destination):
|
||||
attributes += ' aria-current="page"'
|
||||
badge = ""
|
||||
if target_language != language:
|
||||
attributes += ' hreflang="en"'
|
||||
badge = '<span class="nav-badge" lang="en">EN</span>'
|
||||
links.append(f' <a href="{url}"{attributes}>{escape(strings[key])}{badge}</a>')
|
||||
|
||||
entries = []
|
||||
for code, details in LANGUAGES.items():
|
||||
if code == language or state.linkable(code, english):
|
||||
url = href(code, english)
|
||||
elif state.linkable(code, "index.html"):
|
||||
url = href(code, "index.html")
|
||||
else:
|
||||
continue
|
||||
current = ' aria-current="true"' if code == language else ""
|
||||
entries.append(
|
||||
f' <li><a href="{url}" hreflang="{details.tag}" lang="{details.tag}"{current}>{details.name}</a></li>'
|
||||
)
|
||||
language_menu = ""
|
||||
if len(entries) > 1:
|
||||
language_menu = "\n".join((
|
||||
"",
|
||||
' <details class="language-menu">',
|
||||
" <summary>",
|
||||
' <svg width="18" height="18" viewBox="0 0 24 24" aria-hidden="true" focusable="false">',
|
||||
f' <path fill="none" stroke="currentColor" stroke-width="1.6" d="{GLOBE_PATH}" />',
|
||||
" </svg>",
|
||||
f' <span class="sr-only">{escape(strings["language"])}: </span>'
|
||||
f'<span class="language-current">{LANGUAGES[language].name}</span>',
|
||||
" </summary>",
|
||||
' <ul class="language-list">',
|
||||
*entries,
|
||||
" </ul>",
|
||||
" </details>",
|
||||
))
|
||||
|
||||
members = [] if draft else state.alternates(english)
|
||||
hreflang = ""
|
||||
locale_alternates = ""
|
||||
if len(members) > 1:
|
||||
for code in [*members, "x-default"]:
|
||||
tag = LANGUAGES[code].tag if code != "x-default" else code
|
||||
url = language_url(code if code != "x-default" else "en", english)
|
||||
hreflang += f'\n<!-- prettier-ignore -->\n<link rel="alternate" hreflang="{tag}" href="{escape(url, quote=True)}" />'
|
||||
for code in members:
|
||||
if code != language:
|
||||
locale_alternates += (
|
||||
f'\n<!-- prettier-ignore -->\n<meta property="og:locale:alternate" content="{LANGUAGES[code].og_locale}" />'
|
||||
)
|
||||
|
||||
labels = "".join(
|
||||
f'\n data-label-{category}="{escape(label, quote=True)}"'
|
||||
for category, label in state.locales[language]["githubStars"].items()
|
||||
)
|
||||
links.append(f' <a href="{base}{destination}"{attributes}>{label}</a>')
|
||||
values = {
|
||||
"base": base,
|
||||
"brand": localized("index.html")[1],
|
||||
"navigation": "\n" + "\n".join(links) + "\n ",
|
||||
"language_menu": language_menu,
|
||||
"canonical": escape(page_url(site, path), quote=True),
|
||||
"hreflang": hreflang,
|
||||
"og_locale": LANGUAGES[language].og_locale,
|
||||
"og_locale_alternates": locale_alternates,
|
||||
"origin": SITE_ORIGIN,
|
||||
"title": escape(metadata.titles[0].strip(), quote=True),
|
||||
"description": escape(metadata.descriptions[0].strip(), quote=True),
|
||||
"github_star_labels": labels,
|
||||
**{f"t_{key}": escape(value, quote=True) for key, value in strings.items()},
|
||||
}
|
||||
templates = root / "website/templates"
|
||||
return {
|
||||
@@ -346,89 +118,35 @@ def shared_blocks(root, path, source, state=None):
|
||||
}
|
||||
|
||||
|
||||
def translation_notice(state, path, english_source, record):
|
||||
"""Render the stale-translation notice, or nothing while the source hash matches."""
|
||||
if record is None or record.get("source-sha256") == source_digest(english_source):
|
||||
return ""
|
||||
language, english = page_language(state.site, path)
|
||||
strings = state.locales[language]["strings"]
|
||||
depth = len(path.relative_to(state.site).parent.parts)
|
||||
link = (
|
||||
f'<a href="{"../" * depth}{file_route(english)}" hreflang="en">'
|
||||
f'{escape(strings["staleNoticeLink"])}</a>'
|
||||
)
|
||||
message = escape(strings["staleNotice"]).replace("{link}", link)
|
||||
return f' <!-- prettier-ignore -->\n <p class="translation-notice" role="note">{message}</p>\n'
|
||||
|
||||
|
||||
def replace_region(source, start, end, block, label, name):
|
||||
"""Replace the contents of one marked region."""
|
||||
if source.count(start) != 1 or source.count(end) != 1:
|
||||
raise ValueError(f"{label}: needs one {name} region; use new-page or translate for a new page")
|
||||
before, _, remaining = source.partition(start)
|
||||
_, found, after = remaining.partition(end)
|
||||
if not found:
|
||||
raise ValueError(f"{label}: {name} region is out of order")
|
||||
return before + start + block + end + after
|
||||
|
||||
|
||||
def region_contents(source, start, end):
|
||||
"""Return the current contents of a marked region."""
|
||||
return source.partition(start)[2].partition(end)[0]
|
||||
|
||||
|
||||
def sync_pages(root, check=False):
|
||||
"""Update shared regions, or reject drift without changing any authored file.
|
||||
|
||||
Returns the pages that changed. With check=True, structural drift raises, so
|
||||
the returned pages are translations whose only drift is the stale notice;
|
||||
an out-of-date translation is a warning rather than a failure.
|
||||
"""
|
||||
state = SiteState(root)
|
||||
site = state.site
|
||||
"""Update shared regions, or reject drift without changing any authored file."""
|
||||
site = root / "website/public"
|
||||
changes = []
|
||||
notices = []
|
||||
for path in sorted(site.rglob("*.html")):
|
||||
if path.relative_to(site).as_posix() in NOTES_FIXTURES:
|
||||
continue
|
||||
label = path.relative_to(site).as_posix()
|
||||
original = path.read_text(encoding="utf-8")
|
||||
source = original
|
||||
language, english = page_language(site, path)
|
||||
english_source = None
|
||||
if language != "en":
|
||||
english_path = site / english
|
||||
if not english_path.is_file():
|
||||
raise ValueError(f"{label}: translates {english}, which does not exist")
|
||||
english_source = english_path.read_text(encoding="utf-8")
|
||||
source = sync_code_blocks(english_source, source, label)
|
||||
for name, block in shared_blocks(root, path, source, state).items():
|
||||
source = replace_region(
|
||||
source, f" <!-- website:{name}:start -->\n", f" <!-- website:{name}:end -->", block, label, name
|
||||
)
|
||||
if language != "en":
|
||||
notice = translation_notice(state, path, english_source, translation_record(source))
|
||||
structural = replace_region(
|
||||
source, NOTICE_START, NOTICE_END, region_contents(original, NOTICE_START, NOTICE_END), label,
|
||||
"translation-notice",
|
||||
)
|
||||
source = replace_region(source, NOTICE_START, NOTICE_END, notice, label, "translation-notice")
|
||||
if structural == original and source != original:
|
||||
notices.append((path, source))
|
||||
continue
|
||||
for name, block in shared_blocks(root, path, source).items():
|
||||
start = f" <!-- website:{name}:start -->\n"
|
||||
end = f" <!-- website:{name}:end -->"
|
||||
if source.count(start) != 1 or source.count(end) != 1:
|
||||
raise ValueError(f"{path.relative_to(site)}: needs one {name} region; use new-page for a new reading page")
|
||||
before, _, remaining = source.partition(start)
|
||||
_, found, after = remaining.partition(end)
|
||||
if not found:
|
||||
raise ValueError(f"{path.relative_to(site)}: {name} region is out of order")
|
||||
source = before + start + block + end + after
|
||||
if source != original:
|
||||
changes.append((path, source))
|
||||
if check and changes:
|
||||
names = ", ".join(str(path.relative_to(site)) for path, _ in changes)
|
||||
raise ValueError(f"Shared HTML is stale: {names}. Run python3 scripts/website.py sync and commit the updated pages.")
|
||||
if not check:
|
||||
for path, source in changes + notices:
|
||||
for path, source in changes:
|
||||
path.write_text(source, encoding="utf-8")
|
||||
if check:
|
||||
print(f"Checked shared HTML ({len(notices)} translation notices to refresh).")
|
||||
else:
|
||||
print(f"Synced shared HTML ({len(changes) + len(notices)} pages changed).")
|
||||
return [path for path, _ in (notices if check else changes + notices)]
|
||||
print(f"{'Checked' if check else 'Synced'} shared HTML ({len(changes)} pages changed).")
|
||||
return [path for path, _ in changes]
|
||||
|
||||
|
||||
def new_page(root, route, title, description):
|
||||
@@ -436,8 +154,6 @@ def new_page(root, route, title, description):
|
||||
parts = PurePosixPath(route)
|
||||
if not route or parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
|
||||
raise ValueError("Use a relative directory route, such as blog/my-article.")
|
||||
if parts.parts[0] in LANGUAGES:
|
||||
raise ValueError("Create the English page first, then use translate for another language.")
|
||||
site = root / "website/public"
|
||||
path = site.joinpath(*parts.parts, "index.html")
|
||||
if not path.resolve().is_relative_to(site.resolve()):
|
||||
@@ -458,131 +174,11 @@ def new_page(root, route, title, description):
|
||||
return path
|
||||
|
||||
|
||||
def relocate_references(source, english, language, state):
|
||||
"""Rewrite relative references for the translation's directory.
|
||||
|
||||
Pages that already have a version in this language link to it; everything
|
||||
else keeps pointing at the English page or asset.
|
||||
"""
|
||||
english_directory = posixpath.dirname(english)
|
||||
translation_directory = posixpath.join(language, english_directory)
|
||||
|
||||
def rewrite(match):
|
||||
value = match.group(2)
|
||||
url = urlsplit(value)
|
||||
if not value or url.scheme or url.netloc or value.startswith(("#", "/")):
|
||||
return match.group(0)
|
||||
target = posixpath.normpath(posixpath.join(english_directory, url.path))
|
||||
directory = url.path.endswith("/") or target == "."
|
||||
if directory:
|
||||
page = "index.html" if target == "." else f"{target}/index.html"
|
||||
if (language, page) in state.translations:
|
||||
target = posixpath.normpath(posixpath.join(language, target))
|
||||
relative = posixpath.relpath(target, translation_directory or ".")
|
||||
if directory:
|
||||
relative = "./" if relative == "." else relative + "/"
|
||||
return f'{match.group(1)}="{urlunsplit(("", "", relative, url.query, url.fragment))}"'
|
||||
|
||||
return re.sub(r'\b(href|src)="([^"]*)"', rewrite, source)
|
||||
|
||||
|
||||
def new_translation(root, language, route):
|
||||
"""Create a draft translation of an English page, ready to be translated in place."""
|
||||
if language not in LANGUAGES or language == "en":
|
||||
raise ValueError(f"Choose one of: {', '.join(code for code in LANGUAGES if code != 'en')}.")
|
||||
route = route.strip("/")
|
||||
parts = PurePosixPath(route) if route not in ("", ".") else PurePosixPath()
|
||||
if parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
|
||||
raise ValueError("Use the English page's directory route, such as get-started, or / for the home page.")
|
||||
if parts.parts and parts.parts[0] in LANGUAGES:
|
||||
raise ValueError("Name the English route, without a language directory.")
|
||||
english = PurePosixPath(*parts.parts, "index.html").as_posix()
|
||||
state = SiteState(root)
|
||||
site = state.site
|
||||
source_path = site / english
|
||||
if not source_path.is_file() or english in NOTES_FIXTURES:
|
||||
raise ValueError(f"No English page to translate at {english}.")
|
||||
path = site / language / english
|
||||
if path.exists():
|
||||
raise ValueError(f"Translation already exists: {path.relative_to(site)}")
|
||||
source = source_path.read_text(encoding="utf-8")
|
||||
if source.count('<html lang="en">') != 1 or source.count(" <!-- website:sharing:start -->\n") != 1:
|
||||
raise ValueError(f"{english}: needs <html lang=\"en\"> and one sharing region")
|
||||
main = re.search(r"^ {4}<main\b[^>]*>\n", source, re.M)
|
||||
if main is None:
|
||||
raise ValueError(f"{english}: needs a <main> element on its own line")
|
||||
source = source.replace('<html lang="en">', f'<html lang="{LANGUAGES[language].tag}">')
|
||||
source = relocate_references(source, english, language, state)
|
||||
record = (
|
||||
f' <!-- website:translation source-sha256="{source_digest(source_path.read_text(encoding="utf-8"))}" '
|
||||
'status="draft" -->\n'
|
||||
)
|
||||
robots = "" if '<meta name="robots"' in source else ' <meta name="robots" content="noindex" />\n'
|
||||
source = source.replace(" <!-- website:sharing:start -->\n", robots + record + " <!-- website:sharing:start -->\n")
|
||||
main = re.search(r"^ {4}<main\b[^>]*>\n", source, re.M)
|
||||
source = source[: main.end()] + NOTICE_START + NOTICE_END + "\n" + source[main.end():]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(source, encoding="utf-8")
|
||||
sync_pages(root)
|
||||
print(
|
||||
f"Created {path.relative_to(root)} as a noindex draft. Translate its text in place following "
|
||||
"website/translation.md, then run python3 scripts/website.py check."
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def translation_report(root, language=None):
|
||||
"""Describe each translation's review status, freshness and remaining review items."""
|
||||
state = SiteState(root)
|
||||
site = state.site
|
||||
lines = []
|
||||
for (code, english), status in sorted(state.translations.items()):
|
||||
if language and code != language:
|
||||
continue
|
||||
path = site / code / english
|
||||
source = path.read_text(encoding="utf-8")
|
||||
record = translation_record(source) or {}
|
||||
english_path = site / english
|
||||
current = source_digest(english_path.read_text(encoding="utf-8")) if english_path.is_file() else None
|
||||
fresh = record.get("source-sha256") == current
|
||||
reviewed = f" on {record['reviewed']}" if record.get("reviewed") else ""
|
||||
lines.append(f"{code}/{file_route(english) or '(home)'}: {status or 'no record'}{reviewed}, "
|
||||
f"{'up to date' if fresh else 'English source changed'}")
|
||||
if not fresh and current:
|
||||
lines.append(f" after updating the translation, record source-sha256=\"{current}\"")
|
||||
if status != "reviewed":
|
||||
for system, text in re.findall(r'<(?:em|span|strong|kbd)\b[^>]*\bdata-os-ui="([^"]+)"[^>]*>(.*?)</', source, re.S):
|
||||
lines.append(f" confirm on {system}: {' '.join(text.split())}")
|
||||
english_strings = state.locales["en"]["strings"]
|
||||
for code in LANGUAGES:
|
||||
if code == "en" or (language and code != language):
|
||||
continue
|
||||
if not any(key[0] == code for key in state.translations):
|
||||
continue
|
||||
locale = state.locales[code]
|
||||
same = sorted(
|
||||
key for key, value in locale["strings"].items()
|
||||
if value == english_strings[key] and key not in locale.get("sameAsEnglish", [])
|
||||
)
|
||||
if same:
|
||||
lines.append(f"{code}: UI strings still in English: {', '.join(same)}")
|
||||
print("\n".join(lines) or "No translated pages.")
|
||||
return lines
|
||||
|
||||
|
||||
def run_script(root, name, arguments=()):
|
||||
"""Use the active Python environment for the existing preparation and checks."""
|
||||
subprocess.run([sys.executable, str(root / "scripts" / name), *arguments], cwd=root, check=True)
|
||||
|
||||
|
||||
def run_browser_script_tests(root):
|
||||
"""Run the page-script regressions with the repository's Node.js."""
|
||||
try:
|
||||
subprocess.run(["node", str(root / "scripts/test-website-scripts.mjs")], cwd=root, check=True)
|
||||
except FileNotFoundError as error:
|
||||
raise OSError("The website script tests need Node.js; see website/README.md.") from error
|
||||
|
||||
|
||||
def preview_handler(site, prefix):
|
||||
"""Serve one static directory at root or a Pages prefix, preserving redirects."""
|
||||
class Handler(SimpleHTTPRequestHandler):
|
||||
@@ -611,11 +207,6 @@ def main():
|
||||
create.add_argument("route")
|
||||
create.add_argument("--title", required=True)
|
||||
create.add_argument("--description", required=True)
|
||||
translate = commands.add_parser("translate", help="Create a draft translation of an English page")
|
||||
translate.add_argument("language", choices=[code for code in LANGUAGES if code != "en"])
|
||||
translate.add_argument("route", help="The English page's directory route, or / for the home page")
|
||||
report = commands.add_parser("translations", help="List translation status, freshness and review items")
|
||||
report.add_argument("--lang", choices=[code for code in LANGUAGES if code != "en"])
|
||||
serve = commands.add_parser("serve", help="Prepare and preview the static site")
|
||||
serve.add_argument("--port", type=int, default=4173)
|
||||
serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
|
||||
@@ -624,17 +215,12 @@ def main():
|
||||
try:
|
||||
if args.command == "new-page":
|
||||
new_page(root, args.route, args.title, args.description)
|
||||
elif args.command == "translate":
|
||||
new_translation(root, args.language, args.route)
|
||||
elif args.command == "translations":
|
||||
translation_report(root, args.lang)
|
||||
elif args.command == "sync":
|
||||
sync_pages(root, check=args.check)
|
||||
elif args.command == "check":
|
||||
sync_pages(root, check=True)
|
||||
for name in ("prepare-website.py", "verify-website.py", "test-website.py"):
|
||||
run_script(root, name)
|
||||
run_browser_script_tests(root)
|
||||
elif args.command == "links":
|
||||
run_script(root, "audit-website-links.py", ["--report", args.report] if args.report else [])
|
||||
else:
|
||||
|
||||
+2
-2
@@ -8,13 +8,13 @@
|
||||
"url": "https://github.com/morluto/rea",
|
||||
"source": "github"
|
||||
},
|
||||
"version": "6.4.0",
|
||||
"version": "6.3.0",
|
||||
"packages": [
|
||||
{
|
||||
"registryType": "npm",
|
||||
"registryBaseUrl": "https://registry.npmjs.org",
|
||||
"identifier": "rea-agents",
|
||||
"version": "6.4.0",
|
||||
"version": "6.3.0",
|
||||
"runtimeHint": "npx",
|
||||
"transport": {
|
||||
"type": "stdio"
|
||||
|
||||
@@ -4,7 +4,7 @@ import type {
|
||||
AnalysisOperationPort,
|
||||
} from "./AnalysisProvider.js";
|
||||
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { analysisBookmarkSchema } from "../domain/hopperValues.js";
|
||||
import { resolveAnalysisDocument } from "./AnalysisDocument.js";
|
||||
|
||||
@@ -1,21 +1,17 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { lstat, opendir, readFile, realpath, stat } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { parseBinary } from "plist";
|
||||
import { z } from "zod";
|
||||
|
||||
import { preflightPropertyListDecode } from "../artifacts/apple/InterfaceBuilderDecodeBudget.js";
|
||||
import { BinaryTargetError } from "../domain/configurationErrors.js";
|
||||
import { isPathWithinRoot } from "../domain/localPath.js";
|
||||
import {
|
||||
omitPrototypeKeys,
|
||||
parseXmlPropertyList,
|
||||
} from "../domain/propertyListKeys.js";
|
||||
import { parseXmlPropertyList } from "../domain/propertyListKeys.js";
|
||||
import { decodeXmlPlistText } from "../domain/propertyListXmlText.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
/** Decoded representation bound for one binary Info.plist. */
|
||||
const BINARY_PLIST_DECODE_BYTES = 16 * 1024 * 1024;
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Where one app bundle layout keeps its Info.plist and program file. */
|
||||
interface BundleLayout {
|
||||
@@ -197,8 +193,9 @@ const resolveLayoutExecutable = async (
|
||||
try {
|
||||
name =
|
||||
plist.subarray(0, 6).toString("ascii") === "bplist"
|
||||
? await (fileSystem.decodeBinaryPlist?.(plistPath) ??
|
||||
parseBinaryPlistExecutable(plist))
|
||||
? await (fileSystem.decodeBinaryPlist ?? readBinaryPlistExecutable)(
|
||||
plistPath,
|
||||
)
|
||||
: parseXmlPlistExecutable(decodeXmlPlistText(plist));
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
@@ -251,10 +248,6 @@ const resolveLayoutExecutable = async (
|
||||
const parseXmlPlistExecutable = (plist: string): string => {
|
||||
// An unrelated `__proto__` entry must not make the bundle unreadable.
|
||||
const { value } = parseXmlPropertyList(plist);
|
||||
return executableName(value);
|
||||
};
|
||||
|
||||
const executableName = (value: unknown): string => {
|
||||
const executable = executableEntrySchema.safeParse(value);
|
||||
if (!executable.success) throw new Error("CFBundleExecutable is missing");
|
||||
return executable.data.CFBundleExecutable;
|
||||
@@ -262,13 +255,19 @@ const executableName = (value: unknown): string => {
|
||||
|
||||
const executableEntrySchema = z.looseObject({ CFBundleExecutable: z.string() });
|
||||
|
||||
/** Decode a binary plist in-process, so no host plist tool is required. */
|
||||
const parseBinaryPlistExecutable = (plist: Buffer): string => {
|
||||
preflightPropertyListDecode(plist, BINARY_PLIST_DECODE_BYTES);
|
||||
// `plist` assigns a `__proto__` key as the prototype; copying drops it, as
|
||||
// the XML path does.
|
||||
return executableName(omitPrototypeKeys(parseBinary(plist)).value);
|
||||
};
|
||||
const readBinaryPlistExecutable = async (plistPath: string): Promise<string> =>
|
||||
// `-n` strips only the newline plutil appends; it requires macOS 12+.
|
||||
(
|
||||
await execFileAsync("/usr/bin/plutil", [
|
||||
"-extract",
|
||||
"CFBundleExecutable",
|
||||
"raw",
|
||||
"-n",
|
||||
"-o",
|
||||
"-",
|
||||
plistPath,
|
||||
])
|
||||
).stdout;
|
||||
|
||||
const isSafeExecutableName = (name: string): boolean =>
|
||||
name.length > 0 &&
|
||||
|
||||
@@ -12,7 +12,7 @@ import type {
|
||||
CompareWebScreenshotsInput,
|
||||
} from "../domain/webScreenshot.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { ExecutionOptions } from "./AnalysisProvider.js";
|
||||
import type { BrowserObservationPort } from "./BrowserObservationPort.js";
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { ExecutionOptions } from "./AnalysisProvider.js";
|
||||
import type { BrowserScenarioCapturePort } from "./BrowserScenarioCapturePort.js";
|
||||
import { createBrowserScenarioEvidence } from "./BrowserScenarioEvidence.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../domain/evidence.js";
|
||||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import { err, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { AnalysisOperation } from "./AnalysisProvider.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { parseRelatedAddresses } from "../domain/hopperValues.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { ExecutableFormatHint } from "../domain/dosCom.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { EnhancedTools } from "./EnhancedTools.js";
|
||||
import { executeFunctionAnalysisEvidence } from "./FunctionAnalysisEvidence.js";
|
||||
import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js";
|
||||
import type { BinarySession } from "./binary/BinarySession.js";
|
||||
import type { SessionProviderRoute } from "./binary/SessionProviderRouter.js";
|
||||
@@ -16,7 +18,7 @@ import {
|
||||
} from "../contracts/managed/managedToolContracts.js";
|
||||
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { access } from "node:fs/promises";
|
||||
import {
|
||||
readAnalysisSnapshot,
|
||||
@@ -136,7 +138,7 @@ export const runManagedProviderExecution = async (
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<AnalysisExecution, AnalysisError>> =>
|
||||
withProcessCancellation(signal, async (operationSignal) => {
|
||||
const session = await dependencies.createManagedBinarySession();
|
||||
const session = dependencies.createManagedBinarySession();
|
||||
return withSessionCleanup(
|
||||
session,
|
||||
async () => {
|
||||
@@ -192,7 +194,7 @@ const runAnalysis = async (
|
||||
const { logger, signal, snapshotPath } = options;
|
||||
const config = dependencies.readConfiguration();
|
||||
if (!config.ok) return cliError(config.error);
|
||||
const session = await dependencies.createBinarySession(config.value, logger);
|
||||
const session = dependencies.createBinarySession(config.value, logger);
|
||||
return withSessionCleanup(
|
||||
session,
|
||||
async () => {
|
||||
@@ -330,8 +332,6 @@ const executeAnalysisTool = async (input: {
|
||||
}): Promise<{ readonly output: JsonValue; readonly evidence?: Evidence }> => {
|
||||
const { session, tool, signal, evidenceProfile } = input;
|
||||
if (tool === "analyze_function") {
|
||||
const { executeFunctionAnalysisEvidence } =
|
||||
await import("./FunctionAnalysisEvidence.js");
|
||||
const result = await executeFunctionAnalysisEvidence(
|
||||
session,
|
||||
input.arguments,
|
||||
@@ -350,7 +350,6 @@ const executeAnalysisTool = async (input: {
|
||||
tool === "trace_native_values" ||
|
||||
tool === "trace_native_ui_action"
|
||||
) {
|
||||
const { EnhancedTools } = await import("./EnhancedTools.js");
|
||||
const result = await new EnhancedTools(session).execute(
|
||||
tool,
|
||||
input.arguments,
|
||||
@@ -451,7 +450,7 @@ const prepareSnapshot = async (options: {
|
||||
const { path, snapshotPath } = options;
|
||||
if (snapshotPath === undefined || !(await fileExists(snapshotPath)))
|
||||
return ok({});
|
||||
const loaded = await readAnalysisSnapshot(snapshotPath, options.signal);
|
||||
const loaded = await readAnalysisSnapshot(snapshotPath);
|
||||
if (!loaded.ok) return loaded;
|
||||
const target = await parseBinaryTarget(path, {
|
||||
signal: options.signal,
|
||||
|
||||
@@ -10,8 +10,6 @@ export interface DirectAnalysisDependencies {
|
||||
readonly createBinarySession: (
|
||||
config: AppConfig,
|
||||
logger: Logger,
|
||||
) => BinarySession | Promise<BinarySession>;
|
||||
readonly createManagedBinarySession: () =>
|
||||
| BinarySession
|
||||
| Promise<BinarySession>;
|
||||
) => BinarySession;
|
||||
readonly createManagedBinarySession: () => BinarySession;
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ export const runSessionStatus = async (
|
||||
): Promise<JsonValue> => {
|
||||
const config = dependencies.readConfiguration();
|
||||
if (!config.ok) return { error: projectAnalysisError(config.error) };
|
||||
const session = await dependencies.createBinarySession(config.value, logger);
|
||||
const session = dependencies.createBinarySession(config.value, logger);
|
||||
try {
|
||||
return {
|
||||
...jsonObjectSchema.parse(session.status()),
|
||||
|
||||
@@ -36,6 +36,8 @@ import {
|
||||
type DoctorScopeReport,
|
||||
} from "./DoctorScope.js";
|
||||
|
||||
/** Deep provider IDs whose readiness can be selected explicitly by doctor. */
|
||||
export const DOCTOR_PROVIDER_IDS = ["hopper", "ghidra", "ida"] as const;
|
||||
/** One actionable environment diagnostic. */
|
||||
export interface DoctorCheck {
|
||||
readonly name: string;
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
/** Deep-analysis providers whose readiness can be selected explicitly by doctor. */
|
||||
export const DOCTOR_PROVIDER_IDS = ["hopper", "ghidra", "ida"] as const;
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
import type { AnalysisOperation } from "./AnalysisProvider.js";
|
||||
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { z } from "zod";
|
||||
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { err, type Result } from "../domain/result.js";
|
||||
import {
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
AnalysisOutputError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import {
|
||||
addressDistance,
|
||||
functionDossierSchema,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { EvidenceBundle } from "../domain/evidenceBundle.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { jsonValueSchema } from "../domain/jsonValue.js";
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import {
|
||||
createImmutableEvidenceBundle,
|
||||
describeEvidenceBundleFailure,
|
||||
describeValidationFailure,
|
||||
parseEvidenceBundle,
|
||||
@@ -31,29 +30,14 @@ type EvidenceWriteFailure =
|
||||
| AnalysisCancelledError;
|
||||
|
||||
/** Read and validate an evidence bundle at the caller-supplied path. */
|
||||
export const readEvidenceBundle = (
|
||||
export const readEvidenceBundle = async (
|
||||
path: string,
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
|
||||
readBundle(path, parseEvidenceBundle);
|
||||
|
||||
/** Read an authenticated immutable snapshot for a subsequent session import. */
|
||||
export const readImmutableEvidenceBundle = (
|
||||
path: string,
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
|
||||
readBundle(path, (input) => {
|
||||
const bundle = parseEvidenceBundle(input);
|
||||
return createImmutableEvidenceBundle(bundle.records, bundle.unknowns);
|
||||
});
|
||||
|
||||
const readBundle = async (
|
||||
path: string,
|
||||
parseBundle: (input: unknown) => EvidenceBundle,
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> => {
|
||||
const loaded = await readJsonFile(path);
|
||||
if (!loaded.ok) return loaded;
|
||||
let bundle: EvidenceBundle;
|
||||
try {
|
||||
bundle = parseBundle(loaded.value);
|
||||
bundle = parseEvidenceBundle(loaded.value);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence bundle validation failed", {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import {
|
||||
createEvidence,
|
||||
type Evidence,
|
||||
|
||||
@@ -16,11 +16,7 @@ import {
|
||||
AnalysisResourceConstraintError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import {
|
||||
readJsonInputFile,
|
||||
readJsonObjectFile,
|
||||
type JsonObjectMemberReader,
|
||||
} from "./JsonInputFile.js";
|
||||
import { readJsonInputFile } from "./JsonInputFile.js";
|
||||
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
|
||||
|
||||
/** Request control and its owning operation for an interruptible atomic write. */
|
||||
@@ -37,47 +33,13 @@ export const readJsonFile = async (
|
||||
path: string,
|
||||
): Promise<
|
||||
Result<unknown, EvidenceFileError | AnalysisResourceConstraintError>
|
||||
> =>
|
||||
readJsonFileInput(path, (selected) =>
|
||||
readJsonInputFile(selected, "read_evidence_file"),
|
||||
);
|
||||
|
||||
/** Read selected JSON members through the same regular-file error boundary. */
|
||||
export const readJsonObjectMembersFile = (
|
||||
path: string,
|
||||
scans: readonly ReadonlyMap<string, JsonObjectMemberReader>[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<
|
||||
void,
|
||||
EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError
|
||||
>
|
||||
> =>
|
||||
readJsonFileInput(
|
||||
path,
|
||||
(selected) =>
|
||||
readJsonObjectFile(selected, "read_evidence_file", scans, signal),
|
||||
signal,
|
||||
);
|
||||
|
||||
const readJsonFileInput = async <Value>(
|
||||
path: string,
|
||||
read: (
|
||||
selected: string,
|
||||
) => Promise<
|
||||
Result<
|
||||
Value,
|
||||
| import("../domain/analysisErrorCore.js").AnalysisInputError
|
||||
| AnalysisResourceConstraintError
|
||||
>
|
||||
>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<Value, EvidenceFileError | AnalysisResourceConstraintError>
|
||||
> => {
|
||||
const requestedPath = resolve(path);
|
||||
try {
|
||||
const decoded = await read(requestedPath);
|
||||
const decoded = await readJsonInputFile(
|
||||
requestedPath,
|
||||
"read_evidence_file",
|
||||
);
|
||||
if (!decoded.ok) {
|
||||
if (decoded.error instanceof AnalysisResourceConstraintError)
|
||||
return err(decoded.error);
|
||||
@@ -95,7 +57,6 @@ const readJsonFileInput = async <Value>(
|
||||
return ok(decoded.value);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
|
||||
if (signal?.aborted === true) throw cause;
|
||||
return err(
|
||||
new EvidenceFileError(
|
||||
"read",
|
||||
|
||||
+184
-400
@@ -1,14 +1,10 @@
|
||||
import { constants } from "node:buffer";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
|
||||
import { asStream, none } from "stream-chain";
|
||||
import fun from "stream-chain/fun.js";
|
||||
import parser from "stream-json/parser.js";
|
||||
import streamValues from "stream-json/streamers/stream-values.js";
|
||||
import streamArray from "stream-json/streamers/stream-array.js";
|
||||
|
||||
import { RegularFileChangedError } from "../filesystem/RegularFile.js";
|
||||
import { withRegularFile } from "./RegularFileRead.js";
|
||||
@@ -41,196 +37,6 @@ const nativeParseFits = (inputBytes: number): boolean =>
|
||||
inputBytes * 72 + READ_CHUNK_BYTES <=
|
||||
getHeapStatistics().total_available_size;
|
||||
|
||||
/** One selected root member; repeated keys begin again with last-value semantics. */
|
||||
export interface JsonObjectMemberReader {
|
||||
readonly mode: "value" | "array";
|
||||
readonly begin: () => void;
|
||||
readonly accept: (value: unknown) => void | Promise<void>;
|
||||
readonly reject: (cause: unknown) => void;
|
||||
}
|
||||
|
||||
/** Scan selected members without retaining skipped subtrees or prior array items. */
|
||||
export const readJsonObjectFile = (
|
||||
path: string,
|
||||
operation: string,
|
||||
scans: readonly ReadonlyMap<string, JsonObjectMemberReader>[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<void, JsonInputFileFailure>> =>
|
||||
withRegularFile(
|
||||
path,
|
||||
async (handle, stats) => {
|
||||
let previousDigest: string | undefined;
|
||||
try {
|
||||
const native = await readNativeJson(
|
||||
handle,
|
||||
stats.size,
|
||||
path,
|
||||
operation,
|
||||
signal,
|
||||
);
|
||||
if (native !== undefined) {
|
||||
if (!native.ok) return native;
|
||||
const value = native.value;
|
||||
if (
|
||||
typeof value !== "object" ||
|
||||
value === null ||
|
||||
Array.isArray(value)
|
||||
)
|
||||
throw new TypeError("JSON document root must be an object");
|
||||
for (const members of scans)
|
||||
for (const [key, member] of members) {
|
||||
if (!Object.hasOwn(value, key)) continue;
|
||||
signal?.throwIfAborted();
|
||||
member.begin();
|
||||
const selected: unknown = Reflect.get(value, key);
|
||||
if (member.mode === "array" && !Array.isArray(selected)) {
|
||||
member.reject(
|
||||
new TypeError(`JSON member ${key} must be an array`),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const item of member.mode === "array" &&
|
||||
Array.isArray(selected)
|
||||
? selected
|
||||
: [selected]) {
|
||||
signal?.throwIfAborted();
|
||||
try {
|
||||
await member.accept(item);
|
||||
} catch (cause: unknown) {
|
||||
member.reject(cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ok(undefined);
|
||||
}
|
||||
for (const members of scans) {
|
||||
const hash = createHash("sha256");
|
||||
let rootObject = false;
|
||||
let depth = 0;
|
||||
let key: string | undefined;
|
||||
let consume:
|
||||
| ((token: parser.Token) => typeof none | Promise<typeof none>)
|
||||
| undefined;
|
||||
await pipeline(
|
||||
decodedChunks(handle, stats.size, path, signal, (bytes) => {
|
||||
hash.update(bytes);
|
||||
}),
|
||||
asStream(
|
||||
fun(
|
||||
parser({
|
||||
jsonStreaming: false,
|
||||
streamValues: false,
|
||||
packKeys: false,
|
||||
packStrings: false,
|
||||
packNumbers: false,
|
||||
}),
|
||||
packJsonScalars(operation, stats.size, path),
|
||||
(token: parser.Token): typeof none | Promise<typeof none> => {
|
||||
const opens =
|
||||
token.name === "startObject" || token.name === "startArray";
|
||||
const closes =
|
||||
token.name === "endObject" || token.name === "endArray";
|
||||
if (depth === 0) {
|
||||
rootObject = token.name === "startObject";
|
||||
} else if (
|
||||
rootObject &&
|
||||
depth === 1 &&
|
||||
token.name === "keyValue"
|
||||
) {
|
||||
key = token.value;
|
||||
} else if (rootObject && depth === 1 && !closes) {
|
||||
const member =
|
||||
key === undefined ? undefined : members.get(key);
|
||||
consume = undefined;
|
||||
if (member !== undefined) {
|
||||
member.begin();
|
||||
if (
|
||||
member.mode === "array" &&
|
||||
token.name !== "startArray"
|
||||
) {
|
||||
member.reject(
|
||||
new TypeError(`JSON member ${key} must be an array`),
|
||||
);
|
||||
} else {
|
||||
consume = memberConsumer(
|
||||
member,
|
||||
operation,
|
||||
stats.size,
|
||||
path,
|
||||
);
|
||||
}
|
||||
}
|
||||
key = undefined;
|
||||
}
|
||||
const selected = consume;
|
||||
if (opens) depth += 1;
|
||||
if (closes) depth -= 1;
|
||||
if (rootObject && depth === 1 && token.name !== "keyValue")
|
||||
consume = undefined;
|
||||
if (selected === undefined) return none;
|
||||
return selected(token);
|
||||
},
|
||||
),
|
||||
),
|
||||
signal === undefined ? {} : { signal },
|
||||
);
|
||||
if (!rootObject)
|
||||
throw new TypeError("JSON document root must be an object");
|
||||
const digest = hash.digest("hex");
|
||||
if (previousDigest !== undefined && digest !== previousDigest)
|
||||
throw new RegularFileChangedError(path);
|
||||
previousDigest = digest;
|
||||
}
|
||||
return ok(undefined);
|
||||
} catch (cause: unknown) {
|
||||
const failure = jsonReadFailure(cause, operation, stats.size, path);
|
||||
if (failure !== undefined) return err(failure);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const memberConsumer = (
|
||||
member: JsonObjectMemberReader,
|
||||
operation: string,
|
||||
size: number,
|
||||
path: string,
|
||||
) => {
|
||||
let admit = admitJsonAssembly(operation, size, path);
|
||||
const streamer: parser.TokenConsumer<{ readonly value: unknown }> =
|
||||
member.mode === "array" ? streamArray() : streamValues();
|
||||
const consume = fun(
|
||||
(token: parser.Token) => admit(token),
|
||||
streamer,
|
||||
(row: unknown): typeof none | Promise<typeof none> => {
|
||||
if (typeof row !== "object" || row === null || !("value" in row))
|
||||
throw new Error("Unexpected JSON member parser result");
|
||||
// The preceding array item has left the assembler; retain only what the
|
||||
// consumer owns and use live headroom when admitting the next item.
|
||||
admit = admitJsonAssembly(operation, size, path);
|
||||
try {
|
||||
const accepted = member.accept(row.value);
|
||||
if (accepted instanceof Promise)
|
||||
return accepted.then(
|
||||
() => none,
|
||||
(cause: unknown) => {
|
||||
member.reject(cause);
|
||||
return none;
|
||||
},
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
member.reject(cause);
|
||||
}
|
||||
return none;
|
||||
},
|
||||
);
|
||||
return (token: parser.Token): typeof none | Promise<typeof none> => {
|
||||
const consumed = consume(token);
|
||||
return consumed instanceof Promise ? consumed.then(() => none) : none;
|
||||
};
|
||||
};
|
||||
|
||||
/** Parse strict UTF-8 JSON without requiring a whole-document string. */
|
||||
export const readJsonInputFile = (
|
||||
path: string,
|
||||
@@ -243,222 +49,223 @@ export const readJsonInputFile = (
|
||||
let found = false;
|
||||
let value: unknown;
|
||||
try {
|
||||
const native = await readNativeJson(
|
||||
handle,
|
||||
stats.size,
|
||||
path,
|
||||
operation,
|
||||
signal,
|
||||
);
|
||||
if (native !== undefined) return native;
|
||||
const prefix = nativeParseFits(stats.size)
|
||||
? await readPrefix(handle, stats.size + 1, signal)
|
||||
: undefined;
|
||||
if (prefix !== undefined) {
|
||||
if (!prefix.complete || prefix.bytes.length !== stats.size)
|
||||
throw new RegularFileChangedError(path);
|
||||
const parsed = parseUtf8Json(
|
||||
prefix.bytes,
|
||||
operation,
|
||||
path,
|
||||
"json-file-input",
|
||||
);
|
||||
return parsed.ok
|
||||
? ok(parsed.value)
|
||||
: err(invalidJson(operation, parsed.error, parsed.cause));
|
||||
}
|
||||
await pipeline(
|
||||
decodedChunks(handle, stats.size, path, signal),
|
||||
// Fuse synchronous token stages so only file chunks cross the async
|
||||
// stream boundary. Admission still precedes scalar/value assembly.
|
||||
asStream(
|
||||
fun(
|
||||
parser({
|
||||
jsonStreaming: false,
|
||||
streamValues: false,
|
||||
packKeys: false,
|
||||
packStrings: false,
|
||||
packNumbers: false,
|
||||
}),
|
||||
packJsonScalars(operation, stats.size, path),
|
||||
admitJsonAssembly(operation, stats.size, path),
|
||||
streamValues(),
|
||||
(row: unknown): typeof none => {
|
||||
if (
|
||||
found ||
|
||||
typeof row !== "object" ||
|
||||
row === null ||
|
||||
!("key" in row) ||
|
||||
row.key !== 0 ||
|
||||
!("value" in row)
|
||||
)
|
||||
throw new Error("Unexpected JSON parser result");
|
||||
found = true;
|
||||
value = row.value;
|
||||
return none;
|
||||
},
|
||||
),
|
||||
),
|
||||
parser.asStream({
|
||||
jsonStreaming: false,
|
||||
streamValues: false,
|
||||
packKeys: false,
|
||||
packStrings: false,
|
||||
packNumbers: false,
|
||||
}),
|
||||
packJsonScalars(operation, stats.size, path),
|
||||
admitJsonAssembly(operation, stats.size, path),
|
||||
streamValues.asStream(),
|
||||
async (rows: AsyncIterable<unknown>) => {
|
||||
for await (const row of rows) {
|
||||
if (
|
||||
found ||
|
||||
typeof row !== "object" ||
|
||||
row === null ||
|
||||
!("key" in row) ||
|
||||
row.key !== 0 ||
|
||||
!("value" in row)
|
||||
)
|
||||
throw new Error("Unexpected JSON parser result");
|
||||
found = true;
|
||||
value = row.value;
|
||||
}
|
||||
},
|
||||
signal === undefined ? {} : { signal },
|
||||
);
|
||||
return found
|
||||
? ok(value)
|
||||
: err(invalidJson(operation, "JSON document is empty"));
|
||||
} catch (cause: unknown) {
|
||||
const failure = jsonReadFailure(cause, operation, stats.size, path);
|
||||
if (failure !== undefined) return err(failure);
|
||||
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
|
||||
if (
|
||||
cause instanceof TypeError &&
|
||||
"code" in cause &&
|
||||
cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA"
|
||||
)
|
||||
return err(
|
||||
invalidJson(operation, "JSON input is not valid UTF-8", cause),
|
||||
);
|
||||
if (
|
||||
cause instanceof SyntaxError ||
|
||||
(cause instanceof Error && cause.message.startsWith("Parser "))
|
||||
)
|
||||
return err(invalidJson(operation, cause.message, cause));
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
((cause instanceof RangeError &&
|
||||
cause.message === "Invalid string length") ||
|
||||
("code" in cause && cause.code === "ERR_STRING_TOO_LONG"))
|
||||
)
|
||||
return err(
|
||||
new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"An individual JSON string or number token exceeds the runtime string limit",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: stats.size,
|
||||
max_string_code_units: constants.MAX_STRING_LENGTH,
|
||||
},
|
||||
{
|
||||
cause,
|
||||
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
|
||||
},
|
||||
),
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const jsonReadFailure = (
|
||||
cause: unknown,
|
||||
operation: string,
|
||||
size: number,
|
||||
path: string,
|
||||
): JsonInputFileFailure | undefined => {
|
||||
if (cause instanceof AnalysisResourceConstraintError) return cause;
|
||||
if (
|
||||
cause instanceof TypeError &&
|
||||
"code" in cause &&
|
||||
cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA"
|
||||
)
|
||||
return invalidJson(operation, "JSON input is not valid UTF-8", cause);
|
||||
if (
|
||||
cause instanceof SyntaxError ||
|
||||
(cause instanceof Error && cause.message.startsWith("Parser "))
|
||||
)
|
||||
return invalidJson(operation, cause.message, cause);
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
((cause instanceof RangeError &&
|
||||
cause.message === "Invalid string length") ||
|
||||
("code" in cause && cause.code === "ERR_STRING_TOO_LONG"))
|
||||
)
|
||||
return new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"An individual JSON string or number token exceeds the runtime string limit",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: size,
|
||||
max_string_code_units: constants.MAX_STRING_LENGTH,
|
||||
},
|
||||
{
|
||||
cause,
|
||||
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
|
||||
},
|
||||
);
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const packJsonScalars = (
|
||||
operation: string,
|
||||
inputFileBytes: number,
|
||||
path: string,
|
||||
) => {
|
||||
let kind: "keyValue" | "stringValue" | "numberValue" | undefined;
|
||||
let length = 0;
|
||||
let value = "";
|
||||
let parts: string[] = [];
|
||||
let partLength = 0;
|
||||
return (token: parser.Token): parser.Token | typeof none => {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON parser token");
|
||||
switch (token.name) {
|
||||
case "startKey":
|
||||
case "startString":
|
||||
kind = token.name === "startKey" ? "keyValue" : "stringValue";
|
||||
break;
|
||||
case "startNumber":
|
||||
kind = "numberValue";
|
||||
break;
|
||||
case "numberChunk":
|
||||
case "stringChunk":
|
||||
if (
|
||||
kind === undefined ||
|
||||
!("value" in token) ||
|
||||
typeof token.value !== "string"
|
||||
)
|
||||
throw new Error("Unexpected JSON scalar fragment");
|
||||
length += token.value.length;
|
||||
if (length > constants.MAX_STRING_LENGTH)
|
||||
throw new RangeError("Invalid string length");
|
||||
parts.push(token.value);
|
||||
partLength += token.value.length;
|
||||
if (partLength >= PACKED_TOKEN_PART_CODE_UNITS) {
|
||||
// The tokenizer emits short fragments. Coalesce them before retaining
|
||||
// a scalar so millions of substring/rope nodes cannot exhaust the heap
|
||||
// before the native string-length constraint can be reported.
|
||||
) =>
|
||||
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
|
||||
let kind: "keyValue" | "stringValue" | "numberValue" | undefined;
|
||||
let length = 0;
|
||||
let value = "";
|
||||
let parts: string[] = [];
|
||||
let partLength = 0;
|
||||
for await (const token of tokens) {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON parser token");
|
||||
switch (token.name) {
|
||||
case "startKey":
|
||||
case "startString":
|
||||
kind = token.name === "startKey" ? "keyValue" : "stringValue";
|
||||
break;
|
||||
case "startNumber":
|
||||
kind = "numberValue";
|
||||
break;
|
||||
case "numberChunk":
|
||||
case "stringChunk":
|
||||
if (
|
||||
kind === undefined ||
|
||||
!("value" in token) ||
|
||||
typeof token.value !== "string"
|
||||
)
|
||||
throw new Error("Unexpected JSON scalar fragment");
|
||||
length += token.value.length;
|
||||
if (length > constants.MAX_STRING_LENGTH)
|
||||
throw new RangeError("Invalid string length");
|
||||
parts.push(token.value);
|
||||
partLength += token.value.length;
|
||||
if (partLength >= PACKED_TOKEN_PART_CODE_UNITS) {
|
||||
// The tokenizer emits short fragments. Coalesce them before retaining
|
||||
// a scalar so millions of substring/rope nodes cannot exhaust the heap
|
||||
// before the native string-length constraint can be reported.
|
||||
requireStringAssemblyHeadroom(
|
||||
operation,
|
||||
inputFileBytes,
|
||||
path,
|
||||
length,
|
||||
);
|
||||
value += parts.join("");
|
||||
parts = [];
|
||||
partLength = 0;
|
||||
}
|
||||
break;
|
||||
case "endKey":
|
||||
case "endString":
|
||||
case "endNumber":
|
||||
if (kind === undefined) throw new Error("Unexpected JSON scalar end");
|
||||
requireStringAssemblyHeadroom(
|
||||
operation,
|
||||
inputFileBytes,
|
||||
path,
|
||||
length,
|
||||
);
|
||||
value += parts.join("");
|
||||
yield { name: kind, value: value + parts.join("") };
|
||||
kind = undefined;
|
||||
length = 0;
|
||||
value = "";
|
||||
parts = [];
|
||||
partLength = 0;
|
||||
}
|
||||
break;
|
||||
case "endKey":
|
||||
case "endString":
|
||||
case "endNumber": {
|
||||
if (kind === undefined) throw new Error("Unexpected JSON scalar end");
|
||||
requireStringAssemblyHeadroom(operation, inputFileBytes, path, length);
|
||||
const packed = { name: kind, value: value + parts.join("") };
|
||||
kind = undefined;
|
||||
length = 0;
|
||||
value = "";
|
||||
parts = [];
|
||||
partLength = 0;
|
||||
return packed;
|
||||
break;
|
||||
default:
|
||||
yield token;
|
||||
}
|
||||
default:
|
||||
return token;
|
||||
}
|
||||
return none;
|
||||
};
|
||||
};
|
||||
|
||||
/** Admit the retained value graph before the upstream assembler grows it. */
|
||||
const admitJsonAssembly = (
|
||||
operation: string,
|
||||
inputFileBytes: number,
|
||||
path: string,
|
||||
) => {
|
||||
let projectedBytes = 0;
|
||||
let nextCheck = 0;
|
||||
return (token: parser.Token): parser.Token => {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON assembly token");
|
||||
if (token.name === "startObject" || token.name === "startArray") {
|
||||
projectedBytes += 128;
|
||||
} else if (token.name === "keyValue" || token.name === "stringValue") {
|
||||
if (!("value" in token) || typeof token.value !== "string")
|
||||
throw new Error("Unexpected packed JSON string");
|
||||
projectedBytes += 64 + token.value.length * 2;
|
||||
} else if (
|
||||
token.name === "numberValue" ||
|
||||
token.name === "nullValue" ||
|
||||
token.name === "trueValue" ||
|
||||
token.name === "falseValue"
|
||||
) {
|
||||
projectedBytes += 64;
|
||||
) =>
|
||||
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
|
||||
let projectedBytes = 0;
|
||||
let nextCheck = 0;
|
||||
for await (const token of tokens) {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON assembly token");
|
||||
if (token.name === "startObject" || token.name === "startArray") {
|
||||
projectedBytes += 128;
|
||||
} else if (token.name === "keyValue" || token.name === "stringValue") {
|
||||
if (!("value" in token) || typeof token.value !== "string")
|
||||
throw new Error("Unexpected packed JSON string");
|
||||
projectedBytes += 64 + token.value.length * 2;
|
||||
} else if (
|
||||
token.name === "numberValue" ||
|
||||
token.name === "nullValue" ||
|
||||
token.name === "trueValue" ||
|
||||
token.name === "falseValue"
|
||||
) {
|
||||
projectedBytes += 64;
|
||||
}
|
||||
if (projectedBytes >= nextCheck) {
|
||||
const heap = getHeapStatistics();
|
||||
// Conservative capacity for containers, property/array slots and UTF-16
|
||||
// values. Keep another whole projection available for container growth,
|
||||
// plus the next chunk of estimates between samples. This is admission
|
||||
// against live runtime headroom, not a fixed document-size policy.
|
||||
const requiredBytes = projectedBytes + READ_CHUNK_BYTES;
|
||||
if (requiredBytes > heap.total_available_size)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"Insufficient heap headroom to materialize the complete JSON value",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: inputFileBytes,
|
||||
projected_value_bytes: projectedBytes,
|
||||
assembly_headroom_bytes: requiredBytes,
|
||||
available_heap_bytes: heap.total_available_size,
|
||||
heap_size_limit_bytes: heap.heap_size_limit,
|
||||
},
|
||||
{ remediationAction: JSON_INPUT_RESOURCE_REMEDIATION },
|
||||
);
|
||||
nextCheck = projectedBytes + READ_CHUNK_BYTES;
|
||||
}
|
||||
yield token;
|
||||
}
|
||||
if (projectedBytes >= nextCheck) {
|
||||
const heap = getHeapStatistics();
|
||||
// Conservative capacity for containers, property/array slots and UTF-16
|
||||
// values. Keep another whole projection available for container growth,
|
||||
// plus the next chunk of estimates between samples. This is admission
|
||||
// against live runtime headroom, not a fixed document-size policy.
|
||||
const requiredBytes = projectedBytes + READ_CHUNK_BYTES;
|
||||
if (requiredBytes > heap.total_available_size)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"Insufficient heap headroom to materialize the complete JSON value",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: inputFileBytes,
|
||||
projected_value_bytes: projectedBytes,
|
||||
assembly_headroom_bytes: requiredBytes,
|
||||
available_heap_bytes: heap.total_available_size,
|
||||
heap_size_limit_bytes: heap.heap_size_limit,
|
||||
},
|
||||
{ remediationAction: JSON_INPUT_RESOURCE_REMEDIATION },
|
||||
);
|
||||
nextCheck = projectedBytes + READ_CHUNK_BYTES;
|
||||
}
|
||||
return token;
|
||||
};
|
||||
};
|
||||
|
||||
const requireStringAssemblyHeadroom = (
|
||||
operation: string,
|
||||
@@ -495,7 +302,6 @@ async function* decodedChunks(
|
||||
admittedBytes: number,
|
||||
path: string,
|
||||
signal: AbortSignal | undefined,
|
||||
observe?: (bytes: Uint8Array) => void,
|
||||
): AsyncGenerator<string> {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
let leading = true;
|
||||
@@ -514,15 +320,15 @@ async function* decodedChunks(
|
||||
bytes,
|
||||
0,
|
||||
Math.min(bytes.length, admittedBytes - observedBytes + 1),
|
||||
observedBytes,
|
||||
null,
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
if (read.bytesRead === 0) break;
|
||||
observedBytes += read.bytesRead;
|
||||
if (observedBytes > admittedBytes) throw new RegularFileChangedError(path);
|
||||
const chunk = bytes.subarray(0, read.bytesRead);
|
||||
observe?.(chunk);
|
||||
yield checked(decoder.decode(chunk, { stream: true }));
|
||||
yield checked(
|
||||
decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }),
|
||||
);
|
||||
}
|
||||
if (observedBytes !== admittedBytes) throw new RegularFileChangedError(path);
|
||||
yield checked(decoder.decode());
|
||||
@@ -554,28 +360,6 @@ const readPrefix = async (
|
||||
return { bytes, complete: false };
|
||||
};
|
||||
|
||||
const readNativeJson = async (
|
||||
handle: FileHandle,
|
||||
size: number,
|
||||
path: string,
|
||||
operation: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<unknown, JsonInputFileFailure> | undefined> => {
|
||||
if (!nativeParseFits(size)) return undefined;
|
||||
const prefix = await readPrefix(handle, size + 1, signal);
|
||||
if (!prefix.complete || prefix.bytes.length !== size)
|
||||
throw new RegularFileChangedError(path);
|
||||
const parsed = parseUtf8Json(
|
||||
prefix.bytes,
|
||||
operation,
|
||||
path,
|
||||
"json-file-input",
|
||||
);
|
||||
return parsed.ok
|
||||
? ok(parsed.value)
|
||||
: err(invalidJson(operation, parsed.error, parsed.cause));
|
||||
};
|
||||
|
||||
const invalidJson = (
|
||||
operation: string,
|
||||
message: string,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { z } from "zod";
|
||||
|
||||
import { reconstructionCoverageEvaluationInputSchema } from "../domain/reconstructionCoverageInput.js";
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { evaluateReconstructionClosure } from "../domain/reconstructionCoverage.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../domain/evidence.js";
|
||||
import { jsonObjectSchema, jsonValueSchema } from "../domain/jsonValue.js";
|
||||
import {
|
||||
|
||||
@@ -18,9 +18,9 @@ import type {
|
||||
} from "../reference/ReferenceSourceReaderTypes.js";
|
||||
import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js";
|
||||
import { readReferenceSourceVcs } from "./ReferenceSourceVcsAdapter.js";
|
||||
import type {
|
||||
ReferenceSourceImportError,
|
||||
ReferenceSourceImportOptions,
|
||||
import {
|
||||
type ReferenceSourceImportError,
|
||||
type ReferenceSourceImportOptions,
|
||||
} from "./ReferenceSourceImportTypes.js";
|
||||
import {
|
||||
prepareReferenceSourceImport,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { skillDestinations } from "./SetupSkill.js";
|
||||
import type {
|
||||
ClientConfigurationInspection,
|
||||
ClientConfigurationResult,
|
||||
SetupHost,
|
||||
SetupOptions,
|
||||
SetupProviderEnvironment,
|
||||
import {
|
||||
type ClientConfigurationInspection,
|
||||
type ClientConfigurationResult,
|
||||
type SetupHost,
|
||||
type SetupOptions,
|
||||
type SetupProviderEnvironment,
|
||||
} from "./SetupTypes.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
import type { SetupHopperInstallResult } from "./SetupInstallFailure.js";
|
||||
|
||||
@@ -29,7 +29,6 @@ import {
|
||||
supportedClients,
|
||||
type SetupClient,
|
||||
} from "./SupportedClients.js";
|
||||
export { isUninstallFailure } from "./UninstallFailure.js";
|
||||
|
||||
interface ManagedPathStats {
|
||||
readonly uid?: number;
|
||||
@@ -75,6 +74,23 @@ export interface UninstallResult {
|
||||
readonly items: readonly UninstallItem[];
|
||||
}
|
||||
|
||||
/** Whether any requested REA-owned uninstall action failed. */
|
||||
export const isUninstallFailure = (result: UninstallResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "complete":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled uninstall result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Filesystem boundary used by the contained uninstall workflow. */
|
||||
export interface UninstallHost {
|
||||
clients(): Promise<readonly SetupClient[]>;
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
import type { UninstallResult } from "./Uninstall.js";
|
||||
|
||||
/** Whether any requested REA-owned uninstall action failed. */
|
||||
export const isUninstallFailure = (result: UninstallResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "complete":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled uninstall result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -3,7 +3,6 @@ import { gt, valid } from "semver";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import type { Result } from "../domain/result.js";
|
||||
import type { IntegrationMaintenance } from "./UpdateMaintenance.js";
|
||||
export { isUpdateFailure } from "./UpdateFailure.js";
|
||||
|
||||
/** Global npm installation that owns the running package and its entry point. */
|
||||
export interface NpmInstallation {
|
||||
@@ -69,6 +68,24 @@ export type UpdateResult =
|
||||
readonly remediation: string;
|
||||
};
|
||||
|
||||
/** Whether the update command left the requested release unapplied. */
|
||||
export const isUpdateFailure = (result: UpdateResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "current":
|
||||
case "updated":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled update result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Resolve the exact npm command for the identified prefix and release. */
|
||||
export const updateInstallCommand = (
|
||||
installation: NpmInstallation,
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
import type { UpdateResult } from "./Update.js";
|
||||
|
||||
/** Whether the update command left the requested release unapplied. */
|
||||
export const isUpdateFailure = (result: UpdateResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "current":
|
||||
case "updated":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled update result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -7,10 +7,6 @@ import {
|
||||
import { InvestigationRecords } from "./investigation/InvestigationRecords.js";
|
||||
import { UnknownRegistryError } from "../domain/unknownRegistryError.js";
|
||||
import { err } from "../domain/result.js";
|
||||
import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js";
|
||||
import { projectNativeApiInspection } from "./native/NativeApiInspection.js";
|
||||
import { functionDossierSchema } from "../domain/hopperValues.js";
|
||||
import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js";
|
||||
|
||||
describe("workflow residual question retention", () => {
|
||||
it("keeps questions linked to workflow Evidence and records repeats idempotently", () => {
|
||||
@@ -58,47 +54,3 @@ describe("workflow residual question retention", () => {
|
||||
).toEqual(err(error));
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
" Which boundary? ",
|
||||
"\nWhich boundary?\t",
|
||||
"\uFEFFWhich boundary?\uFEFF",
|
||||
])(
|
||||
"retains contract-accepted question %j without rewriting workflow Evidence",
|
||||
(question) => {
|
||||
const boundary = nativeApiBoundarySchema.parse({
|
||||
available: false,
|
||||
reason: "Synthetic provider unavailable boundary",
|
||||
residual_unknowns: [question],
|
||||
});
|
||||
const dossier = functionDossierSchema.parse(ghidraFunctionDossier());
|
||||
const result = projectNativeApiInspection({
|
||||
...dossier,
|
||||
native_api: boundary,
|
||||
});
|
||||
const evidence = createWorkflowEvidence({
|
||||
target: undefined,
|
||||
operation: "inspect_native_api",
|
||||
parameters: { procedure: "0x1000" },
|
||||
result,
|
||||
upstreamProfile: undefined,
|
||||
});
|
||||
const records = new InvestigationRecords();
|
||||
expect(records.recordEvidence(evidence).ok).toBe(true);
|
||||
expect(
|
||||
recordWorkflowUnknowns({
|
||||
name: "inspect_native_api",
|
||||
result,
|
||||
evidenceId: evidence.evidence_id,
|
||||
recordUnknown: (input) => records.recordUnknown(input),
|
||||
}).ok,
|
||||
).toBe(true);
|
||||
expect(records.listUnknowns()).toMatchObject([
|
||||
{ question: "Which boundary?" },
|
||||
]);
|
||||
expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence);
|
||||
expect(evidence.normalized_result).toMatchObject({
|
||||
residual_unknowns: [question],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
AnalysisSnapshotMismatchError,
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { NoBinaryOpenError } from "../../domain/configurationErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
analysisQueryId,
|
||||
@@ -18,7 +18,6 @@ import {
|
||||
snapshotMatchesTarget,
|
||||
snapshotTarget,
|
||||
parseAnalysisSnapshot,
|
||||
immutableAnalysisSnapshot,
|
||||
type AnalysisSnapshot,
|
||||
type AnalysisSnapshotEntry,
|
||||
type AnalysisSnapshotWorkflowEntry,
|
||||
@@ -108,7 +107,6 @@ export class AnalysisSnapshotCache {
|
||||
|
||||
/** Merge already-validated entries and return the new-entry count. */
|
||||
stage(snapshot: AnalysisSnapshot): number {
|
||||
snapshot = immutableAnalysisSnapshot(snapshot);
|
||||
if (
|
||||
(this.#target !== undefined &&
|
||||
JSON.stringify(this.#target) !== JSON.stringify(snapshot.target)) ||
|
||||
@@ -125,12 +123,12 @@ export class AnalysisSnapshotCache {
|
||||
for (const entry of snapshot.entries) {
|
||||
if (!this.#entries.has(entry.query_id) && !this.#hasCapacity()) continue;
|
||||
if (!this.#entries.has(entry.query_id)) imported += 1;
|
||||
this.#entries.set(entry.query_id, entry);
|
||||
this.#entries.set(entry.query_id, structuredClone(entry));
|
||||
}
|
||||
for (const entry of snapshot.workflow_entries) {
|
||||
if (!this.#workflowEntries.has(entry.query_id) && !this.#hasCapacity())
|
||||
continue;
|
||||
this.#workflowEntries.set(entry.query_id, entry);
|
||||
this.#workflowEntries.set(entry.query_id, structuredClone(entry));
|
||||
}
|
||||
return imported;
|
||||
}
|
||||
@@ -188,7 +186,7 @@ export class AnalysisSnapshotCache {
|
||||
): Result<number, AnalysisError> {
|
||||
let validated: AnalysisSnapshot;
|
||||
try {
|
||||
validated = immutableAnalysisSnapshot(snapshot);
|
||||
validated = parseAnalysisSnapshot(snapshot);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Analysis snapshot validation failed", {
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
|
||||
import {
|
||||
prepareSnapshotEntry,
|
||||
parsePreparedAnalysisSnapshot,
|
||||
type PreparedSnapshotEntry,
|
||||
parseAnalysisSnapshot,
|
||||
serializeAnalysisSnapshot,
|
||||
type AnalysisSnapshot,
|
||||
} from "../../domain/analysisSnapshot.js";
|
||||
@@ -13,106 +9,27 @@ import {
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import { readJsonObjectMembersFile, writeTextFile } from "../JsonFiles.js";
|
||||
import type { JsonObjectMemberReader } from "../JsonInputFile.js";
|
||||
import { readJsonFile, writeTextFile } from "../JsonFiles.js";
|
||||
|
||||
type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError;
|
||||
|
||||
/** Read and validate an analysis snapshot from the caller's path. */
|
||||
export const readAnalysisSnapshot = async (
|
||||
path: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<
|
||||
AnalysisSnapshot,
|
||||
SnapshotFailure | AnalysisResourceConstraintError | AnalysisCancelledError
|
||||
>
|
||||
Result<AnalysisSnapshot, SnapshotFailure | AnalysisResourceConstraintError>
|
||||
> => {
|
||||
const entries = snapshotEntryReader("provider");
|
||||
const workflows = snapshotEntryReader("workflow");
|
||||
const header: Record<string, unknown> = {};
|
||||
const metadata = new Map<string, JsonObjectMemberReader>(
|
||||
["target", "binding", "evidence_bundle"].map((name) => [
|
||||
name,
|
||||
{
|
||||
mode: "value",
|
||||
begin: () => {
|
||||
delete header[name];
|
||||
},
|
||||
accept: (value) => {
|
||||
header[name] = value;
|
||||
},
|
||||
reject: (cause) => {
|
||||
throw cause;
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
const loaded = await readJsonFile(path);
|
||||
if (!loaded.ok) return loaded;
|
||||
try {
|
||||
const loaded = await readJsonObjectMembersFile(
|
||||
path,
|
||||
[
|
||||
new Map([
|
||||
["entries", entries.reader],
|
||||
["workflow_entries", workflows.reader],
|
||||
]),
|
||||
metadata,
|
||||
],
|
||||
signal,
|
||||
);
|
||||
if (!loaded.ok) return loaded;
|
||||
signal?.throwIfAborted();
|
||||
const snapshot = parsePreparedAnalysisSnapshot(
|
||||
header,
|
||||
entries.finish(),
|
||||
workflows.finish(),
|
||||
);
|
||||
// Deliver queued cancellation before handing a complete cache to the session.
|
||||
await setImmediate();
|
||||
signal?.throwIfAborted();
|
||||
return ok(snapshot);
|
||||
return ok(parseAnalysisSnapshot(loaded.value));
|
||||
} catch (cause: unknown) {
|
||||
if (signal?.aborted === true)
|
||||
return err(new AnalysisCancelledError("read_evidence_file", { cause }));
|
||||
return err(
|
||||
new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const snapshotEntryReader = (kind: "provider" | "workflow") => {
|
||||
let seen = false;
|
||||
let entries: PreparedSnapshotEntry[] = [];
|
||||
let failure: { readonly cause: unknown } | undefined;
|
||||
const reader: JsonObjectMemberReader = {
|
||||
mode: "array",
|
||||
begin: () => {
|
||||
seen = true;
|
||||
entries = [];
|
||||
failure = undefined;
|
||||
},
|
||||
accept: (value) => {
|
||||
if (failure === undefined)
|
||||
entries.push(prepareSnapshotEntry(value, kind));
|
||||
},
|
||||
reject: (cause) => {
|
||||
failure ??= { cause };
|
||||
},
|
||||
};
|
||||
return {
|
||||
reader,
|
||||
finish: (): PreparedSnapshotEntry[] => {
|
||||
if (!seen)
|
||||
throw new TypeError(
|
||||
`Analysis snapshot ${kind === "provider" ? "entries" : "workflow_entries"} is missing`,
|
||||
);
|
||||
if (failure !== undefined) throw failure.cause;
|
||||
return entries;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/** Atomically write a deterministic analysis snapshot to the caller's path. */
|
||||
export const writeAnalysisSnapshot = async (
|
||||
snapshot: AnalysisSnapshot,
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
type AnalysisProfileCommitment,
|
||||
} from "../../domain/analysisProfile.js";
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { createEvidence } from "../../domain/evidence.js";
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import { NoBinaryOpenError } from "../../domain/configurationErrors.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import type { EvidenceSubjectTarget } from "../../domain/evidence.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
|
||||
@@ -8,12 +8,12 @@ import {
|
||||
} from "../../domain/analysisSnapshot.js";
|
||||
import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js";
|
||||
import { parseBinaryTarget } from "../BinaryTargetResolver.js";
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import { type BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import {
|
||||
EvidenceIntegrityError,
|
||||
AnalysisSnapshotMismatchError,
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, type Result } from "../../domain/result.js";
|
||||
import type { SessionProviderRoute } from "./SessionProviderRouter.js";
|
||||
import { SessionProviderRouter } from "./SessionProviderRouter.js";
|
||||
|
||||
@@ -9,8 +9,8 @@ import {
|
||||
EvidenceIntegrityError,
|
||||
AnalysisSnapshotMismatchError,
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { UnknownRegistryError } from "../../domain/unknownRegistryError.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type UnknownRegistryError } from "../../domain/unknownRegistryError.js";
|
||||
import type {
|
||||
RecordUnknownInput,
|
||||
ResidualUnknown,
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { AnalysisProfileCommitment } from "../../domain/analysisProfile.js"
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import { ProviderSelectionError } from "../../domain/providerSelectionError.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import type {
|
||||
AnalysisClient,
|
||||
|
||||
@@ -417,9 +417,8 @@ export class EvidenceLedger {
|
||||
unknowns: ReadonlyMap<string, ResidualUnknown>,
|
||||
): Result<void, EvidenceIntegrityError | UnknownRegistryError> {
|
||||
try {
|
||||
createImmutableEvidenceBundle(
|
||||
[...records.values()],
|
||||
[...unknowns.values()],
|
||||
parseEvidenceBundle(
|
||||
createEvidenceBundle([...records.values()], [...unknowns.values()]),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
return err(new UnknownRegistryError("integrity", { cause }));
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
import type { EvidenceBundle } from "../../domain/evidenceBundle.js";
|
||||
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
summarizeEvidenceBundle,
|
||||
type EvidenceBundleSummary,
|
||||
type EvidenceRecordFilters,
|
||||
} from "../../domain/evidenceRecordSummary.js";
|
||||
|
||||
/** Select complete bundle delivery or discovery metadata without analysis payloads. */
|
||||
export const inspectEvidenceBundle = (
|
||||
bundle: EvidenceBundle,
|
||||
options: {
|
||||
readonly detail: "complete" | "summary";
|
||||
readonly filters?: EvidenceRecordFilters | undefined;
|
||||
},
|
||||
): Result<EvidenceBundle | EvidenceBundleSummary, AnalysisInputError> => {
|
||||
if (options.detail === "complete" && options.filters !== undefined)
|
||||
return err(
|
||||
new AnalysisInputError("get_evidence_bundle", undefined, [
|
||||
{
|
||||
path: ["filters"],
|
||||
reason: "invalid_value",
|
||||
message:
|
||||
"Record filters require detail: summary; complete returns the canonical bundle.",
|
||||
expected: "detail: summary",
|
||||
},
|
||||
]),
|
||||
);
|
||||
return ok(
|
||||
options.detail === "summary"
|
||||
? summarizeEvidenceBundle(bundle, options.filters)
|
||||
: bundle,
|
||||
);
|
||||
};
|
||||
@@ -61,10 +61,7 @@ describe("investigation record ownership", () => {
|
||||
value: {
|
||||
scope_digest: target.sha256,
|
||||
mutation_evidence_ids: [
|
||||
unknownMutationEvidence(target, {
|
||||
operation: "record_unknown",
|
||||
input,
|
||||
}).evidence_id,
|
||||
unknownMutationEvidence(target, input).evidence_id,
|
||||
],
|
||||
},
|
||||
});
|
||||
@@ -74,19 +71,14 @@ describe("investigation record ownership", () => {
|
||||
value: {
|
||||
scope_digest: null,
|
||||
mutation_evidence_ids: [
|
||||
unknownMutationEvidence(undefined, {
|
||||
operation: "record_unknown",
|
||||
input,
|
||||
}).evidence_id,
|
||||
unknownMutationEvidence(undefined, input).evidence_id,
|
||||
],
|
||||
},
|
||||
});
|
||||
const mutation = records.evidenceById(
|
||||
unknownMutationEvidence(undefined, { operation: "record_unknown", input })
|
||||
.evidence_id,
|
||||
unknownMutationEvidence(undefined, input).evidence_id,
|
||||
);
|
||||
expect(mutation?.subject).toBeNull();
|
||||
expect(mutation?.parameters).toEqual(input);
|
||||
expect(records.listUnknowns()).toHaveLength(2);
|
||||
});
|
||||
|
||||
@@ -132,10 +124,6 @@ describe("investigation record ownership", () => {
|
||||
bundle.records.find(({ operation }) => operation === "update_unknown")
|
||||
?.subject?.digest.sha256,
|
||||
).toBe(target.sha256);
|
||||
expect(
|
||||
bundle.records.find(({ operation }) => operation === "update_unknown")
|
||||
?.parameters,
|
||||
).toEqual(input);
|
||||
expect(records.updateUnknown(input, target).ok).toBe(false);
|
||||
expect(records.exportEvidenceBundle()).toEqual(bundle);
|
||||
});
|
||||
@@ -174,59 +162,3 @@ describe("investigation record ownership", () => {
|
||||
expect(records.exportEvidenceBundle()).toEqual(createEvidenceBundle([]));
|
||||
});
|
||||
});
|
||||
|
||||
it("retains complete mutation intent and distinguishes independent update requests", () => {
|
||||
const source = new InvestigationRecords();
|
||||
expect(source.recordEvidence(evidence).ok).toBe(true);
|
||||
const input = {
|
||||
...unknownInput("Which replay establishes the answer?"),
|
||||
supporting_evidence_ids: [evidence.evidence_id],
|
||||
required_environment: {
|
||||
id: "fixture",
|
||||
platform: "linux",
|
||||
architecture: "x86_64",
|
||||
isolation: "process" as const,
|
||||
},
|
||||
recommended_probes: [
|
||||
{ operation: "inspect_native_api", rationale: "Replay" },
|
||||
],
|
||||
};
|
||||
const created = source.recordUnknown(input);
|
||||
if (!created.ok) throw created.error;
|
||||
const base = source.exportEvidenceBundle();
|
||||
const creation = base.records.find(
|
||||
({ operation }) => operation === "record_unknown",
|
||||
);
|
||||
expect(creation?.parameters).toEqual(input);
|
||||
expect(creation?.evidence_links).toEqual([evidence.evidence_id]);
|
||||
|
||||
const { question: _question, domain: _domain, ...details } = input;
|
||||
const mutations = (["high", "low"] as const).map((severity) => {
|
||||
const records = new InvestigationRecords();
|
||||
expect(records.mergeEvidenceBundle(base).ok).toBe(true);
|
||||
const command = updateUnknownInputSchema.parse({
|
||||
...details,
|
||||
unknown_id: created.value.unknown_id,
|
||||
expected_revision: 1,
|
||||
status: "resolved",
|
||||
severity,
|
||||
resolution: {
|
||||
disposition: "withdrawn",
|
||||
rationale: "Caller withdrew this question",
|
||||
evidence_ids: [evidence.evidence_id],
|
||||
},
|
||||
});
|
||||
const updated = records.updateUnknown(command);
|
||||
if (!updated.ok) throw updated.error;
|
||||
const mutation = records
|
||||
.exportEvidenceBundle()
|
||||
.records.find(({ operation }) => operation === "update_unknown");
|
||||
expect(mutation?.parameters).toEqual(command);
|
||||
expect(mutation?.evidence_links).toEqual([evidence.evidence_id]);
|
||||
expect(updated.value.mutation_evidence_ids).toContain(
|
||||
mutation?.evidence_id,
|
||||
);
|
||||
return mutation;
|
||||
});
|
||||
expect(mutations[0]?.evidence_id).not.toBe(mutations[1]?.evidence_id);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import type { EvidenceBundle } from "../../domain/evidenceBundle.js";
|
||||
import type { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
@@ -12,7 +12,11 @@ import type {
|
||||
} from "../../domain/residualUnknown.js";
|
||||
import type { Result } from "../../domain/result.js";
|
||||
import { EvidenceLedger, type EvidenceImportDelta } from "./EvidenceLedger.js";
|
||||
import { unknownMutationEvidence } from "./UnknownEvidence.js";
|
||||
import {
|
||||
UNKNOWN_REGISTRY_PROVIDER,
|
||||
unknownEvidenceLinks,
|
||||
unknownMutationEvidence,
|
||||
} from "./UnknownEvidence.js";
|
||||
import type {
|
||||
EvidenceReader,
|
||||
EvidenceWriter,
|
||||
@@ -79,10 +83,11 @@ export class InvestigationRecords
|
||||
input: RecordUnknownInput,
|
||||
target?: BinaryTarget,
|
||||
): Result<ResidualUnknown, AnalysisError> {
|
||||
return this.#evidence.recordUnknown(
|
||||
const recorded = this.#evidence.recordUnknown(
|
||||
input,
|
||||
unknownMutationEvidence(target, { operation: "record_unknown", input }),
|
||||
unknownMutationEvidence(target, input),
|
||||
);
|
||||
return recorded;
|
||||
}
|
||||
|
||||
/** Commit derived Evidence and its Unknown atomically without an implicit subject. */
|
||||
@@ -90,14 +95,12 @@ export class InvestigationRecords
|
||||
evidence: Evidence,
|
||||
input: RecordUnknownInput,
|
||||
): Result<ResidualUnknown | null, AnalysisError> {
|
||||
return this.#evidence.recordWithUnknown(
|
||||
const recorded = this.#evidence.recordWithUnknown(
|
||||
evidence,
|
||||
input,
|
||||
unknownMutationEvidence(undefined, {
|
||||
operation: "record_unknown",
|
||||
input,
|
||||
}),
|
||||
unknownMutationEvidence(undefined, input),
|
||||
);
|
||||
return recorded;
|
||||
}
|
||||
|
||||
/** Update an optimistic revision with an explicitly supplied Evidence subject. */
|
||||
@@ -105,11 +108,23 @@ export class InvestigationRecords
|
||||
input: UpdateUnknownInput,
|
||||
target?: BinaryTarget,
|
||||
): Result<ResidualUnknown, AnalysisError> {
|
||||
const evidence = unknownMutationEvidence(target, {
|
||||
const evidence = createEvidence(target, UNKNOWN_REGISTRY_PROVIDER, {
|
||||
predicateType: "rea.residual-unknown-mutation",
|
||||
operation: "update_unknown",
|
||||
input,
|
||||
parameters: {
|
||||
unknown_id: input.unknown_id,
|
||||
expected_revision: input.expected_revision,
|
||||
},
|
||||
result: { action: "update", status: input.status },
|
||||
confidence: "derived",
|
||||
authority: "analyst-inference",
|
||||
evidenceLinks: unknownEvidenceLinks(input),
|
||||
limitations: [
|
||||
"Registry mutation evidence records analyst intent, not proof of the answer.",
|
||||
],
|
||||
});
|
||||
return this.#evidence.updateUnknown(input, evidence);
|
||||
const updated = this.#evidence.updateUnknown(input, evidence);
|
||||
return updated;
|
||||
}
|
||||
|
||||
/** Return detached current Unknown revisions matching the selected filters. */
|
||||
|
||||
@@ -7,54 +7,43 @@ import type {
|
||||
import type { ProviderIdentity } from "../AnalysisProvider.js";
|
||||
|
||||
/** Provider identity for analyst-authored residual-unknown mutations. */
|
||||
const UNKNOWN_REGISTRY_PROVIDER: ProviderIdentity = {
|
||||
export const UNKNOWN_REGISTRY_PROVIDER: ProviderIdentity = {
|
||||
id: "rea-unknown-registry",
|
||||
name: "REA residual unknown registry",
|
||||
version: "1",
|
||||
};
|
||||
|
||||
type UnknownMutation =
|
||||
| { readonly operation: "record_unknown"; readonly input: RecordUnknownInput }
|
||||
| {
|
||||
readonly operation: "update_unknown";
|
||||
readonly input: UpdateUnknownInput;
|
||||
};
|
||||
|
||||
/** Create immutable evidence for one approved residual-unknown mutation. */
|
||||
export const unknownMutationEvidence = (
|
||||
target: BinaryTarget | undefined,
|
||||
mutation: UnknownMutation,
|
||||
input: RecordUnknownInput,
|
||||
): Evidence =>
|
||||
createEvidence(target, UNKNOWN_REGISTRY_PROVIDER, {
|
||||
predicateType: "rea.residual-unknown-mutation",
|
||||
operation: mutation.operation,
|
||||
parameters: { ...mutation.input },
|
||||
result:
|
||||
mutation.operation === "record_unknown"
|
||||
? {
|
||||
action: "record",
|
||||
question: mutation.input.question,
|
||||
required_authority: mutation.input.required_authority,
|
||||
required_confidence: mutation.input.required_confidence,
|
||||
}
|
||||
: { action: "update", status: mutation.input.status },
|
||||
operation: "record_unknown",
|
||||
parameters: { domain: input.domain, severity: input.severity },
|
||||
result: {
|
||||
action: "record",
|
||||
question: input.question,
|
||||
required_authority: input.required_authority,
|
||||
required_confidence: input.required_confidence,
|
||||
},
|
||||
confidence: "derived",
|
||||
authority: "analyst-inference",
|
||||
evidenceLinks: unknownEvidenceLinks(mutation.input),
|
||||
evidenceLinks: unknownEvidenceLinks(input),
|
||||
limitations: [
|
||||
"Registry mutation evidence records analyst intent, not proof of the answer.",
|
||||
],
|
||||
});
|
||||
|
||||
/** Deduplicate evidence links carried by an unknown mutation. */
|
||||
const unknownEvidenceLinks = (
|
||||
export const unknownEvidenceLinks = (
|
||||
input: RecordUnknownInput | UpdateUnknownInput,
|
||||
): string[] => [
|
||||
...new Set([
|
||||
): string[] =>
|
||||
[
|
||||
...input.supporting_evidence_ids,
|
||||
...input.contradicting_evidence_ids,
|
||||
...("resolution" in input && input.resolution !== null
|
||||
? input.resolution.evidence_ids
|
||||
: []),
|
||||
]),
|
||||
];
|
||||
].filter((id, index, values) => values.indexOf(id) === index);
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
AnalysisCapabilityUnavailableError,
|
||||
AnalysisInputError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import type { ElectronActiveObservationInput } from "../../domain/javascript/electronActiveObservation.js";
|
||||
import { err, type Result } from "../../domain/result.js";
|
||||
|
||||
@@ -6,7 +6,7 @@ import type {
|
||||
ListElectronTargetsInput,
|
||||
} from "../../domain/javascript/electronObservation.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { createElectronEvidence } from "./ElectronEvidence.js";
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
|
||||
import { compareJavaScriptApplicationVersions } from "../../domain/javascript/javascriptApplicationVersionComparison.js";
|
||||
|
||||
@@ -26,7 +26,6 @@ import {
|
||||
staticInferenceEvidence,
|
||||
} from "./JavaScriptArtifactGraphEvidence.js";
|
||||
import { resolveArtifactPathByContext } from "./JavaScriptArtifactPathResolution.js";
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
|
||||
interface PackageRoleInput {
|
||||
readonly packageNode: ApplicationNode;
|
||||
@@ -282,7 +281,6 @@ export const addJavaScriptPackageNodes = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
): ApplicationNode[] => {
|
||||
const roots: ApplicationNode[] = [];
|
||||
const rootPath = applicationManifestPath(context);
|
||||
for (const packageValue of context.analysis.packages) {
|
||||
const file = context.filesByPath.get(packageValue.path);
|
||||
if (file === undefined) continue;
|
||||
@@ -311,7 +309,7 @@ export const addJavaScriptPackageNodes = (
|
||||
},
|
||||
],
|
||||
});
|
||||
if (file.path === rootPath) {
|
||||
if (roots.length === 0) {
|
||||
roots.push(node);
|
||||
context.accumulator.addEdge({
|
||||
source_node_id: node.node_id,
|
||||
@@ -332,7 +330,6 @@ export const addJavaScriptPackageNodes = (
|
||||
file,
|
||||
operation: "inventory-package",
|
||||
});
|
||||
if (dependencyManifest(file.path)) continue;
|
||||
addPackageRole(context, {
|
||||
packageNode: node,
|
||||
packageFile: file,
|
||||
@@ -349,24 +346,6 @@ export const addJavaScriptPackageNodes = (
|
||||
return roots;
|
||||
};
|
||||
|
||||
/** Dependency-only artifacts retain their inventory root. */
|
||||
const applicationManifestPath = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
): string | undefined =>
|
||||
context.analysis.packages
|
||||
.map(({ path }) => path)
|
||||
.filter(
|
||||
(path) => context.filesByPath.has(path) && !dependencyManifest(path),
|
||||
)
|
||||
.sort(
|
||||
(left, right) =>
|
||||
left.split("/").length - right.split("/").length ||
|
||||
compareUnicodeCodePoints(left, right),
|
||||
)[0];
|
||||
|
||||
const dependencyManifest = (path: string): boolean =>
|
||||
path.split("/").slice(0, -1).includes("node_modules");
|
||||
|
||||
const createFileTarget = (
|
||||
context: JavaScriptArtifactGraphContext,
|
||||
file: JavaScriptArtifactFile,
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
AnalysisCancelledError,
|
||||
AnalysisCapabilityUnavailableError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type {
|
||||
ListJavaScriptRuntimeTargetsInput,
|
||||
ObserveJavaScriptRuntimeInput,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { resolve } from "node:path";
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
|
||||
import {
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
AnalysisInputError,
|
||||
AnalysisProtocolError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
applicationGraphEvidenceInputError,
|
||||
parseApplicationGraphEvidence,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
createEvidence,
|
||||
type Evidence,
|
||||
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
import { parseManagedMemberEvidence } from "../../domain/managed/managedMemberComparisonMatch.js";
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import { parseBinaryTarget } from "../BinaryTargetResolver.js";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import {
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
AnalysisUnsupportedTargetError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { ArtifactOperationError } from "../domain/artifactOperationError.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { interfaceBuilderLimitsSchema } from "../domain/apple/interfaceBuilderGraph.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -8,151 +6,9 @@ import { describe, expect, it, onTestFinished } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { CPU, fatImage, machoImage } from "./apple/MachoImage.fixture.js";
|
||||
import { MachOSliceArtifactReader } from "./MachOSliceArtifactReader.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import { ok } from "../domain/result.js";
|
||||
import { openRegularFile } from "../filesystem/RegularFile.js";
|
||||
import { ProviderProcessSupervisor } from "../process/ProviderProcess.js";
|
||||
import type { NativeCommandRunner } from "../native/CommandRunner.js";
|
||||
import { NativeCommandFailure } from "../native/CommandRunner.js";
|
||||
|
||||
const cleanupOwner = (runId: string): ProviderProcessSupervisor => {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
stdout: new PassThrough(),
|
||||
stderr: new PassThrough(),
|
||||
exitCode: null as number | null,
|
||||
signalCode: null as NodeJS.Signals | null,
|
||||
kill: () => true,
|
||||
});
|
||||
return new ProviderProcessSupervisor({
|
||||
process: child,
|
||||
ownsProcessLifetime: true,
|
||||
ownership: {
|
||||
runId,
|
||||
leaderPid: 45_000,
|
||||
processGroupId: 45_000,
|
||||
expectedParentPid: process.pid,
|
||||
},
|
||||
cleanup: async () => ({ cleaned: false, reason: "still running" }),
|
||||
});
|
||||
};
|
||||
|
||||
describe("Mach-O cleanup exceptions", () => {
|
||||
it("retains an undefined cleanup rejection as an explicit failure", async () => {
|
||||
const reader = new MachOSliceArtifactReader(
|
||||
"/unused/macho",
|
||||
{},
|
||||
{
|
||||
run: async () => {
|
||||
throw new Error("Unexpected command");
|
||||
},
|
||||
retryCleanup: async () => {
|
||||
throw undefined;
|
||||
},
|
||||
},
|
||||
);
|
||||
await expect(reader.close()).rejects.toMatchObject({
|
||||
cleanup: {
|
||||
reason: "undefined",
|
||||
resources: ["native-command-process-cleanup"],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Mach-O command cleanup lifecycle", () => {
|
||||
it("waits for active lipo capture before cleanup and preserves its failure", async () => {
|
||||
const root = await createTestTempDirectory("rea-slices-close-race-");
|
||||
const binary = join(root, "input");
|
||||
await writeFile(binary, Buffer.from("fixture"));
|
||||
let finishCommand!: (
|
||||
result: Awaited<ReturnType<NativeCommandRunner["run"]>>,
|
||||
) => void;
|
||||
let commandStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
commandStarted = resolve;
|
||||
});
|
||||
const command = new Promise<
|
||||
Awaited<ReturnType<NativeCommandRunner["run"]>>
|
||||
>((resolve) => {
|
||||
finishCommand = resolve;
|
||||
});
|
||||
let cleanupRetries = 0;
|
||||
const runner: NativeCommandRunner = {
|
||||
run: () => {
|
||||
commandStarted();
|
||||
return command;
|
||||
},
|
||||
retryCleanup: async () => {
|
||||
cleanupRetries += 1;
|
||||
if (cleanupRetries > 1) return [];
|
||||
return [
|
||||
{
|
||||
owner: cleanupOwner("run:first"),
|
||||
reason: "first process group remains live",
|
||||
resources: ["run:first", "process-group:1234"],
|
||||
},
|
||||
{
|
||||
owner: cleanupOwner("run:second"),
|
||||
reason: "second process group remains live",
|
||||
resources: ["run:second", "process-group:5678"],
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
const reader = new MachOSliceArtifactReader(binary, {}, runner);
|
||||
const iteration = reader.entries();
|
||||
const firstEntry = iteration[Symbol.asyncIterator]().next();
|
||||
await started;
|
||||
const closing = reader.close();
|
||||
await Promise.resolve();
|
||||
expect(cleanupRetries).toBe(0);
|
||||
finishCommand(
|
||||
err(
|
||||
new NativeCommandFailure(
|
||||
"lipo",
|
||||
"timeout",
|
||||
null,
|
||||
undefined,
|
||||
{
|
||||
stdout: "partial lipo output",
|
||||
stderr: "partial lipo diagnostic",
|
||||
stdoutBytes: 19,
|
||||
stderrBytes: 23,
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
truncated: false,
|
||||
},
|
||||
"capture process group remains unverified",
|
||||
["capture-group:9999"],
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
await expect(firstEntry).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
cause: expect.objectContaining({
|
||||
cleanupFailure: "capture process group remains unverified",
|
||||
}),
|
||||
cleanup: {
|
||||
reason: "capture process group remains unverified",
|
||||
resources: ["capture-group:9999"],
|
||||
},
|
||||
});
|
||||
await expect(closing).rejects.toMatchObject({
|
||||
cleanup: {
|
||||
reason:
|
||||
"run:first, process-group:1234: first process group remains live; run:second, process-group:5678: second process group remains live",
|
||||
resources: [
|
||||
"run:first",
|
||||
"process-group:1234",
|
||||
"run:second",
|
||||
"process-group:5678",
|
||||
],
|
||||
},
|
||||
});
|
||||
await reader.close();
|
||||
expect(cleanupRetries).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MachOSliceArtifactReader", () => {
|
||||
it("keeps a borrowed source usable after a cancelled slice read", async () => {
|
||||
|
||||
@@ -39,7 +39,6 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
#ownedSource: OwnedFileHandle | undefined;
|
||||
#pendingSource: Promise<StableRegularFileDescriptor> | undefined;
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
readonly #pendingCommands = new Set<Promise<unknown>>();
|
||||
readonly #ownsSource: boolean;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
@@ -60,46 +59,16 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
this.#entries.clear();
|
||||
const source = await this.#ensureSource();
|
||||
await this.#verifySource(source);
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Universal Mach-O reader is closed",
|
||||
);
|
||||
const command = this.runner.run(
|
||||
const captured = await this.runner.run(
|
||||
"lipo",
|
||||
["-detailed_info", this.path],
|
||||
signal === undefined ? {} : { signal },
|
||||
);
|
||||
this.#pendingCommands.add(command);
|
||||
let captured: Awaited<typeof command>;
|
||||
try {
|
||||
captured = await command;
|
||||
} finally {
|
||||
this.#pendingCommands.delete(command);
|
||||
}
|
||||
if (!captured.ok)
|
||||
throw new ArtifactReaderFailure(
|
||||
captured.error.reason === "cancelled" ? "cancelled" : "unavailable",
|
||||
"lipo could not enumerate universal Mach-O slices",
|
||||
{
|
||||
cause: captured.error,
|
||||
...(captured.error.cleanupFailure === null &&
|
||||
captured.error.cleanupResources.length === 0
|
||||
? {}
|
||||
: {
|
||||
cleanup: {
|
||||
reason:
|
||||
captured.error.cleanupFailure ??
|
||||
"Native command cleanup remains unverified",
|
||||
resources: [...captured.error.cleanupResources],
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Universal Mach-O reader closed during slice discovery",
|
||||
{ cause: captured.error },
|
||||
);
|
||||
this.#command = {
|
||||
tool: captured.value.tool,
|
||||
@@ -293,49 +262,13 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
this.#closePromise ??= this.#releaseResources().catch((cause: unknown) => {
|
||||
this.#closePromise ??= this.#closeSource().catch((cause: unknown) => {
|
||||
this.#closePromise = undefined;
|
||||
throw cause;
|
||||
});
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #releaseResources(): Promise<void> {
|
||||
await Promise.allSettled([...this.#pendingCommands]);
|
||||
let cleanup: ArtifactReaderFailure["cleanup"];
|
||||
try {
|
||||
const failures = (await this.runner.retryCleanup?.()) ?? [];
|
||||
if (failures.length > 0)
|
||||
cleanup = {
|
||||
reason: failures
|
||||
.map(
|
||||
({ reason, resources }) => `${resources.join(", ")}: ${reason}`,
|
||||
)
|
||||
.join("; "),
|
||||
resources: [
|
||||
...new Set(failures.flatMap(({ resources }) => resources)),
|
||||
],
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
cleanup = ArtifactReaderFailure.cleanupObservation(
|
||||
cause,
|
||||
"native-command-process-cleanup",
|
||||
);
|
||||
}
|
||||
try {
|
||||
await this.#closeSource();
|
||||
} catch (cause: unknown) {
|
||||
if (cleanup === undefined) throw cause;
|
||||
throw ArtifactReaderFailure.withCleanup(cause, cleanup);
|
||||
}
|
||||
if (cleanup !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Native command process cleanup remains incomplete",
|
||||
{ cleanup },
|
||||
);
|
||||
}
|
||||
|
||||
async #ensureSource(): Promise<StableRegularFileDescriptor> {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
|
||||
@@ -147,17 +147,8 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
const filenames = new Set<string>();
|
||||
for await (const entry of this.#reader.getEntriesGenerator()) {
|
||||
abortIfNeeded(signal);
|
||||
// Each enumeration can replay valid entries, but a second record with
|
||||
// the same name must not replace an already emitted source identity.
|
||||
if (filenames.has(entry.filename))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact path collision: ${entry.filename}`,
|
||||
);
|
||||
filenames.add(entry.filename);
|
||||
const metadata = projectZipEntry(entry);
|
||||
this.#entries.set(entry.filename, {
|
||||
source: entry,
|
||||
|
||||
@@ -27,9 +27,9 @@ import {
|
||||
type IntegrityContradiction,
|
||||
} from "../../domain/artifactGraph.js";
|
||||
import { AnalysisUnsupportedTargetError } from "../../domain/analysisErrorCore.js";
|
||||
import type {
|
||||
ArtifactResourceOwner,
|
||||
ArtifactResourceScope,
|
||||
import {
|
||||
type ArtifactResourceOwner,
|
||||
type ArtifactResourceScope,
|
||||
} from "../ArtifactResourceScope.js";
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
zipPackageFormatForPath,
|
||||
} from "../../domain/zipPackageFormat.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js";
|
||||
import { type ArtifactResourceOwner } from "../ArtifactResourceScope.js";
|
||||
import type { HashResult } from "../ArtifactHash.js";
|
||||
import {
|
||||
hashStableRootArtifactHandle,
|
||||
|
||||
@@ -34,7 +34,7 @@ import {
|
||||
artifactManifestId,
|
||||
} from "../../domain/artifactIdentity.js";
|
||||
import { classifyAndHashRootForInventory } from "./classify.js";
|
||||
import type { HashResult } from "../ArtifactHash.js";
|
||||
import { type HashResult } from "../ArtifactHash.js";
|
||||
import {
|
||||
hashStableRootArtifact,
|
||||
hashStableRootArtifactHandle,
|
||||
|
||||
@@ -43,10 +43,9 @@ import {
|
||||
type WebScreenshotDiff,
|
||||
} from "../domain/webScreenshot.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import {
|
||||
discoverCdpEndpoint,
|
||||
@@ -315,12 +314,6 @@ export class CdpBrowserProvider implements BrowserObservationPort {
|
||||
try {
|
||||
return ok(webScreenshotDiffSchema.parse(comparePngScreenshots(input)));
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof TypeError)
|
||||
return err(
|
||||
new AnalysisInputError("compare_web_screenshots", { cause }, [
|
||||
{ path: [], reason: "invalid_format", message: cause.message },
|
||||
]),
|
||||
);
|
||||
return err(providerError(cause, "compare_web_screenshots"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -606,402 +606,3 @@ describe("SVG xlink:href context", () => {
|
||||
expect(dangling.urls).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
const SVG_NAMESPACE = "http://www.w3.org/2000/svg";
|
||||
const XLINK_NAMESPACE = "http://www.w3.org/1999/xlink";
|
||||
const XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml";
|
||||
|
||||
const xhtmlRoot = (
|
||||
attributes: readonly (readonly [string, string])[] = [
|
||||
["xmlns", XHTML_NAMESPACE],
|
||||
["xmlns:svg", SVG_NAMESPACE],
|
||||
["xmlns:xlink", XLINK_NAMESPACE],
|
||||
],
|
||||
): SvgSnapshotNode => ({
|
||||
name: "html",
|
||||
parent: 0,
|
||||
attributes,
|
||||
});
|
||||
|
||||
const approvedHref = (
|
||||
nodeIndex: number,
|
||||
url: string,
|
||||
): {
|
||||
readonly node_index: number;
|
||||
readonly attribute: "href";
|
||||
readonly url: string;
|
||||
readonly destination_scope: "approved";
|
||||
} => ({
|
||||
node_index: nodeIndex,
|
||||
attribute: "href",
|
||||
url,
|
||||
destination_scope: "approved",
|
||||
});
|
||||
|
||||
describe("XML SVG roots and namespace scope restoration", () => {
|
||||
it.each(["svg", "document"])(
|
||||
"resolves namespaces under an unprefixed XML %s root",
|
||||
(name) => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{
|
||||
name,
|
||||
parent: 0,
|
||||
attributes: [
|
||||
["xmlns", SVG_NAMESPACE],
|
||||
["xmlns:p", XLINK_NAMESPACE],
|
||||
],
|
||||
},
|
||||
{ name: "a", parent: 1, attributes: [["p:href", "/linked"]] },
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([approvedHref(2, `${origin}/linked`)]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([2, 3, 99])(
|
||||
"rejects self-parent, cyclic and dangling XML ancestry (%s)",
|
||||
(parent) => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot(),
|
||||
{
|
||||
name: "svg:a",
|
||||
parent,
|
||||
attributes: [
|
||||
["xmlns:svg", SVG_NAMESPACE],
|
||||
["xmlns:p", XLINK_NAMESPACE],
|
||||
["p:href", "/invalid"],
|
||||
],
|
||||
},
|
||||
{ name: "svg:g", parent: 2 },
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it("restores namespace declarations and undeclarations before visiting siblings", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot(),
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["xmlns:xlink", ""],
|
||||
["xlink:href", "/undeclared"],
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "/inherited"]],
|
||||
},
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["xmlns:svg", "other"],
|
||||
["xlink:href", "/rebound"],
|
||||
],
|
||||
},
|
||||
{ name: "svg:a", parent: 1, attributes: [["xlink:href", "/restored"]] },
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
approvedHref(3, `${origin}/inherited`),
|
||||
approvedHref(5, `${origin}/restored`),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("XHTML and XML SVG namespace prefixes", () => {
|
||||
it("reports prefixed svg:a, svg:image, and an svg:a with no svg parent", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot(),
|
||||
{ name: "svg:svg", parent: 1 },
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 2,
|
||||
attributes: [
|
||||
["xlink:href", "/xl-pa1"],
|
||||
["xlink:href", "/later"],
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "svg:image",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "guide"]],
|
||||
},
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "/xl-orphan"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.nodes[3]?.node_name).toBe("svg:a");
|
||||
expect(result.nodes[3]?.attribute_names).toEqual(["xlink:href"]);
|
||||
expect(result.urls).toEqual([
|
||||
approvedHref(3, `${origin}/xl-pa1`),
|
||||
approvedHref(4, `${origin}/screens/assets/guide`),
|
||||
approvedHref(5, `${origin}/xl-orphan`),
|
||||
]);
|
||||
});
|
||||
|
||||
it("reports svg:a under svg:foreignObject and skips XHTML anchors there", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot([
|
||||
["xmlns", XHTML_NAMESPACE],
|
||||
["xmlns:svg", SVG_NAMESPACE],
|
||||
["xmlns:xlink", XLINK_NAMESPACE],
|
||||
["xmlns:h", XHTML_NAMESPACE],
|
||||
]),
|
||||
{ name: "svg:svg", parent: 1 },
|
||||
{ name: "svg:foreignObject", parent: 2 },
|
||||
{
|
||||
name: "a",
|
||||
parent: 3,
|
||||
attributes: [["xlink:href", "/xl-html-fo"]],
|
||||
},
|
||||
{
|
||||
name: "h:a",
|
||||
parent: 3,
|
||||
attributes: [["xlink:href", "/xl-h"]],
|
||||
},
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 3,
|
||||
attributes: [["xlink:href", "/xl-fo-svg"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(urlFor(result, 4)).toBeUndefined();
|
||||
expect(urlFor(result, 5)).toBeUndefined();
|
||||
expect(result.urls).toEqual([approvedHref(6, `${origin}/xl-fo-svg`)]);
|
||||
});
|
||||
|
||||
it("reads XLink by namespace, including a prefix other than xlink", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot([["xmlns", XHTML_NAMESPACE]]),
|
||||
{
|
||||
name: "svg",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["xmlns", SVG_NAMESPACE],
|
||||
["xmlns:xlink", "http://example.com/not-xlink"],
|
||||
["xmlns:x", XLINK_NAMESPACE],
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "a",
|
||||
parent: 2,
|
||||
attributes: [
|
||||
["xlink:href", "/not-xlink"],
|
||||
["x:href", "/xl-custom"],
|
||||
["x:href", "/later"],
|
||||
],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.nodes[3]?.attribute_names).toEqual(["xlink:href", "x:href"]);
|
||||
expect(result.urls).toEqual([approvedHref(3, `${origin}/xl-custom`)]);
|
||||
});
|
||||
|
||||
it("keeps plain href ahead of a namespaced XLink href, including an empty href", () => {
|
||||
const baseUrl = `${origin}/screens/assets/`;
|
||||
const anchor = (
|
||||
parent: number,
|
||||
attributes: readonly (readonly [string, string])[],
|
||||
): SvgSnapshotNode => ({
|
||||
name: "svg:a",
|
||||
parent,
|
||||
attributes,
|
||||
});
|
||||
const result = captureSvg(
|
||||
svgDocument(
|
||||
[
|
||||
xhtmlRoot(),
|
||||
{ name: "svg:svg", parent: 1 },
|
||||
anchor(2, [
|
||||
["xlink:href", "/xlink-loses"],
|
||||
["href", "/href-wins"],
|
||||
]),
|
||||
anchor(2, [
|
||||
["href", "/href-wins"],
|
||||
["xlink:href", "/xlink-loses"],
|
||||
]),
|
||||
anchor(2, [
|
||||
["href", ""],
|
||||
["xlink:href", "/xlink-should-lose"],
|
||||
]),
|
||||
],
|
||||
baseUrl,
|
||||
),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
approvedHref(3, `${origin}/href-wins`),
|
||||
approvedHref(4, `${origin}/href-wins`),
|
||||
approvedHref(5, baseUrl),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("XHTML SVG exclusions, HTML documents, and prefix scope", () => {
|
||||
it("classifies a prefixed SVG xlink:href outside the allowed origins as outside_policy", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot(),
|
||||
{ name: "svg:svg", parent: 1 },
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "https://cdn.example.test/out"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([
|
||||
{
|
||||
node_index: 3,
|
||||
attribute: "href",
|
||||
url: null,
|
||||
destination_scope: "outside_policy",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not treat bogus:svg in another namespace as SVG", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
xhtmlRoot([
|
||||
["xmlns", XHTML_NAMESPACE],
|
||||
["xmlns:svg", SVG_NAMESPACE],
|
||||
["xmlns:xlink", XLINK_NAMESPACE],
|
||||
["xmlns:bogus", "http://example.com/not-svg"],
|
||||
]),
|
||||
{
|
||||
name: "bogus:svg",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "/xl-bogus"]],
|
||||
},
|
||||
{
|
||||
name: "bogus:a",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "/xl-bogus-a"]],
|
||||
},
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "/xl-kept"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.nodes[2]?.node_name).toBe("bogus:svg");
|
||||
expect(result.urls).toEqual([approvedHref(4, `${origin}/xl-kept`)]);
|
||||
});
|
||||
|
||||
it("leaves HTML documents on literal svg and xlink:href names", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "HTML", parent: 0 },
|
||||
{
|
||||
name: "svg",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["xmlns", SVG_NAMESPACE],
|
||||
["xmlns:x", XLINK_NAMESPACE],
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "a",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "/xl-literal"]],
|
||||
},
|
||||
{
|
||||
name: "a",
|
||||
parent: 2,
|
||||
attributes: [["x:href", "/xl-custom-html"]],
|
||||
},
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["xmlns:svg", SVG_NAMESPACE],
|
||||
["xmlns:xlink", XLINK_NAMESPACE],
|
||||
["xlink:href", "/xl-prefixed-html"],
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "A",
|
||||
parent: 1,
|
||||
attributes: [["xlink:href", "/xl-html-anchor"]],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([approvedHref(3, `${origin}/xl-literal`)]);
|
||||
});
|
||||
|
||||
it("applies an element's own xmlns declarations to its name and href", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{ name: "html", parent: 0, attributes: [["xmlns", XHTML_NAMESPACE]] },
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 1,
|
||||
attributes: [
|
||||
["xmlns:svg", SVG_NAMESPACE],
|
||||
["xmlns:xlink", XLINK_NAMESPACE],
|
||||
["xlink:href", "/xl-self"],
|
||||
],
|
||||
},
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([approvedHref(2, `${origin}/xl-self`)]);
|
||||
});
|
||||
|
||||
it("resolves a prefix declared on a later ancestor", () => {
|
||||
const result = captureSvg(
|
||||
svgDocument([
|
||||
{
|
||||
name: "svg:a",
|
||||
parent: 2,
|
||||
attributes: [["xlink:href", "/forward"]],
|
||||
},
|
||||
{ name: "svg:svg", parent: 3 },
|
||||
xhtmlRoot(),
|
||||
]),
|
||||
);
|
||||
expect(result.urls).toEqual([approvedHref(1, `${origin}/forward`)]);
|
||||
});
|
||||
|
||||
it("resolves a deep chain with distinct namespace declarations without quadratic expansion", () => {
|
||||
const depth = 12_000;
|
||||
const elements: SvgSnapshotNode[] = [
|
||||
xhtmlRoot(),
|
||||
{ name: "svg:svg", parent: 1 },
|
||||
];
|
||||
let parent = 2;
|
||||
for (let index = 0; index < depth; index += 1) {
|
||||
elements.push({
|
||||
name: "svg:g",
|
||||
parent,
|
||||
attributes: [[`xmlns:p${index}`, `urn:prefix:${index}`]],
|
||||
});
|
||||
parent = elements.length;
|
||||
}
|
||||
elements.push({
|
||||
name: "svg:a",
|
||||
parent,
|
||||
attributes: [["xlink:href", "/deep"]],
|
||||
});
|
||||
const snapshot = svgDocument(elements);
|
||||
const result = captureSvg(snapshot);
|
||||
expect(result.urls).toEqual([
|
||||
approvedHref(elements.length, `${origin}/deep`),
|
||||
]);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -11,7 +11,6 @@ import {
|
||||
} from "./CdpCaptureValues.js";
|
||||
import { exclusionReasonForUrl } from "./CdpCaptureEventHelpers.js";
|
||||
import type { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
|
||||
import { createDocumentSvgAncestry, domSvgLink } from "./CdpCaptureDomSvg.js";
|
||||
|
||||
export type CapturedResource = Omit<
|
||||
WebPageInspection["resources"][number],
|
||||
@@ -172,20 +171,24 @@ export const captureDom = (
|
||||
? documentNodes.attributes
|
||||
: [];
|
||||
const baseIndex = nodes.length;
|
||||
const ancestry = createDocumentSvgAncestry({
|
||||
nodeTypes,
|
||||
nodeNames,
|
||||
parents,
|
||||
strings,
|
||||
attributes,
|
||||
});
|
||||
const svgContexts = new Map<number, boolean>();
|
||||
total += nodeTypes.length;
|
||||
for (let index = 0; index < nodeTypes.length; index += 1) {
|
||||
const attributeIndexes = numberArray(attributes[index]);
|
||||
const parent = Math.trunc(parents[index] ?? -1);
|
||||
const nodeIndex = nodes.length;
|
||||
const nodeName = indexedString(strings, nodeNames[index]);
|
||||
const attributeNames = uniqueAttributeNames(attributeIndexes, strings);
|
||||
const rawAttributeNames = attributeIndexes
|
||||
.filter((_value, attributeIndex) => attributeIndex % 2 === 0)
|
||||
.map((value) => indexedString(strings, value));
|
||||
const seenAttributeNames = new Set<string>();
|
||||
const attributeNames: string[] = [];
|
||||
for (const name of rawAttributeNames) {
|
||||
if (!seenAttributeNames.has(name)) {
|
||||
seenAttributeNames.add(name);
|
||||
attributeNames.push(name);
|
||||
}
|
||||
}
|
||||
nodes.push({
|
||||
index: nodeIndex,
|
||||
parent_index: parent < 0 ? -1 : baseIndex + parent,
|
||||
@@ -203,7 +206,13 @@ export const captureDom = (
|
||||
nodeIndex,
|
||||
nodeName,
|
||||
allowedOrigins,
|
||||
...domSvgLink(ancestry, index),
|
||||
svgElement: elementInSvgContext(
|
||||
index,
|
||||
nodeNames,
|
||||
parents,
|
||||
strings,
|
||||
svgContexts,
|
||||
),
|
||||
});
|
||||
for (const url of metadata.urls) {
|
||||
urls.push(url);
|
||||
@@ -358,21 +367,6 @@ const indexedString = (strings: readonly string[], index: unknown): string => {
|
||||
return integer === undefined ? "" : (strings[Math.trunc(integer)] ?? "");
|
||||
};
|
||||
|
||||
const uniqueAttributeNames = (
|
||||
attributeIndexes: readonly number[],
|
||||
strings: readonly string[],
|
||||
): string[] => {
|
||||
const names: string[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (let index = 0; index < attributeIndexes.length; index += 2) {
|
||||
const name = indexedString(strings, attributeIndexes[index]);
|
||||
if (seen.has(name)) continue;
|
||||
seen.add(name);
|
||||
names.push(name);
|
||||
}
|
||||
return names;
|
||||
};
|
||||
|
||||
const numberArray = (value: unknown): readonly number[] =>
|
||||
Array.isArray(value)
|
||||
? value.flatMap((item) => {
|
||||
@@ -390,22 +384,41 @@ interface DomMetadataOptions {
|
||||
readonly nodeName: string;
|
||||
readonly allowedOrigins: ReadonlySet<string>;
|
||||
readonly svgElement: boolean;
|
||||
readonly xmlDocument: boolean;
|
||||
readonly xlinkHref: string | undefined;
|
||||
}
|
||||
|
||||
// Chrome reads an SVG XLink href only when href itself is absent, and an
|
||||
// empty href still wins. HTML keeps the literal attribute name `xlink:href`.
|
||||
// XML uses any prefix bound to the XLink namespace. Both are stored as href.
|
||||
const hrefAttributeValue = (
|
||||
pairs: ReadonlyMap<string, string>,
|
||||
attribute: string,
|
||||
options: DomMetadataOptions,
|
||||
): string | undefined => {
|
||||
const declared = pairs.get(attribute);
|
||||
if (declared !== undefined) return declared;
|
||||
if (attribute !== "href" || !options.svgElement) return undefined;
|
||||
return options.xmlDocument ? options.xlinkHref : pairs.get("xlink:href");
|
||||
// DOMSnapshot carries no namespace URI. Chrome's HTML parser records SVG
|
||||
// local names (`svg`, `foreignObject`). Descendants of foreignObject are HTML
|
||||
// until a nested `svg`.
|
||||
const elementInSvgContext = (
|
||||
index: number,
|
||||
nodeNames: readonly number[],
|
||||
parents: readonly number[],
|
||||
strings: readonly string[],
|
||||
contexts: Map<number, boolean>,
|
||||
): boolean => {
|
||||
let current = index;
|
||||
const seen = new Set<number>();
|
||||
let inSvg = false;
|
||||
while (current >= 0 && current < nodeNames.length && !seen.has(current)) {
|
||||
const cached = contexts.get(current);
|
||||
if (cached !== undefined) {
|
||||
inSvg = cached;
|
||||
break;
|
||||
}
|
||||
seen.add(current);
|
||||
if (indexedString(strings, nodeNames[current]) === "svg") {
|
||||
inSvg = true;
|
||||
break;
|
||||
}
|
||||
const parent = Math.trunc(parents[current] ?? -1);
|
||||
if (parent < 0 || parent === current) break;
|
||||
if (indexedString(strings, nodeNames[parent]) === "foreignObject") break;
|
||||
current = parent;
|
||||
}
|
||||
// Cache the entire examined chain, including forward references and cycles,
|
||||
// so shared ancestors are examined once per document.
|
||||
for (const node of seen) contexts.set(node, inSvg);
|
||||
return inSvg;
|
||||
};
|
||||
|
||||
const domMetadata = (
|
||||
@@ -422,6 +435,7 @@ const domMetadata = (
|
||||
nodeIndex,
|
||||
nodeName,
|
||||
allowedOrigins,
|
||||
svgElement,
|
||||
} = options;
|
||||
const pairs = new Map<string, string>();
|
||||
for (let index = 0; index + 1 < attributes.length; index += 2) {
|
||||
@@ -433,7 +447,13 @@ const domMetadata = (
|
||||
}
|
||||
const urls: WebPageInspection["metadata"]["dom_urls"] = [];
|
||||
for (const attribute of domUrlAttributes) {
|
||||
const value = hrefAttributeValue(pairs, attribute, options);
|
||||
// Chrome reads SVG xlink:href only when href itself is absent, and an
|
||||
// empty href still wins. Report the fallback as href.
|
||||
const value =
|
||||
pairs.get(attribute) ??
|
||||
(attribute === "href" && svgElement
|
||||
? pairs.get("xlink:href")
|
||||
: undefined);
|
||||
if (value === undefined) continue;
|
||||
const tagName = nodeName.toLowerCase();
|
||||
// Chrome strips HTML whitespace for FORM.action, while submit controls
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
import { numberValue } from "./CdpCaptureValues.js";
|
||||
|
||||
const SVG_NAMESPACE = "http://www.w3.org/2000/svg";
|
||||
const XLINK_NAMESPACE = "http://www.w3.org/1999/xlink";
|
||||
|
||||
interface DomSnapshotNodes {
|
||||
readonly nodeTypes: readonly number[];
|
||||
readonly nodeNames: readonly number[];
|
||||
readonly parents: readonly number[];
|
||||
readonly strings: readonly string[];
|
||||
readonly attributes: readonly unknown[];
|
||||
}
|
||||
|
||||
interface XmlSvgLink {
|
||||
readonly svgElement: boolean;
|
||||
readonly xlinkHref: string | undefined;
|
||||
}
|
||||
|
||||
interface DocumentSvgAncestry {
|
||||
readonly xml: boolean;
|
||||
readonly nodes: DomSnapshotNodes;
|
||||
readonly svgContexts: Map<number, boolean>;
|
||||
readonly xmlLinks: Map<number, XmlSvgLink>;
|
||||
}
|
||||
|
||||
interface DomSvgLink {
|
||||
readonly svgElement: boolean;
|
||||
readonly xmlDocument: boolean;
|
||||
readonly xlinkHref: string | undefined;
|
||||
}
|
||||
|
||||
const UNRESOLVED: XmlSvgLink = {
|
||||
svgElement: false,
|
||||
xlinkHref: undefined,
|
||||
};
|
||||
|
||||
const stringAt = (strings: readonly string[], index: unknown): string => {
|
||||
const integer = numberValue(index);
|
||||
return integer === undefined ? "" : (strings[Math.trunc(integer)] ?? "");
|
||||
};
|
||||
|
||||
const numberArray = (value: unknown): readonly number[] =>
|
||||
Array.isArray(value)
|
||||
? value.flatMap((item) => {
|
||||
const number = numberValue(item);
|
||||
return number === undefined ? [] : [number];
|
||||
})
|
||||
: [];
|
||||
|
||||
// DOMSnapshot has no namespace URI. HTML documents use uppercase `HTML` and
|
||||
// the parser's local names. XML documents preserve prefixed names and xmlns
|
||||
// declarations, including those on standalone SVG roots.
|
||||
const xmlDocument = (nodes: DomSnapshotNodes): boolean => {
|
||||
const documentIndex = nodes.nodeTypes.findIndex(
|
||||
(nodeType) => Math.trunc(nodeType) === 9,
|
||||
);
|
||||
if (documentIndex < 0) return false;
|
||||
for (let index = 0; index < nodes.nodeTypes.length; index += 1) {
|
||||
if (Math.trunc(nodes.nodeTypes[index] ?? 0) !== 1) continue;
|
||||
if (Math.trunc(nodes.parents[index] ?? -1) !== documentIndex) continue;
|
||||
const name = stringAt(nodes.strings, nodes.nodeNames[index]);
|
||||
if (name === "HTML") return false;
|
||||
if (name === "html" || name.includes(":")) return true;
|
||||
// Explicit namespaces also identify standalone SVG and other XML roots.
|
||||
// Namespace-free synthetic snapshots retain the legacy HTML behavior.
|
||||
const attributes = numberArray(nodes.attributes[index]);
|
||||
return attributes.some(
|
||||
(value, offset) =>
|
||||
offset % 2 === 0 &&
|
||||
xmlnsPrefix(stringAt(nodes.strings, value)) !== undefined,
|
||||
);
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
export const createDocumentSvgAncestry = (
|
||||
nodes: DomSnapshotNodes,
|
||||
): DocumentSvgAncestry => {
|
||||
const xml = xmlDocument(nodes);
|
||||
return {
|
||||
xml,
|
||||
nodes,
|
||||
svgContexts: new Map(),
|
||||
xmlLinks: xml ? resolveXmlElements(nodes) : new Map(),
|
||||
};
|
||||
};
|
||||
|
||||
export const domSvgLink = (
|
||||
ancestry: DocumentSvgAncestry,
|
||||
index: number,
|
||||
): DomSvgLink => {
|
||||
if (!ancestry.xml) {
|
||||
return {
|
||||
svgElement: elementInSvgContext(
|
||||
index,
|
||||
ancestry.nodes,
|
||||
ancestry.svgContexts,
|
||||
),
|
||||
xmlDocument: false,
|
||||
xlinkHref: undefined,
|
||||
};
|
||||
}
|
||||
const resolved = ancestry.xmlLinks.get(index) ?? UNRESOLVED;
|
||||
return {
|
||||
svgElement: resolved.svgElement,
|
||||
xmlDocument: true,
|
||||
xlinkHref: resolved.xlinkHref,
|
||||
};
|
||||
};
|
||||
|
||||
// Chrome's HTML parser records SVG local names (`svg`, `foreignObject`).
|
||||
// Descendants of foreignObject are HTML until a nested `svg`.
|
||||
const elementInSvgContext = (
|
||||
index: number,
|
||||
nodes: DomSnapshotNodes,
|
||||
contexts: Map<number, boolean>,
|
||||
): boolean => {
|
||||
let current = index;
|
||||
const seen = new Set<number>();
|
||||
let inSvg = false;
|
||||
while (
|
||||
current >= 0 &&
|
||||
current < nodes.nodeNames.length &&
|
||||
!seen.has(current)
|
||||
) {
|
||||
const cached = contexts.get(current);
|
||||
if (cached !== undefined) {
|
||||
inSvg = cached;
|
||||
break;
|
||||
}
|
||||
seen.add(current);
|
||||
if (stringAt(nodes.strings, nodes.nodeNames[current]) === "svg") {
|
||||
inSvg = true;
|
||||
break;
|
||||
}
|
||||
const parent = Math.trunc(nodes.parents[current] ?? -1);
|
||||
if (parent < 0 || parent === current) break;
|
||||
if (stringAt(nodes.strings, nodes.nodeNames[parent]) === "foreignObject")
|
||||
break;
|
||||
current = parent;
|
||||
}
|
||||
// Cache the entire examined chain, including forward references and cycles,
|
||||
// so shared ancestors are examined once per document.
|
||||
for (const node of seen) contexts.set(node, inSvg);
|
||||
return inSvg;
|
||||
};
|
||||
|
||||
interface NamespaceRestore {
|
||||
readonly prefix: string;
|
||||
readonly previous: string | undefined;
|
||||
}
|
||||
|
||||
type XmlWalkStep =
|
||||
| { readonly node: number }
|
||||
| { readonly restore: readonly NamespaceRestore[] };
|
||||
|
||||
// Walk parent-before-child with one mutable namespace environment. Restore
|
||||
// declarations on exit so siblings retain their own scope. Only declarations
|
||||
// on the active path are retained, rather than cumulative maps at every node.
|
||||
const resolveXmlElements = (
|
||||
nodes: DomSnapshotNodes,
|
||||
): Map<number, XmlSvgLink> => {
|
||||
const children = new Map<number, number[]>();
|
||||
const pending: XmlWalkStep[] = [];
|
||||
for (let node = 0; node < nodes.nodeNames.length; node += 1) {
|
||||
const parent = Math.trunc(nodes.parents[node] ?? -1);
|
||||
if (parent < 0) pending.push({ node });
|
||||
else if (parent < nodes.nodeNames.length && parent !== node) {
|
||||
const siblings = children.get(parent);
|
||||
if (siblings === undefined) children.set(parent, [node]);
|
||||
else siblings.push(node);
|
||||
}
|
||||
}
|
||||
const bindings = new Map<string, string>();
|
||||
const resolved = new Map<number, XmlSvgLink>();
|
||||
while (pending.length > 0) {
|
||||
const step = pending.pop();
|
||||
if (step === undefined) break;
|
||||
if ("restore" in step) {
|
||||
for (const { prefix, previous } of step.restore) {
|
||||
if (previous === undefined) bindings.delete(prefix);
|
||||
else bindings.set(prefix, previous);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const { node } = step;
|
||||
const attributes = numberArray(nodes.attributes[node]);
|
||||
const restore = applyDeclarations(bindings, attributes, nodes.strings);
|
||||
const svgElement =
|
||||
Math.trunc(nodes.nodeTypes[node] ?? 0) === 1 &&
|
||||
elementNamespace(
|
||||
stringAt(nodes.strings, nodes.nodeNames[node]),
|
||||
bindings,
|
||||
) === SVG_NAMESPACE;
|
||||
resolved.set(node, {
|
||||
svgElement,
|
||||
xlinkHref: svgElement
|
||||
? xlinkHrefValue(attributes, nodes.strings, bindings)
|
||||
: undefined,
|
||||
});
|
||||
pending.push({ restore });
|
||||
for (const child of children.get(node) ?? []) pending.push({ node: child });
|
||||
}
|
||||
// Cycles (including self-parents) and dangling chains cannot reach a root,
|
||||
// so remain unresolved instead of deriving URLs from an invalid ancestry.
|
||||
return resolved;
|
||||
};
|
||||
|
||||
const applyDeclarations = (
|
||||
bindings: Map<string, string>,
|
||||
attributes: readonly number[],
|
||||
strings: readonly string[],
|
||||
): readonly NamespaceRestore[] => {
|
||||
const restore: NamespaceRestore[] = [];
|
||||
const seen = new Set<string>();
|
||||
for (let index = 0; index + 1 < attributes.length; index += 2) {
|
||||
const prefix = xmlnsPrefix(stringAt(strings, attributes[index]));
|
||||
if (prefix === undefined || seen.has(prefix)) continue;
|
||||
seen.add(prefix);
|
||||
restore.push({ prefix, previous: bindings.get(prefix) });
|
||||
const value = stringAt(strings, attributes[index + 1]);
|
||||
if (value.length === 0) bindings.delete(prefix);
|
||||
else bindings.set(prefix, value);
|
||||
}
|
||||
return restore;
|
||||
};
|
||||
|
||||
const xmlnsPrefix = (name: string): string | undefined => {
|
||||
if (name === "xmlns") return "";
|
||||
if (!name.startsWith("xmlns:")) return undefined;
|
||||
const prefix = name.slice("xmlns:".length);
|
||||
if (prefix.length === 0 || prefix === "xmlns" || prefix.includes(":"))
|
||||
return undefined;
|
||||
return prefix;
|
||||
};
|
||||
|
||||
const elementNamespace = (
|
||||
nodeName: string,
|
||||
bindings: ReadonlyMap<string, string>,
|
||||
): string => {
|
||||
const colon = nodeName.indexOf(":");
|
||||
if (colon < 0) return bindings.get("") ?? "";
|
||||
const prefix = nodeName.slice(0, colon);
|
||||
const local = nodeName.slice(colon + 1);
|
||||
if (
|
||||
prefix.length === 0 ||
|
||||
local.length === 0 ||
|
||||
local.includes(":") ||
|
||||
prefix === "xmlns"
|
||||
)
|
||||
return "";
|
||||
return bindings.get(prefix) ?? "";
|
||||
};
|
||||
|
||||
// First prefixed href bound to the XLink namespace. Duplicate names keep the
|
||||
// earlier value. Plain href is applied by the caller and wins even when empty.
|
||||
const xlinkHrefValue = (
|
||||
attributes: readonly number[],
|
||||
strings: readonly string[],
|
||||
bindings: ReadonlyMap<string, string>,
|
||||
): string | undefined => {
|
||||
const seen = new Set<string>();
|
||||
for (let index = 0; index + 1 < attributes.length; index += 2) {
|
||||
const name = stringAt(strings, attributes[index]);
|
||||
if (seen.has(name)) continue;
|
||||
seen.add(name);
|
||||
const colon = name.indexOf(":");
|
||||
if (colon <= 0) continue;
|
||||
const prefix = name.slice(0, colon);
|
||||
const local = name.slice(colon + 1);
|
||||
if (local !== "href" || bindings.get(prefix) !== XLINK_NAMESPACE) continue;
|
||||
return stringAt(strings, attributes[index + 1]);
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user