Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c865e20a0b | ||
|
|
0fd4021842 | ||
|
|
4db6ff461f | ||
|
|
ce58aebc0e | ||
|
|
80f6365523 | ||
|
|
dabcd783d5 | ||
|
|
3d34185171 | ||
|
|
21650455a9 | ||
|
|
844fa52772 | ||
|
|
e36c82becd | ||
|
|
092de8f4b3 | ||
|
|
3f4ea32189 | ||
|
|
966dc3acde | ||
|
|
91f6840c35 | ||
|
|
d9b4ae182d | ||
|
|
8be7ad496c | ||
|
|
01762b10c6 | ||
|
|
29ff887f65 | ||
|
|
693ed332b5 | ||
|
|
d3088c3795 | ||
|
|
a82d14e4db | ||
|
|
5f4c7ff6df | ||
|
|
13e03e9436 | ||
|
|
4353089c67 | ||
|
|
57c5acac2e | ||
|
|
0ecf3f2438 |
+43
-31
@@ -115,14 +115,8 @@ jobs:
|
||||
- run: npm ci
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rea-dist
|
||||
path: dist
|
||||
- name: Generate test artifacts from the downloaded runtime
|
||||
run: |
|
||||
node scripts/generate-mcp-tool-catalog.mjs
|
||||
node scripts/generate-skill-metadata.mjs
|
||||
node scripts/generate-product-catalog.mjs
|
||||
node scripts/generate-completion-ledger.mjs
|
||||
name: rea-test-runtime
|
||||
path: .
|
||||
- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -146,11 +140,17 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run build:cached
|
||||
- run: npm run test:prepare
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rea-dist
|
||||
path: dist
|
||||
name: rea-test-runtime
|
||||
path: |
|
||||
dist/
|
||||
skills/
|
||||
.cache/mcp-tool-catalog.json
|
||||
docs/public/product-catalog.json
|
||||
docs/verification/managed-conformance-*.json
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
@@ -274,7 +274,10 @@ jobs:
|
||||
if: needs.changes.outputs.runtime == 'true'
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 2
|
||||
# The four platform jobs use independent hosted runners; keep each
|
||||
# runner's existing heap and threadpool limits while avoiding a queued
|
||||
# second platform wave.
|
||||
max-parallel: 4
|
||||
matrix: ${{ fromJSON(needs.changes.outputs.matrix) }}
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 20
|
||||
@@ -320,25 +323,34 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npm run verify:keyed-archive
|
||||
- run: npm run verify:asset-catalog
|
||||
- run: npm run verify:macos-bundle
|
||||
- name: Legacy simulator platform verification
|
||||
run: npm run verify:legacy-simulator
|
||||
- name: Native installer and compiled NIB hierarchy regressions
|
||||
run: npm run test:focused -- tests/boundary/filesystem/installScript.test.ts tests/boundary/filesystem/nibNativeHierarchy.test.ts tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
- name: Real binary metadata and Interface Builder integration
|
||||
run: npm run verify:apple-dispatch && npm run verify:interface-builder
|
||||
- name: Native plist CLI selection regressions
|
||||
run: npm run test:focused -- tests/boundary/cli/nativePlistCli.test.ts tests/boundary/cli/nativePlistPrototypeKeyCli.test.ts tests/boundary/cli/nativePlistSelectionFailureCli.test.ts
|
||||
- name: Native Swift demangling CLI and MCP regressions
|
||||
run: npm run test:focused -- tests/boundary/cli/nativeDemangleCli.test.ts tests/boundary/mcp/nativeDemangleMcp.test.ts
|
||||
- name: Native signature target binding and CLI/MCP recovery regressions
|
||||
run: npm run test:focused -- tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts tests/boundary/cli/nativeSignatureCli.test.ts tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts
|
||||
- name: Process filesystem snapshot descriptor regressions
|
||||
run: npm run test:focused -- src/process/capture/FilesystemSnapshot.test.ts
|
||||
- name: Native Darwin ownership and process cleanup regressions
|
||||
run: npm run test:focused -- tests/boundary/process/darwinProcessRunTokenReader.test.ts tests/boundary/process/directAnalysisSignals.test.ts tests/boundary/process/processCaptureLifecycle.test.ts tests/boundary/process/processCli.test.ts tests/boundary/mcp/sessionMcp.test.ts
|
||||
- name: Verify Apple artifact formats and metadata
|
||||
run: |
|
||||
npm run verify:keyed-archive
|
||||
npm run verify:asset-catalog
|
||||
npm run verify:macos-bundle
|
||||
npm run verify:legacy-simulator
|
||||
npm run verify:apple-dispatch
|
||||
npm run verify:interface-builder
|
||||
- name: Run native Apple and process boundary regressions
|
||||
run: >-
|
||||
npm run test:focused --
|
||||
tests/boundary/filesystem/installScript.test.ts
|
||||
tests/boundary/filesystem/nibNativeHierarchy.test.ts
|
||||
tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
tests/boundary/cli/nativePlistCli.test.ts
|
||||
tests/boundary/cli/nativePlistPrototypeKeyCli.test.ts
|
||||
tests/boundary/cli/nativePlistSelectionFailureCli.test.ts
|
||||
tests/boundary/cli/nativeDemangleCli.test.ts
|
||||
tests/boundary/mcp/nativeDemangleMcp.test.ts
|
||||
tests/boundary/providers/native/nativeSignatureTargetBinding.test.ts
|
||||
tests/boundary/cli/nativeSignatureCli.test.ts
|
||||
tests/boundary/mcp/nativeSignatureTargetBindingMcp.test.ts
|
||||
src/process/capture/FilesystemSnapshot.test.ts
|
||||
tests/boundary/process/darwinProcessRunTokenReader.test.ts
|
||||
tests/boundary/process/directAnalysisSignals.test.ts
|
||||
tests/boundary/process/processCaptureLifecycle.test.ts
|
||||
tests/boundary/process/processCli.test.ts
|
||||
tests/boundary/mcp/sessionMcp.test.ts
|
||||
|
||||
reconstruction-readiness:
|
||||
name: Reconstruction readiness
|
||||
|
||||
@@ -30,5 +30,9 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
|
||||
python3 -m pip install -r website/requirements.txt
|
||||
- name: Set up Node.js for website script tests
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Check shared HTML, links, assets and contributor workflows
|
||||
run: python3 scripts/website.py check
|
||||
|
||||
@@ -59,8 +59,15 @@ jobs:
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y libcairo2 fonts-dejavu-core
|
||||
python3 -m pip install -r website/requirements.txt
|
||||
- name: Set up Node.js for website script tests
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Check shared HTML, links, assets and contributor workflows
|
||||
run: python3 scripts/website.py check
|
||||
# An out-of-date translation only warns in checks; publish it with its notice.
|
||||
- name: Refresh translation notices
|
||||
run: python3 scripts/website.py sync
|
||||
- name: Record the published commit
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
|
||||
+2
-1
@@ -22,7 +22,8 @@
|
||||
],
|
||||
"max-params": ["warn", 4],
|
||||
"no-var": "warn",
|
||||
"no-var-requires": "error"
|
||||
"no-var-requires": "error",
|
||||
"typescript/no-import-type-side-effects": "error"
|
||||
},
|
||||
"overrides": [
|
||||
{
|
||||
|
||||
+12
-32
@@ -68,23 +68,15 @@ selects the npm release, and persistent MCP registrations are pinned to the
|
||||
version that performed setup. Installing newer instructions does not update a
|
||||
running server or its registration.
|
||||
|
||||
The npm release checked on **2026-10-11** was **6.3.0**, published from
|
||||
[`74cf9e14`](https://github.com/morluto/rea/commit/74cf9e1401fcd89482fe39c27cc898f5858e9b72),
|
||||
with bundled skill version 34. It does not include automatic Qwen Code, Pi, or
|
||||
Hermes registration: `setup --client qwen_code`, `setup --client pi`, and
|
||||
`setup --client hermes` reject those IDs before writing configuration. It also
|
||||
does not honor `OPENCODE_CONFIG_DIR` when selecting OpenCode's configuration
|
||||
directory. These workflows below require a source build or a later release
|
||||
containing the relevant changes; `@latest` alone cannot select unpublished
|
||||
changes.
|
||||
|
||||
The earlier release checked on **2026-10-07** was **5.0.0** (133 MCP tools), published
|
||||
from the fixed checkpoint
|
||||
[`b33236ec`](https://github.com/morluto/rea/releases/tag/rea-agents-5.0.0).
|
||||
The public CLI, MCP catalog and target-free session, and isolated update from
|
||||
4.1.0 to 5.0.0 were verified through npm. The artifact includes Windows native
|
||||
controls, Android/JADX and firmware tools, Ghidra function annotations, and
|
||||
retained application-Evidence references.
|
||||
The npm release checked on **2026-10-11** was **6.4.0**, published from
|
||||
[`a56f7bb6`](https://github.com/morluto/rea/commit/a56f7bb668ab23fec8d33f7af353ac7ed2fe1895).
|
||||
The actual registry package accepted dry-run setup plans for all 17 client IDs
|
||||
listed below, including Qwen Code, Pi and Hermes. OpenCode's plan selected
|
||||
`OPENCODE_CONFIG_DIR` even when that directory did not yet exist. These checks
|
||||
used an isolated home on macOS with Node 24; they verify planning and argument
|
||||
admission, without writing client configuration or establishing live integration
|
||||
with every client. This resolves the npm 6.3.0 setup gaps tracked in
|
||||
[#1650](https://github.com/morluto/rea/issues/1650).
|
||||
|
||||
Main's catalog describes the current code. A source build or a subsequent
|
||||
release containing changes after this checkpoint is required for newer
|
||||
@@ -172,9 +164,6 @@ schema constraints; REA still validates complete canonical inputs. Full-profile
|
||||
Windows remain unverified; see [native client verification](testing.md).
|
||||
|
||||
For [Pi](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/mcp.md),
|
||||
automatic setup requires a build containing the integration; npm 6.3.0 does not
|
||||
include it. See [released package and main](#released-package-and-main).
|
||||
In a supported build,
|
||||
setup writes a `type: "stdio"` entry under `mcpServers.rea` in the user-level
|
||||
`~/.pi/agent/mcp.json` and installs the bundled skill under `~/.agents/skills`.
|
||||
Use `rea setup --client pi --dry-run --json` to inspect the plan, then
|
||||
@@ -201,10 +190,7 @@ but unavailable in a project. An extension that registers `/mcp` can replace
|
||||
Pi's built-in MCP support and use its own configuration instead. Run `/reload`
|
||||
in an existing Pi session after changing servers outside the session.
|
||||
|
||||
For Qwen Code, automatic setup requires a build containing the integration;
|
||||
npm 6.3.0 does not include it. See
|
||||
[released package and main](#released-package-and-main). In a supported build,
|
||||
setup registers `rea` in the `mcpServers` table of the
|
||||
For Qwen Code, setup registers `rea` in the `mcpServers` table of the
|
||||
user-level `~/.qwen/settings.json`, or `$QWEN_HOME/settings.json` when configured.
|
||||
Setup and doctor follow Qwen Code's tilde and working-directory-relative home
|
||||
overrides. Qwen Code also discovers personal skills in `~/.agents/skills`,
|
||||
@@ -236,10 +222,7 @@ registration. Doctor treats an `enabled: false` entry as active when
|
||||
`enabledServers` lists `rea`, as OMP does, unless `disabledServers` also lists
|
||||
it. Run setup under each profile that should load REA.
|
||||
|
||||
For Hermes, automatic setup also requires a build containing the integration;
|
||||
npm 6.3.0 does not include it. See
|
||||
[released package and main](#released-package-and-main).
|
||||
In a supported build, setup writes a `mcp_servers.rea` entry to `config.yaml`, preserving
|
||||
For Hermes, setup writes a `mcp_servers.rea` entry to `config.yaml`, preserving
|
||||
comments and unrelated settings. It resolves that file from `HERMES_HOME`, else
|
||||
from the platform default Hermes itself uses — `%LOCALAPPDATA%\hermes` on
|
||||
Windows, `~/.hermes` elsewhere — including any `HERMES_DATA_DIR_SUFFIX`. Hermes
|
||||
@@ -293,10 +276,7 @@ for its native configuration contract.
|
||||
|
||||
### Other client settings
|
||||
|
||||
For OpenCode, npm 6.3.0 does not honor `OPENCODE_CONFIG_DIR`; use a source build
|
||||
or a later release containing the override support when selecting a custom
|
||||
configuration directory. See [released package and main](#released-package-and-main).
|
||||
In a supported build, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2
|
||||
For OpenCode, setup writes the V1 `mcp.rea` entry, which OpenCode V1 and V2
|
||||
both load. If the configuration already uses OpenCode V2's native
|
||||
`mcp.servers` table, setup registers REA there instead and replaces any earlier
|
||||
`mcp.rea` entry from REA. Setup honors `OPENCODE_CONFIG_DIR` and
|
||||
|
||||
+29
-8
@@ -23,6 +23,11 @@ cases. Share setup without hiding inputs or expected evidence. For successful
|
||||
`.ok` assertion adds no coverage. Protocol fixtures must reject unmodeled
|
||||
commands and preserve actual producer reply shapes.
|
||||
|
||||
Stateless source-analysis matrices may share a file-owned stdio MCP connection
|
||||
and retain representative compiled CLI parity checks. Every distinct source
|
||||
scenario still runs through the public workflow; session-lifecycle scenarios
|
||||
need their own sessions and cleanup assertions.
|
||||
|
||||
Report unavailable authority as a named skip. Await owned cleanup before a
|
||||
verifier reports success. Native process inspectors are released at Vitest
|
||||
worker teardown; fork termination does not run Node exit hooks, and per-file
|
||||
@@ -823,11 +828,24 @@ CI exercises the pinned Node.js runtime on native hosted runners:
|
||||
|
||||
Package lanes assert the actual Node platform/architecture before verification
|
||||
and record those values with the Node version. The four-host package matrix
|
||||
runs at most two jobs concurrently with explicit timeouts and Node heap/thread
|
||||
limits; Windows retains its curated capability lane. Package checks cover
|
||||
installation, CLI/MCP discovery, target-free
|
||||
analysis, configuration backups/recovery, Evidence and owned lifecycle; Inspector
|
||||
checks execute source-owned loopback targets and special filename cases.
|
||||
runs up to four jobs concurrently on independent hosted runners with explicit
|
||||
timeouts and per-runner Node heap/thread limits; Windows retains its curated
|
||||
capability lane. Package checks cover installation, CLI/MCP discovery,
|
||||
target-free analysis, configuration backups/recovery, Evidence and owned
|
||||
lifecycle; Inspector checks execute source-owned loopback targets and special
|
||||
filename cases.
|
||||
|
||||
The Apple artifact CLI/MCP lane uses one `macos-15` job to avoid adding demand
|
||||
for scarce macOS runner capacity. It runs the artifact-format and metadata
|
||||
verifiers, followed by one consolidated focused invocation containing all
|
||||
existing native and process-boundary regression paths.
|
||||
|
||||
The Linux source-test shards download a shared runtime and generated test
|
||||
metadata snapshot produced by the build job with `npm run test:prepare`. The
|
||||
snapshot includes `dist/`, generated skills, the MCP tool catalog, product
|
||||
catalog, and managed-conformance metadata. Each shard still installs the locked
|
||||
dependencies and runs its assigned coverage-enabled tests; it no longer
|
||||
regenerates the same metadata independently.
|
||||
|
||||
These native baseline checks complement the Linux source-test shards and the
|
||||
separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,
|
||||
@@ -930,9 +948,12 @@ provider evidence.
|
||||
Vitest projects use up to two workers, bounded by host parallelism.
|
||||
`process-boundary` runs later with serial files because process-tree sampling
|
||||
shares host resources. Acceptance and process-global cases use isolated forks
|
||||
without serial scheduling. Pure domain/service and MCP boundary projects share
|
||||
immutable module graphs; each MCP session still owns its resources. Other
|
||||
projects retain per-file isolation. See `vitest.config.ts` for current settings.
|
||||
without serial scheduling. Pure domain/service, composition, conformance,
|
||||
evaluation, and MCP boundary projects share module graphs; their tests avoid
|
||||
process-global state and keep resource cleanup test-scoped. Each MCP session
|
||||
still owns its resources. Adapter, non-MCP boundary, acceptance, and
|
||||
process-global projects retain per-file isolation. See `vitest.config.ts` for
|
||||
current settings.
|
||||
|
||||
Build/documentation writers hold checkout-local output locks. Tests do not
|
||||
hold a broad command lock; `check:pr` finishes tests before document validation.
|
||||
|
||||
Generated
+1
@@ -39,6 +39,7 @@
|
||||
"plist": "5.0.0",
|
||||
"semver": "7.8.5",
|
||||
"smol-toml": "1.9.0",
|
||||
"stream-chain": "4.2.6",
|
||||
"stream-json": "3.7.0",
|
||||
"write-file-atomic": "7.0.0",
|
||||
"ws": "8.21.0",
|
||||
|
||||
@@ -292,6 +292,7 @@
|
||||
"plist": "5.0.0",
|
||||
"semver": "7.8.5",
|
||||
"smol-toml": "1.9.0",
|
||||
"stream-chain": "4.2.6",
|
||||
"stream-json": "3.7.0",
|
||||
"write-file-atomic": "7.0.0",
|
||||
"ws": "8.21.0",
|
||||
|
||||
@@ -222,6 +222,7 @@ export const SCOPES = {
|
||||
website: [
|
||||
"website/**",
|
||||
"scripts/*website*.py",
|
||||
"scripts/test-website-scripts.mjs",
|
||||
".github/workflows/website-check.yml",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,481 @@
|
||||
// Run the website's browser scripts against a small document model, so missing
|
||||
// page-owned messages are checked without a browser.
|
||||
import assert from "node:assert/strict";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { test } from "node:test";
|
||||
import { setImmediate as nextTurn } from "node:timers/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const ASSETS = new URL("../website/public/assets/", import.meta.url);
|
||||
|
||||
class Element {
|
||||
constructor(tag, attributes = {}, children = [], text = "") {
|
||||
this.tagName = tag.toUpperCase();
|
||||
this.attributes = new Map(Object.entries(attributes));
|
||||
this.children = [];
|
||||
this.parent = null;
|
||||
this.listeners = new Map();
|
||||
this.ownText = text;
|
||||
this.hidden = "hidden" in attributes;
|
||||
this.disabled = "disabled" in attributes;
|
||||
this.checked = "checked" in attributes;
|
||||
this.value = attributes.value ?? "";
|
||||
for (const child of children) this.append(child);
|
||||
}
|
||||
|
||||
append(child) {
|
||||
child.parent = this;
|
||||
this.children.push(child);
|
||||
}
|
||||
|
||||
get textContent() {
|
||||
return (
|
||||
this.ownText + this.children.map((child) => child.textContent).join("")
|
||||
);
|
||||
}
|
||||
|
||||
set textContent(value) {
|
||||
this.ownText = String(value);
|
||||
this.children = [];
|
||||
}
|
||||
|
||||
get id() {
|
||||
return this.getAttribute("id") ?? "";
|
||||
}
|
||||
|
||||
get lang() {
|
||||
return this.getAttribute("lang") ?? "";
|
||||
}
|
||||
|
||||
get dataset() {
|
||||
const values = {};
|
||||
for (const [name, value] of this.attributes) {
|
||||
if (!name.startsWith("data-")) continue;
|
||||
const key = name
|
||||
.slice(5)
|
||||
.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase());
|
||||
values[key] = value;
|
||||
}
|
||||
return values;
|
||||
}
|
||||
|
||||
getAttribute(name) {
|
||||
return this.attributes.has(name) ? this.attributes.get(name) : null;
|
||||
}
|
||||
|
||||
setAttribute(name, value) {
|
||||
this.attributes.set(name, String(value));
|
||||
}
|
||||
|
||||
removeAttribute(name) {
|
||||
this.attributes.delete(name);
|
||||
}
|
||||
|
||||
addEventListener(type, listener) {
|
||||
this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]);
|
||||
}
|
||||
|
||||
dispatch(type, event = {}) {
|
||||
for (const listener of this.listeners.get(type) ?? []) {
|
||||
listener({ preventDefault() {}, repeat: false, ...event });
|
||||
}
|
||||
}
|
||||
|
||||
click() {
|
||||
this.dispatch("click");
|
||||
}
|
||||
|
||||
focus() {}
|
||||
|
||||
descendants() {
|
||||
return this.children.flatMap((child) => [child, ...child.descendants()]);
|
||||
}
|
||||
|
||||
matches(selector) {
|
||||
return selector
|
||||
.split(",")
|
||||
.some((part) => matchesComplex(this, part.trim()));
|
||||
}
|
||||
|
||||
closest(selector) {
|
||||
for (let node = this; node !== null; node = node.parent) {
|
||||
if (node.matches(selector)) return node;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
querySelectorAll(selector) {
|
||||
return this.descendants().filter((node) => node.matches(selector));
|
||||
}
|
||||
|
||||
querySelector(selector) {
|
||||
return this.querySelectorAll(selector)[0] ?? null;
|
||||
}
|
||||
|
||||
getContext() {
|
||||
return this.context ?? null;
|
||||
}
|
||||
}
|
||||
|
||||
// Supports the selector forms the website scripts use: tag, #id, .class and
|
||||
// [attribute], [attribute="value"] or [attribute^="value"], joined by spaces.
|
||||
function matchesCompound(node, compound) {
|
||||
const pattern =
|
||||
/^([a-z0-9]*)((?:#[\w-]+|\.[\w-]+|\[[\w-]+(?:\^?="[^"]*")?\])*)$/i;
|
||||
const parsed = pattern.exec(compound);
|
||||
if (parsed === null) throw new Error(`Unsupported selector: ${compound}`);
|
||||
const [, tag, rest] = parsed;
|
||||
if (tag && node.tagName !== tag.toUpperCase()) return false;
|
||||
for (const [, id, className, name, operator, value] of rest.matchAll(
|
||||
/#([\w-]+)|\.([\w-]+)|\[([\w-]+)(?:(\^?=)"([^"]*)")?\]/g,
|
||||
)) {
|
||||
if (id !== undefined) {
|
||||
if (node.id !== id) return false;
|
||||
continue;
|
||||
}
|
||||
if (className !== undefined) {
|
||||
const classes = (node.getAttribute("class") ?? "").split(/\s+/);
|
||||
if (!classes.includes(className)) return false;
|
||||
continue;
|
||||
}
|
||||
const actual = node.getAttribute(name);
|
||||
if (actual === null) return false;
|
||||
if (operator === "=" && actual !== value) return false;
|
||||
if (operator === "^=" && !actual.startsWith(value)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function matchesComplex(node, selector) {
|
||||
const parts = selector.split(/\s+/);
|
||||
if (!matchesCompound(node, parts.at(-1))) return false;
|
||||
let ancestor = node.parent;
|
||||
for (const part of parts.slice(0, -1).reverse()) {
|
||||
while (ancestor !== null && !matchesCompound(ancestor, part))
|
||||
ancestor = ancestor.parent;
|
||||
if (ancestor === null) return false;
|
||||
ancestor = ancestor.parent;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
const h = (tag, attributes, ...children) => {
|
||||
const text = children.filter((child) => typeof child === "string").join("");
|
||||
return new Element(
|
||||
tag,
|
||||
attributes ?? {},
|
||||
children.filter((child) => child instanceof Element),
|
||||
text,
|
||||
);
|
||||
};
|
||||
|
||||
// Install a fresh document and browser globals, then run one script.
|
||||
async function run(
|
||||
script,
|
||||
body,
|
||||
{ lang = "en", clipboard = "resolve", stars = 1234 } = {},
|
||||
) {
|
||||
const root = h("html", { lang }, h("body", {}, ...body));
|
||||
const timers = [];
|
||||
const written = [];
|
||||
const canvasText = [];
|
||||
for (const canvas of root.querySelectorAll("canvas")) {
|
||||
canvas.context = new Proxy(
|
||||
{ fillText: (text) => canvasText.push(text) },
|
||||
{ get: (target, name) => target[name] ?? (() => {}), set: () => true },
|
||||
);
|
||||
}
|
||||
const document = {
|
||||
documentElement: root,
|
||||
hidden: false,
|
||||
getElementById: (id) =>
|
||||
root.descendants().find((node) => node.id === id) ?? null,
|
||||
querySelector: (selector) => root.querySelector(selector),
|
||||
querySelectorAll: (selector) => root.querySelectorAll(selector),
|
||||
createRange: () => ({ selectNodeContents() {} }),
|
||||
addEventListener() {},
|
||||
};
|
||||
const window = {
|
||||
scrollY: 0,
|
||||
innerHeight: 900,
|
||||
location: { hash: "" },
|
||||
addEventListener() {},
|
||||
getSelection: () => ({ removeAllRanges() {}, addRange() {} }),
|
||||
matchMedia: () => ({ matches: false, addEventListener() {} }),
|
||||
requestAnimationFrame: () => 1,
|
||||
scrollTo() {},
|
||||
setTimeout: (callback) => timers.push(callback),
|
||||
};
|
||||
Object.assign(globalThis, {
|
||||
document,
|
||||
window,
|
||||
requestAnimationFrame: () => 1,
|
||||
cancelAnimationFrame() {},
|
||||
Path2D: class {},
|
||||
fetch: async () => ({
|
||||
ok: true,
|
||||
json: async () => ({ stargazers_count: stars }),
|
||||
}),
|
||||
});
|
||||
Object.defineProperty(globalThis, "navigator", {
|
||||
configurable: true,
|
||||
value: {
|
||||
clipboard: {
|
||||
writeText: async (text) => {
|
||||
if (clipboard === "reject") throw new Error("denied");
|
||||
written.push(text);
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
await import(
|
||||
`${pathToFileURL(new URL(script, ASSETS).pathname)}?load=${randomUUID()}`
|
||||
);
|
||||
await nextTurn();
|
||||
return { root, timers, written, canvasText };
|
||||
}
|
||||
|
||||
const copyBlock = () => [
|
||||
h("button", { "data-copy": "command" }, "Copy"),
|
||||
h("pre", {}, h("code", { id: "command" }, "npx rea-agents setup")),
|
||||
];
|
||||
const copyStatus = (messages = {}) =>
|
||||
h("p", {
|
||||
id: "copy-status",
|
||||
...Object.fromEntries(
|
||||
Object.entries(messages).map(([key, value]) => [`data-${key}`, value]),
|
||||
),
|
||||
});
|
||||
const englishCopyMessages = {
|
||||
copied: "Copied",
|
||||
selected: "Selected",
|
||||
"status-copied": "Code copied to clipboard.",
|
||||
"status-selected": "Code selected. Use your keyboard to copy it.",
|
||||
};
|
||||
|
||||
test("copy buttons report and restore localized labels", async () => {
|
||||
const { root, timers, written } = await run("main.js", [
|
||||
...copyBlock(),
|
||||
copyStatus(englishCopyMessages),
|
||||
]);
|
||||
const button = root.querySelector("[data-copy]");
|
||||
button.click();
|
||||
await nextTurn();
|
||||
assert.deepEqual(written, ["npx rea-agents setup"]);
|
||||
assert.equal(button.textContent, "Copied");
|
||||
assert.equal(
|
||||
root.querySelector("#copy-status").textContent,
|
||||
"Code copied to clipboard.",
|
||||
);
|
||||
timers.forEach((callback) => callback());
|
||||
assert.equal(button.textContent, "Copy");
|
||||
});
|
||||
|
||||
test("copy buttons keep their label when messages are missing", async () => {
|
||||
for (const [body, clipboard] of [
|
||||
[[...copyBlock(), copyStatus()], "resolve"],
|
||||
[[...copyBlock(), copyStatus()], "reject"],
|
||||
[
|
||||
[...copyBlock(), copyStatus({ copied: "", "status-copied": "" })],
|
||||
"resolve",
|
||||
],
|
||||
[copyBlock(), "resolve"],
|
||||
]) {
|
||||
const { root, timers, written } = await run("main.js", body, { clipboard });
|
||||
const button = root.querySelector("[data-copy]");
|
||||
button.click();
|
||||
await nextTurn();
|
||||
assert.equal(written.length, clipboard === "resolve" ? 1 : 0);
|
||||
assert.equal(button.textContent, "Copy");
|
||||
assert.equal(root.querySelector("#copy-status")?.textContent ?? "", "");
|
||||
timers.forEach((callback) => callback());
|
||||
assert.equal(button.textContent, "Copy");
|
||||
}
|
||||
});
|
||||
|
||||
const githubLink = (labels = {}) =>
|
||||
h(
|
||||
"a",
|
||||
{
|
||||
class: "nav-github",
|
||||
"aria-label": "REA on GitHub",
|
||||
title: "REA on GitHub",
|
||||
...labels,
|
||||
},
|
||||
h("span", { class: "nav-github-count", hidden: "" }),
|
||||
);
|
||||
|
||||
test("the star label follows the page language and plural rules", async () => {
|
||||
const labels = {
|
||||
"data-label-one": "REA on GitHub ({count} star)",
|
||||
"data-label-other": "REA on GitHub ({count} stars)",
|
||||
};
|
||||
for (const [stars, lang, expected] of [
|
||||
[1, "en", "REA on GitHub (1 star)"],
|
||||
[1234, "en", "REA on GitHub (1,234 stars)"],
|
||||
[1234, "de", "REA on GitHub (1.234 stars)"],
|
||||
]) {
|
||||
const { root } = await run("main.js", [githubLink(labels)], {
|
||||
stars,
|
||||
lang,
|
||||
});
|
||||
await nextTurn();
|
||||
assert.equal(
|
||||
root.querySelector(".nav-github").getAttribute("aria-label"),
|
||||
expected,
|
||||
);
|
||||
}
|
||||
// Languages without a "one" category use the "other" label.
|
||||
const { root } = await run(
|
||||
"main.js",
|
||||
[githubLink({ "data-label-other": "{count}" })],
|
||||
{ stars: 1, lang: "ja" },
|
||||
);
|
||||
await nextTurn();
|
||||
assert.equal(
|
||||
root.querySelector(".nav-github").getAttribute("aria-label"),
|
||||
"1",
|
||||
);
|
||||
});
|
||||
|
||||
test("the star label stays unchanged without a localized label", async () => {
|
||||
const { root } = await run("main.js", [githubLink()], { stars: 1 });
|
||||
await nextTurn();
|
||||
const link = root.querySelector(".nav-github");
|
||||
assert.equal(link.getAttribute("aria-label"), "REA on GitHub");
|
||||
assert.equal(link.getAttribute("title"), "REA on GitHub");
|
||||
assert.equal(link.querySelector(".nav-github-count").hidden, true);
|
||||
});
|
||||
|
||||
const percentDemo = (messages) =>
|
||||
h(
|
||||
"figure",
|
||||
{ "data-percent-demo": "", ...messages },
|
||||
h(
|
||||
"div",
|
||||
{ class: "percent-operation", hidden: "" },
|
||||
h(
|
||||
"button",
|
||||
{ "data-percent-operation": "add", "aria-pressed": "true" },
|
||||
"200 + 10%",
|
||||
),
|
||||
h(
|
||||
"button",
|
||||
{ "data-percent-operation": "multiply", "aria-pressed": "false" },
|
||||
"200 × 10%",
|
||||
),
|
||||
),
|
||||
h("span", { "data-percent-expression": "" }, "200 +"),
|
||||
h("strong", { "data-percent-display": "" }, "20"),
|
||||
h(
|
||||
"div",
|
||||
{ class: "percent-controls", hidden: "" },
|
||||
h("button", { "data-percent-key": "percent" }, "%"),
|
||||
h("button", { "data-percent-key": "equals", disabled: "" }, "="),
|
||||
h("button", { "data-percent-key": "reset" }, "Reset"),
|
||||
),
|
||||
h(
|
||||
"p",
|
||||
{ "data-percent-status": "" },
|
||||
"10% of 200 = 20. Then 200 + 20 = 220.",
|
||||
),
|
||||
);
|
||||
|
||||
test("the percent demo uses page messages and keeps text when one is missing", async () => {
|
||||
const { root } = await run("home.js", [
|
||||
percentDemo({
|
||||
"data-msg-input": "Press % to see what happens to 10.",
|
||||
"data-msg-result": "200 {operator} {percent} = {result}.",
|
||||
}),
|
||||
]);
|
||||
const status = root.querySelector("[data-percent-status]");
|
||||
const key = (name) =>
|
||||
root.querySelector(`[data-percent-key="${name}"]`).click();
|
||||
assert.equal(status.textContent, "Press % to see what happens to 10.");
|
||||
key("percent");
|
||||
assert.equal(status.textContent, "Press % to see what happens to 10.");
|
||||
key("equals");
|
||||
assert.equal(status.textContent, "200 + 20 = 220.");
|
||||
|
||||
const bare = await run("home.js", [percentDemo({})]);
|
||||
const bareStatus = bare.root.querySelector("[data-percent-status]");
|
||||
for (const name of ["percent", "equals", "reset"]) {
|
||||
bare.root.querySelector(`[data-percent-key="${name}"]`).click();
|
||||
assert.equal(
|
||||
bareStatus.textContent,
|
||||
"10% of 200 = 20. Then 200 + 20 = 220.",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
const dinoDemo = (messages) =>
|
||||
h(
|
||||
"figure",
|
||||
{ "data-dino-demo": "", ...messages },
|
||||
h("svg", { class: "dino-fallback" }),
|
||||
h("canvas", { hidden: "" }),
|
||||
h(
|
||||
"div",
|
||||
{ class: "dino-controls", hidden: "" },
|
||||
h("button", { "data-dino-start": "" }, "Play"),
|
||||
h("button", { "data-dino-jump": "" }, "Jump ↑"),
|
||||
h("button", { "data-dino-reset": "" }, "Restart"),
|
||||
h("output", { "data-dino-speed": "" }, "6.0"),
|
||||
h("input", { "data-dino-slider": "", value: "6" }),
|
||||
h("input", { "data-dino-auto": "", checked: "" }),
|
||||
),
|
||||
h(
|
||||
"p",
|
||||
{ "data-dino-status": "" },
|
||||
"Original rule: start at 6, then increase toward 13.",
|
||||
),
|
||||
);
|
||||
|
||||
test("the dinosaur game never shows a missing message", async () => {
|
||||
const { root, canvasText } = await run("dino-demo.js", [
|
||||
dinoDemo({}),
|
||||
h("button", { "data-speed-check": "", hidden: "" }, "Replay"),
|
||||
h("p", { "data-speed-check-status": "" }, "Not replayed yet."),
|
||||
]);
|
||||
const start = root.querySelector("[data-dino-start]");
|
||||
const status = root.querySelector("[data-dino-status]");
|
||||
const slider = root.querySelector("[data-dino-slider]");
|
||||
start.click();
|
||||
start.click();
|
||||
slider.value = "9.5";
|
||||
slider.dispatch("input");
|
||||
root.querySelector("[data-dino-reset]").click();
|
||||
assert.equal(start.textContent, "Play");
|
||||
assert.equal(
|
||||
status.textContent,
|
||||
"Original rule: start at 6, then increase toward 13.",
|
||||
);
|
||||
assert.deepEqual(canvasText, []);
|
||||
root.querySelector("[data-speed-check]").click();
|
||||
assert.equal(
|
||||
root.querySelector("[data-speed-check-status]").textContent,
|
||||
"Not replayed yet.",
|
||||
);
|
||||
});
|
||||
|
||||
test("the dinosaur game fills placeholders from page messages", async () => {
|
||||
const { root, canvasText } = await run("dino-demo.js", [
|
||||
dinoDemo({
|
||||
"data-msg-pause": "Pause",
|
||||
"data-msg-fixed-running":
|
||||
"Fixed speed: {speed}. Space or Jump clears a cactus.",
|
||||
"data-msg-distance": "DISTANCE {distance}",
|
||||
}),
|
||||
]);
|
||||
const start = root.querySelector("[data-dino-start]");
|
||||
const slider = root.querySelector("[data-dino-slider]");
|
||||
start.click();
|
||||
assert.equal(start.textContent, "Pause");
|
||||
slider.value = "9.5";
|
||||
slider.dispatch("input");
|
||||
assert.equal(
|
||||
root.querySelector("[data-dino-status]").textContent,
|
||||
"Fixed speed: 9.5. Space or Jump clears a cactus.",
|
||||
);
|
||||
assert.ok(canvasText.includes("DISTANCE 0000"));
|
||||
});
|
||||
+212
-2
@@ -2,7 +2,10 @@
|
||||
|
||||
import contextlib
|
||||
import importlib.util
|
||||
from html import escape
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
@@ -137,7 +140,7 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
def test_shared_edits_preserve_body_and_catch_drift_before_rewriting(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates"):
|
||||
for name in ("public", "templates", "locales"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
page = root / "website/public/blog/touhou-reconstruction/index.html"
|
||||
original = page.read_text()
|
||||
@@ -159,7 +162,7 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
def test_new_page_escapes_metadata_stays_a_draft_and_joins_sitemap_when_ready(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates"):
|
||||
for name in ("public", "templates", "locales"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
site = root / "website/public"
|
||||
page = WEBSITE.new_page(root, "blog/new-example", 'A <test> & "result"', 'Check "quotes" & <markup>.')
|
||||
@@ -242,5 +245,212 @@ class WebsiteMaintenanceChecks(unittest.TestCase):
|
||||
worker.join()
|
||||
|
||||
|
||||
|
||||
def site_copy(directory):
|
||||
"""Copy the authored website inputs without existing translated pages."""
|
||||
root = Path(directory)
|
||||
for name in ("public", "templates", "locales"):
|
||||
shutil.copytree(ROOT / "website" / name, root / "website" / name)
|
||||
for code in WEBSITE.LANGUAGES:
|
||||
if code != "en":
|
||||
shutil.rmtree(root / "website/public" / code, ignore_errors=True)
|
||||
return root
|
||||
|
||||
|
||||
def zh_hant(key):
|
||||
"""Return one zh-Hant UI string, so assertions follow the locale file."""
|
||||
strings = json.loads((ROOT / "website/locales/zh-hant.json").read_text())["strings"]
|
||||
return escape(strings[key])
|
||||
|
||||
|
||||
def quietly(function, *arguments):
|
||||
"""Run a command helper without its progress output."""
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
return function(*arguments)
|
||||
|
||||
|
||||
def mark_reviewed(path):
|
||||
"""Apply the reviewer's change: reviewed status and date, without noindex.
|
||||
|
||||
A reviewed translation has its own search title, so give the test page one.
|
||||
"""
|
||||
source = re.sub(r"<title>(.*?)</title>", r"<title>\1 (zh-Hant)</title>", path.read_text(), count=1)
|
||||
source = source.replace('status="draft"', 'status="reviewed" reviewed="2026-10-11"', 1)
|
||||
path.write_text(source.replace(' <meta name="robots" content="noindex" />\n', "", 1))
|
||||
|
||||
|
||||
def translation_errors(root):
|
||||
"""Return the verifier's translation errors and warnings for a site copy."""
|
||||
return VERIFY.check_translations(root, VERIFY.parse_pages(root / "website/public"))
|
||||
|
||||
|
||||
class WebsiteTranslationChecks(unittest.TestCase):
|
||||
"""Create translations through the contributor command and check what readers get."""
|
||||
|
||||
def test_translate_creates_an_unlinked_noindex_draft(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
site = root / "website/public"
|
||||
english = (site / "get-started/index.html").read_text()
|
||||
page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
|
||||
source = page.read_text()
|
||||
self.assertEqual(page, site / "zh-hant/get-started/index.html")
|
||||
self.assertIn('<html lang="zh-Hant">', source)
|
||||
self.assertIn('<meta name="robots" content="noindex" />', source)
|
||||
self.assertEqual(
|
||||
WEBSITE.translation_record(source),
|
||||
{"source-sha256": WEBSITE.source_digest(english), "status": "draft"},
|
||||
)
|
||||
self.assertIn('href="../../assets/styles.css"', source)
|
||||
self.assertIn('<a class="brand" href="../../"', source)
|
||||
self.assertIn(
|
||||
'<a href="../../zh-hant/get-started/" class="nav-start" aria-current="page">'
|
||||
f'{zh_hant("navGetStarted")}</a>',
|
||||
source,
|
||||
)
|
||||
self.assertIn(
|
||||
f'<a href="../../showcase/" hreflang="en">{zh_hant("navShowcases")}'
|
||||
'<span class="nav-badge" lang="en">EN</span></a>',
|
||||
source,
|
||||
)
|
||||
self.assertIn('<li><a href="../../get-started/" hreflang="en" lang="en">English</a></li>', source)
|
||||
self.assertIn('<meta property="og:locale" content="zh_TW" />', source)
|
||||
self.assertNotIn('rel="alternate"', source)
|
||||
self.assertIn(WEBSITE.NOTICE_START + WEBSITE.NOTICE_END, source)
|
||||
self.assertEqual(english, (site / "get-started/index.html").read_text())
|
||||
errors, warnings = translation_errors(root)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertLessEqual({file for file, _ in warnings}, {"website/locales/zh-hant.json"})
|
||||
self.assertEqual(VERIFY.check_site(site)[3], [])
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
PREPARE.prepare_sitemap(site)
|
||||
self.assertNotIn("zh-hant", (site / "sitemap.xml").read_text())
|
||||
for language, route, message in (
|
||||
("zh-hant", "get-started", "already exists"),
|
||||
("en", "faq", "Choose one of"),
|
||||
("ja", "missing", "No English page"),
|
||||
("ja", "zh-hant/get-started", "without a language directory"),
|
||||
("ja", "../outside", "directory route"),
|
||||
):
|
||||
with self.subTest(language=language, route=route):
|
||||
with self.assertRaisesRegex(ValueError, message):
|
||||
quietly(WEBSITE.new_translation, root, language, route)
|
||||
|
||||
def test_reviewed_translations_join_menus_hreflang_and_the_sitemap(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
site = root / "website/public"
|
||||
mark_reviewed(quietly(WEBSITE.new_translation, root, "zh-hant", "get-started"))
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
english = (site / "get-started/index.html").read_text()
|
||||
for tag, url in (
|
||||
("en", "https://rea.tools/get-started/"),
|
||||
("zh-Hant", "https://rea.tools/zh-hant/get-started/"),
|
||||
("x-default", "https://rea.tools/get-started/"),
|
||||
):
|
||||
self.assertIn(f'<link rel="alternate" hreflang="{tag}" href="{url}" />', english)
|
||||
self.assertIn('<meta property="og:locale:alternate" content="zh_TW" />', english)
|
||||
self.assertIn(
|
||||
'<li><a href="../zh-hant/get-started/" hreflang="zh-Hant" lang="zh-Hant">繁體中文</a></li>', english
|
||||
)
|
||||
self.assertNotIn("language-menu", (site / "faq/index.html").read_text())
|
||||
translated = (site / "zh-hant/get-started/index.html").read_text()
|
||||
self.assertIn('<meta property="og:locale:alternate" content="en_US" />', translated)
|
||||
|
||||
# A reviewed home page becomes the menu's fallback for untranslated pages.
|
||||
mark_reviewed(quietly(WEBSITE.new_translation, root, "zh-hant", "/"))
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
self.assertIn(
|
||||
'<li><a href="../zh-hant/" hreflang="zh-Hant" lang="zh-Hant">繁體中文</a></li>',
|
||||
(site / "faq/index.html").read_text(),
|
||||
)
|
||||
translated = (site / "zh-hant/get-started/index.html").read_text()
|
||||
self.assertIn('<a class="brand" href="../../zh-hant/"', translated)
|
||||
home = (site / "zh-hant/index.html").read_text()
|
||||
self.assertIn('href="get-started/"', home)
|
||||
self.assertIn('href="../showcase/"', home)
|
||||
self.assertIn('href="../assets/styles.css"', home)
|
||||
with contextlib.redirect_stdout(io.StringIO()):
|
||||
PREPARE.prepare_sitemap(site)
|
||||
sitemap = (site / "sitemap.xml").read_text()
|
||||
self.assertIn("https://rea.tools/zh-hant/get-started/", sitemap)
|
||||
self.assertIn("https://rea.tools/zh-hant/", sitemap)
|
||||
self.assertEqual(VERIFY.check_site(site)[3], [])
|
||||
self.assertEqual(translation_errors(root)[0], [])
|
||||
|
||||
def test_english_commands_sync_and_text_changes_show_a_notice(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
site = root / "website/public"
|
||||
english_path = site / "get-started/index.html"
|
||||
page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
|
||||
english = english_path.read_text()
|
||||
identifier = WEBSITE.synced_code_ids(english)[0]
|
||||
block = WEBSITE.code_block(identifier)
|
||||
|
||||
# A command-only change is copied and keeps the translation current.
|
||||
command = block.search(english).group(2) + " --test-flag"
|
||||
english_path.write_text(block.sub(lambda match: match.group(1) + command + match.group(3), english))
|
||||
with self.assertRaisesRegex(ValueError, "Shared HTML is stale: zh-hant/get-started/index.html"):
|
||||
quietly(WEBSITE.sync_pages, root, True)
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
self.assertEqual(block.search(page.read_text()).group(2), command)
|
||||
self.assertNotIn('class="translation-notice"', page.read_text())
|
||||
self.assertEqual(translation_errors(root), ([], []))
|
||||
|
||||
# A text change marks the translation out of date.
|
||||
unchanged = english_path.read_text()
|
||||
changed = unchanged.replace(" </main>", " <p>A new English paragraph.</p>\n </main>", 1)
|
||||
english_path.write_text(changed)
|
||||
self.assertEqual(quietly(WEBSITE.sync_pages, root, True), [page])
|
||||
errors, warnings = translation_errors(root)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertTrue(any("has changed since this translation" in message for _, message in warnings))
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
link = f'<a href="../../get-started/" hreflang="en">{zh_hant("staleNoticeLink")}</a>'
|
||||
self.assertIn(
|
||||
f'<p class="translation-notice" role="note">{zh_hant("staleNotice").replace("{link}", link)}</p>',
|
||||
page.read_text(),
|
||||
)
|
||||
report = quietly(WEBSITE.translation_report, root)
|
||||
self.assertIn(f' after updating the translation, record source-sha256="{WEBSITE.source_digest(changed)}"', report)
|
||||
|
||||
english_path.write_text(unchanged)
|
||||
quietly(WEBSITE.sync_pages, root)
|
||||
self.assertNotIn('class="translation-notice"', page.read_text())
|
||||
|
||||
page.write_text(block.sub(lambda match: match.group(1) + "edited" + match.group(3), page.read_text()))
|
||||
self.assertTrue(any(f"{identifier} must match" in error for error in translation_errors(root)[0]))
|
||||
|
||||
def test_translations_keep_structure_and_cjk_spacing(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = site_copy(directory)
|
||||
page = quietly(WEBSITE.new_translation, root, "zh-hant", "get-started")
|
||||
original = page.read_text()
|
||||
first, second = WEBSITE.copy_targets(original)[:2]
|
||||
|
||||
def added(paragraph):
|
||||
return (" </main>", f" {paragraph}\n </main>")
|
||||
|
||||
cases = (
|
||||
(*added("<p>完成後\n 重新啟動。</p>"), "renders as a gap"),
|
||||
('<main id="main"', '<main id="moved"', "element IDs differ"),
|
||||
(f'data-copy="{first}"', f'data-copy="{second}"', "copy buttons differ"),
|
||||
(' <meta name="robots" content="noindex" />\n', "", "draft translation needs"),
|
||||
('status="draft"', 'status="final"', "status must be draft or reviewed"),
|
||||
)
|
||||
for old, new, message in cases:
|
||||
with self.subTest(message=message):
|
||||
self.assertIn(old, original)
|
||||
page.write_text(original.replace(old, new, 1))
|
||||
self.assertTrue(any(message in error for error in translation_errors(root)[0]))
|
||||
page.write_text(original.replace(*added("<p>重新啟動 REA 用戶端。</p>"), 1))
|
||||
self.assertEqual(translation_errors(root)[0], [])
|
||||
page.write_text(original.replace(*added("<p><code>extra-command</code></p>"), 1))
|
||||
errors, warnings = translation_errors(root)
|
||||
self.assertEqual(errors, [])
|
||||
self.assertTrue(any("inline code differs" in message for _, message in warnings))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { access, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { access, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
@@ -25,8 +24,6 @@ const electronPath = join(fixtureRoot, "electron");
|
||||
const environment = {
|
||||
...process.env,
|
||||
};
|
||||
const temporaryRoot = await mkdtemp(join(tmpdir(), "rea-readiness-"));
|
||||
|
||||
await access(nativePath);
|
||||
const nativeEvidence = await runCli(["inspect-artifact", nativePath, "--json"]);
|
||||
const javascriptCliEvidence = await runCli([
|
||||
@@ -135,7 +132,6 @@ try {
|
||||
);
|
||||
} finally {
|
||||
await client.close();
|
||||
await rm(temporaryRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
async function runCli(arguments_) {
|
||||
|
||||
+221
-8
@@ -1,6 +1,8 @@
|
||||
"""Check website references, search metadata and the sole production publisher."""
|
||||
|
||||
from collections import Counter
|
||||
from html.parser import HTMLParser
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
@@ -10,7 +12,21 @@ from urllib.parse import unquote, urlsplit
|
||||
import xml.etree.ElementTree as ET
|
||||
from zipfile import BadZipFile, ZipFile
|
||||
|
||||
from website import NOTES_FIXTURES, SITE_ORIGIN, authored_markdown, page_url, sync_pages
|
||||
from website import (
|
||||
LANGUAGES,
|
||||
NOTES_FIXTURES,
|
||||
SITE_ORIGIN,
|
||||
SiteState,
|
||||
authored_markdown,
|
||||
code_block,
|
||||
copy_targets,
|
||||
page_language,
|
||||
page_url,
|
||||
source_digest,
|
||||
sync_pages,
|
||||
synced_code_ids,
|
||||
translation_record,
|
||||
)
|
||||
|
||||
LEGACY_ORIGIN = "morluto.github.io"
|
||||
SITE_TEXT_URL = re.compile(
|
||||
@@ -167,8 +183,9 @@ def check_search_metadata(site, pages):
|
||||
title = page.titles[0].strip() if len(page.titles) == 1 else ""
|
||||
if not title:
|
||||
errors.append(f"{label}: needs exactly one nonempty head title")
|
||||
if page.lang != "en" or page.h1_count != 1:
|
||||
errors.append(f"{label}: needs lang=en and exactly one h1")
|
||||
tag = LANGUAGES[page_language(site, path)[0]].tag
|
||||
if page.lang != tag or page.h1_count != 1:
|
||||
errors.append(f"{label}: needs lang={tag} and exactly one h1")
|
||||
if label in NOTES_FIXTURES and not page.noindex():
|
||||
errors.append(
|
||||
f"{label}: example application must have a head noindex directive"
|
||||
@@ -280,15 +297,21 @@ def check_search_metadata(site, pages):
|
||||
return len(indexable), errors
|
||||
|
||||
|
||||
def check_site(site):
|
||||
"""Check assets, HTML fragment destinations and SVG syntax."""
|
||||
def parse_pages(site):
|
||||
"""Parse every HTML page once for reference, metadata and translation checks."""
|
||||
pages = {}
|
||||
errors = []
|
||||
references = 0
|
||||
for path in sorted(site.rglob("*.html")):
|
||||
parser = HtmlReferences()
|
||||
parser.feed(path.read_text(encoding="utf-8"))
|
||||
pages[path] = parser
|
||||
return pages
|
||||
|
||||
|
||||
def check_site(site):
|
||||
"""Check assets, HTML fragment destinations and SVG syntax."""
|
||||
pages = parse_pages(site)
|
||||
errors = []
|
||||
references = 0
|
||||
if site / "index.html" not in pages:
|
||||
errors.append("website/public/index.html is missing")
|
||||
|
||||
@@ -346,6 +369,190 @@ def check_site(site):
|
||||
return len(pages), references, indexable, errors
|
||||
|
||||
|
||||
# Han, kana and CJK punctuation; a space between two of them renders as a gap.
|
||||
CJK_GAP = re.compile(
|
||||
r"([、-〿-ヿ㐀-䶿一-鿿豈-!-])"
|
||||
r"\s+([、-〿-ヿ㐀-䶿一-鿿豈-!-])"
|
||||
)
|
||||
CODE_TAGS = {"code", "kbd", "pre", "script", "style"}
|
||||
|
||||
|
||||
class TranslationText(HTMLParser):
|
||||
"""Collect prose, inline code and message attributes for comparing translations."""
|
||||
|
||||
def __init__(self, synced):
|
||||
super().__init__()
|
||||
self.synced = set(synced)
|
||||
self.stack = []
|
||||
self.prose = []
|
||||
self.inline_code = Counter()
|
||||
self.messages = Counter()
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attributes = dict(attrs)
|
||||
self.messages.update(name for name in attributes if name.startswith("data-msg-"))
|
||||
if tag in CODE_TAGS:
|
||||
self.stack.append([tag, attributes.get("id"), ""])
|
||||
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
attributes = dict(attrs)
|
||||
self.messages.update(name for name in attributes if name.startswith("data-msg-"))
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if self.stack and self.stack[-1][0] == tag:
|
||||
name, identifier, text = self.stack.pop()
|
||||
if name in ("code", "kbd") and identifier not in self.synced:
|
||||
self.inline_code[" ".join(text.split())] += 1
|
||||
if self.stack:
|
||||
self.stack[-1][2] += text
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.stack:
|
||||
self.stack[-1][2] += data
|
||||
else:
|
||||
self.prose.append(data)
|
||||
|
||||
|
||||
def translation_warning(path, root, message):
|
||||
"""Return a warning tied to a repository file."""
|
||||
return (path.relative_to(root).as_posix(), message)
|
||||
|
||||
|
||||
def check_translations(root, pages):
|
||||
"""Check translation records, copied code, structure parity and CJK spacing.
|
||||
|
||||
A changed English source is a warning: its translation keeps working, shows a
|
||||
notice after sync and is updated later. Errors cover what sync or the
|
||||
translator controls directly.
|
||||
"""
|
||||
site = root / "website/public"
|
||||
errors = []
|
||||
warnings = []
|
||||
try:
|
||||
state = SiteState(root)
|
||||
except ValueError as error:
|
||||
return [str(error)], []
|
||||
for path, page in pages.items():
|
||||
language, english = page_language(site, path)
|
||||
if language == "en":
|
||||
continue
|
||||
label = path.relative_to(site).as_posix()
|
||||
source = path.read_text(encoding="utf-8")
|
||||
record = translation_record(source)
|
||||
english_path = site / english
|
||||
if not english_path.is_file():
|
||||
errors.append(f"{label}: translates {english}, which does not exist")
|
||||
continue
|
||||
english_source = english_path.read_text(encoding="utf-8")
|
||||
if record is None:
|
||||
errors.append(f"{label}: needs one <!-- website:translation source-sha256=... status=... --> record")
|
||||
continue
|
||||
status = record.get("status")
|
||||
if status not in ("draft", "reviewed"):
|
||||
errors.append(f"{label}: translation status must be draft or reviewed")
|
||||
if status == "draft" and not page.noindex():
|
||||
errors.append(f"{label}: a draft translation needs <meta name=\"robots\" content=\"noindex\" />")
|
||||
if status == "reviewed":
|
||||
if page.noindex():
|
||||
errors.append(f"{label}: remove noindex from a reviewed translation")
|
||||
if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", record.get("reviewed", "")):
|
||||
errors.append(f"{label}: a reviewed translation needs reviewed=\"YYYY-MM-DD\"")
|
||||
fresh = record.get("source-sha256") == source_digest(english_source)
|
||||
if not fresh:
|
||||
warnings.append(translation_warning(
|
||||
path, root,
|
||||
f"The English page {english} has changed since this translation. Update the translation, "
|
||||
"then set source-sha256 to the new value from python3 scripts/website.py translations.",
|
||||
))
|
||||
|
||||
synced = synced_code_ids(english_source)
|
||||
for identifier in synced:
|
||||
expected = code_block(identifier).search(english_source).group(2)
|
||||
found = code_block(identifier).findall(source)
|
||||
if len(found) != 1 or found[0][1] != expected:
|
||||
errors.append(f"{label}: code block {identifier} must match the English page; run sync")
|
||||
|
||||
english_page = pages[english_path]
|
||||
differences = []
|
||||
if page.ids != english_page.ids:
|
||||
differences.append(
|
||||
f"element IDs differ (missing {sorted(english_page.ids - page.ids)}, extra {sorted(page.ids - english_page.ids)})"
|
||||
)
|
||||
if copy_targets(source) != copy_targets(english_source):
|
||||
differences.append("copy buttons differ from the English page")
|
||||
translated = TranslationText(synced)
|
||||
translated.feed(source)
|
||||
original = TranslationText(synced)
|
||||
original.feed(english_source)
|
||||
if translated.messages != original.messages:
|
||||
differences.append("data-msg-* message attributes differ from the English page")
|
||||
for difference in differences:
|
||||
if fresh:
|
||||
errors.append(f"{label}: {difference}")
|
||||
else:
|
||||
warnings.append(translation_warning(path, root, difference))
|
||||
if fresh and translated.inline_code != original.inline_code:
|
||||
changed = sorted((translated.inline_code - original.inline_code) + (original.inline_code - translated.inline_code))
|
||||
warnings.append(translation_warning(
|
||||
path, root, f"inline code differs from the English page: {', '.join(repr(text) for text in changed[:5])}",
|
||||
))
|
||||
|
||||
if language in ("ja", "zh-hans", "zh-hant"):
|
||||
for text in translated.prose:
|
||||
for match in CJK_GAP.finditer(text):
|
||||
errors.append(
|
||||
f"{label}: whitespace between {match.group(1)!r} and {match.group(2)!r} renders as a gap; "
|
||||
"keep CJK text on one line without spaces"
|
||||
)
|
||||
|
||||
for reference in page.references:
|
||||
url = urlsplit(reference)
|
||||
if url.scheme or url.netloc or not url.path or reference.startswith("/"):
|
||||
continue
|
||||
target = (path.parent / unquote(url.path)).resolve()
|
||||
if target.is_dir():
|
||||
target /= "index.html"
|
||||
if not target.is_relative_to(site.resolve()):
|
||||
continue
|
||||
target_language, target_english = page_language(site.resolve(), target)
|
||||
if (
|
||||
target_language == "en"
|
||||
and target_english != english
|
||||
and state.translations.get((language, target_english)) == "reviewed"
|
||||
):
|
||||
warnings.append(translation_warning(
|
||||
path, root, f"links to the English {target_english}; link to its reviewed translation instead",
|
||||
))
|
||||
|
||||
english_strings = state.locales["en"]["strings"]
|
||||
for code in LANGUAGES:
|
||||
if code == "en" or not any(key[0] == code for key in state.translations):
|
||||
continue
|
||||
locale = state.locales[code]
|
||||
same = sorted(
|
||||
key for key, value in locale["strings"].items()
|
||||
if value == english_strings[key] and key not in locale.get("sameAsEnglish", [])
|
||||
)
|
||||
if same:
|
||||
warnings.append((f"website/locales/{code}.json", f"UI strings still in English: {', '.join(same)}"))
|
||||
return errors, warnings
|
||||
|
||||
|
||||
def report_warnings(warnings):
|
||||
"""Print warnings, as GitHub annotations and a job summary when running in Actions."""
|
||||
for file, message in warnings:
|
||||
print(f"Warning: {file}: {message}")
|
||||
if os.environ.get("GITHUB_ACTIONS") != "true" or not warnings:
|
||||
return
|
||||
for file, message in warnings:
|
||||
print(f"::warning file={file}::{message}")
|
||||
summary = os.environ.get("GITHUB_STEP_SUMMARY")
|
||||
if summary:
|
||||
with open(summary, "a", encoding="utf-8") as output:
|
||||
output.write("### Website warnings\n\n")
|
||||
output.writelines(f"- `{file}`: {message}\n" for file, message in warnings)
|
||||
|
||||
|
||||
def check_publisher(root):
|
||||
"""Detect a second Pages action that could replace the website."""
|
||||
expected = root / ".github/workflows/website-pages.yml"
|
||||
@@ -404,10 +611,16 @@ def main():
|
||||
errors.extend(check_example_archive(root / "website/public"))
|
||||
errors.extend(check_publisher(root))
|
||||
errors.extend(check_repository_links(root))
|
||||
translation_errors, warnings = check_translations(root, parse_pages(root / "website/public"))
|
||||
errors.extend(translation_errors)
|
||||
try:
|
||||
sync_pages(root, check=True)
|
||||
for path in sync_pages(root, check=True):
|
||||
warnings.append(translation_warning(
|
||||
path, root, "the out-of-date notice needs refreshing; run python3 scripts/website.py sync",
|
||||
))
|
||||
except (OSError, ValueError, KeyError) as error:
|
||||
errors.append(str(error))
|
||||
report_warnings(warnings)
|
||||
if errors:
|
||||
print("Website checks failed:\n" + "\n".join(errors), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
+442
-28
@@ -1,26 +1,53 @@
|
||||
"""Maintain shared HTML and run the static website's contributor commands."""
|
||||
|
||||
import argparse
|
||||
from collections import namedtuple
|
||||
from functools import partial
|
||||
from hashlib import sha256
|
||||
from html import escape
|
||||
from html.parser import HTMLParser
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
from pathlib import Path, PurePosixPath
|
||||
import posixpath
|
||||
import re
|
||||
from string import Template
|
||||
import subprocess
|
||||
import sys
|
||||
from textwrap import indent
|
||||
from urllib.parse import quote, urlsplit
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
|
||||
|
||||
SITE_ORIGIN = "https://rea.tools"
|
||||
NOTES_FIXTURES = {"examples/notes-electron/index.html", "examples/notes-web/index.html"}
|
||||
# Each label is a key in website/locales/<language>.json.
|
||||
NAVIGATION = (
|
||||
("Showcases", "showcase/"),
|
||||
("Blog", "blog/"),
|
||||
("Get started", "get-started/"),
|
||||
("Guides", "guides/"),
|
||||
("FAQ", "faq/"),
|
||||
("navShowcases", "showcase/"),
|
||||
("navBlog", "blog/"),
|
||||
("navGetStarted", "get-started/"),
|
||||
("navGuides", "guides/"),
|
||||
("navFaq", "faq/"),
|
||||
)
|
||||
Language = namedtuple("Language", "tag name og_locale")
|
||||
# English stays at the root; every other language lives under its directory.
|
||||
# The order is the language menu's order.
|
||||
LANGUAGES = {
|
||||
"en": Language("en", "English", "en_US"),
|
||||
"zh-hans": Language("zh-Hans", "简体中文", "zh_CN"),
|
||||
"zh-hant": Language("zh-Hant", "繁體中文", "zh_TW"),
|
||||
"ja": Language("ja", "日本語", "ja_JP"),
|
||||
"ko": Language("ko", "한국어", "ko_KR"),
|
||||
"de": Language("de", "Deutsch", "de_DE"),
|
||||
"fr": Language("fr", "Français", "fr_FR"),
|
||||
}
|
||||
TRANSLATION_STATUSES = ("draft", "reviewed")
|
||||
TRANSLATION_RECORD = re.compile(r"<!-- website:translation((?:\s+[\w-]+=\"[^\"]*\")*)\s*-->")
|
||||
SHARED_REGION = re.compile(r"<!-- website:([\w-]+):start -->.*?<!-- website:\1:end -->", re.S)
|
||||
NOTICE_START = " <!-- website:translation-notice:start -->\n"
|
||||
NOTICE_END = " <!-- website:translation-notice:end -->"
|
||||
GLOBE_PATH = (
|
||||
"M12 3a9 9 0 1 0 0 18a9 9 0 1 0 0-18Z M3.6 9h16.8 M3.6 15h16.8 "
|
||||
"M12 3c2.4 2.5 3.6 5.5 3.6 9s-1.2 6.5-3.6 9c-2.4-2.5-3.6-5.5-3.6-9s1.2-6.5 3.6-9Z"
|
||||
)
|
||||
|
||||
|
||||
@@ -75,10 +102,14 @@ class PageMetadata(HTMLParser):
|
||||
self.titles[-1] += data
|
||||
|
||||
|
||||
def file_route(relative):
|
||||
"""Map a site-relative HTML file to its public route."""
|
||||
return relative.removesuffix("index.html") if PurePosixPath(relative).name == "index.html" else relative
|
||||
|
||||
|
||||
def page_route(site, path):
|
||||
"""Map an HTML file to its public route without a separate page registry."""
|
||||
route = path.relative_to(site).as_posix()
|
||||
return route.removesuffix("index.html") if path.name == "index.html" else route
|
||||
return file_route(path.relative_to(site).as_posix())
|
||||
|
||||
|
||||
def page_url(site, path):
|
||||
@@ -86,30 +117,227 @@ def page_url(site, path):
|
||||
return f"{SITE_ORIGIN}/{quote(page_route(site, path), safe='/')}"
|
||||
|
||||
|
||||
def shared_blocks(root, path, source):
|
||||
def page_language(site, path):
|
||||
"""Return a page's language directory and the site-relative file of its English source."""
|
||||
relative = path.relative_to(site).as_posix()
|
||||
first, separator, rest = relative.partition("/")
|
||||
if separator and first in LANGUAGES and first != "en":
|
||||
return first, rest
|
||||
return "en", relative
|
||||
|
||||
|
||||
def load_locales(root):
|
||||
"""Read the UI strings for every language and require the same keys as English."""
|
||||
directory = root / "website/locales"
|
||||
locales = {}
|
||||
for code in LANGUAGES:
|
||||
path = directory / f"{code}.json"
|
||||
try:
|
||||
locale = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as error:
|
||||
raise ValueError(f"website/locales/{code}.json: cannot read UI strings: {error}") from error
|
||||
locales[code] = locale
|
||||
expected = set(locales["en"]["strings"])
|
||||
for code, locale in locales.items():
|
||||
label = f"website/locales/{code}.json"
|
||||
strings = locale.get("strings")
|
||||
if not isinstance(strings, dict) or set(strings) != expected:
|
||||
missing = sorted(expected - set(strings or {}))
|
||||
extra = sorted(set(strings or {}) - expected)
|
||||
raise ValueError(f"{label}: strings must match en.json (missing {missing}, extra {extra})")
|
||||
if any(not isinstance(value, str) or not value.strip() for value in strings.values()):
|
||||
raise ValueError(f"{label}: every UI string must be nonempty text")
|
||||
stars = locale.get("githubStars")
|
||||
if not isinstance(stars, dict) or "other" not in stars or not set(stars) <= {"zero", "one", "two", "few", "many", "other"}:
|
||||
raise ValueError(f"{label}: githubStars needs an 'other' label and CLDR plural categories only")
|
||||
if any("{count}" not in value for value in stars.values()):
|
||||
raise ValueError(f"{label}: every githubStars label needs a {{count}} placeholder")
|
||||
if "{link}" not in strings["staleNotice"]:
|
||||
raise ValueError(f"{label}: staleNotice needs a {{link}} placeholder")
|
||||
same = locale.get("sameAsEnglish", [])
|
||||
if not isinstance(same, list) or not set(same) <= expected:
|
||||
raise ValueError(f"{label}: sameAsEnglish must list UI string keys")
|
||||
return locales
|
||||
|
||||
|
||||
def translation_record(source):
|
||||
"""Return the attributes of a translation's provenance comment, or None."""
|
||||
records = TRANSLATION_RECORD.findall(source)
|
||||
if len(records) != 1:
|
||||
return None
|
||||
return dict(re.findall(r'([\w-]+)="([^"]*)"', records[0]))
|
||||
|
||||
|
||||
def copy_targets(source):
|
||||
"""Return the IDs referenced by copy buttons, in page order."""
|
||||
return list(dict.fromkeys(re.findall(r'\bdata-copy="([^"]+)"', source)))
|
||||
|
||||
|
||||
def code_block(identifier):
|
||||
"""Match a code element by ID, capturing its opening tag, contents and end tag."""
|
||||
return re.compile(rf'(<code\b[^>]*\bid="{re.escape(identifier)}"[^>]*>)(.*?)(</code>)', re.S)
|
||||
|
||||
|
||||
def synced_code_ids(source):
|
||||
"""Return the copy targets that are code blocks; translations take them from English."""
|
||||
return [identifier for identifier in copy_targets(source) if code_block(identifier).search(source)]
|
||||
|
||||
|
||||
def source_digest(source):
|
||||
"""Hash an English page's translatable content.
|
||||
|
||||
Shared regions and copied code blocks are left out, so template edits and
|
||||
command-only changes do not mark translations out of date.
|
||||
"""
|
||||
content = SHARED_REGION.sub("", source)
|
||||
for identifier in synced_code_ids(source):
|
||||
content = code_block(identifier).sub(r"\1\3", content)
|
||||
return sha256(" ".join(content.split()).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def sync_code_blocks(english, source, label):
|
||||
"""Copy every synced code block's contents from the English source, matched by ID."""
|
||||
for identifier in synced_code_ids(english):
|
||||
contents = code_block(identifier).search(english).group(2)
|
||||
pattern = code_block(identifier)
|
||||
if len(pattern.findall(source)) != 1:
|
||||
raise ValueError(f"{label}: needs one <code id=\"{identifier}\"> block copied from the English page")
|
||||
source = pattern.sub(lambda match: match.group(1) + contents + match.group(3), source)
|
||||
return source
|
||||
|
||||
|
||||
class SiteState:
|
||||
"""Read the locales and every translation's status once for a sync or check."""
|
||||
|
||||
def __init__(self, root):
|
||||
self.root = root
|
||||
self.site = root / "website/public"
|
||||
self.locales = load_locales(root)
|
||||
self.translations = {}
|
||||
for path in sorted(self.site.rglob("*.html")):
|
||||
language, english = page_language(self.site, path)
|
||||
if language != "en":
|
||||
record = translation_record(path.read_text(encoding="utf-8")) or {}
|
||||
self.translations[(language, english)] = record.get("status")
|
||||
|
||||
def linkable(self, language, english, from_draft=False):
|
||||
"""Report whether other pages may link to this language's version of a page."""
|
||||
if language == "en":
|
||||
return (self.site / english).is_file()
|
||||
status = self.translations.get((language, english))
|
||||
return status == "reviewed" or (from_draft and status == "draft")
|
||||
|
||||
def alternates(self, english):
|
||||
"""Return the languages whose versions of a page join its hreflang cluster."""
|
||||
return [code for code in LANGUAGES if code == "en" or self.translations.get((code, english)) == "reviewed"]
|
||||
|
||||
|
||||
def language_url(language, english):
|
||||
"""Return the canonical URL of one language's version of a page."""
|
||||
prefix = "" if language == "en" else f"{language}/"
|
||||
return f"{SITE_ORIGIN}/{quote(prefix + file_route(english), safe='/')}"
|
||||
|
||||
|
||||
def shared_blocks(root, path, source, state=None):
|
||||
"""Render shared regions while keeping the page's body and metadata authored."""
|
||||
site = root / "website/public"
|
||||
state = state or SiteState(root)
|
||||
site = state.site
|
||||
metadata = PageMetadata()
|
||||
metadata.feed(source)
|
||||
for name, values in (("title", metadata.titles), ("description", metadata.descriptions)):
|
||||
if len(values) != 1 or not values[0].strip():
|
||||
raise ValueError(f"{path.relative_to(site)}: needs one nonempty head {name}")
|
||||
language, english = page_language(site, path)
|
||||
strings = state.locales[language]["strings"]
|
||||
depth = len(path.relative_to(site).parent.parts)
|
||||
base = "../" * depth or "./"
|
||||
route = page_route(site, path)
|
||||
route = file_route(english)
|
||||
draft = language != "en" and state.translations.get((language, english)) == "draft"
|
||||
|
||||
def href(target_language, target):
|
||||
prefix = "" if target_language == "en" else f"{target_language}/"
|
||||
return base + file_route(prefix + target)
|
||||
|
||||
def localized(target):
|
||||
"""Prefer this language's version of a target page, else the English page."""
|
||||
if language != "en" and state.linkable(language, target, from_draft=draft):
|
||||
return language, href(language, target)
|
||||
return "en", href("en", target)
|
||||
|
||||
links = []
|
||||
for label, destination in NAVIGATION:
|
||||
for key, destination in NAVIGATION:
|
||||
target_language, url = localized(destination + "index.html")
|
||||
attributes = ' class="nav-start"' if destination == "get-started/" else ""
|
||||
if route.startswith(destination):
|
||||
attributes += ' aria-current="page"'
|
||||
links.append(f' <a href="{base}{destination}"{attributes}>{label}</a>')
|
||||
badge = ""
|
||||
if target_language != language:
|
||||
attributes += ' hreflang="en"'
|
||||
badge = '<span class="nav-badge" lang="en">EN</span>'
|
||||
links.append(f' <a href="{url}"{attributes}>{escape(strings[key])}{badge}</a>')
|
||||
|
||||
entries = []
|
||||
for code, details in LANGUAGES.items():
|
||||
if code == language or state.linkable(code, english):
|
||||
url = href(code, english)
|
||||
elif state.linkable(code, "index.html"):
|
||||
url = href(code, "index.html")
|
||||
else:
|
||||
continue
|
||||
current = ' aria-current="true"' if code == language else ""
|
||||
entries.append(
|
||||
f' <li><a href="{url}" hreflang="{details.tag}" lang="{details.tag}"{current}>{details.name}</a></li>'
|
||||
)
|
||||
language_menu = ""
|
||||
if len(entries) > 1:
|
||||
language_menu = "\n".join((
|
||||
"",
|
||||
' <details class="language-menu">',
|
||||
" <summary>",
|
||||
' <svg width="18" height="18" viewBox="0 0 24 24" aria-hidden="true" focusable="false">',
|
||||
f' <path fill="none" stroke="currentColor" stroke-width="1.6" d="{GLOBE_PATH}" />',
|
||||
" </svg>",
|
||||
f' <span class="sr-only">{escape(strings["language"])}: </span>'
|
||||
f'<span class="language-current">{LANGUAGES[language].name}</span>',
|
||||
" </summary>",
|
||||
' <ul class="language-list">',
|
||||
*entries,
|
||||
" </ul>",
|
||||
" </details>",
|
||||
))
|
||||
|
||||
members = [] if draft else state.alternates(english)
|
||||
hreflang = ""
|
||||
locale_alternates = ""
|
||||
if len(members) > 1:
|
||||
for code in [*members, "x-default"]:
|
||||
tag = LANGUAGES[code].tag if code != "x-default" else code
|
||||
url = language_url(code if code != "x-default" else "en", english)
|
||||
hreflang += f'\n<!-- prettier-ignore -->\n<link rel="alternate" hreflang="{tag}" href="{escape(url, quote=True)}" />'
|
||||
for code in members:
|
||||
if code != language:
|
||||
locale_alternates += (
|
||||
f'\n<!-- prettier-ignore -->\n<meta property="og:locale:alternate" content="{LANGUAGES[code].og_locale}" />'
|
||||
)
|
||||
|
||||
labels = "".join(
|
||||
f'\n data-label-{category}="{escape(label, quote=True)}"'
|
||||
for category, label in state.locales[language]["githubStars"].items()
|
||||
)
|
||||
values = {
|
||||
"base": base,
|
||||
"brand": localized("index.html")[1],
|
||||
"navigation": "\n" + "\n".join(links) + "\n ",
|
||||
"language_menu": language_menu,
|
||||
"canonical": escape(page_url(site, path), quote=True),
|
||||
"hreflang": hreflang,
|
||||
"og_locale": LANGUAGES[language].og_locale,
|
||||
"og_locale_alternates": locale_alternates,
|
||||
"origin": SITE_ORIGIN,
|
||||
"title": escape(metadata.titles[0].strip(), quote=True),
|
||||
"description": escape(metadata.descriptions[0].strip(), quote=True),
|
||||
"github_star_labels": labels,
|
||||
**{f"t_{key}": escape(value, quote=True) for key, value in strings.items()},
|
||||
}
|
||||
templates = root / "website/templates"
|
||||
return {
|
||||
@@ -118,35 +346,89 @@ def shared_blocks(root, path, source):
|
||||
}
|
||||
|
||||
|
||||
def translation_notice(state, path, english_source, record):
|
||||
"""Render the stale-translation notice, or nothing while the source hash matches."""
|
||||
if record is None or record.get("source-sha256") == source_digest(english_source):
|
||||
return ""
|
||||
language, english = page_language(state.site, path)
|
||||
strings = state.locales[language]["strings"]
|
||||
depth = len(path.relative_to(state.site).parent.parts)
|
||||
link = (
|
||||
f'<a href="{"../" * depth}{file_route(english)}" hreflang="en">'
|
||||
f'{escape(strings["staleNoticeLink"])}</a>'
|
||||
)
|
||||
message = escape(strings["staleNotice"]).replace("{link}", link)
|
||||
return f' <!-- prettier-ignore -->\n <p class="translation-notice" role="note">{message}</p>\n'
|
||||
|
||||
|
||||
def replace_region(source, start, end, block, label, name):
|
||||
"""Replace the contents of one marked region."""
|
||||
if source.count(start) != 1 or source.count(end) != 1:
|
||||
raise ValueError(f"{label}: needs one {name} region; use new-page or translate for a new page")
|
||||
before, _, remaining = source.partition(start)
|
||||
_, found, after = remaining.partition(end)
|
||||
if not found:
|
||||
raise ValueError(f"{label}: {name} region is out of order")
|
||||
return before + start + block + end + after
|
||||
|
||||
|
||||
def region_contents(source, start, end):
|
||||
"""Return the current contents of a marked region."""
|
||||
return source.partition(start)[2].partition(end)[0]
|
||||
|
||||
|
||||
def sync_pages(root, check=False):
|
||||
"""Update shared regions, or reject drift without changing any authored file."""
|
||||
site = root / "website/public"
|
||||
"""Update shared regions, or reject drift without changing any authored file.
|
||||
|
||||
Returns the pages that changed. With check=True, structural drift raises, so
|
||||
the returned pages are translations whose only drift is the stale notice;
|
||||
an out-of-date translation is a warning rather than a failure.
|
||||
"""
|
||||
state = SiteState(root)
|
||||
site = state.site
|
||||
changes = []
|
||||
notices = []
|
||||
for path in sorted(site.rglob("*.html")):
|
||||
if path.relative_to(site).as_posix() in NOTES_FIXTURES:
|
||||
continue
|
||||
label = path.relative_to(site).as_posix()
|
||||
original = path.read_text(encoding="utf-8")
|
||||
source = original
|
||||
for name, block in shared_blocks(root, path, source).items():
|
||||
start = f" <!-- website:{name}:start -->\n"
|
||||
end = f" <!-- website:{name}:end -->"
|
||||
if source.count(start) != 1 or source.count(end) != 1:
|
||||
raise ValueError(f"{path.relative_to(site)}: needs one {name} region; use new-page for a new reading page")
|
||||
before, _, remaining = source.partition(start)
|
||||
_, found, after = remaining.partition(end)
|
||||
if not found:
|
||||
raise ValueError(f"{path.relative_to(site)}: {name} region is out of order")
|
||||
source = before + start + block + end + after
|
||||
language, english = page_language(site, path)
|
||||
english_source = None
|
||||
if language != "en":
|
||||
english_path = site / english
|
||||
if not english_path.is_file():
|
||||
raise ValueError(f"{label}: translates {english}, which does not exist")
|
||||
english_source = english_path.read_text(encoding="utf-8")
|
||||
source = sync_code_blocks(english_source, source, label)
|
||||
for name, block in shared_blocks(root, path, source, state).items():
|
||||
source = replace_region(
|
||||
source, f" <!-- website:{name}:start -->\n", f" <!-- website:{name}:end -->", block, label, name
|
||||
)
|
||||
if language != "en":
|
||||
notice = translation_notice(state, path, english_source, translation_record(source))
|
||||
structural = replace_region(
|
||||
source, NOTICE_START, NOTICE_END, region_contents(original, NOTICE_START, NOTICE_END), label,
|
||||
"translation-notice",
|
||||
)
|
||||
source = replace_region(source, NOTICE_START, NOTICE_END, notice, label, "translation-notice")
|
||||
if structural == original and source != original:
|
||||
notices.append((path, source))
|
||||
continue
|
||||
if source != original:
|
||||
changes.append((path, source))
|
||||
if check and changes:
|
||||
names = ", ".join(str(path.relative_to(site)) for path, _ in changes)
|
||||
raise ValueError(f"Shared HTML is stale: {names}. Run python3 scripts/website.py sync and commit the updated pages.")
|
||||
if not check:
|
||||
for path, source in changes:
|
||||
for path, source in changes + notices:
|
||||
path.write_text(source, encoding="utf-8")
|
||||
print(f"{'Checked' if check else 'Synced'} shared HTML ({len(changes)} pages changed).")
|
||||
return [path for path, _ in changes]
|
||||
if check:
|
||||
print(f"Checked shared HTML ({len(notices)} translation notices to refresh).")
|
||||
else:
|
||||
print(f"Synced shared HTML ({len(changes) + len(notices)} pages changed).")
|
||||
return [path for path, _ in (notices if check else changes + notices)]
|
||||
|
||||
|
||||
def new_page(root, route, title, description):
|
||||
@@ -154,6 +436,8 @@ def new_page(root, route, title, description):
|
||||
parts = PurePosixPath(route)
|
||||
if not route or parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
|
||||
raise ValueError("Use a relative directory route, such as blog/my-article.")
|
||||
if parts.parts[0] in LANGUAGES:
|
||||
raise ValueError("Create the English page first, then use translate for another language.")
|
||||
site = root / "website/public"
|
||||
path = site.joinpath(*parts.parts, "index.html")
|
||||
if not path.resolve().is_relative_to(site.resolve()):
|
||||
@@ -174,11 +458,131 @@ def new_page(root, route, title, description):
|
||||
return path
|
||||
|
||||
|
||||
def relocate_references(source, english, language, state):
|
||||
"""Rewrite relative references for the translation's directory.
|
||||
|
||||
Pages that already have a version in this language link to it; everything
|
||||
else keeps pointing at the English page or asset.
|
||||
"""
|
||||
english_directory = posixpath.dirname(english)
|
||||
translation_directory = posixpath.join(language, english_directory)
|
||||
|
||||
def rewrite(match):
|
||||
value = match.group(2)
|
||||
url = urlsplit(value)
|
||||
if not value or url.scheme or url.netloc or value.startswith(("#", "/")):
|
||||
return match.group(0)
|
||||
target = posixpath.normpath(posixpath.join(english_directory, url.path))
|
||||
directory = url.path.endswith("/") or target == "."
|
||||
if directory:
|
||||
page = "index.html" if target == "." else f"{target}/index.html"
|
||||
if (language, page) in state.translations:
|
||||
target = posixpath.normpath(posixpath.join(language, target))
|
||||
relative = posixpath.relpath(target, translation_directory or ".")
|
||||
if directory:
|
||||
relative = "./" if relative == "." else relative + "/"
|
||||
return f'{match.group(1)}="{urlunsplit(("", "", relative, url.query, url.fragment))}"'
|
||||
|
||||
return re.sub(r'\b(href|src)="([^"]*)"', rewrite, source)
|
||||
|
||||
|
||||
def new_translation(root, language, route):
|
||||
"""Create a draft translation of an English page, ready to be translated in place."""
|
||||
if language not in LANGUAGES or language == "en":
|
||||
raise ValueError(f"Choose one of: {', '.join(code for code in LANGUAGES if code != 'en')}.")
|
||||
route = route.strip("/")
|
||||
parts = PurePosixPath(route) if route not in ("", ".") else PurePosixPath()
|
||||
if parts.is_absolute() or ".." in parts.parts or "\\" in route or parts.suffix:
|
||||
raise ValueError("Use the English page's directory route, such as get-started, or / for the home page.")
|
||||
if parts.parts and parts.parts[0] in LANGUAGES:
|
||||
raise ValueError("Name the English route, without a language directory.")
|
||||
english = PurePosixPath(*parts.parts, "index.html").as_posix()
|
||||
state = SiteState(root)
|
||||
site = state.site
|
||||
source_path = site / english
|
||||
if not source_path.is_file() or english in NOTES_FIXTURES:
|
||||
raise ValueError(f"No English page to translate at {english}.")
|
||||
path = site / language / english
|
||||
if path.exists():
|
||||
raise ValueError(f"Translation already exists: {path.relative_to(site)}")
|
||||
source = source_path.read_text(encoding="utf-8")
|
||||
if source.count('<html lang="en">') != 1 or source.count(" <!-- website:sharing:start -->\n") != 1:
|
||||
raise ValueError(f"{english}: needs <html lang=\"en\"> and one sharing region")
|
||||
main = re.search(r"^ {4}<main\b[^>]*>\n", source, re.M)
|
||||
if main is None:
|
||||
raise ValueError(f"{english}: needs a <main> element on its own line")
|
||||
source = source.replace('<html lang="en">', f'<html lang="{LANGUAGES[language].tag}">')
|
||||
source = relocate_references(source, english, language, state)
|
||||
record = (
|
||||
f' <!-- website:translation source-sha256="{source_digest(source_path.read_text(encoding="utf-8"))}" '
|
||||
'status="draft" -->\n'
|
||||
)
|
||||
robots = "" if '<meta name="robots"' in source else ' <meta name="robots" content="noindex" />\n'
|
||||
source = source.replace(" <!-- website:sharing:start -->\n", robots + record + " <!-- website:sharing:start -->\n")
|
||||
main = re.search(r"^ {4}<main\b[^>]*>\n", source, re.M)
|
||||
source = source[: main.end()] + NOTICE_START + NOTICE_END + "\n" + source[main.end():]
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(source, encoding="utf-8")
|
||||
sync_pages(root)
|
||||
print(
|
||||
f"Created {path.relative_to(root)} as a noindex draft. Translate its text in place following "
|
||||
"website/translation.md, then run python3 scripts/website.py check."
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def translation_report(root, language=None):
|
||||
"""Describe each translation's review status, freshness and remaining review items."""
|
||||
state = SiteState(root)
|
||||
site = state.site
|
||||
lines = []
|
||||
for (code, english), status in sorted(state.translations.items()):
|
||||
if language and code != language:
|
||||
continue
|
||||
path = site / code / english
|
||||
source = path.read_text(encoding="utf-8")
|
||||
record = translation_record(source) or {}
|
||||
english_path = site / english
|
||||
current = source_digest(english_path.read_text(encoding="utf-8")) if english_path.is_file() else None
|
||||
fresh = record.get("source-sha256") == current
|
||||
reviewed = f" on {record['reviewed']}" if record.get("reviewed") else ""
|
||||
lines.append(f"{code}/{file_route(english) or '(home)'}: {status or 'no record'}{reviewed}, "
|
||||
f"{'up to date' if fresh else 'English source changed'}")
|
||||
if not fresh and current:
|
||||
lines.append(f" after updating the translation, record source-sha256=\"{current}\"")
|
||||
if status != "reviewed":
|
||||
for system, text in re.findall(r'<(?:em|span|strong|kbd)\b[^>]*\bdata-os-ui="([^"]+)"[^>]*>(.*?)</', source, re.S):
|
||||
lines.append(f" confirm on {system}: {' '.join(text.split())}")
|
||||
english_strings = state.locales["en"]["strings"]
|
||||
for code in LANGUAGES:
|
||||
if code == "en" or (language and code != language):
|
||||
continue
|
||||
if not any(key[0] == code for key in state.translations):
|
||||
continue
|
||||
locale = state.locales[code]
|
||||
same = sorted(
|
||||
key for key, value in locale["strings"].items()
|
||||
if value == english_strings[key] and key not in locale.get("sameAsEnglish", [])
|
||||
)
|
||||
if same:
|
||||
lines.append(f"{code}: UI strings still in English: {', '.join(same)}")
|
||||
print("\n".join(lines) or "No translated pages.")
|
||||
return lines
|
||||
|
||||
|
||||
def run_script(root, name, arguments=()):
|
||||
"""Use the active Python environment for the existing preparation and checks."""
|
||||
subprocess.run([sys.executable, str(root / "scripts" / name), *arguments], cwd=root, check=True)
|
||||
|
||||
|
||||
def run_browser_script_tests(root):
|
||||
"""Run the page-script regressions with the repository's Node.js."""
|
||||
try:
|
||||
subprocess.run(["node", str(root / "scripts/test-website-scripts.mjs")], cwd=root, check=True)
|
||||
except FileNotFoundError as error:
|
||||
raise OSError("The website script tests need Node.js; see website/README.md.") from error
|
||||
|
||||
|
||||
def preview_handler(site, prefix):
|
||||
"""Serve one static directory at root or a Pages prefix, preserving redirects."""
|
||||
class Handler(SimpleHTTPRequestHandler):
|
||||
@@ -207,6 +611,11 @@ def main():
|
||||
create.add_argument("route")
|
||||
create.add_argument("--title", required=True)
|
||||
create.add_argument("--description", required=True)
|
||||
translate = commands.add_parser("translate", help="Create a draft translation of an English page")
|
||||
translate.add_argument("language", choices=[code for code in LANGUAGES if code != "en"])
|
||||
translate.add_argument("route", help="The English page's directory route, or / for the home page")
|
||||
report = commands.add_parser("translations", help="List translation status, freshness and review items")
|
||||
report.add_argument("--lang", choices=[code for code in LANGUAGES if code != "en"])
|
||||
serve = commands.add_parser("serve", help="Prepare and preview the static site")
|
||||
serve.add_argument("--port", type=int, default=4173)
|
||||
serve.add_argument("--base-path", choices=("/", "/rea/"), default="/")
|
||||
@@ -215,12 +624,17 @@ def main():
|
||||
try:
|
||||
if args.command == "new-page":
|
||||
new_page(root, args.route, args.title, args.description)
|
||||
elif args.command == "translate":
|
||||
new_translation(root, args.language, args.route)
|
||||
elif args.command == "translations":
|
||||
translation_report(root, args.lang)
|
||||
elif args.command == "sync":
|
||||
sync_pages(root, check=args.check)
|
||||
elif args.command == "check":
|
||||
sync_pages(root, check=True)
|
||||
for name in ("prepare-website.py", "verify-website.py", "test-website.py"):
|
||||
run_script(root, name)
|
||||
run_browser_script_tests(root)
|
||||
elif args.command == "links":
|
||||
run_script(root, "audit-website-links.py", ["--report", args.report] if args.report else [])
|
||||
else:
|
||||
|
||||
@@ -4,7 +4,7 @@ import type {
|
||||
AnalysisOperationPort,
|
||||
} from "./AnalysisProvider.js";
|
||||
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { analysisBookmarkSchema } from "../domain/hopperValues.js";
|
||||
import { resolveAnalysisDocument } from "./AnalysisDocument.js";
|
||||
|
||||
@@ -12,7 +12,7 @@ import type {
|
||||
CompareWebScreenshotsInput,
|
||||
} from "../domain/webScreenshot.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type { ExecutionOptions } from "./AnalysisProvider.js";
|
||||
import type { BrowserObservationPort } from "./BrowserObservationPort.js";
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { ExecutionOptions } from "./AnalysisProvider.js";
|
||||
import type { BrowserScenarioCapturePort } from "./BrowserScenarioCapturePort.js";
|
||||
import { createBrowserScenarioEvidence } from "./BrowserScenarioEvidence.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../domain/evidence.js";
|
||||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import { err, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { AnalysisOperation } from "./AnalysisProvider.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { parseRelatedAddresses } from "../domain/hopperValues.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
import type { ExecutableFormatHint } from "../domain/dosCom.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { EnhancedTools } from "./EnhancedTools.js";
|
||||
import { executeFunctionAnalysisEvidence } from "./FunctionAnalysisEvidence.js";
|
||||
import type { DirectAnalysisDependencies } from "./DirectAnalysisDependencies.js";
|
||||
import type { BinarySession } from "./binary/BinarySession.js";
|
||||
import type { SessionProviderRoute } from "./binary/SessionProviderRouter.js";
|
||||
@@ -18,7 +16,7 @@ import {
|
||||
} from "../contracts/managed/managedToolContracts.js";
|
||||
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { access } from "node:fs/promises";
|
||||
import {
|
||||
readAnalysisSnapshot,
|
||||
@@ -332,6 +330,8 @@ const executeAnalysisTool = async (input: {
|
||||
}): Promise<{ readonly output: JsonValue; readonly evidence?: Evidence }> => {
|
||||
const { session, tool, signal, evidenceProfile } = input;
|
||||
if (tool === "analyze_function") {
|
||||
const { executeFunctionAnalysisEvidence } =
|
||||
await import("./FunctionAnalysisEvidence.js");
|
||||
const result = await executeFunctionAnalysisEvidence(
|
||||
session,
|
||||
input.arguments,
|
||||
@@ -350,6 +350,7 @@ const executeAnalysisTool = async (input: {
|
||||
tool === "trace_native_values" ||
|
||||
tool === "trace_native_ui_action"
|
||||
) {
|
||||
const { EnhancedTools } = await import("./EnhancedTools.js");
|
||||
const result = await new EnhancedTools(session).execute(
|
||||
tool,
|
||||
input.arguments,
|
||||
@@ -450,7 +451,7 @@ const prepareSnapshot = async (options: {
|
||||
const { path, snapshotPath } = options;
|
||||
if (snapshotPath === undefined || !(await fileExists(snapshotPath)))
|
||||
return ok({});
|
||||
const loaded = await readAnalysisSnapshot(snapshotPath);
|
||||
const loaded = await readAnalysisSnapshot(snapshotPath, options.signal);
|
||||
if (!loaded.ok) return loaded;
|
||||
const target = await parseBinaryTarget(path, {
|
||||
signal: options.signal,
|
||||
|
||||
@@ -36,8 +36,6 @@ import {
|
||||
type DoctorScopeReport,
|
||||
} from "./DoctorScope.js";
|
||||
|
||||
/** Deep provider IDs whose readiness can be selected explicitly by doctor. */
|
||||
export const DOCTOR_PROVIDER_IDS = ["hopper", "ghidra", "ida"] as const;
|
||||
/** One actionable environment diagnostic. */
|
||||
export interface DoctorCheck {
|
||||
readonly name: string;
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
/** Deep-analysis providers whose readiness can be selected explicitly by doctor. */
|
||||
export const DOCTOR_PROVIDER_IDS = ["hopper", "ghidra", "ida"] as const;
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
import type { AnalysisOperation } from "./AnalysisProvider.js";
|
||||
import { AnalysisOutputError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import type {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { z } from "zod";
|
||||
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { err, type Result } from "../domain/result.js";
|
||||
import {
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
AnalysisOutputError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import {
|
||||
addressDistance,
|
||||
functionDossierSchema,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { EvidenceIntegrityError } from "../domain/evidenceErrors.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { EvidenceBundle } from "../domain/evidenceBundle.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { jsonValueSchema } from "../domain/jsonValue.js";
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
createImmutableEvidenceBundle,
|
||||
describeEvidenceBundleFailure,
|
||||
describeValidationFailure,
|
||||
parseEvidenceBundle,
|
||||
@@ -30,14 +31,29 @@ type EvidenceWriteFailure =
|
||||
| AnalysisCancelledError;
|
||||
|
||||
/** Read and validate an evidence bundle at the caller-supplied path. */
|
||||
export const readEvidenceBundle = async (
|
||||
export const readEvidenceBundle = (
|
||||
path: string,
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
|
||||
readBundle(path, parseEvidenceBundle);
|
||||
|
||||
/** Read an authenticated immutable snapshot for a subsequent session import. */
|
||||
export const readImmutableEvidenceBundle = (
|
||||
path: string,
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> =>
|
||||
readBundle(path, (input) => {
|
||||
const bundle = parseEvidenceBundle(input);
|
||||
return createImmutableEvidenceBundle(bundle.records, bundle.unknowns);
|
||||
});
|
||||
|
||||
const readBundle = async (
|
||||
path: string,
|
||||
parseBundle: (input: unknown) => EvidenceBundle,
|
||||
): Promise<Result<EvidenceBundle, EvidenceReadFailure>> => {
|
||||
const loaded = await readJsonFile(path);
|
||||
if (!loaded.ok) return loaded;
|
||||
let bundle: EvidenceBundle;
|
||||
try {
|
||||
bundle = parseEvidenceBundle(loaded.value);
|
||||
bundle = parseBundle(loaded.value);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Evidence bundle validation failed", {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import {
|
||||
createEvidence,
|
||||
type Evidence,
|
||||
|
||||
@@ -16,7 +16,11 @@ import {
|
||||
AnalysisResourceConstraintError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { readJsonInputFile } from "./JsonInputFile.js";
|
||||
import {
|
||||
readJsonInputFile,
|
||||
readJsonObjectFile,
|
||||
type JsonObjectMemberReader,
|
||||
} from "./JsonInputFile.js";
|
||||
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
|
||||
|
||||
/** Request control and its owning operation for an interruptible atomic write. */
|
||||
@@ -33,13 +37,47 @@ export const readJsonFile = async (
|
||||
path: string,
|
||||
): Promise<
|
||||
Result<unknown, EvidenceFileError | AnalysisResourceConstraintError>
|
||||
> =>
|
||||
readJsonFileInput(path, (selected) =>
|
||||
readJsonInputFile(selected, "read_evidence_file"),
|
||||
);
|
||||
|
||||
/** Read selected JSON members through the same regular-file error boundary. */
|
||||
export const readJsonObjectMembersFile = (
|
||||
path: string,
|
||||
scans: readonly ReadonlyMap<string, JsonObjectMemberReader>[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<
|
||||
void,
|
||||
EvidenceFileError | AnalysisResourceConstraintError | AnalysisCancelledError
|
||||
>
|
||||
> =>
|
||||
readJsonFileInput(
|
||||
path,
|
||||
(selected) =>
|
||||
readJsonObjectFile(selected, "read_evidence_file", scans, signal),
|
||||
signal,
|
||||
);
|
||||
|
||||
const readJsonFileInput = async <Value>(
|
||||
path: string,
|
||||
read: (
|
||||
selected: string,
|
||||
) => Promise<
|
||||
Result<
|
||||
Value,
|
||||
| import("../domain/analysisErrorCore.js").AnalysisInputError
|
||||
| AnalysisResourceConstraintError
|
||||
>
|
||||
>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<Value, EvidenceFileError | AnalysisResourceConstraintError>
|
||||
> => {
|
||||
const requestedPath = resolve(path);
|
||||
try {
|
||||
const decoded = await readJsonInputFile(
|
||||
requestedPath,
|
||||
"read_evidence_file",
|
||||
);
|
||||
const decoded = await read(requestedPath);
|
||||
if (!decoded.ok) {
|
||||
if (decoded.error instanceof AnalysisResourceConstraintError)
|
||||
return err(decoded.error);
|
||||
@@ -57,6 +95,7 @@ export const readJsonFile = async (
|
||||
return ok(decoded.value);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
|
||||
if (signal?.aborted === true) throw cause;
|
||||
return err(
|
||||
new EvidenceFileError(
|
||||
"read",
|
||||
|
||||
+400
-184
@@ -1,10 +1,14 @@
|
||||
import { constants } from "node:buffer";
|
||||
import { createHash } from "node:crypto";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { pipeline } from "node:stream/promises";
|
||||
import { getHeapStatistics } from "node:v8";
|
||||
|
||||
import { asStream, none } from "stream-chain";
|
||||
import fun from "stream-chain/fun.js";
|
||||
import parser from "stream-json/parser.js";
|
||||
import streamValues from "stream-json/streamers/stream-values.js";
|
||||
import streamArray from "stream-json/streamers/stream-array.js";
|
||||
|
||||
import { RegularFileChangedError } from "../filesystem/RegularFile.js";
|
||||
import { withRegularFile } from "./RegularFileRead.js";
|
||||
@@ -37,6 +41,196 @@ const nativeParseFits = (inputBytes: number): boolean =>
|
||||
inputBytes * 72 + READ_CHUNK_BYTES <=
|
||||
getHeapStatistics().total_available_size;
|
||||
|
||||
/** One selected root member; repeated keys begin again with last-value semantics. */
|
||||
export interface JsonObjectMemberReader {
|
||||
readonly mode: "value" | "array";
|
||||
readonly begin: () => void;
|
||||
readonly accept: (value: unknown) => void | Promise<void>;
|
||||
readonly reject: (cause: unknown) => void;
|
||||
}
|
||||
|
||||
/** Scan selected members without retaining skipped subtrees or prior array items. */
|
||||
export const readJsonObjectFile = (
|
||||
path: string,
|
||||
operation: string,
|
||||
scans: readonly ReadonlyMap<string, JsonObjectMemberReader>[],
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<void, JsonInputFileFailure>> =>
|
||||
withRegularFile(
|
||||
path,
|
||||
async (handle, stats) => {
|
||||
let previousDigest: string | undefined;
|
||||
try {
|
||||
const native = await readNativeJson(
|
||||
handle,
|
||||
stats.size,
|
||||
path,
|
||||
operation,
|
||||
signal,
|
||||
);
|
||||
if (native !== undefined) {
|
||||
if (!native.ok) return native;
|
||||
const value = native.value;
|
||||
if (
|
||||
typeof value !== "object" ||
|
||||
value === null ||
|
||||
Array.isArray(value)
|
||||
)
|
||||
throw new TypeError("JSON document root must be an object");
|
||||
for (const members of scans)
|
||||
for (const [key, member] of members) {
|
||||
if (!Object.hasOwn(value, key)) continue;
|
||||
signal?.throwIfAborted();
|
||||
member.begin();
|
||||
const selected: unknown = Reflect.get(value, key);
|
||||
if (member.mode === "array" && !Array.isArray(selected)) {
|
||||
member.reject(
|
||||
new TypeError(`JSON member ${key} must be an array`),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const item of member.mode === "array" &&
|
||||
Array.isArray(selected)
|
||||
? selected
|
||||
: [selected]) {
|
||||
signal?.throwIfAborted();
|
||||
try {
|
||||
await member.accept(item);
|
||||
} catch (cause: unknown) {
|
||||
member.reject(cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
return ok(undefined);
|
||||
}
|
||||
for (const members of scans) {
|
||||
const hash = createHash("sha256");
|
||||
let rootObject = false;
|
||||
let depth = 0;
|
||||
let key: string | undefined;
|
||||
let consume:
|
||||
| ((token: parser.Token) => typeof none | Promise<typeof none>)
|
||||
| undefined;
|
||||
await pipeline(
|
||||
decodedChunks(handle, stats.size, path, signal, (bytes) => {
|
||||
hash.update(bytes);
|
||||
}),
|
||||
asStream(
|
||||
fun(
|
||||
parser({
|
||||
jsonStreaming: false,
|
||||
streamValues: false,
|
||||
packKeys: false,
|
||||
packStrings: false,
|
||||
packNumbers: false,
|
||||
}),
|
||||
packJsonScalars(operation, stats.size, path),
|
||||
(token: parser.Token): typeof none | Promise<typeof none> => {
|
||||
const opens =
|
||||
token.name === "startObject" || token.name === "startArray";
|
||||
const closes =
|
||||
token.name === "endObject" || token.name === "endArray";
|
||||
if (depth === 0) {
|
||||
rootObject = token.name === "startObject";
|
||||
} else if (
|
||||
rootObject &&
|
||||
depth === 1 &&
|
||||
token.name === "keyValue"
|
||||
) {
|
||||
key = token.value;
|
||||
} else if (rootObject && depth === 1 && !closes) {
|
||||
const member =
|
||||
key === undefined ? undefined : members.get(key);
|
||||
consume = undefined;
|
||||
if (member !== undefined) {
|
||||
member.begin();
|
||||
if (
|
||||
member.mode === "array" &&
|
||||
token.name !== "startArray"
|
||||
) {
|
||||
member.reject(
|
||||
new TypeError(`JSON member ${key} must be an array`),
|
||||
);
|
||||
} else {
|
||||
consume = memberConsumer(
|
||||
member,
|
||||
operation,
|
||||
stats.size,
|
||||
path,
|
||||
);
|
||||
}
|
||||
}
|
||||
key = undefined;
|
||||
}
|
||||
const selected = consume;
|
||||
if (opens) depth += 1;
|
||||
if (closes) depth -= 1;
|
||||
if (rootObject && depth === 1 && token.name !== "keyValue")
|
||||
consume = undefined;
|
||||
if (selected === undefined) return none;
|
||||
return selected(token);
|
||||
},
|
||||
),
|
||||
),
|
||||
signal === undefined ? {} : { signal },
|
||||
);
|
||||
if (!rootObject)
|
||||
throw new TypeError("JSON document root must be an object");
|
||||
const digest = hash.digest("hex");
|
||||
if (previousDigest !== undefined && digest !== previousDigest)
|
||||
throw new RegularFileChangedError(path);
|
||||
previousDigest = digest;
|
||||
}
|
||||
return ok(undefined);
|
||||
} catch (cause: unknown) {
|
||||
const failure = jsonReadFailure(cause, operation, stats.size, path);
|
||||
if (failure !== undefined) return err(failure);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const memberConsumer = (
|
||||
member: JsonObjectMemberReader,
|
||||
operation: string,
|
||||
size: number,
|
||||
path: string,
|
||||
) => {
|
||||
let admit = admitJsonAssembly(operation, size, path);
|
||||
const streamer: parser.TokenConsumer<{ readonly value: unknown }> =
|
||||
member.mode === "array" ? streamArray() : streamValues();
|
||||
const consume = fun(
|
||||
(token: parser.Token) => admit(token),
|
||||
streamer,
|
||||
(row: unknown): typeof none | Promise<typeof none> => {
|
||||
if (typeof row !== "object" || row === null || !("value" in row))
|
||||
throw new Error("Unexpected JSON member parser result");
|
||||
// The preceding array item has left the assembler; retain only what the
|
||||
// consumer owns and use live headroom when admitting the next item.
|
||||
admit = admitJsonAssembly(operation, size, path);
|
||||
try {
|
||||
const accepted = member.accept(row.value);
|
||||
if (accepted instanceof Promise)
|
||||
return accepted.then(
|
||||
() => none,
|
||||
(cause: unknown) => {
|
||||
member.reject(cause);
|
||||
return none;
|
||||
},
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
member.reject(cause);
|
||||
}
|
||||
return none;
|
||||
},
|
||||
);
|
||||
return (token: parser.Token): typeof none | Promise<typeof none> => {
|
||||
const consumed = consume(token);
|
||||
return consumed instanceof Promise ? consumed.then(() => none) : none;
|
||||
};
|
||||
};
|
||||
|
||||
/** Parse strict UTF-8 JSON without requiring a whole-document string. */
|
||||
export const readJsonInputFile = (
|
||||
path: string,
|
||||
@@ -49,223 +243,222 @@ export const readJsonInputFile = (
|
||||
let found = false;
|
||||
let value: unknown;
|
||||
try {
|
||||
const prefix = nativeParseFits(stats.size)
|
||||
? await readPrefix(handle, stats.size + 1, signal)
|
||||
: undefined;
|
||||
if (prefix !== undefined) {
|
||||
if (!prefix.complete || prefix.bytes.length !== stats.size)
|
||||
throw new RegularFileChangedError(path);
|
||||
const parsed = parseUtf8Json(
|
||||
prefix.bytes,
|
||||
operation,
|
||||
path,
|
||||
"json-file-input",
|
||||
);
|
||||
return parsed.ok
|
||||
? ok(parsed.value)
|
||||
: err(invalidJson(operation, parsed.error, parsed.cause));
|
||||
}
|
||||
const native = await readNativeJson(
|
||||
handle,
|
||||
stats.size,
|
||||
path,
|
||||
operation,
|
||||
signal,
|
||||
);
|
||||
if (native !== undefined) return native;
|
||||
await pipeline(
|
||||
decodedChunks(handle, stats.size, path, signal),
|
||||
parser.asStream({
|
||||
jsonStreaming: false,
|
||||
streamValues: false,
|
||||
packKeys: false,
|
||||
packStrings: false,
|
||||
packNumbers: false,
|
||||
}),
|
||||
packJsonScalars(operation, stats.size, path),
|
||||
admitJsonAssembly(operation, stats.size, path),
|
||||
streamValues.asStream(),
|
||||
async (rows: AsyncIterable<unknown>) => {
|
||||
for await (const row of rows) {
|
||||
if (
|
||||
found ||
|
||||
typeof row !== "object" ||
|
||||
row === null ||
|
||||
!("key" in row) ||
|
||||
row.key !== 0 ||
|
||||
!("value" in row)
|
||||
)
|
||||
throw new Error("Unexpected JSON parser result");
|
||||
found = true;
|
||||
value = row.value;
|
||||
}
|
||||
},
|
||||
// Fuse synchronous token stages so only file chunks cross the async
|
||||
// stream boundary. Admission still precedes scalar/value assembly.
|
||||
asStream(
|
||||
fun(
|
||||
parser({
|
||||
jsonStreaming: false,
|
||||
streamValues: false,
|
||||
packKeys: false,
|
||||
packStrings: false,
|
||||
packNumbers: false,
|
||||
}),
|
||||
packJsonScalars(operation, stats.size, path),
|
||||
admitJsonAssembly(operation, stats.size, path),
|
||||
streamValues(),
|
||||
(row: unknown): typeof none => {
|
||||
if (
|
||||
found ||
|
||||
typeof row !== "object" ||
|
||||
row === null ||
|
||||
!("key" in row) ||
|
||||
row.key !== 0 ||
|
||||
!("value" in row)
|
||||
)
|
||||
throw new Error("Unexpected JSON parser result");
|
||||
found = true;
|
||||
value = row.value;
|
||||
return none;
|
||||
},
|
||||
),
|
||||
),
|
||||
signal === undefined ? {} : { signal },
|
||||
);
|
||||
return found
|
||||
? ok(value)
|
||||
: err(invalidJson(operation, "JSON document is empty"));
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
|
||||
if (
|
||||
cause instanceof TypeError &&
|
||||
"code" in cause &&
|
||||
cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA"
|
||||
)
|
||||
return err(
|
||||
invalidJson(operation, "JSON input is not valid UTF-8", cause),
|
||||
);
|
||||
if (
|
||||
cause instanceof SyntaxError ||
|
||||
(cause instanceof Error && cause.message.startsWith("Parser "))
|
||||
)
|
||||
return err(invalidJson(operation, cause.message, cause));
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
((cause instanceof RangeError &&
|
||||
cause.message === "Invalid string length") ||
|
||||
("code" in cause && cause.code === "ERR_STRING_TOO_LONG"))
|
||||
)
|
||||
return err(
|
||||
new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"An individual JSON string or number token exceeds the runtime string limit",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: stats.size,
|
||||
max_string_code_units: constants.MAX_STRING_LENGTH,
|
||||
},
|
||||
{
|
||||
cause,
|
||||
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
|
||||
},
|
||||
),
|
||||
);
|
||||
const failure = jsonReadFailure(cause, operation, stats.size, path);
|
||||
if (failure !== undefined) return err(failure);
|
||||
throw cause;
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
|
||||
const jsonReadFailure = (
|
||||
cause: unknown,
|
||||
operation: string,
|
||||
size: number,
|
||||
path: string,
|
||||
): JsonInputFileFailure | undefined => {
|
||||
if (cause instanceof AnalysisResourceConstraintError) return cause;
|
||||
if (
|
||||
cause instanceof TypeError &&
|
||||
"code" in cause &&
|
||||
cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA"
|
||||
)
|
||||
return invalidJson(operation, "JSON input is not valid UTF-8", cause);
|
||||
if (
|
||||
cause instanceof SyntaxError ||
|
||||
(cause instanceof Error && cause.message.startsWith("Parser "))
|
||||
)
|
||||
return invalidJson(operation, cause.message, cause);
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
((cause instanceof RangeError &&
|
||||
cause.message === "Invalid string length") ||
|
||||
("code" in cause && cause.code === "ERR_STRING_TOO_LONG"))
|
||||
)
|
||||
return new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"An individual JSON string or number token exceeds the runtime string limit",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: size,
|
||||
max_string_code_units: constants.MAX_STRING_LENGTH,
|
||||
},
|
||||
{
|
||||
cause,
|
||||
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
|
||||
},
|
||||
);
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const packJsonScalars = (
|
||||
operation: string,
|
||||
inputFileBytes: number,
|
||||
path: string,
|
||||
) =>
|
||||
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
|
||||
let kind: "keyValue" | "stringValue" | "numberValue" | undefined;
|
||||
let length = 0;
|
||||
let value = "";
|
||||
let parts: string[] = [];
|
||||
let partLength = 0;
|
||||
for await (const token of tokens) {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON parser token");
|
||||
switch (token.name) {
|
||||
case "startKey":
|
||||
case "startString":
|
||||
kind = token.name === "startKey" ? "keyValue" : "stringValue";
|
||||
break;
|
||||
case "startNumber":
|
||||
kind = "numberValue";
|
||||
break;
|
||||
case "numberChunk":
|
||||
case "stringChunk":
|
||||
if (
|
||||
kind === undefined ||
|
||||
!("value" in token) ||
|
||||
typeof token.value !== "string"
|
||||
)
|
||||
throw new Error("Unexpected JSON scalar fragment");
|
||||
length += token.value.length;
|
||||
if (length > constants.MAX_STRING_LENGTH)
|
||||
throw new RangeError("Invalid string length");
|
||||
parts.push(token.value);
|
||||
partLength += token.value.length;
|
||||
if (partLength >= PACKED_TOKEN_PART_CODE_UNITS) {
|
||||
// The tokenizer emits short fragments. Coalesce them before retaining
|
||||
// a scalar so millions of substring/rope nodes cannot exhaust the heap
|
||||
// before the native string-length constraint can be reported.
|
||||
requireStringAssemblyHeadroom(
|
||||
operation,
|
||||
inputFileBytes,
|
||||
path,
|
||||
length,
|
||||
);
|
||||
value += parts.join("");
|
||||
parts = [];
|
||||
partLength = 0;
|
||||
}
|
||||
break;
|
||||
case "endKey":
|
||||
case "endString":
|
||||
case "endNumber":
|
||||
if (kind === undefined) throw new Error("Unexpected JSON scalar end");
|
||||
) => {
|
||||
let kind: "keyValue" | "stringValue" | "numberValue" | undefined;
|
||||
let length = 0;
|
||||
let value = "";
|
||||
let parts: string[] = [];
|
||||
let partLength = 0;
|
||||
return (token: parser.Token): parser.Token | typeof none => {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON parser token");
|
||||
switch (token.name) {
|
||||
case "startKey":
|
||||
case "startString":
|
||||
kind = token.name === "startKey" ? "keyValue" : "stringValue";
|
||||
break;
|
||||
case "startNumber":
|
||||
kind = "numberValue";
|
||||
break;
|
||||
case "numberChunk":
|
||||
case "stringChunk":
|
||||
if (
|
||||
kind === undefined ||
|
||||
!("value" in token) ||
|
||||
typeof token.value !== "string"
|
||||
)
|
||||
throw new Error("Unexpected JSON scalar fragment");
|
||||
length += token.value.length;
|
||||
if (length > constants.MAX_STRING_LENGTH)
|
||||
throw new RangeError("Invalid string length");
|
||||
parts.push(token.value);
|
||||
partLength += token.value.length;
|
||||
if (partLength >= PACKED_TOKEN_PART_CODE_UNITS) {
|
||||
// The tokenizer emits short fragments. Coalesce them before retaining
|
||||
// a scalar so millions of substring/rope nodes cannot exhaust the heap
|
||||
// before the native string-length constraint can be reported.
|
||||
requireStringAssemblyHeadroom(
|
||||
operation,
|
||||
inputFileBytes,
|
||||
path,
|
||||
length,
|
||||
);
|
||||
yield { name: kind, value: value + parts.join("") };
|
||||
kind = undefined;
|
||||
length = 0;
|
||||
value = "";
|
||||
value += parts.join("");
|
||||
parts = [];
|
||||
partLength = 0;
|
||||
break;
|
||||
default:
|
||||
yield token;
|
||||
}
|
||||
break;
|
||||
case "endKey":
|
||||
case "endString":
|
||||
case "endNumber": {
|
||||
if (kind === undefined) throw new Error("Unexpected JSON scalar end");
|
||||
requireStringAssemblyHeadroom(operation, inputFileBytes, path, length);
|
||||
const packed = { name: kind, value: value + parts.join("") };
|
||||
kind = undefined;
|
||||
length = 0;
|
||||
value = "";
|
||||
parts = [];
|
||||
partLength = 0;
|
||||
return packed;
|
||||
}
|
||||
default:
|
||||
return token;
|
||||
}
|
||||
return none;
|
||||
};
|
||||
};
|
||||
|
||||
/** Admit the retained value graph before the upstream assembler grows it. */
|
||||
const admitJsonAssembly = (
|
||||
operation: string,
|
||||
inputFileBytes: number,
|
||||
path: string,
|
||||
) =>
|
||||
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
|
||||
let projectedBytes = 0;
|
||||
let nextCheck = 0;
|
||||
for await (const token of tokens) {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON assembly token");
|
||||
if (token.name === "startObject" || token.name === "startArray") {
|
||||
projectedBytes += 128;
|
||||
} else if (token.name === "keyValue" || token.name === "stringValue") {
|
||||
if (!("value" in token) || typeof token.value !== "string")
|
||||
throw new Error("Unexpected packed JSON string");
|
||||
projectedBytes += 64 + token.value.length * 2;
|
||||
} else if (
|
||||
token.name === "numberValue" ||
|
||||
token.name === "nullValue" ||
|
||||
token.name === "trueValue" ||
|
||||
token.name === "falseValue"
|
||||
) {
|
||||
projectedBytes += 64;
|
||||
}
|
||||
if (projectedBytes >= nextCheck) {
|
||||
const heap = getHeapStatistics();
|
||||
// Conservative capacity for containers, property/array slots and UTF-16
|
||||
// values. Keep another whole projection available for container growth,
|
||||
// plus the next chunk of estimates between samples. This is admission
|
||||
// against live runtime headroom, not a fixed document-size policy.
|
||||
const requiredBytes = projectedBytes + READ_CHUNK_BYTES;
|
||||
if (requiredBytes > heap.total_available_size)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"Insufficient heap headroom to materialize the complete JSON value",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: inputFileBytes,
|
||||
projected_value_bytes: projectedBytes,
|
||||
assembly_headroom_bytes: requiredBytes,
|
||||
available_heap_bytes: heap.total_available_size,
|
||||
heap_size_limit_bytes: heap.heap_size_limit,
|
||||
},
|
||||
{ remediationAction: JSON_INPUT_RESOURCE_REMEDIATION },
|
||||
);
|
||||
nextCheck = projectedBytes + READ_CHUNK_BYTES;
|
||||
}
|
||||
yield token;
|
||||
) => {
|
||||
let projectedBytes = 0;
|
||||
let nextCheck = 0;
|
||||
return (token: parser.Token): parser.Token => {
|
||||
if (typeof token !== "object" || token === null || !("name" in token))
|
||||
throw new Error("Unexpected JSON assembly token");
|
||||
if (token.name === "startObject" || token.name === "startArray") {
|
||||
projectedBytes += 128;
|
||||
} else if (token.name === "keyValue" || token.name === "stringValue") {
|
||||
if (!("value" in token) || typeof token.value !== "string")
|
||||
throw new Error("Unexpected packed JSON string");
|
||||
projectedBytes += 64 + token.value.length * 2;
|
||||
} else if (
|
||||
token.name === "numberValue" ||
|
||||
token.name === "nullValue" ||
|
||||
token.name === "trueValue" ||
|
||||
token.name === "falseValue"
|
||||
) {
|
||||
projectedBytes += 64;
|
||||
}
|
||||
if (projectedBytes >= nextCheck) {
|
||||
const heap = getHeapStatistics();
|
||||
// Conservative capacity for containers, property/array slots and UTF-16
|
||||
// values. Keep another whole projection available for container growth,
|
||||
// plus the next chunk of estimates between samples. This is admission
|
||||
// against live runtime headroom, not a fixed document-size policy.
|
||||
const requiredBytes = projectedBytes + READ_CHUNK_BYTES;
|
||||
if (requiredBytes > heap.total_available_size)
|
||||
throw new AnalysisResourceConstraintError(
|
||||
operation,
|
||||
"memory",
|
||||
"Insufficient heap headroom to materialize the complete JSON value",
|
||||
{
|
||||
input_path: path,
|
||||
input_file_bytes: inputFileBytes,
|
||||
projected_value_bytes: projectedBytes,
|
||||
assembly_headroom_bytes: requiredBytes,
|
||||
available_heap_bytes: heap.total_available_size,
|
||||
heap_size_limit_bytes: heap.heap_size_limit,
|
||||
},
|
||||
{ remediationAction: JSON_INPUT_RESOURCE_REMEDIATION },
|
||||
);
|
||||
nextCheck = projectedBytes + READ_CHUNK_BYTES;
|
||||
}
|
||||
return token;
|
||||
};
|
||||
};
|
||||
|
||||
const requireStringAssemblyHeadroom = (
|
||||
operation: string,
|
||||
@@ -302,6 +495,7 @@ async function* decodedChunks(
|
||||
admittedBytes: number,
|
||||
path: string,
|
||||
signal: AbortSignal | undefined,
|
||||
observe?: (bytes: Uint8Array) => void,
|
||||
): AsyncGenerator<string> {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
let leading = true;
|
||||
@@ -320,15 +514,15 @@ async function* decodedChunks(
|
||||
bytes,
|
||||
0,
|
||||
Math.min(bytes.length, admittedBytes - observedBytes + 1),
|
||||
null,
|
||||
observedBytes,
|
||||
);
|
||||
signal?.throwIfAborted();
|
||||
if (read.bytesRead === 0) break;
|
||||
observedBytes += read.bytesRead;
|
||||
if (observedBytes > admittedBytes) throw new RegularFileChangedError(path);
|
||||
yield checked(
|
||||
decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }),
|
||||
);
|
||||
const chunk = bytes.subarray(0, read.bytesRead);
|
||||
observe?.(chunk);
|
||||
yield checked(decoder.decode(chunk, { stream: true }));
|
||||
}
|
||||
if (observedBytes !== admittedBytes) throw new RegularFileChangedError(path);
|
||||
yield checked(decoder.decode());
|
||||
@@ -360,6 +554,28 @@ const readPrefix = async (
|
||||
return { bytes, complete: false };
|
||||
};
|
||||
|
||||
const readNativeJson = async (
|
||||
handle: FileHandle,
|
||||
size: number,
|
||||
path: string,
|
||||
operation: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Result<unknown, JsonInputFileFailure> | undefined> => {
|
||||
if (!nativeParseFits(size)) return undefined;
|
||||
const prefix = await readPrefix(handle, size + 1, signal);
|
||||
if (!prefix.complete || prefix.bytes.length !== size)
|
||||
throw new RegularFileChangedError(path);
|
||||
const parsed = parseUtf8Json(
|
||||
prefix.bytes,
|
||||
operation,
|
||||
path,
|
||||
"json-file-input",
|
||||
);
|
||||
return parsed.ok
|
||||
? ok(parsed.value)
|
||||
: err(invalidJson(operation, parsed.error, parsed.cause));
|
||||
};
|
||||
|
||||
const invalidJson = (
|
||||
operation: string,
|
||||
message: string,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { z } from "zod";
|
||||
|
||||
import { reconstructionCoverageEvaluationInputSchema } from "../domain/reconstructionCoverageInput.js";
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { evaluateReconstructionClosure } from "../domain/reconstructionCoverage.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../domain/evidence.js";
|
||||
import { jsonObjectSchema, jsonValueSchema } from "../domain/jsonValue.js";
|
||||
import {
|
||||
|
||||
@@ -18,9 +18,9 @@ import type {
|
||||
} from "../reference/ReferenceSourceReaderTypes.js";
|
||||
import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js";
|
||||
import { readReferenceSourceVcs } from "./ReferenceSourceVcsAdapter.js";
|
||||
import {
|
||||
type ReferenceSourceImportError,
|
||||
type ReferenceSourceImportOptions,
|
||||
import type {
|
||||
ReferenceSourceImportError,
|
||||
ReferenceSourceImportOptions,
|
||||
} from "./ReferenceSourceImportTypes.js";
|
||||
import {
|
||||
prepareReferenceSourceImport,
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { skillDestinations } from "./SetupSkill.js";
|
||||
import {
|
||||
type ClientConfigurationInspection,
|
||||
type ClientConfigurationResult,
|
||||
type SetupHost,
|
||||
type SetupOptions,
|
||||
type SetupProviderEnvironment,
|
||||
import type {
|
||||
ClientConfigurationInspection,
|
||||
ClientConfigurationResult,
|
||||
SetupHost,
|
||||
SetupOptions,
|
||||
SetupProviderEnvironment,
|
||||
} from "./SetupTypes.js";
|
||||
import type { SetupClient } from "./SupportedClients.js";
|
||||
import type { SetupHopperInstallResult } from "./SetupInstallFailure.js";
|
||||
|
||||
@@ -29,6 +29,7 @@ import {
|
||||
supportedClients,
|
||||
type SetupClient,
|
||||
} from "./SupportedClients.js";
|
||||
export { isUninstallFailure } from "./UninstallFailure.js";
|
||||
|
||||
interface ManagedPathStats {
|
||||
readonly uid?: number;
|
||||
@@ -74,23 +75,6 @@ export interface UninstallResult {
|
||||
readonly items: readonly UninstallItem[];
|
||||
}
|
||||
|
||||
/** Whether any requested REA-owned uninstall action failed. */
|
||||
export const isUninstallFailure = (result: UninstallResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "complete":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled uninstall result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Filesystem boundary used by the contained uninstall workflow. */
|
||||
export interface UninstallHost {
|
||||
clients(): Promise<readonly SetupClient[]>;
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { UninstallResult } from "./Uninstall.js";
|
||||
|
||||
/** Whether any requested REA-owned uninstall action failed. */
|
||||
export const isUninstallFailure = (result: UninstallResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "complete":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled uninstall result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -3,6 +3,7 @@ import { gt, valid } from "semver";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import type { Result } from "../domain/result.js";
|
||||
import type { IntegrationMaintenance } from "./UpdateMaintenance.js";
|
||||
export { isUpdateFailure } from "./UpdateFailure.js";
|
||||
|
||||
/** Global npm installation that owns the running package and its entry point. */
|
||||
export interface NpmInstallation {
|
||||
@@ -68,24 +69,6 @@ export type UpdateResult =
|
||||
readonly remediation: string;
|
||||
};
|
||||
|
||||
/** Whether the update command left the requested release unapplied. */
|
||||
export const isUpdateFailure = (result: UpdateResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "current":
|
||||
case "updated":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled update result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/** Resolve the exact npm command for the identified prefix and release. */
|
||||
export const updateInstallCommand = (
|
||||
installation: NpmInstallation,
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import type { UpdateResult } from "./Update.js";
|
||||
|
||||
/** Whether the update command left the requested release unapplied. */
|
||||
export const isUpdateFailure = (result: UpdateResult): boolean => {
|
||||
const status = result.status;
|
||||
switch (status) {
|
||||
case "current":
|
||||
case "updated":
|
||||
return false;
|
||||
case "failed":
|
||||
return true;
|
||||
default: {
|
||||
const exhaustive: never = status;
|
||||
throw new TypeError(
|
||||
`Unhandled update result status: ${String(exhaustive)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -7,6 +7,10 @@ import {
|
||||
import { InvestigationRecords } from "./investigation/InvestigationRecords.js";
|
||||
import { UnknownRegistryError } from "../domain/unknownRegistryError.js";
|
||||
import { err } from "../domain/result.js";
|
||||
import { nativeApiBoundarySchema } from "../domain/native/nativeApiBoundary.js";
|
||||
import { projectNativeApiInspection } from "./native/NativeApiInspection.js";
|
||||
import { functionDossierSchema } from "../domain/hopperValues.js";
|
||||
import { ghidraFunctionDossier } from "../domain/ghidraValues.fixture.js";
|
||||
|
||||
describe("workflow residual question retention", () => {
|
||||
it("keeps questions linked to workflow Evidence and records repeats idempotently", () => {
|
||||
@@ -54,3 +58,47 @@ describe("workflow residual question retention", () => {
|
||||
).toEqual(err(error));
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
" Which boundary? ",
|
||||
"\nWhich boundary?\t",
|
||||
"\uFEFFWhich boundary?\uFEFF",
|
||||
])(
|
||||
"retains contract-accepted question %j without rewriting workflow Evidence",
|
||||
(question) => {
|
||||
const boundary = nativeApiBoundarySchema.parse({
|
||||
available: false,
|
||||
reason: "Synthetic provider unavailable boundary",
|
||||
residual_unknowns: [question],
|
||||
});
|
||||
const dossier = functionDossierSchema.parse(ghidraFunctionDossier());
|
||||
const result = projectNativeApiInspection({
|
||||
...dossier,
|
||||
native_api: boundary,
|
||||
});
|
||||
const evidence = createWorkflowEvidence({
|
||||
target: undefined,
|
||||
operation: "inspect_native_api",
|
||||
parameters: { procedure: "0x1000" },
|
||||
result,
|
||||
upstreamProfile: undefined,
|
||||
});
|
||||
const records = new InvestigationRecords();
|
||||
expect(records.recordEvidence(evidence).ok).toBe(true);
|
||||
expect(
|
||||
recordWorkflowUnknowns({
|
||||
name: "inspect_native_api",
|
||||
result,
|
||||
evidenceId: evidence.evidence_id,
|
||||
recordUnknown: (input) => records.recordUnknown(input),
|
||||
}).ok,
|
||||
).toBe(true);
|
||||
expect(records.listUnknowns()).toMatchObject([
|
||||
{ question: "Which boundary?" },
|
||||
]);
|
||||
expect(records.evidenceById(evidence.evidence_id)).toEqual(evidence);
|
||||
expect(evidence.normalized_result).toMatchObject({
|
||||
residual_unknowns: [question],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
AnalysisSnapshotMismatchError,
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { NoBinaryOpenError } from "../../domain/configurationErrors.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import {
|
||||
analysisQueryId,
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
snapshotMatchesTarget,
|
||||
snapshotTarget,
|
||||
parseAnalysisSnapshot,
|
||||
immutableAnalysisSnapshot,
|
||||
type AnalysisSnapshot,
|
||||
type AnalysisSnapshotEntry,
|
||||
type AnalysisSnapshotWorkflowEntry,
|
||||
@@ -107,6 +108,7 @@ export class AnalysisSnapshotCache {
|
||||
|
||||
/** Merge already-validated entries and return the new-entry count. */
|
||||
stage(snapshot: AnalysisSnapshot): number {
|
||||
snapshot = immutableAnalysisSnapshot(snapshot);
|
||||
if (
|
||||
(this.#target !== undefined &&
|
||||
JSON.stringify(this.#target) !== JSON.stringify(snapshot.target)) ||
|
||||
@@ -123,12 +125,12 @@ export class AnalysisSnapshotCache {
|
||||
for (const entry of snapshot.entries) {
|
||||
if (!this.#entries.has(entry.query_id) && !this.#hasCapacity()) continue;
|
||||
if (!this.#entries.has(entry.query_id)) imported += 1;
|
||||
this.#entries.set(entry.query_id, structuredClone(entry));
|
||||
this.#entries.set(entry.query_id, entry);
|
||||
}
|
||||
for (const entry of snapshot.workflow_entries) {
|
||||
if (!this.#workflowEntries.has(entry.query_id) && !this.#hasCapacity())
|
||||
continue;
|
||||
this.#workflowEntries.set(entry.query_id, structuredClone(entry));
|
||||
this.#workflowEntries.set(entry.query_id, entry);
|
||||
}
|
||||
return imported;
|
||||
}
|
||||
@@ -186,7 +188,7 @@ export class AnalysisSnapshotCache {
|
||||
): Result<number, AnalysisError> {
|
||||
let validated: AnalysisSnapshot;
|
||||
try {
|
||||
validated = parseAnalysisSnapshot(snapshot);
|
||||
validated = immutableAnalysisSnapshot(snapshot);
|
||||
} catch (cause: unknown) {
|
||||
return err(
|
||||
new EvidenceIntegrityError("Analysis snapshot validation failed", {
|
||||
|
||||
@@ -1,5 +1,9 @@
|
||||
import { setImmediate } from "node:timers/promises";
|
||||
|
||||
import {
|
||||
parseAnalysisSnapshot,
|
||||
prepareSnapshotEntry,
|
||||
parsePreparedAnalysisSnapshot,
|
||||
type PreparedSnapshotEntry,
|
||||
serializeAnalysisSnapshot,
|
||||
type AnalysisSnapshot,
|
||||
} from "../../domain/analysisSnapshot.js";
|
||||
@@ -9,27 +13,106 @@ import {
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js";
|
||||
import { readJsonFile, writeTextFile } from "../JsonFiles.js";
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import { readJsonObjectMembersFile, writeTextFile } from "../JsonFiles.js";
|
||||
import type { JsonObjectMemberReader } from "../JsonInputFile.js";
|
||||
|
||||
type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError;
|
||||
|
||||
/** Read and validate an analysis snapshot from the caller's path. */
|
||||
export const readAnalysisSnapshot = async (
|
||||
path: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<
|
||||
Result<AnalysisSnapshot, SnapshotFailure | AnalysisResourceConstraintError>
|
||||
Result<
|
||||
AnalysisSnapshot,
|
||||
SnapshotFailure | AnalysisResourceConstraintError | AnalysisCancelledError
|
||||
>
|
||||
> => {
|
||||
const loaded = await readJsonFile(path);
|
||||
if (!loaded.ok) return loaded;
|
||||
const entries = snapshotEntryReader("provider");
|
||||
const workflows = snapshotEntryReader("workflow");
|
||||
const header: Record<string, unknown> = {};
|
||||
const metadata = new Map<string, JsonObjectMemberReader>(
|
||||
["target", "binding", "evidence_bundle"].map((name) => [
|
||||
name,
|
||||
{
|
||||
mode: "value",
|
||||
begin: () => {
|
||||
delete header[name];
|
||||
},
|
||||
accept: (value) => {
|
||||
header[name] = value;
|
||||
},
|
||||
reject: (cause) => {
|
||||
throw cause;
|
||||
},
|
||||
},
|
||||
]),
|
||||
);
|
||||
try {
|
||||
return ok(parseAnalysisSnapshot(loaded.value));
|
||||
const loaded = await readJsonObjectMembersFile(
|
||||
path,
|
||||
[
|
||||
new Map([
|
||||
["entries", entries.reader],
|
||||
["workflow_entries", workflows.reader],
|
||||
]),
|
||||
metadata,
|
||||
],
|
||||
signal,
|
||||
);
|
||||
if (!loaded.ok) return loaded;
|
||||
signal?.throwIfAborted();
|
||||
const snapshot = parsePreparedAnalysisSnapshot(
|
||||
header,
|
||||
entries.finish(),
|
||||
workflows.finish(),
|
||||
);
|
||||
// Deliver queued cancellation before handing a complete cache to the session.
|
||||
await setImmediate();
|
||||
signal?.throwIfAborted();
|
||||
return ok(snapshot);
|
||||
} catch (cause: unknown) {
|
||||
if (signal?.aborted === true)
|
||||
return err(new AnalysisCancelledError("read_evidence_file", { cause }));
|
||||
return err(
|
||||
new EvidenceIntegrityError(snapshotValidationMessage(cause), { cause }),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const snapshotEntryReader = (kind: "provider" | "workflow") => {
|
||||
let seen = false;
|
||||
let entries: PreparedSnapshotEntry[] = [];
|
||||
let failure: { readonly cause: unknown } | undefined;
|
||||
const reader: JsonObjectMemberReader = {
|
||||
mode: "array",
|
||||
begin: () => {
|
||||
seen = true;
|
||||
entries = [];
|
||||
failure = undefined;
|
||||
},
|
||||
accept: (value) => {
|
||||
if (failure === undefined)
|
||||
entries.push(prepareSnapshotEntry(value, kind));
|
||||
},
|
||||
reject: (cause) => {
|
||||
failure ??= { cause };
|
||||
},
|
||||
};
|
||||
return {
|
||||
reader,
|
||||
finish: (): PreparedSnapshotEntry[] => {
|
||||
if (!seen)
|
||||
throw new TypeError(
|
||||
`Analysis snapshot ${kind === "provider" ? "entries" : "workflow_entries"} is missing`,
|
||||
);
|
||||
if (failure !== undefined) throw failure.cause;
|
||||
return entries;
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
/** Atomically write a deterministic analysis snapshot to the caller's path. */
|
||||
export const writeAnalysisSnapshot = async (
|
||||
snapshot: AnalysisSnapshot,
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
type AnalysisProfileCommitment,
|
||||
} from "../../domain/analysisProfile.js";
|
||||
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { createEvidence } from "../../domain/evidence.js";
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import { NoBinaryOpenError } from "../../domain/configurationErrors.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import type { EvidenceSubjectTarget } from "../../domain/evidence.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
|
||||
@@ -8,12 +8,12 @@ import {
|
||||
} from "../../domain/analysisSnapshot.js";
|
||||
import type { AnalysisSnapshot } from "../../domain/analysisSnapshot.js";
|
||||
import { parseBinaryTarget } from "../BinaryTargetResolver.js";
|
||||
import { type BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import {
|
||||
EvidenceIntegrityError,
|
||||
AnalysisSnapshotMismatchError,
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, type Result } from "../../domain/result.js";
|
||||
import type { SessionProviderRoute } from "./SessionProviderRouter.js";
|
||||
import { SessionProviderRouter } from "./SessionProviderRouter.js";
|
||||
|
||||
@@ -9,8 +9,8 @@ import {
|
||||
EvidenceIntegrityError,
|
||||
AnalysisSnapshotMismatchError,
|
||||
} from "../../domain/evidenceErrors.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { type UnknownRegistryError } from "../../domain/unknownRegistryError.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { UnknownRegistryError } from "../../domain/unknownRegistryError.js";
|
||||
import type {
|
||||
RecordUnknownInput,
|
||||
ResidualUnknown,
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { AnalysisProfileCommitment } from "../../domain/analysisProfile.js"
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import { ProviderSelectionError } from "../../domain/providerSelectionError.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import type {
|
||||
AnalysisClient,
|
||||
|
||||
@@ -417,8 +417,9 @@ export class EvidenceLedger {
|
||||
unknowns: ReadonlyMap<string, ResidualUnknown>,
|
||||
): Result<void, EvidenceIntegrityError | UnknownRegistryError> {
|
||||
try {
|
||||
parseEvidenceBundle(
|
||||
createEvidenceBundle([...records.values()], [...unknowns.values()]),
|
||||
createImmutableEvidenceBundle(
|
||||
[...records.values()],
|
||||
[...unknowns.values()],
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
return err(new UnknownRegistryError("integrity", { cause }));
|
||||
|
||||
@@ -61,7 +61,10 @@ describe("investigation record ownership", () => {
|
||||
value: {
|
||||
scope_digest: target.sha256,
|
||||
mutation_evidence_ids: [
|
||||
unknownMutationEvidence(target, input).evidence_id,
|
||||
unknownMutationEvidence(target, {
|
||||
operation: "record_unknown",
|
||||
input,
|
||||
}).evidence_id,
|
||||
],
|
||||
},
|
||||
});
|
||||
@@ -71,14 +74,19 @@ describe("investigation record ownership", () => {
|
||||
value: {
|
||||
scope_digest: null,
|
||||
mutation_evidence_ids: [
|
||||
unknownMutationEvidence(undefined, input).evidence_id,
|
||||
unknownMutationEvidence(undefined, {
|
||||
operation: "record_unknown",
|
||||
input,
|
||||
}).evidence_id,
|
||||
],
|
||||
},
|
||||
});
|
||||
const mutation = records.evidenceById(
|
||||
unknownMutationEvidence(undefined, input).evidence_id,
|
||||
unknownMutationEvidence(undefined, { operation: "record_unknown", input })
|
||||
.evidence_id,
|
||||
);
|
||||
expect(mutation?.subject).toBeNull();
|
||||
expect(mutation?.parameters).toEqual(input);
|
||||
expect(records.listUnknowns()).toHaveLength(2);
|
||||
});
|
||||
|
||||
@@ -124,6 +132,10 @@ describe("investigation record ownership", () => {
|
||||
bundle.records.find(({ operation }) => operation === "update_unknown")
|
||||
?.subject?.digest.sha256,
|
||||
).toBe(target.sha256);
|
||||
expect(
|
||||
bundle.records.find(({ operation }) => operation === "update_unknown")
|
||||
?.parameters,
|
||||
).toEqual(input);
|
||||
expect(records.updateUnknown(input, target).ok).toBe(false);
|
||||
expect(records.exportEvidenceBundle()).toEqual(bundle);
|
||||
});
|
||||
@@ -162,3 +174,59 @@ describe("investigation record ownership", () => {
|
||||
expect(records.exportEvidenceBundle()).toEqual(createEvidenceBundle([]));
|
||||
});
|
||||
});
|
||||
|
||||
it("retains complete mutation intent and distinguishes independent update requests", () => {
|
||||
const source = new InvestigationRecords();
|
||||
expect(source.recordEvidence(evidence).ok).toBe(true);
|
||||
const input = {
|
||||
...unknownInput("Which replay establishes the answer?"),
|
||||
supporting_evidence_ids: [evidence.evidence_id],
|
||||
required_environment: {
|
||||
id: "fixture",
|
||||
platform: "linux",
|
||||
architecture: "x86_64",
|
||||
isolation: "process" as const,
|
||||
},
|
||||
recommended_probes: [
|
||||
{ operation: "inspect_native_api", rationale: "Replay" },
|
||||
],
|
||||
};
|
||||
const created = source.recordUnknown(input);
|
||||
if (!created.ok) throw created.error;
|
||||
const base = source.exportEvidenceBundle();
|
||||
const creation = base.records.find(
|
||||
({ operation }) => operation === "record_unknown",
|
||||
);
|
||||
expect(creation?.parameters).toEqual(input);
|
||||
expect(creation?.evidence_links).toEqual([evidence.evidence_id]);
|
||||
|
||||
const { question: _question, domain: _domain, ...details } = input;
|
||||
const mutations = (["high", "low"] as const).map((severity) => {
|
||||
const records = new InvestigationRecords();
|
||||
expect(records.mergeEvidenceBundle(base).ok).toBe(true);
|
||||
const command = updateUnknownInputSchema.parse({
|
||||
...details,
|
||||
unknown_id: created.value.unknown_id,
|
||||
expected_revision: 1,
|
||||
status: "resolved",
|
||||
severity,
|
||||
resolution: {
|
||||
disposition: "withdrawn",
|
||||
rationale: "Caller withdrew this question",
|
||||
evidence_ids: [evidence.evidence_id],
|
||||
},
|
||||
});
|
||||
const updated = records.updateUnknown(command);
|
||||
if (!updated.ok) throw updated.error;
|
||||
const mutation = records
|
||||
.exportEvidenceBundle()
|
||||
.records.find(({ operation }) => operation === "update_unknown");
|
||||
expect(mutation?.parameters).toEqual(command);
|
||||
expect(mutation?.evidence_links).toEqual([evidence.evidence_id]);
|
||||
expect(updated.value.mutation_evidence_ids).toContain(
|
||||
mutation?.evidence_id,
|
||||
);
|
||||
return mutation;
|
||||
});
|
||||
expect(mutations[0]?.evidence_id).not.toBe(mutations[1]?.evidence_id);
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import type { EvidenceBundle } from "../../domain/evidenceBundle.js";
|
||||
import type { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
@@ -12,11 +12,7 @@ import type {
|
||||
} from "../../domain/residualUnknown.js";
|
||||
import type { Result } from "../../domain/result.js";
|
||||
import { EvidenceLedger, type EvidenceImportDelta } from "./EvidenceLedger.js";
|
||||
import {
|
||||
UNKNOWN_REGISTRY_PROVIDER,
|
||||
unknownEvidenceLinks,
|
||||
unknownMutationEvidence,
|
||||
} from "./UnknownEvidence.js";
|
||||
import { unknownMutationEvidence } from "./UnknownEvidence.js";
|
||||
import type {
|
||||
EvidenceReader,
|
||||
EvidenceWriter,
|
||||
@@ -83,11 +79,10 @@ export class InvestigationRecords
|
||||
input: RecordUnknownInput,
|
||||
target?: BinaryTarget,
|
||||
): Result<ResidualUnknown, AnalysisError> {
|
||||
const recorded = this.#evidence.recordUnknown(
|
||||
return this.#evidence.recordUnknown(
|
||||
input,
|
||||
unknownMutationEvidence(target, input),
|
||||
unknownMutationEvidence(target, { operation: "record_unknown", input }),
|
||||
);
|
||||
return recorded;
|
||||
}
|
||||
|
||||
/** Commit derived Evidence and its Unknown atomically without an implicit subject. */
|
||||
@@ -95,12 +90,14 @@ export class InvestigationRecords
|
||||
evidence: Evidence,
|
||||
input: RecordUnknownInput,
|
||||
): Result<ResidualUnknown | null, AnalysisError> {
|
||||
const recorded = this.#evidence.recordWithUnknown(
|
||||
return this.#evidence.recordWithUnknown(
|
||||
evidence,
|
||||
input,
|
||||
unknownMutationEvidence(undefined, input),
|
||||
unknownMutationEvidence(undefined, {
|
||||
operation: "record_unknown",
|
||||
input,
|
||||
}),
|
||||
);
|
||||
return recorded;
|
||||
}
|
||||
|
||||
/** Update an optimistic revision with an explicitly supplied Evidence subject. */
|
||||
@@ -108,23 +105,11 @@ export class InvestigationRecords
|
||||
input: UpdateUnknownInput,
|
||||
target?: BinaryTarget,
|
||||
): Result<ResidualUnknown, AnalysisError> {
|
||||
const evidence = createEvidence(target, UNKNOWN_REGISTRY_PROVIDER, {
|
||||
predicateType: "rea.residual-unknown-mutation",
|
||||
const evidence = unknownMutationEvidence(target, {
|
||||
operation: "update_unknown",
|
||||
parameters: {
|
||||
unknown_id: input.unknown_id,
|
||||
expected_revision: input.expected_revision,
|
||||
},
|
||||
result: { action: "update", status: input.status },
|
||||
confidence: "derived",
|
||||
authority: "analyst-inference",
|
||||
evidenceLinks: unknownEvidenceLinks(input),
|
||||
limitations: [
|
||||
"Registry mutation evidence records analyst intent, not proof of the answer.",
|
||||
],
|
||||
input,
|
||||
});
|
||||
const updated = this.#evidence.updateUnknown(input, evidence);
|
||||
return updated;
|
||||
return this.#evidence.updateUnknown(input, evidence);
|
||||
}
|
||||
|
||||
/** Return detached current Unknown revisions matching the selected filters. */
|
||||
|
||||
@@ -7,43 +7,54 @@ import type {
|
||||
import type { ProviderIdentity } from "../AnalysisProvider.js";
|
||||
|
||||
/** Provider identity for analyst-authored residual-unknown mutations. */
|
||||
export const UNKNOWN_REGISTRY_PROVIDER: ProviderIdentity = {
|
||||
const UNKNOWN_REGISTRY_PROVIDER: ProviderIdentity = {
|
||||
id: "rea-unknown-registry",
|
||||
name: "REA residual unknown registry",
|
||||
version: "1",
|
||||
};
|
||||
|
||||
type UnknownMutation =
|
||||
| { readonly operation: "record_unknown"; readonly input: RecordUnknownInput }
|
||||
| {
|
||||
readonly operation: "update_unknown";
|
||||
readonly input: UpdateUnknownInput;
|
||||
};
|
||||
|
||||
/** Create immutable evidence for one approved residual-unknown mutation. */
|
||||
export const unknownMutationEvidence = (
|
||||
target: BinaryTarget | undefined,
|
||||
input: RecordUnknownInput,
|
||||
mutation: UnknownMutation,
|
||||
): Evidence =>
|
||||
createEvidence(target, UNKNOWN_REGISTRY_PROVIDER, {
|
||||
predicateType: "rea.residual-unknown-mutation",
|
||||
operation: "record_unknown",
|
||||
parameters: { domain: input.domain, severity: input.severity },
|
||||
result: {
|
||||
action: "record",
|
||||
question: input.question,
|
||||
required_authority: input.required_authority,
|
||||
required_confidence: input.required_confidence,
|
||||
},
|
||||
operation: mutation.operation,
|
||||
parameters: { ...mutation.input },
|
||||
result:
|
||||
mutation.operation === "record_unknown"
|
||||
? {
|
||||
action: "record",
|
||||
question: mutation.input.question,
|
||||
required_authority: mutation.input.required_authority,
|
||||
required_confidence: mutation.input.required_confidence,
|
||||
}
|
||||
: { action: "update", status: mutation.input.status },
|
||||
confidence: "derived",
|
||||
authority: "analyst-inference",
|
||||
evidenceLinks: unknownEvidenceLinks(input),
|
||||
evidenceLinks: unknownEvidenceLinks(mutation.input),
|
||||
limitations: [
|
||||
"Registry mutation evidence records analyst intent, not proof of the answer.",
|
||||
],
|
||||
});
|
||||
|
||||
/** Deduplicate evidence links carried by an unknown mutation. */
|
||||
export const unknownEvidenceLinks = (
|
||||
const unknownEvidenceLinks = (
|
||||
input: RecordUnknownInput | UpdateUnknownInput,
|
||||
): string[] =>
|
||||
[
|
||||
): string[] => [
|
||||
...new Set([
|
||||
...input.supporting_evidence_ids,
|
||||
...input.contradicting_evidence_ids,
|
||||
...("resolution" in input && input.resolution !== null
|
||||
? input.resolution.evidence_ids
|
||||
: []),
|
||||
].filter((id, index, values) => values.indexOf(id) === index);
|
||||
]),
|
||||
];
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
AnalysisCapabilityUnavailableError,
|
||||
AnalysisInputError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import type { ElectronActiveObservationInput } from "../../domain/javascript/electronActiveObservation.js";
|
||||
import { err, type Result } from "../../domain/result.js";
|
||||
|
||||
@@ -6,7 +6,7 @@ import type {
|
||||
ListElectronTargetsInput,
|
||||
} from "../../domain/javascript/electronObservation.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { createElectronEvidence } from "./ElectronEvidence.js";
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
|
||||
import { compareJavaScriptApplicationVersions } from "../../domain/javascript/javascriptApplicationVersionComparison.js";
|
||||
|
||||
@@ -6,7 +6,7 @@ import {
|
||||
AnalysisCancelledError,
|
||||
AnalysisCapabilityUnavailableError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type {
|
||||
ListJavaScriptRuntimeTargetsInput,
|
||||
ObserveJavaScriptRuntimeInput,
|
||||
|
||||
@@ -2,7 +2,7 @@ import { resolve } from "node:path";
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../../domain/evidence.js";
|
||||
import { analysisInputErrorFromIssues } from "../../domain/inputIssueProjection.js";
|
||||
import {
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
AnalysisInputError,
|
||||
AnalysisProtocolError,
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
applicationGraphEvidenceInputError,
|
||||
parseApplicationGraphEvidence,
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import {
|
||||
createEvidence,
|
||||
type Evidence,
|
||||
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
import { parseManagedMemberEvidence } from "../../domain/managed/managedMemberComparisonMatch.js";
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { EvidenceIntegrityError } from "../../domain/evidenceErrors.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import { parseBinaryTarget } from "../BinaryTargetResolver.js";
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { AnalysisProtocolError } from "../../domain/analysisErrorCore.js";
|
||||
import { type AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
|
||||
import { createEvidence, type Evidence } from "../../domain/evidence.js";
|
||||
import { jsonValueSchema } from "../../domain/jsonValue.js";
|
||||
import {
|
||||
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
AnalysisUnsupportedTargetError,
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { ArtifactOperationError } from "../domain/artifactOperationError.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { interfaceBuilderLimitsSchema } from "../domain/apple/interfaceBuilderGraph.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import { EventEmitter } from "node:events";
|
||||
import { PassThrough } from "node:stream";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -6,9 +8,151 @@ import { describe, expect, it, onTestFinished } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { CPU, fatImage, machoImage } from "./apple/MachoImage.fixture.js";
|
||||
import { MachOSliceArtifactReader } from "./MachOSliceArtifactReader.js";
|
||||
import { ok } from "../domain/result.js";
|
||||
import { err, ok } from "../domain/result.js";
|
||||
import { openRegularFile } from "../filesystem/RegularFile.js";
|
||||
import { ProviderProcessSupervisor } from "../process/ProviderProcess.js";
|
||||
import type { NativeCommandRunner } from "../native/CommandRunner.js";
|
||||
import { NativeCommandFailure } from "../native/CommandRunner.js";
|
||||
|
||||
const cleanupOwner = (runId: string): ProviderProcessSupervisor => {
|
||||
const child = Object.assign(new EventEmitter(), {
|
||||
stdout: new PassThrough(),
|
||||
stderr: new PassThrough(),
|
||||
exitCode: null as number | null,
|
||||
signalCode: null as NodeJS.Signals | null,
|
||||
kill: () => true,
|
||||
});
|
||||
return new ProviderProcessSupervisor({
|
||||
process: child,
|
||||
ownsProcessLifetime: true,
|
||||
ownership: {
|
||||
runId,
|
||||
leaderPid: 45_000,
|
||||
processGroupId: 45_000,
|
||||
expectedParentPid: process.pid,
|
||||
},
|
||||
cleanup: async () => ({ cleaned: false, reason: "still running" }),
|
||||
});
|
||||
};
|
||||
|
||||
describe("Mach-O cleanup exceptions", () => {
|
||||
it("retains an undefined cleanup rejection as an explicit failure", async () => {
|
||||
const reader = new MachOSliceArtifactReader(
|
||||
"/unused/macho",
|
||||
{},
|
||||
{
|
||||
run: async () => {
|
||||
throw new Error("Unexpected command");
|
||||
},
|
||||
retryCleanup: async () => {
|
||||
throw undefined;
|
||||
},
|
||||
},
|
||||
);
|
||||
await expect(reader.close()).rejects.toMatchObject({
|
||||
cleanup: {
|
||||
reason: "undefined",
|
||||
resources: ["native-command-process-cleanup"],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("Mach-O command cleanup lifecycle", () => {
|
||||
it("waits for active lipo capture before cleanup and preserves its failure", async () => {
|
||||
const root = await createTestTempDirectory("rea-slices-close-race-");
|
||||
const binary = join(root, "input");
|
||||
await writeFile(binary, Buffer.from("fixture"));
|
||||
let finishCommand!: (
|
||||
result: Awaited<ReturnType<NativeCommandRunner["run"]>>,
|
||||
) => void;
|
||||
let commandStarted!: () => void;
|
||||
const started = new Promise<void>((resolve) => {
|
||||
commandStarted = resolve;
|
||||
});
|
||||
const command = new Promise<
|
||||
Awaited<ReturnType<NativeCommandRunner["run"]>>
|
||||
>((resolve) => {
|
||||
finishCommand = resolve;
|
||||
});
|
||||
let cleanupRetries = 0;
|
||||
const runner: NativeCommandRunner = {
|
||||
run: () => {
|
||||
commandStarted();
|
||||
return command;
|
||||
},
|
||||
retryCleanup: async () => {
|
||||
cleanupRetries += 1;
|
||||
if (cleanupRetries > 1) return [];
|
||||
return [
|
||||
{
|
||||
owner: cleanupOwner("run:first"),
|
||||
reason: "first process group remains live",
|
||||
resources: ["run:first", "process-group:1234"],
|
||||
},
|
||||
{
|
||||
owner: cleanupOwner("run:second"),
|
||||
reason: "second process group remains live",
|
||||
resources: ["run:second", "process-group:5678"],
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
const reader = new MachOSliceArtifactReader(binary, {}, runner);
|
||||
const iteration = reader.entries();
|
||||
const firstEntry = iteration[Symbol.asyncIterator]().next();
|
||||
await started;
|
||||
const closing = reader.close();
|
||||
await Promise.resolve();
|
||||
expect(cleanupRetries).toBe(0);
|
||||
finishCommand(
|
||||
err(
|
||||
new NativeCommandFailure(
|
||||
"lipo",
|
||||
"timeout",
|
||||
null,
|
||||
undefined,
|
||||
{
|
||||
stdout: "partial lipo output",
|
||||
stderr: "partial lipo diagnostic",
|
||||
stdoutBytes: 19,
|
||||
stderrBytes: 23,
|
||||
exitCode: null,
|
||||
signal: null,
|
||||
truncated: false,
|
||||
},
|
||||
"capture process group remains unverified",
|
||||
["capture-group:9999"],
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
await expect(firstEntry).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
cause: expect.objectContaining({
|
||||
cleanupFailure: "capture process group remains unverified",
|
||||
}),
|
||||
cleanup: {
|
||||
reason: "capture process group remains unverified",
|
||||
resources: ["capture-group:9999"],
|
||||
},
|
||||
});
|
||||
await expect(closing).rejects.toMatchObject({
|
||||
cleanup: {
|
||||
reason:
|
||||
"run:first, process-group:1234: first process group remains live; run:second, process-group:5678: second process group remains live",
|
||||
resources: [
|
||||
"run:first",
|
||||
"process-group:1234",
|
||||
"run:second",
|
||||
"process-group:5678",
|
||||
],
|
||||
},
|
||||
});
|
||||
await reader.close();
|
||||
expect(cleanupRetries).toBe(2);
|
||||
});
|
||||
});
|
||||
|
||||
describe("MachOSliceArtifactReader", () => {
|
||||
it("keeps a borrowed source usable after a cancelled slice read", async () => {
|
||||
|
||||
@@ -39,6 +39,7 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
#ownedSource: OwnedFileHandle | undefined;
|
||||
#pendingSource: Promise<StableRegularFileDescriptor> | undefined;
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
readonly #pendingCommands = new Set<Promise<unknown>>();
|
||||
readonly #ownsSource: boolean;
|
||||
#closed = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
@@ -59,16 +60,46 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
this.#entries.clear();
|
||||
const source = await this.#ensureSource();
|
||||
await this.#verifySource(source);
|
||||
const captured = await this.runner.run(
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Universal Mach-O reader is closed",
|
||||
);
|
||||
const command = this.runner.run(
|
||||
"lipo",
|
||||
["-detailed_info", this.path],
|
||||
signal === undefined ? {} : { signal },
|
||||
);
|
||||
this.#pendingCommands.add(command);
|
||||
let captured: Awaited<typeof command>;
|
||||
try {
|
||||
captured = await command;
|
||||
} finally {
|
||||
this.#pendingCommands.delete(command);
|
||||
}
|
||||
if (!captured.ok)
|
||||
throw new ArtifactReaderFailure(
|
||||
captured.error.reason === "cancelled" ? "cancelled" : "unavailable",
|
||||
"lipo could not enumerate universal Mach-O slices",
|
||||
{ cause: captured.error },
|
||||
{
|
||||
cause: captured.error,
|
||||
...(captured.error.cleanupFailure === null &&
|
||||
captured.error.cleanupResources.length === 0
|
||||
? {}
|
||||
: {
|
||||
cleanup: {
|
||||
reason:
|
||||
captured.error.cleanupFailure ??
|
||||
"Native command cleanup remains unverified",
|
||||
resources: [...captured.error.cleanupResources],
|
||||
},
|
||||
}),
|
||||
},
|
||||
);
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Universal Mach-O reader closed during slice discovery",
|
||||
);
|
||||
this.#command = {
|
||||
tool: captured.value.tool,
|
||||
@@ -262,13 +293,49 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
this.#closePromise ??= this.#closeSource().catch((cause: unknown) => {
|
||||
this.#closePromise ??= this.#releaseResources().catch((cause: unknown) => {
|
||||
this.#closePromise = undefined;
|
||||
throw cause;
|
||||
});
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #releaseResources(): Promise<void> {
|
||||
await Promise.allSettled([...this.#pendingCommands]);
|
||||
let cleanup: ArtifactReaderFailure["cleanup"];
|
||||
try {
|
||||
const failures = (await this.runner.retryCleanup?.()) ?? [];
|
||||
if (failures.length > 0)
|
||||
cleanup = {
|
||||
reason: failures
|
||||
.map(
|
||||
({ reason, resources }) => `${resources.join(", ")}: ${reason}`,
|
||||
)
|
||||
.join("; "),
|
||||
resources: [
|
||||
...new Set(failures.flatMap(({ resources }) => resources)),
|
||||
],
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
cleanup = ArtifactReaderFailure.cleanupObservation(
|
||||
cause,
|
||||
"native-command-process-cleanup",
|
||||
);
|
||||
}
|
||||
try {
|
||||
await this.#closeSource();
|
||||
} catch (cause: unknown) {
|
||||
if (cleanup === undefined) throw cause;
|
||||
throw ArtifactReaderFailure.withCleanup(cause, cleanup);
|
||||
}
|
||||
if (cleanup !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
"Native command process cleanup remains incomplete",
|
||||
{ cleanup },
|
||||
);
|
||||
}
|
||||
|
||||
async #ensureSource(): Promise<StableRegularFileDescriptor> {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure(
|
||||
|
||||
@@ -147,8 +147,17 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
const filenames = new Set<string>();
|
||||
for await (const entry of this.#reader.getEntriesGenerator()) {
|
||||
abortIfNeeded(signal);
|
||||
// Each enumeration can replay valid entries, but a second record with
|
||||
// the same name must not replace an already emitted source identity.
|
||||
if (filenames.has(entry.filename))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Artifact path collision: ${entry.filename}`,
|
||||
);
|
||||
filenames.add(entry.filename);
|
||||
const metadata = projectZipEntry(entry);
|
||||
this.#entries.set(entry.filename, {
|
||||
source: entry,
|
||||
|
||||
@@ -27,9 +27,9 @@ import {
|
||||
type IntegrityContradiction,
|
||||
} from "../../domain/artifactGraph.js";
|
||||
import { AnalysisUnsupportedTargetError } from "../../domain/analysisErrorCore.js";
|
||||
import {
|
||||
type ArtifactResourceOwner,
|
||||
type ArtifactResourceScope,
|
||||
import type {
|
||||
ArtifactResourceOwner,
|
||||
ArtifactResourceScope,
|
||||
} from "../ArtifactResourceScope.js";
|
||||
import type { BinaryTarget } from "../../domain/binaryTargetTypes.js";
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
zipPackageFormatForPath,
|
||||
} from "../../domain/zipPackageFormat.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import { type ArtifactResourceOwner } from "../ArtifactResourceScope.js";
|
||||
import type { ArtifactResourceOwner } from "../ArtifactResourceScope.js";
|
||||
import type { HashResult } from "../ArtifactHash.js";
|
||||
import {
|
||||
hashStableRootArtifactHandle,
|
||||
|
||||
@@ -34,7 +34,7 @@ import {
|
||||
artifactManifestId,
|
||||
} from "../../domain/artifactIdentity.js";
|
||||
import { classifyAndHashRootForInventory } from "./classify.js";
|
||||
import { type HashResult } from "../ArtifactHash.js";
|
||||
import type { HashResult } from "../ArtifactHash.js";
|
||||
import {
|
||||
hashStableRootArtifact,
|
||||
hashStableRootArtifactHandle,
|
||||
|
||||
@@ -46,7 +46,7 @@ import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { AnalysisInputError } from "../domain/analysisErrorCore.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import {
|
||||
discoverCdpEndpoint,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
|
||||
import { createWebTextArtifact } from "../domain/webContentArtifact.js";
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { CdpCommandRejection } from "./CdpCommandRejection.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { safeParseJson } from "../domain/safeJson.js";
|
||||
import { WEB_RUNTIME_LIMITS } from "../domain/webRuntime.js";
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import {
|
||||
discoverCdpEndpoint,
|
||||
|
||||
@@ -5,7 +5,7 @@ import { z } from "zod";
|
||||
import { AnalysisCancelledError } from "../domain/analysisErrorCore.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
|
||||
import { safeParseJson } from "../domain/safeJson.js";
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { InspectWebPageInput } from "../domain/browserObservation.js";
|
||||
import type { WebPageInspection } from "../domain/browserObservationSchemas.js";
|
||||
import type { ProgressReporter } from "../application/ProgressReporter.js";
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { stableWebResources } from "../domain/webInventory.js";
|
||||
import type { CdpEndpointDiscovery, CdpEndpointTarget } from "./CdpEndpoint.js";
|
||||
import { CdpConnection } from "./CdpConnection.js";
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { BrowserObservationError } from "../domain/browserObservationError.js";
|
||||
import { type BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import type { BrowserObservationOperation } from "../domain/browserObservationErrors.js";
|
||||
import { CdpConnection } from "./CdpConnection.js";
|
||||
import {
|
||||
cdpTargetWebSocket,
|
||||
|
||||
@@ -3,7 +3,7 @@ import { dirname } from "node:path";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { _electron as electron } from "playwright-core";
|
||||
import type { _electron as electron } from "playwright-core";
|
||||
|
||||
import type {
|
||||
ExecutionOptions,
|
||||
@@ -60,8 +60,9 @@ type ElectronPaths = {
|
||||
export class PlaywrightElectronActiveProvider implements ElectronActiveObservationPort {
|
||||
constructor(
|
||||
private readonly environment: Readonly<Record<string, string | undefined>>,
|
||||
private readonly launch: typeof electron.launch = (options) =>
|
||||
electron.launch(options),
|
||||
// Load Playwright only when a capture launches; MCP startup stays off its module graph.
|
||||
private readonly launch: typeof electron.launch = async (options) =>
|
||||
(await import("playwright-core"))._electron.launch(options),
|
||||
) {}
|
||||
|
||||
identity(): ProviderIdentity {
|
||||
|
||||
@@ -2,12 +2,7 @@ import { chmod, mkdtemp, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import {
|
||||
chromium,
|
||||
type Browser,
|
||||
type BrowserContext,
|
||||
type Page,
|
||||
} from "playwright-core";
|
||||
import type { Browser, BrowserContext, Page, chromium } from "playwright-core";
|
||||
|
||||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
@@ -479,7 +474,9 @@ export const openPlaywrightScenarioBrowser = async (
|
||||
readonly retainCleanup?: (close: () => Promise<unknown>) => void;
|
||||
} = {},
|
||||
): Promise<OpenedScenarioBrowser> => {
|
||||
const launcher = options.launcher ?? chromium;
|
||||
// Load Playwright only when a scenario opens; MCP startup stays off its module graph.
|
||||
const launcher =
|
||||
options.launcher ?? (await import("playwright-core")).chromium;
|
||||
if (scenario.browser.mode === "connect")
|
||||
return openAttachedScenarioBrowser(
|
||||
scenario.browser,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { type BrowserContext, type Page } from "playwright-core";
|
||||
import type { BrowserContext, Page } from "playwright-core";
|
||||
|
||||
import type { BrowserScenario } from "../domain/browserScenario.js";
|
||||
import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
|
||||
|
||||
@@ -16,7 +16,7 @@ import { CLI_COMMANDS } from "../cliCommandNames.js";
|
||||
import { parseCliJsonInput } from "../cliJsonInput.js";
|
||||
import { logCliCommand } from "../cliLogging.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import { androidApplicationProjectionInputSchema } from "../domain/android/androidApplication.js";
|
||||
import { appleApplicationProjectionInputSchema } from "../domain/apple/appleApplication.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
|
||||
|
||||
+32
-25
@@ -1,24 +1,14 @@
|
||||
import { z } from "incur";
|
||||
|
||||
import {
|
||||
DOCTOR_PROVIDER_IDS,
|
||||
runDoctor,
|
||||
type DoctorScope,
|
||||
} from "../application/Doctor.js";
|
||||
import type { DoctorScope } from "../application/Doctor.js";
|
||||
import { DOCTOR_PROVIDER_IDS } from "../application/DoctorProviderIds.js";
|
||||
import {
|
||||
isSetupFailure,
|
||||
type SetupOptions,
|
||||
} from "../application/SetupTypes.js";
|
||||
import { runSetup } from "../application/Setup.js";
|
||||
import { systemSetupHost } from "../application/SetupHost.js";
|
||||
import {
|
||||
isUninstallFailure,
|
||||
runUninstall,
|
||||
systemUninstallHost,
|
||||
} from "../application/Uninstall.js";
|
||||
import { isUninstallFailure } from "../application/UninstallFailure.js";
|
||||
import { homeDirectoryFromEnvironment } from "../config/homeDirectory.js";
|
||||
import { isUpdateFailure, runUpdate } from "../application/Update.js";
|
||||
import { systemUpdateHost } from "../application/UpdateRuntime.js";
|
||||
import { isUpdateFailure } from "../application/UpdateFailure.js";
|
||||
import {
|
||||
confirmInteractiveSetup,
|
||||
renderInteractiveSetupResult,
|
||||
@@ -26,7 +16,6 @@ import {
|
||||
} from "./interactiveSetup.js";
|
||||
import { PRODUCT_IDENTITY } from "../identity.js";
|
||||
import { logCliCommand } from "../cliLogging.js";
|
||||
import { createSystemDoctorHost } from "../doctorRuntime.js";
|
||||
import { CLI_COMMANDS } from "../cliCommandNames.js";
|
||||
import type { Logger } from "pino";
|
||||
import type { CliInstance } from "./types.js";
|
||||
@@ -130,13 +119,17 @@ const registerDoctorCommand = (
|
||||
.describe("Require the installed REA skill identity to be aligned"),
|
||||
}),
|
||||
run: ({ options }) =>
|
||||
logCliCommand(logger, "doctor", () =>
|
||||
runDoctor(
|
||||
logCliCommand(logger, "doctor", async () => {
|
||||
const [{ runDoctor }, { createSystemDoctorHost }] = await Promise.all([
|
||||
import("../application/Doctor.js"),
|
||||
import("../doctorRuntime.js"),
|
||||
]);
|
||||
return runDoctor(
|
||||
options.target,
|
||||
createSystemDoctorHost(environment),
|
||||
doctorScope(options),
|
||||
),
|
||||
),
|
||||
);
|
||||
}),
|
||||
});
|
||||
};
|
||||
|
||||
@@ -157,15 +150,18 @@ const registerMaintenanceCommands = (
|
||||
logCliCommand(
|
||||
logger,
|
||||
"uninstall",
|
||||
() =>
|
||||
runUninstall(
|
||||
async () => {
|
||||
const { runUninstall, systemUninstallHost } =
|
||||
await import("../application/Uninstall.js");
|
||||
return runUninstall(
|
||||
options.purgeData,
|
||||
systemUninstallHost(
|
||||
homeDirectoryFromEnvironment(environment, process.platform),
|
||||
undefined,
|
||||
environment,
|
||||
),
|
||||
),
|
||||
);
|
||||
},
|
||||
isUninstallFailure,
|
||||
),
|
||||
});
|
||||
@@ -176,12 +172,17 @@ const registerMaintenanceCommands = (
|
||||
logCliCommand(
|
||||
logger,
|
||||
CLI_COMMANDS.update,
|
||||
() =>
|
||||
runUpdate(
|
||||
async () => {
|
||||
const [{ runUpdate }, { systemUpdateHost }] = await Promise.all([
|
||||
import("../application/Update.js"),
|
||||
import("../application/UpdateRuntime.js"),
|
||||
]);
|
||||
return runUpdate(
|
||||
PRODUCT_IDENTITY.packageVersion,
|
||||
systemUpdateHost(undefined, undefined, environment),
|
||||
formatExplicit ? "structured" : "human",
|
||||
),
|
||||
);
|
||||
},
|
||||
isUpdateFailure,
|
||||
),
|
||||
});
|
||||
@@ -208,6 +209,12 @@ const runSetupCommand = async (
|
||||
const { options } = input;
|
||||
const interactive = setupIsInteractive(options, input.formatExplicit);
|
||||
const hasExplicitScope = setupHasExplicitScope(options);
|
||||
const [{ runSetup }, { systemSetupHost }, { createSystemDoctorHost }] =
|
||||
await Promise.all([
|
||||
import("../application/Setup.js"),
|
||||
import("../application/SetupHost.js"),
|
||||
import("../doctorRuntime.js"),
|
||||
]);
|
||||
const result = await runSetup(
|
||||
setupRunOptions(input, interactive, hasExplicitScope),
|
||||
systemSetupHost(createSystemDoctorHost(environment), environment),
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { constants } from "node:buffer";
|
||||
import { type Writable } from "node:stream";
|
||||
import type { Writable } from "node:stream";
|
||||
|
||||
import { Cli, Filter } from "incur";
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ import type { Logger } from "pino";
|
||||
import { CLI_COMMANDS } from "./cliCommandNames.js";
|
||||
import type { JsonValue } from "./domain/jsonValue.js";
|
||||
import { projectAnalysisError } from "./domain/analysisErrorProjection.js";
|
||||
import { type AnalysisError } from "./domain/analysisErrorBase.js";
|
||||
import type { AnalysisError } from "./domain/analysisErrorBase.js";
|
||||
|
||||
/** Register caller-path Evidence commands. */
|
||||
export const registerEvidenceCommands = (
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
import { snapshotEnvironment } from "../process/snapshotEnvironment.js";
|
||||
import { parseConfig } from "../config/parseConfig.js";
|
||||
import {
|
||||
runDirectAnalysis as executeDirectAnalysis,
|
||||
runProviderAnalysis as executeProviderAnalysis,
|
||||
runManagedProviderExecution as executeManagedProvider,
|
||||
} from "../application/DirectAnalysis.js";
|
||||
import { runSessionStatus as executeSessionStatus } from "../application/DirectAnalysisStatus.js";
|
||||
import type { DirectAnalysisDependencies } from "../application/DirectAnalysisDependencies.js";
|
||||
|
||||
type DirectAnalysisRuntime = typeof import("../application/DirectAnalysis.js");
|
||||
type DirectAnalysisStatusRuntime =
|
||||
typeof import("../application/DirectAnalysisStatus.js");
|
||||
|
||||
type BoundOperation<Operation> = Operation extends (
|
||||
dependencies: infer _Dependencies,
|
||||
...args: infer Arguments
|
||||
) => infer Result
|
||||
? (...args: Arguments) => Result
|
||||
: never;
|
||||
|
||||
/** Bind one-shot analysis and status commands to one caller-selected environment. */
|
||||
export const createDirectAnalysis = (
|
||||
selectedEnvironment: Readonly<Record<string, string | undefined>>,
|
||||
@@ -24,10 +29,31 @@ export const createDirectAnalysis = (
|
||||
return createManagedBinarySession();
|
||||
},
|
||||
};
|
||||
const runDirectAnalysis: BoundOperation<
|
||||
DirectAnalysisRuntime["runDirectAnalysis"]
|
||||
> = async (...args) => {
|
||||
const { runDirectAnalysis } =
|
||||
await import("../application/DirectAnalysis.js");
|
||||
return runDirectAnalysis(dependencies, ...args);
|
||||
};
|
||||
const runProviderAnalysis: BoundOperation<
|
||||
DirectAnalysisRuntime["runProviderAnalysis"]
|
||||
> = async (...args) => {
|
||||
const { runProviderAnalysis } =
|
||||
await import("../application/DirectAnalysis.js");
|
||||
return runProviderAnalysis(dependencies, ...args);
|
||||
};
|
||||
const runSessionStatus: BoundOperation<
|
||||
DirectAnalysisStatusRuntime["runSessionStatus"]
|
||||
> = async (...args) => {
|
||||
const { runSessionStatus } =
|
||||
await import("../application/DirectAnalysisStatus.js");
|
||||
return runSessionStatus(dependencies, ...args);
|
||||
};
|
||||
return {
|
||||
runDirectAnalysis: executeDirectAnalysis.bind(undefined, dependencies),
|
||||
runProviderAnalysis: executeProviderAnalysis.bind(undefined, dependencies),
|
||||
runSessionStatus: executeSessionStatus.bind(undefined, dependencies),
|
||||
runDirectAnalysis,
|
||||
runProviderAnalysis,
|
||||
runSessionStatus,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -35,12 +61,18 @@ export const createDirectAnalysis = (
|
||||
export type DirectAnalysis = ReturnType<typeof createDirectAnalysis>;
|
||||
|
||||
/** Execute managed metadata without configuration or native provider candidates. */
|
||||
export const runManagedProviderExecution = executeManagedProvider.bind(
|
||||
undefined,
|
||||
{
|
||||
createManagedBinarySession: async () => {
|
||||
const { createManagedBinarySession } = await import("./binary.js");
|
||||
return createManagedBinarySession();
|
||||
export const runManagedProviderExecution: BoundOperation<
|
||||
DirectAnalysisRuntime["runManagedProviderExecution"]
|
||||
> = async (...args) => {
|
||||
const { runManagedProviderExecution } =
|
||||
await import("../application/DirectAnalysis.js");
|
||||
return runManagedProviderExecution(
|
||||
{
|
||||
createManagedBinarySession: async () => {
|
||||
const { createManagedBinarySession } = await import("./binary.js");
|
||||
return createManagedBinarySession();
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
...args,
|
||||
);
|
||||
};
|
||||
|
||||
@@ -27,11 +27,8 @@ import {
|
||||
import { ProviderSelectionError } from "./providerSelectionError.js";
|
||||
import { UnknownRegistryError } from "./unknownRegistryError.js";
|
||||
import { providerRetryAction } from "./providerOperationHealth.js";
|
||||
import {
|
||||
type AnalysisError,
|
||||
type AnalysisErrorTag,
|
||||
} from "./analysisErrorBase.js";
|
||||
import { type AnalysisErrorProjection } from "./analysisErrorProjection.js";
|
||||
import type { AnalysisError, AnalysisErrorTag } from "./analysisErrorBase.js";
|
||||
import type { AnalysisErrorProjection } from "./analysisErrorProjection.js";
|
||||
|
||||
export const analysisErrorRemediationAction = (
|
||||
error: AnalysisError,
|
||||
|
||||
@@ -40,10 +40,7 @@ import { ProviderAdapterError } from "./providerAdapterError.js";
|
||||
import { ProviderSelectionError } from "./providerSelectionError.js";
|
||||
import { UnknownRegistryError } from "./unknownRegistryError.js";
|
||||
import { providerRetryAction } from "./providerOperationHealth.js";
|
||||
import {
|
||||
type AnalysisError,
|
||||
type AnalysisErrorTag,
|
||||
} from "./analysisErrorBase.js";
|
||||
import type { AnalysisError, AnalysisErrorTag } from "./analysisErrorBase.js";
|
||||
|
||||
/** Project expected failures into exhaustive, secret-safe caller fields. */
|
||||
export const projectAnalysisError = (
|
||||
|
||||
+340
-199
@@ -3,7 +3,10 @@ import { createHash } from "node:crypto";
|
||||
import canonicalize from "canonicalize";
|
||||
import { z } from "zod";
|
||||
|
||||
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
|
||||
import {
|
||||
compareUnicodeCodePoints,
|
||||
compositeKey,
|
||||
} from "./unicodeCodePointOrder.js";
|
||||
|
||||
import {
|
||||
analysisProfileSchema,
|
||||
@@ -19,6 +22,7 @@ import {
|
||||
evidenceBundleForTarget,
|
||||
evidenceBundleSchema,
|
||||
parseEvidenceBundle,
|
||||
parseImmutableEvidenceBundle,
|
||||
} from "./evidenceBundle.js";
|
||||
import {
|
||||
evidenceLocationSchema,
|
||||
@@ -33,6 +37,8 @@ import {
|
||||
} from "./jsonValue.js";
|
||||
import { digestSchema } from "./../domain/digests.js";
|
||||
import { prefixedDigestSchema } from "./../domain/digests.js";
|
||||
import { canonicalJsonDigestSteps } from "./canonicalJsonDigestSteps.js";
|
||||
import { freezeJsonSnapshot } from "./immutableJson.js";
|
||||
|
||||
const architectureSchema = z.enum(BINARY_ARCHITECTURES);
|
||||
const formatSchema = z.enum([
|
||||
@@ -136,6 +142,146 @@ export type AnalysisSnapshotWorkflowEntry = z.infer<typeof workflowEntrySchema>;
|
||||
export type AnalysisSnapshotTarget = z.infer<typeof targetSchema>;
|
||||
export type AnalysisSnapshotBinding = z.infer<typeof bindingSchema>;
|
||||
|
||||
// These handles only represent entries parsed in this realm. File readers can
|
||||
// release their payloads while retaining an authenticated comparison operand.
|
||||
const preparedEntryBrand = Symbol("prepared-snapshot-entry");
|
||||
export interface PreparedSnapshotEntry {
|
||||
readonly [preparedEntryBrand]: true;
|
||||
}
|
||||
type EntryMetadata = Omit<AnalysisSnapshotEntry, "execution"> & {
|
||||
readonly execution: Omit<
|
||||
AnalysisSnapshotEntry["execution"],
|
||||
"result" | "raw_result"
|
||||
>;
|
||||
};
|
||||
type WorkflowMetadata = Omit<AnalysisSnapshotWorkflowEntry, "execution"> & {
|
||||
readonly execution: Omit<
|
||||
AnalysisSnapshotWorkflowEntry["execution"],
|
||||
"result" | "raw_result"
|
||||
>;
|
||||
};
|
||||
type PreparedEntry =
|
||||
| {
|
||||
readonly kind: "provider";
|
||||
readonly entry: EntryMetadata;
|
||||
readonly digest: string;
|
||||
}
|
||||
| {
|
||||
readonly kind: "workflow";
|
||||
readonly entry: WorkflowMetadata;
|
||||
readonly digest: string;
|
||||
};
|
||||
const preparedEntries = new WeakMap<PreparedSnapshotEntry, PreparedEntry>();
|
||||
|
||||
/** Capture a cache entry without retaining a second copy of its full payload. */
|
||||
export const prepareSnapshotEntry = (
|
||||
input: unknown,
|
||||
kind: "provider" | "workflow",
|
||||
): PreparedSnapshotEntry => {
|
||||
const handle: PreparedSnapshotEntry = Object.freeze({
|
||||
[preparedEntryBrand]: true as const,
|
||||
});
|
||||
if (kind === "provider") {
|
||||
const entry = entrySchema.parse(input);
|
||||
const { result, raw_result, ...execution } = entry.execution;
|
||||
preparedEntries.set(handle, {
|
||||
kind,
|
||||
entry: { ...entry, execution },
|
||||
digest: snapshotContentDigest(result, raw_result, execution),
|
||||
});
|
||||
} else {
|
||||
const entry = workflowEntrySchema.parse(input);
|
||||
const { result, raw_result, ...execution } = entry.execution;
|
||||
preparedEntries.set(handle, {
|
||||
kind,
|
||||
entry: { ...entry, execution },
|
||||
digest: snapshotContentDigest(result, raw_result, execution),
|
||||
});
|
||||
}
|
||||
return handle;
|
||||
};
|
||||
|
||||
const snapshotFileHeaderSchema = z.object({
|
||||
target: targetSchema,
|
||||
binding: bindingSchema,
|
||||
evidence_bundle: z.unknown(),
|
||||
});
|
||||
|
||||
/** Reconstruct file entries only from fully authenticated matching Evidence. */
|
||||
export const parsePreparedAnalysisSnapshot = (
|
||||
input: unknown,
|
||||
entries: readonly PreparedSnapshotEntry[],
|
||||
workflowEntries: readonly PreparedSnapshotEntry[],
|
||||
): AnalysisSnapshot => {
|
||||
const header = snapshotFileHeaderSchema.parse(input);
|
||||
const bundle = parseImmutableEvidenceBundle(header.evidence_bundle);
|
||||
const providerRows = entries.map((handle) => {
|
||||
const row = preparedEntries.get(handle);
|
||||
if (row?.kind !== "provider")
|
||||
throw new TypeError(
|
||||
"Analysis snapshot entry was not parsed in this realm",
|
||||
);
|
||||
return row;
|
||||
});
|
||||
const workflowRows = workflowEntries.map((handle) => {
|
||||
const row = preparedEntries.get(handle);
|
||||
if (row?.kind !== "workflow")
|
||||
throw new TypeError(
|
||||
"Analysis snapshot workflow entry was not parsed in this realm",
|
||||
);
|
||||
return row;
|
||||
});
|
||||
const restored = sealSnapshot(
|
||||
restoreSnapshot(header, bundle, providerRows, workflowRows),
|
||||
);
|
||||
for (const handle of [...entries, ...workflowEntries])
|
||||
preparedEntries.delete(handle);
|
||||
return restored;
|
||||
};
|
||||
|
||||
const contentDigest = (value: JsonValue): string => {
|
||||
const steps = canonicalJsonDigestSteps(value);
|
||||
let step = steps.next();
|
||||
while (!step.done) step = steps.next();
|
||||
return step.value;
|
||||
};
|
||||
|
||||
const snapshotContentDigest = (
|
||||
result: JsonValue,
|
||||
rawResult: JsonValue | null,
|
||||
execution: Pick<
|
||||
AnalysisSnapshotEntry["execution"],
|
||||
"limitations" | "locations" | "subject"
|
||||
>,
|
||||
): string => {
|
||||
const metadata = jsonObjectSchema.parse({
|
||||
limitations: execution.limitations,
|
||||
locations: execution.locations,
|
||||
subject:
|
||||
execution.subject === null
|
||||
? null
|
||||
: {
|
||||
local_path: execution.subject.path,
|
||||
sha256: execution.subject.sha256,
|
||||
format: execution.subject.format,
|
||||
architecture: execution.subject.architecture,
|
||||
},
|
||||
});
|
||||
return contentDigest({
|
||||
...metadata,
|
||||
normalized_result: result,
|
||||
raw_result: rawResult,
|
||||
});
|
||||
};
|
||||
|
||||
const evidenceContentDigest = (evidence: Evidence): string => {
|
||||
return contentDigest({
|
||||
...jsonObjectSchema.parse(evidenceContentMetadata(evidence)),
|
||||
normalized_result: evidence.normalized_result,
|
||||
raw_result: evidence.raw_result,
|
||||
});
|
||||
};
|
||||
|
||||
interface SnapshotExecution {
|
||||
readonly result: JsonValue;
|
||||
readonly rawResult: JsonValue | null;
|
||||
@@ -305,22 +451,10 @@ const evidenceMatchesEntry = (
|
||||
snapshot: Pick<AnalysisSnapshot, "target" | "binding">,
|
||||
): boolean =>
|
||||
isCorrespondingEvidence(evidence, snapshot) &&
|
||||
canonicalJson({
|
||||
query: evidenceQueryKey(evidence),
|
||||
normalized_result: evidence.normalized_result,
|
||||
raw_result: evidence.raw_result,
|
||||
limitations: evidence.limitations,
|
||||
locations: evidence.locations,
|
||||
subject:
|
||||
evidence.subject === null
|
||||
? null
|
||||
: {
|
||||
local_path: evidence.subject.local_path,
|
||||
sha256: evidence.subject.digest.sha256,
|
||||
format: evidence.subject.format,
|
||||
architecture: evidence.subject.architecture,
|
||||
},
|
||||
}) === entryEvidenceKey(entry, snapshot.binding);
|
||||
compositeKey([
|
||||
evidenceQueryKey(evidence),
|
||||
evidenceContentDigest(evidence),
|
||||
]) === entryEvidenceKey(entry, snapshot.binding);
|
||||
|
||||
/** Compute the stable lookup key for one provider/profile-specific query. */
|
||||
export const analysisQueryId = (
|
||||
@@ -373,94 +507,189 @@ export const createAnalysisSnapshotEntry = (input: {
|
||||
};
|
||||
};
|
||||
|
||||
const validatedSnapshots = new WeakMap<object, AnalysisSnapshot>();
|
||||
|
||||
/** Parse a provider/profile-exact snapshot and verify its entries and evidence. */
|
||||
export const parseAnalysisSnapshot = (input: unknown): AnalysisSnapshot => {
|
||||
if (typeof input === "object" && input !== null) {
|
||||
const owned = validatedSnapshots.get(input);
|
||||
if (owned !== undefined) return owned;
|
||||
}
|
||||
const parsed = analysisSnapshotSchema.parse(input);
|
||||
parseEvidenceBundle(parsed.evidence_bundle);
|
||||
const scopedEvidence = evidenceBundleForTarget(
|
||||
parsed.evidence_bundle,
|
||||
parsed.target.sha256,
|
||||
restoreSnapshot(
|
||||
parsed,
|
||||
parseEvidenceBundle(parsed.evidence_bundle),
|
||||
parsed.entries.map((entry) => ({
|
||||
kind: "provider",
|
||||
entry,
|
||||
digest: snapshotContentDigest(
|
||||
entry.execution.result,
|
||||
entry.execution.raw_result,
|
||||
entry.execution,
|
||||
),
|
||||
})),
|
||||
parsed.workflow_entries.map((entry) => ({
|
||||
kind: "workflow",
|
||||
entry,
|
||||
digest: snapshotContentDigest(
|
||||
entry.execution.result,
|
||||
entry.execution.raw_result,
|
||||
entry.execution,
|
||||
),
|
||||
})),
|
||||
);
|
||||
// The validated canonical bundle is filtered, never extended. Equal counts
|
||||
// establish that no record or unknown was removed without serializing results.
|
||||
return parsed;
|
||||
};
|
||||
|
||||
/** Own an authenticated immutable cache while preserving mutable caller exports. */
|
||||
export const immutableAnalysisSnapshot = (input: unknown): AnalysisSnapshot =>
|
||||
sealSnapshot(parseAnalysisSnapshot(input));
|
||||
|
||||
const sealSnapshot = (snapshot: AnalysisSnapshot): AnalysisSnapshot => {
|
||||
for (const entry of [...snapshot.entries, ...snapshot.workflow_entries]) {
|
||||
freezeJsonSnapshot(entry.execution.result);
|
||||
freezeJsonSnapshot(entry.execution.raw_result);
|
||||
}
|
||||
freezeJsonSnapshot(snapshot);
|
||||
validatedSnapshots.set(snapshot, snapshot);
|
||||
return snapshot;
|
||||
};
|
||||
|
||||
const restoreSnapshot = (
|
||||
header: Pick<AnalysisSnapshot, "target" | "binding">,
|
||||
bundle: AnalysisSnapshot["evidence_bundle"],
|
||||
providerRows: readonly Extract<PreparedEntry, { kind: "provider" }>[],
|
||||
workflowRows: readonly Extract<PreparedEntry, { kind: "workflow" }>[],
|
||||
): AnalysisSnapshot => {
|
||||
const scopedEvidence = evidenceBundleForTarget(bundle, header.target.sha256);
|
||||
if (
|
||||
scopedEvidence.records.length !== parsed.evidence_bundle.records.length ||
|
||||
scopedEvidence.unknowns.length !== parsed.evidence_bundle.unknowns.length
|
||||
scopedEvidence.records.length !== bundle.records.length ||
|
||||
scopedEvidence.unknowns.length !== bundle.unknowns.length
|
||||
)
|
||||
throw new TypeError(
|
||||
"Analysis snapshot evidence contains records for another target",
|
||||
);
|
||||
if (!hasCanonicalQueryOrder(parsed.entries))
|
||||
if (!hasCanonicalQueryOrder(providerRows.map(({ entry }) => entry)))
|
||||
throw new TypeError("Analysis snapshot entries are not canonical");
|
||||
if (!hasCanonicalQueryOrder(parsed.workflow_entries))
|
||||
if (!hasCanonicalQueryOrder(workflowRows.map(({ entry }) => entry)))
|
||||
throw new TypeError("Analysis snapshot workflow entries are not canonical");
|
||||
const ids = new Set<string>();
|
||||
const index = createEvidenceIndex(parsed);
|
||||
const workflowIndex = createWorkflowEvidenceIndex(parsed);
|
||||
for (const entry of parsed.entries) {
|
||||
const corresponding = new Set<string>();
|
||||
const matching = new Map<string, Evidence>();
|
||||
for (const evidence of bundle.records) {
|
||||
if (
|
||||
entry.execution.provider.id !== parsed.binding.provider.id ||
|
||||
entry.execution.provider.name !== parsed.binding.provider.name ||
|
||||
entry.execution.provider.version !== parsed.binding.provider.version
|
||||
evidence.predicate_type !== "rea.analysis" ||
|
||||
evidence.authority !== "shipped-artifact" ||
|
||||
evidence.subject?.digest.sha256 !== header.target.sha256 ||
|
||||
evidence.analysis_profile === null ||
|
||||
(evidence.confidence !== "observed" && evidence.confidence !== "derived")
|
||||
)
|
||||
throw new TypeError("Analysis snapshot entry provider does not match");
|
||||
continue;
|
||||
if (
|
||||
evidence.confidence === "observed" &&
|
||||
!isCorrespondingEvidence(evidence, header)
|
||||
)
|
||||
continue;
|
||||
const query = compositeKey([
|
||||
evidence.confidence,
|
||||
evidenceQueryKey(evidence),
|
||||
]);
|
||||
corresponding.add(query);
|
||||
matching.set(
|
||||
compositeKey([query, evidenceContentDigest(evidence)]),
|
||||
evidence,
|
||||
);
|
||||
}
|
||||
const find = (
|
||||
query: string,
|
||||
digest: string,
|
||||
label: string,
|
||||
operation: string,
|
||||
): Evidence => {
|
||||
const evidence = matching.get(compositeKey([query, digest]));
|
||||
if (evidence !== undefined) return evidence;
|
||||
throw new TypeError(
|
||||
corresponding.has(query)
|
||||
? `Analysis snapshot ${label} ${operation} differs from its Evidence record`
|
||||
: `Analysis snapshot ${label} ${operation} has no profile-bound Evidence record`,
|
||||
);
|
||||
};
|
||||
const checkId = (
|
||||
entry: EntryMetadata | WorkflowMetadata,
|
||||
workflow: boolean,
|
||||
): void => {
|
||||
const expected = analysisQueryId(
|
||||
parsed.target,
|
||||
parsed.binding,
|
||||
header.target,
|
||||
header.binding,
|
||||
entry.operation,
|
||||
entry.parameters,
|
||||
);
|
||||
if (entry.query_id !== expected)
|
||||
throw new TypeError("Analysis snapshot query identifier does not match");
|
||||
throw new TypeError(
|
||||
workflow
|
||||
? "Analysis snapshot workflow query identifier does not match"
|
||||
: "Analysis snapshot query identifier does not match",
|
||||
);
|
||||
if (ids.has(entry.query_id))
|
||||
throw new TypeError("Analysis snapshot contains duplicate queries");
|
||||
ids.add(entry.query_id);
|
||||
if (!entryHasMatchingEvidence(entry, parsed, index)) {
|
||||
if (entryHasCorrespondingEvidence(entry, parsed, index))
|
||||
throw new TypeError(
|
||||
`Analysis snapshot entry ${entry.operation} differs from its Evidence record`,
|
||||
);
|
||||
throw new TypeError(
|
||||
`Analysis snapshot entry ${entry.operation} has no profile-bound Evidence record`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const entry of parsed.workflow_entries) {
|
||||
};
|
||||
const entries = providerRows.map(({ entry, digest }) => {
|
||||
if (
|
||||
entry.execution.provider.id !==
|
||||
entry.execution.analysis_profile.provider.id ||
|
||||
entry.execution.provider.name !==
|
||||
entry.execution.analysis_profile.provider.name ||
|
||||
entry.execution.provider.version !==
|
||||
entry.execution.analysis_profile.provider.version
|
||||
entry.execution.provider.id !== header.binding.provider.id ||
|
||||
entry.execution.provider.name !== header.binding.provider.name ||
|
||||
entry.execution.provider.version !== header.binding.provider.version
|
||||
)
|
||||
throw new TypeError("Analysis snapshot entry provider does not match");
|
||||
checkId(entry, false);
|
||||
const evidence = find(
|
||||
compositeKey(["observed", entryQueryKey(entry, header.binding)]),
|
||||
digest,
|
||||
"entry",
|
||||
entry.operation,
|
||||
);
|
||||
return {
|
||||
...entry,
|
||||
execution: {
|
||||
...entry.execution,
|
||||
result: evidence.normalized_result,
|
||||
raw_result: evidence.raw_result,
|
||||
},
|
||||
};
|
||||
});
|
||||
const workflow_entries = workflowRows.map(({ entry, digest }) => {
|
||||
const provider = entry.execution.analysis_profile.provider;
|
||||
if (
|
||||
entry.execution.provider.id !== provider.id ||
|
||||
entry.execution.provider.name !== provider.name ||
|
||||
entry.execution.provider.version !== provider.version
|
||||
)
|
||||
throw new TypeError(
|
||||
"Analysis snapshot workflow entry provider does not match its profile",
|
||||
);
|
||||
const expected = analysisQueryId(
|
||||
parsed.target,
|
||||
parsed.binding,
|
||||
checkId(entry, true);
|
||||
const evidence = find(
|
||||
compositeKey(["derived", workflowEntryQueryKey(entry)]),
|
||||
digest,
|
||||
"workflow entry",
|
||||
entry.operation,
|
||||
entry.parameters,
|
||||
);
|
||||
if (entry.query_id !== expected)
|
||||
throw new TypeError(
|
||||
"Analysis snapshot workflow query identifier does not match",
|
||||
);
|
||||
if (ids.has(entry.query_id))
|
||||
throw new TypeError("Analysis snapshot contains duplicate queries");
|
||||
ids.add(entry.query_id);
|
||||
if (!workflowEntryHasMatchingEvidence(entry, workflowIndex)) {
|
||||
if (workflowEntryHasCorrespondingEvidence(entry, workflowIndex))
|
||||
throw new TypeError(
|
||||
`Analysis snapshot workflow entry ${entry.operation} differs from its Evidence record`,
|
||||
);
|
||||
throw new TypeError(
|
||||
`Analysis snapshot workflow entry ${entry.operation} has no profile-bound Evidence record`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return parsed;
|
||||
return {
|
||||
...entry,
|
||||
execution: {
|
||||
...entry.execution,
|
||||
result: evidence.normalized_result,
|
||||
raw_result: evidence.raw_result,
|
||||
},
|
||||
};
|
||||
});
|
||||
return {
|
||||
target: header.target,
|
||||
binding: header.binding,
|
||||
entries,
|
||||
workflow_entries,
|
||||
evidence_bundle: bundle,
|
||||
};
|
||||
};
|
||||
|
||||
const hasCanonicalQueryOrder = (
|
||||
@@ -478,56 +707,6 @@ const hasCanonicalQueryOrder = (
|
||||
return true;
|
||||
};
|
||||
|
||||
/** Check that a cached provider execution is represented by bundled Evidence. */
|
||||
interface EvidenceIndex {
|
||||
readonly matching: ReadonlySet<string>;
|
||||
readonly corresponding: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
const createEvidenceIndex = (
|
||||
snapshot: Pick<AnalysisSnapshot, "target" | "binding" | "evidence_bundle">,
|
||||
): EvidenceIndex => {
|
||||
const matching = new Set<string>();
|
||||
const corresponding = new Set<string>();
|
||||
for (const evidence of snapshot.evidence_bundle.records) {
|
||||
if (!isCorrespondingEvidence(evidence, snapshot)) continue;
|
||||
const query = evidenceQueryKey(evidence);
|
||||
corresponding.add(query);
|
||||
matching.add(
|
||||
canonicalJson({
|
||||
query,
|
||||
normalized_result: evidence.normalized_result,
|
||||
raw_result: evidence.raw_result,
|
||||
limitations: evidence.limitations,
|
||||
locations: evidence.locations,
|
||||
subject:
|
||||
evidence.subject === null
|
||||
? null
|
||||
: {
|
||||
local_path: evidence.subject.local_path,
|
||||
sha256: evidence.subject.digest.sha256,
|
||||
format: evidence.subject.format,
|
||||
architecture: evidence.subject.architecture,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
const result = { matching, corresponding };
|
||||
return result;
|
||||
};
|
||||
|
||||
const entryHasMatchingEvidence = (
|
||||
entry: AnalysisSnapshotEntry,
|
||||
snapshot: Pick<AnalysisSnapshot, "target" | "binding" | "evidence_bundle">,
|
||||
index: EvidenceIndex,
|
||||
): boolean => index.matching.has(entryEvidenceKey(entry, snapshot.binding));
|
||||
|
||||
const entryHasCorrespondingEvidence = (
|
||||
entry: AnalysisSnapshotEntry,
|
||||
snapshot: Pick<AnalysisSnapshot, "target" | "binding" | "evidence_bundle">,
|
||||
index: EvidenceIndex,
|
||||
): boolean => index.corresponding.has(entryQueryKey(entry, snapshot.binding));
|
||||
|
||||
const isCorrespondingEvidence = (
|
||||
evidence: Evidence,
|
||||
snapshot: Pick<AnalysisSnapshot, "target" | "binding">,
|
||||
@@ -543,7 +722,7 @@ const isCorrespondingEvidence = (
|
||||
);
|
||||
|
||||
const evidenceQueryKey = (evidence: Evidence): string =>
|
||||
canonicalJson({
|
||||
contentDigest({
|
||||
operation: evidence.operation,
|
||||
parameters: evidence.parameters,
|
||||
provider: evidence.provider,
|
||||
@@ -551,10 +730,12 @@ const evidenceQueryKey = (evidence: Evidence): string =>
|
||||
});
|
||||
|
||||
const entryQueryKey = (
|
||||
entry: AnalysisSnapshotEntry,
|
||||
entry: Pick<AnalysisSnapshotEntry, "operation" | "parameters"> & {
|
||||
readonly execution: Pick<AnalysisSnapshotEntry["execution"], "provider">;
|
||||
},
|
||||
binding: AnalysisSnapshotBinding,
|
||||
): string =>
|
||||
canonicalJson({
|
||||
contentDigest({
|
||||
operation: entry.operation,
|
||||
parameters: entry.parameters,
|
||||
provider: entry.execution.provider,
|
||||
@@ -565,25 +746,24 @@ const entryEvidenceKey = (
|
||||
entry: AnalysisSnapshotEntry,
|
||||
binding: AnalysisSnapshotBinding,
|
||||
): string =>
|
||||
canonicalJson({
|
||||
query: entryQueryKey(entry, binding),
|
||||
normalized_result: entry.execution.result,
|
||||
raw_result: entry.execution.raw_result,
|
||||
limitations: entry.execution.limitations,
|
||||
locations: entry.execution.locations,
|
||||
subject:
|
||||
entry.execution.subject === null
|
||||
? null
|
||||
: {
|
||||
local_path: entry.execution.subject.path,
|
||||
sha256: entry.execution.subject.sha256,
|
||||
format: entry.execution.subject.format,
|
||||
architecture: entry.execution.subject.architecture,
|
||||
},
|
||||
});
|
||||
compositeKey([
|
||||
entryQueryKey(entry, binding),
|
||||
snapshotContentDigest(
|
||||
entry.execution.result,
|
||||
entry.execution.raw_result,
|
||||
entry.execution,
|
||||
),
|
||||
]);
|
||||
|
||||
const workflowEntryQueryKey = (entry: AnalysisSnapshotWorkflowEntry): string =>
|
||||
canonicalJson({
|
||||
const workflowEntryQueryKey = (
|
||||
entry: Pick<AnalysisSnapshotWorkflowEntry, "operation" | "parameters"> & {
|
||||
readonly execution: Pick<
|
||||
AnalysisSnapshotWorkflowEntry["execution"],
|
||||
"provider" | "analysis_profile"
|
||||
>;
|
||||
},
|
||||
): string =>
|
||||
contentDigest({
|
||||
operation: entry.operation,
|
||||
parameters: entry.parameters,
|
||||
provider: entry.execution.provider,
|
||||
@@ -593,22 +773,14 @@ const workflowEntryQueryKey = (entry: AnalysisSnapshotWorkflowEntry): string =>
|
||||
const workflowEntryEvidenceKey = (
|
||||
entry: AnalysisSnapshotWorkflowEntry,
|
||||
): string =>
|
||||
canonicalJson({
|
||||
query: workflowEntryQueryKey(entry),
|
||||
normalized_result: entry.execution.result,
|
||||
raw_result: entry.execution.raw_result,
|
||||
limitations: entry.execution.limitations,
|
||||
locations: entry.execution.locations,
|
||||
subject:
|
||||
entry.execution.subject === null
|
||||
? null
|
||||
: {
|
||||
local_path: entry.execution.subject.path,
|
||||
sha256: entry.execution.subject.sha256,
|
||||
format: entry.execution.subject.format,
|
||||
architecture: entry.execution.subject.architecture,
|
||||
},
|
||||
});
|
||||
compositeKey([
|
||||
workflowEntryQueryKey(entry),
|
||||
snapshotContentDigest(
|
||||
entry.execution.result,
|
||||
entry.execution.raw_result,
|
||||
entry.execution,
|
||||
),
|
||||
]);
|
||||
|
||||
const isWorkflowEvidence = (
|
||||
evidence: Evidence,
|
||||
@@ -628,49 +800,18 @@ const isWorkflowEvidence = (
|
||||
evidence.provider.name === entry.execution.provider.name &&
|
||||
evidence.provider.version === entry.execution.provider.version;
|
||||
|
||||
const createWorkflowEvidenceIndex = (
|
||||
snapshot: Pick<AnalysisSnapshot, "target" | "evidence_bundle">,
|
||||
): EvidenceIndex => {
|
||||
const matching = new Set<string>();
|
||||
const corresponding = new Set<string>();
|
||||
for (const evidence of snapshot.evidence_bundle.records) {
|
||||
if (
|
||||
evidence.predicate_type !== "rea.analysis" ||
|
||||
evidence.confidence !== "derived" ||
|
||||
evidence.authority !== "shipped-artifact" ||
|
||||
evidence.subject?.digest.sha256 !== snapshot.target.sha256 ||
|
||||
evidence.analysis_profile === null
|
||||
)
|
||||
continue;
|
||||
corresponding.add(evidenceQueryKey(evidence));
|
||||
matching.add(canonicalJson(evidenceContentProjection(evidence)));
|
||||
}
|
||||
return { matching, corresponding };
|
||||
};
|
||||
|
||||
const workflowEntryHasMatchingEvidence = (
|
||||
entry: AnalysisSnapshotWorkflowEntry,
|
||||
index: EvidenceIndex,
|
||||
): boolean => index.matching.has(workflowEntryEvidenceKey(entry));
|
||||
|
||||
const workflowEntryHasCorrespondingEvidence = (
|
||||
entry: AnalysisSnapshotWorkflowEntry,
|
||||
index: EvidenceIndex,
|
||||
): boolean => index.corresponding.has(workflowEntryQueryKey(entry));
|
||||
|
||||
const evidenceMatchesWorkflowEntry = (
|
||||
evidence: Evidence,
|
||||
entry: AnalysisSnapshotWorkflowEntry,
|
||||
snapshot: Pick<AnalysisSnapshot, "target">,
|
||||
): boolean =>
|
||||
isWorkflowEvidence(evidence, snapshot, entry) &&
|
||||
canonicalJson(evidenceContentProjection(evidence)) ===
|
||||
workflowEntryEvidenceKey(entry);
|
||||
compositeKey([
|
||||
evidenceQueryKey(evidence),
|
||||
evidenceContentDigest(evidence),
|
||||
]) === workflowEntryEvidenceKey(entry);
|
||||
|
||||
const evidenceContentProjection = (evidence: Evidence) => ({
|
||||
query: evidenceQueryKey(evidence),
|
||||
normalized_result: evidence.normalized_result,
|
||||
raw_result: evidence.raw_result,
|
||||
const evidenceContentMetadata = (evidence: Evidence) => ({
|
||||
limitations: evidence.limitations,
|
||||
locations: evidence.locations,
|
||||
subject:
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
type ArtifactInventoryResult,
|
||||
type ArtifactNode,
|
||||
type ArtifactOccurrence,
|
||||
import type {
|
||||
ArtifactInventoryResult,
|
||||
ArtifactNode,
|
||||
ArtifactOccurrence,
|
||||
} from "./artifactGraph.js";
|
||||
import { evidenceSchema } from "./evidence.js";
|
||||
import {
|
||||
|
||||
@@ -234,11 +234,33 @@ export const describeValidationFailure = (cause: unknown): string => {
|
||||
|
||||
/** Parse records, verify semantic IDs, and reject inconsistent manifests. */
|
||||
export const parseEvidenceBundle = (input: unknown): EvidenceBundle => {
|
||||
return parseBundle(input, false);
|
||||
};
|
||||
|
||||
/** Authenticate and seal file-owned payloads once for downstream zero-copy reuse. */
|
||||
export const parseImmutableEvidenceBundle = (
|
||||
input: unknown,
|
||||
): EvidenceBundle => {
|
||||
return parseBundle(input, true);
|
||||
};
|
||||
|
||||
const parseBundle = (input: unknown, seal: boolean): EvidenceBundle => {
|
||||
if (typeof input === "object" && input !== null) {
|
||||
const owned = immutableBundles.get(input);
|
||||
if (owned !== undefined) return owned;
|
||||
}
|
||||
const parsed = evidenceBundleSchema.parse(input);
|
||||
return authenticateBundle(evidenceBundleSchema.parse(input), seal);
|
||||
};
|
||||
|
||||
/** Authenticate evidence and bundle invariants after the bundle shape is parsed. */
|
||||
export const authenticateParsedEvidenceBundle = (
|
||||
parsed: EvidenceBundle,
|
||||
): EvidenceBundle => authenticateBundle(parsed, false);
|
||||
|
||||
const authenticateBundle = (
|
||||
parsed: EvidenceBundle,
|
||||
seal: boolean,
|
||||
): EvidenceBundle => {
|
||||
const recordIds = parsed.records.map(({ evidence_id: id }) => id);
|
||||
if (new Set(recordIds).size !== recordIds.length)
|
||||
throw new TypeError("Evidence bundle contains duplicate record IDs");
|
||||
@@ -246,7 +268,12 @@ export const parseEvidenceBundle = (input: unknown): EvidenceBundle => {
|
||||
const canonical = createEvidenceBundle(
|
||||
parsed.records.map((record, index) => {
|
||||
try {
|
||||
return parseEvidence(record);
|
||||
if (!seal) return parseEvidence(record);
|
||||
// Schema parsing gave this boundary exclusive ownership. Register only
|
||||
// validated payloads before authenticating the complete record envelope.
|
||||
record.normalized_result = freezeJsonSnapshot(record.normalized_result);
|
||||
record.raw_result = freezeJsonSnapshot(record.raw_result);
|
||||
return immutableEvidence(record);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof z.ZodError)
|
||||
throw new z.ZodError(
|
||||
@@ -262,6 +289,10 @@ export const parseEvidenceBundle = (input: unknown): EvidenceBundle => {
|
||||
);
|
||||
if (JSON.stringify(parsed) !== JSON.stringify(canonical))
|
||||
throw new TypeError("Evidence bundle manifests are not canonical");
|
||||
if (seal) {
|
||||
freezeJsonSnapshot(canonical);
|
||||
immutableBundles.set(canonical, canonical);
|
||||
}
|
||||
return canonical;
|
||||
};
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js";
|
||||
import type { Evidence } from "../evidence.js";
|
||||
import {
|
||||
type ApplicationEdge,
|
||||
type ApplicationNode,
|
||||
import type {
|
||||
ApplicationEdge,
|
||||
ApplicationNode,
|
||||
} from "./javascriptApplicationGraphSchemas.js";
|
||||
import type { ApplicationFeatureTraceResult } from "./javascriptFeatureTraceSchemas.js";
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { type JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js";
|
||||
import type { JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js";
|
||||
import { digestCanonicalValue } from "../canonicalDigest.js";
|
||||
import { uniqueSorted } from "../canonicalOrdering.js";
|
||||
import type { StaticLoadStateProjection } from "./javascriptRuntimeLoadState.js";
|
||||
|
||||
@@ -143,7 +143,7 @@ export function* analyzeParsedJavaScriptSemanticsSteps(
|
||||
const parserPartial = file.errors.length > 0;
|
||||
const bindings = immutableSemanticBindings(state);
|
||||
yield;
|
||||
const callables = collectSemanticReturns(file.program, state, parserPartial);
|
||||
const callables = collectSemanticReturns(state, parserPartial);
|
||||
yield;
|
||||
const moduleLinks = resolveSemanticModuleCallables(state, callables);
|
||||
const derived = yield* collectJavaScriptDerivedSemanticsSteps(
|
||||
|
||||
@@ -26,7 +26,6 @@ import {
|
||||
import { evaluateSemanticExpression } from "./javascriptSemanticValues.js";
|
||||
import { range } from "./javascriptStaticAnalysisHelpers.js";
|
||||
import { semanticReturnCoverage } from "./javascriptSemanticCoverage.js";
|
||||
import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";
|
||||
import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js";
|
||||
|
||||
interface ReturnExpression {
|
||||
@@ -36,13 +35,11 @@ interface ReturnExpression {
|
||||
|
||||
/** Evaluate direct returns while excluding every nested callable. */
|
||||
export const collectSemanticReturns = (
|
||||
program: t.Program,
|
||||
state: JavaScriptSemanticAnalysisState,
|
||||
parserPartial: boolean,
|
||||
): JavaScriptSemanticCallable[] => {
|
||||
const nodes = callableNodes(program);
|
||||
return state.callables.map((callable) => {
|
||||
const node = nodes.get(callable.callableId);
|
||||
const node = state.callableNodesById.get(callable.callableId);
|
||||
if (node === undefined)
|
||||
return {
|
||||
...callable,
|
||||
@@ -117,17 +114,6 @@ export const resolveSemanticModuleCallables = (
|
||||
});
|
||||
};
|
||||
|
||||
const callableNodes = (program: t.Program): Map<string, t.Node> => {
|
||||
const output = new Map<string, t.Node>();
|
||||
traverseJavaScriptAst(program, {
|
||||
enter: (node) => {
|
||||
const id = semanticCallableIdForNode(node);
|
||||
if (id !== null) output.set(id, node);
|
||||
},
|
||||
});
|
||||
return output;
|
||||
};
|
||||
|
||||
const directReturnExpressions = (callable: t.Node): ReturnExpression[] =>
|
||||
semanticCallableResultExpressions(callable)
|
||||
.filter(({ kind }) => kind === "return")
|
||||
|
||||
@@ -8,10 +8,10 @@ import type { ApplicationNode } from "../javascript/javascriptApplicationGraphSc
|
||||
import { managedSourceCoverage } from "./managedApplicationGraphCoverage.js";
|
||||
import type { Evidence } from "../evidence.js";
|
||||
import { managedTokenScopesMatch } from "./managedInspectionEvidence.js";
|
||||
import {
|
||||
type ManagedArtifactInspection,
|
||||
type ManagedMemberInspection,
|
||||
type ManagedNativeBoundaryInspection,
|
||||
import type {
|
||||
ManagedArtifactInspection,
|
||||
ManagedMemberInspection,
|
||||
ManagedNativeBoundaryInspection,
|
||||
} from "./managedArtifact.js";
|
||||
import type {
|
||||
GraphBuildState,
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { ProcessCapture } from "./processCaptureParsing.js";
|
||||
import { comparableTerminalFrame } from "./processObservation.js";
|
||||
import { AnalysisInputError } from "../analysisErrorCore.js";
|
||||
import { jsonValueSchema } from "../jsonValue.js";
|
||||
import { type ProcessTraceSpecification } from "./processTraceSpecification.js";
|
||||
import type { ProcessTraceSpecification } from "./processTraceSpecification.js";
|
||||
import { processTraceComparisonResultSchema } from "./processTraceEvaluation.js";
|
||||
import {
|
||||
dimensionsForTraceSources,
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
parseProcessCapture,
|
||||
type ProcessCapture,
|
||||
} from "./processCaptureParsing.js";
|
||||
import { type ProcessTraceSpecification } from "./processTraceSpecification.js";
|
||||
import type { ProcessTraceSpecification } from "./processTraceSpecification.js";
|
||||
|
||||
export const emptyCapture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
|
||||
export const terminal = { sequence: 0, at_ms: 900, data: "Ready" };
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseProcessCapture } from "./processCaptureParsing.js";
|
||||
import { type ProcessTraceSpecification } from "./processTraceSpecification.js";
|
||||
import type { ProcessTraceSpecification } from "./processTraceSpecification.js";
|
||||
import { processTraceComparisonResultSchema } from "./processTraceEvaluation.js";
|
||||
import {
|
||||
capture,
|
||||
|
||||
@@ -14,6 +14,45 @@ const input = (
|
||||
...overrides,
|
||||
});
|
||||
|
||||
describe("readiness bundle authentication", () => {
|
||||
it("authenticates each Evidence record after parsing the readiness contract", () => {
|
||||
const value = input();
|
||||
const record = value.evidence_bundle.records[0];
|
||||
if (record === undefined) throw new TypeError("Missing Evidence record");
|
||||
value.evidence_bundle.records[0] = {
|
||||
...record,
|
||||
evidence_id: `ev_${"0".repeat(64)}`,
|
||||
};
|
||||
|
||||
expect(() => createReconstructionReadinessReport(value)).toThrow(
|
||||
/Evidence semantic identifier does not match its record/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a noncanonical bundle manifest after readiness parsing", () => {
|
||||
const value = input();
|
||||
const provider = value.evidence_bundle.providers[0];
|
||||
if (provider === undefined)
|
||||
throw new TypeError("Missing provider manifest");
|
||||
value.evidence_bundle.providers.push(provider);
|
||||
|
||||
expect(() => createReconstructionReadinessReport(value)).toThrow(
|
||||
/Evidence bundle manifests are not canonical/u,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects duplicate Evidence record IDs after readiness parsing", () => {
|
||||
const value = input();
|
||||
const record = value.evidence_bundle.records[0];
|
||||
if (record === undefined) throw new TypeError("Missing Evidence record");
|
||||
value.evidence_bundle.records.push(record);
|
||||
|
||||
expect(() => createReconstructionReadinessReport(value)).toThrow(
|
||||
/Evidence bundle contains duplicate record IDs/u,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reconstruction readiness report", () => {
|
||||
it("accepts every caller supplied capability", () => {
|
||||
const capability = RECONSTRUCTION_READINESS_EXAMPLE.capabilities[0];
|
||||
|
||||
@@ -1,8 +1,5 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import canonicalize from "canonicalize";
|
||||
|
||||
import { parseEvidenceBundle } from "./evidenceBundle.js";
|
||||
import { digestCanonicalValue } from "./canonicalDigest.js";
|
||||
import { authenticateParsedEvidenceBundle } from "./evidenceBundle.js";
|
||||
import { readinessFindingOrder } from "./reconstructionReadinessFindingHelpers.js";
|
||||
import {
|
||||
READINESS_STAGE_IDS,
|
||||
@@ -24,7 +21,7 @@ export const createReconstructionReadinessReport = (
|
||||
const input = reconstructionReadinessInputSchema.parse(inputValue);
|
||||
const sourceDigest = digest(sourceValue(input));
|
||||
const evidenceIds = new Set(
|
||||
parseEvidenceBundle(input.evidence_bundle).records.map(
|
||||
authenticateParsedEvidenceBundle(input.evidence_bundle).records.map(
|
||||
({ evidence_id: evidenceId }) => evidenceId,
|
||||
),
|
||||
);
|
||||
@@ -185,8 +182,5 @@ const ratio = (numerator: number, denominator: number): number =>
|
||||
denominator === 0 ? 1 : numerator / denominator;
|
||||
|
||||
const digest = (value: unknown): string => {
|
||||
const encoded = canonicalize(value);
|
||||
if (encoded === undefined)
|
||||
throw new TypeError("Reconstruction readiness value is not canonical JSON");
|
||||
return createHash("sha256").update(encoded).digest("hex");
|
||||
return digestCanonicalValue(value, "Reconstruction readiness");
|
||||
};
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { recordUnknownInputSchema } from "./residualUnknown.js";
|
||||
import {
|
||||
createResidualUnknown,
|
||||
recordUnknownInputSchema,
|
||||
residualUnknownSchema,
|
||||
updateResidualUnknown,
|
||||
} from "./residualUnknown.js";
|
||||
|
||||
it("accepts complete unknown questions, environment details and probe guidance", () => {
|
||||
const longText = "x".repeat(4_097);
|
||||
@@ -23,3 +28,103 @@ it("accepts complete unknown questions, environment details and probe guidance",
|
||||
expect(parsed.question).toHaveLength(4_097);
|
||||
expect(parsed.recommended_probes[0]?.operation).toHaveLength(4_097);
|
||||
});
|
||||
|
||||
const input = recordUnknownInputSchema.parse({
|
||||
question: "Fixture 0",
|
||||
domain: "canonical-constructor",
|
||||
severity: "medium",
|
||||
required_authority: null,
|
||||
required_confidence: "derived",
|
||||
required_environment: null,
|
||||
recommended_probes: [],
|
||||
relationships: [],
|
||||
});
|
||||
const mutationId = `ev_${"a".repeat(64)}`;
|
||||
const scopeDigest = "b".repeat(64);
|
||||
|
||||
it("preserves persisted canonical identity and digests while authenticating imported changes", () => {
|
||||
const created = createResidualUnknown(input, mutationId, scopeDigest);
|
||||
expect(created.unknown_id).toBe(
|
||||
"unk_139dc8e6c9feb0ba370584d190003eaa5064ec693cf1dfd5ec4be7496d59559c",
|
||||
);
|
||||
expect(created.revision_digest).toBe(
|
||||
"7f41f8ddcfdb364f7b05bda5d9ff75ea3a83fe8f3a29f07dfadfe00141b72c25",
|
||||
);
|
||||
const updated = updateResidualUnknown(
|
||||
created,
|
||||
{
|
||||
...input,
|
||||
unknown_id: created.unknown_id,
|
||||
expected_revision: 1,
|
||||
status: "investigating",
|
||||
severity: "high",
|
||||
required_confidence: "observed",
|
||||
recommended_probes: [
|
||||
{ operation: "inspect_native_api", rationale: "Fixture probe" },
|
||||
],
|
||||
resolution: null,
|
||||
},
|
||||
mutationId,
|
||||
);
|
||||
expect(updated.revision_digest).toBe(
|
||||
"8f2d8c1985f143724b087524ed997d13602d343db6f444b4c37bb8bc9e91dcbb",
|
||||
);
|
||||
expect(residualUnknownSchema.parse(updated)).toEqual(updated);
|
||||
expect(
|
||||
residualUnknownSchema.safeParse({ ...updated, severity: "low" }).success,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("normalizes producer text before identity and revision authentication", () => {
|
||||
const raw = {
|
||||
...input,
|
||||
question: "\uFEFFFixture\t0\n",
|
||||
domain: " canonical-constructor ",
|
||||
recommended_probes: [
|
||||
{ operation: "inspect_native_api", rationale: " Replay " },
|
||||
],
|
||||
};
|
||||
const created = createResidualUnknown(raw, mutationId, scopeDigest);
|
||||
const canonical = createResidualUnknown(
|
||||
recordUnknownInputSchema.parse(raw),
|
||||
mutationId,
|
||||
scopeDigest,
|
||||
);
|
||||
expect(created).toEqual(canonical);
|
||||
expect(created.unknown_id).toBe(
|
||||
createResidualUnknown(input, mutationId, scopeDigest).unknown_id,
|
||||
);
|
||||
expect(residualUnknownSchema.parse(created)).toEqual(created);
|
||||
});
|
||||
|
||||
it("keeps generated revisions subject to state and relationship validation", () => {
|
||||
const created = createResidualUnknown(input, mutationId, scopeDigest);
|
||||
const command = {
|
||||
...input,
|
||||
unknown_id: created.unknown_id,
|
||||
expected_revision: 1,
|
||||
status: "resolved" as const,
|
||||
resolution: null,
|
||||
};
|
||||
expect(() => updateResidualUnknown(created, command, mutationId)).toThrow(
|
||||
/Only resolved unknowns carry a resolution/u,
|
||||
);
|
||||
expect(() =>
|
||||
updateResidualUnknown(
|
||||
created,
|
||||
{ ...command, status: "contradicted" },
|
||||
mutationId,
|
||||
),
|
||||
).toThrow(/Contradicted status requires contradicting evidence/u);
|
||||
expect(() =>
|
||||
updateResidualUnknown(
|
||||
created,
|
||||
{
|
||||
...command,
|
||||
status: "open",
|
||||
relationships: [{ type: "related-to", unknown_id: created.unknown_id }],
|
||||
},
|
||||
mutationId,
|
||||
),
|
||||
).toThrow(/An unknown cannot relate to itself/u);
|
||||
});
|
||||
|
||||
@@ -82,6 +82,70 @@ const residualUnknownObjectSchema = z.object({
|
||||
mutation_evidence_ids: z.array(evidenceIdSchema).min(1),
|
||||
});
|
||||
|
||||
type UnknownRevision = Omit<
|
||||
z.infer<typeof residualUnknownObjectSchema>,
|
||||
"revision_digest"
|
||||
>;
|
||||
|
||||
const refineRevision = (value: UnknownRevision, context: z.RefinementCtx) => {
|
||||
if ((value.revision === 1) !== (value.previous_revision_digest === null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Only the first revision has no predecessor",
|
||||
path: ["previous_revision_digest"],
|
||||
});
|
||||
if ((value.status === "resolved") !== (value.resolution !== null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Only resolved unknowns carry a resolution",
|
||||
path: ["resolution"],
|
||||
});
|
||||
if (
|
||||
value.status === "contradicted" &&
|
||||
value.contradicting_evidence_ids.length === 0
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Contradicted status requires contradicting evidence",
|
||||
path: ["contradicting_evidence_ids"],
|
||||
});
|
||||
if (
|
||||
value.relationships.some(
|
||||
(relationship) => relationship.unknown_id === value.unknown_id,
|
||||
)
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "An unknown cannot relate to itself",
|
||||
path: ["relationships"],
|
||||
});
|
||||
for (const ids of [
|
||||
value.supporting_evidence_ids,
|
||||
value.contradicting_evidence_ids,
|
||||
value.mutation_evidence_ids,
|
||||
])
|
||||
if (new Set(ids).size !== ids.length)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Evidence IDs must be unique",
|
||||
});
|
||||
const contradictingEvidence = new Set(value.contradicting_evidence_ids);
|
||||
if (value.supporting_evidence_ids.some((id) => contradictingEvidence.has(id)))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Evidence cannot simultaneously support and contradict",
|
||||
});
|
||||
const relationships = value.relationships.map(
|
||||
(relationship) => `${relationship.type}:${relationship.unknown_id}`,
|
||||
);
|
||||
if (new Set(relationships).size !== relationships.length)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Relationships must be unique",
|
||||
path: ["relationships"],
|
||||
});
|
||||
};
|
||||
|
||||
export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
|
||||
(value, context) => {
|
||||
if (computeRevisionDigest(value) !== value.revision_digest)
|
||||
@@ -90,67 +154,14 @@ export const residualUnknownSchema = residualUnknownObjectSchema.superRefine(
|
||||
message: "Residual unknown revision digest does not match",
|
||||
path: ["revision_digest"],
|
||||
});
|
||||
if ((value.revision === 1) !== (value.previous_revision_digest === null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Only the first revision has no predecessor",
|
||||
path: ["previous_revision_digest"],
|
||||
});
|
||||
if ((value.status === "resolved") !== (value.resolution !== null))
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Only resolved unknowns carry a resolution",
|
||||
path: ["resolution"],
|
||||
});
|
||||
if (
|
||||
value.status === "contradicted" &&
|
||||
value.contradicting_evidence_ids.length === 0
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Contradicted status requires contradicting evidence",
|
||||
path: ["contradicting_evidence_ids"],
|
||||
});
|
||||
if (
|
||||
value.relationships.some(
|
||||
(relationship) => relationship.unknown_id === value.unknown_id,
|
||||
)
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "An unknown cannot relate to itself",
|
||||
path: ["relationships"],
|
||||
});
|
||||
for (const ids of [
|
||||
value.supporting_evidence_ids,
|
||||
value.contradicting_evidence_ids,
|
||||
value.mutation_evidence_ids,
|
||||
])
|
||||
if (new Set(ids).size !== ids.length)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Evidence IDs must be unique",
|
||||
});
|
||||
const contradictingEvidence = new Set(value.contradicting_evidence_ids);
|
||||
if (
|
||||
value.supporting_evidence_ids.some((id) => contradictingEvidence.has(id))
|
||||
)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Evidence cannot simultaneously support and contradict",
|
||||
});
|
||||
const relationships = value.relationships.map(
|
||||
(relationship) => `${relationship.type}:${relationship.unknown_id}`,
|
||||
);
|
||||
if (new Set(relationships).size !== relationships.length)
|
||||
context.addIssue({
|
||||
code: "custom",
|
||||
message: "Relationships must be unique",
|
||||
path: ["relationships"],
|
||||
});
|
||||
refineRevision(value, context);
|
||||
},
|
||||
);
|
||||
|
||||
const generatedRevisionSchema = residualUnknownObjectSchema
|
||||
.omit({ revision_digest: true })
|
||||
.superRefine(refineRevision);
|
||||
|
||||
export type ResidualUnknown = z.infer<typeof residualUnknownSchema>;
|
||||
export type UnknownStatus = ResidualUnknown["status"];
|
||||
|
||||
@@ -200,9 +211,10 @@ export const createResidualUnknown = (
|
||||
mutationEvidenceId: string,
|
||||
scopeDigest: string | null,
|
||||
): ResidualUnknown => {
|
||||
const question = input.question.replace(/\s+/gu, " ");
|
||||
const question = input.question.replace(/\s+/gu, " ").trim();
|
||||
const domain = input.domain.trim();
|
||||
const identity = canonicalJson({
|
||||
domain: input.domain,
|
||||
domain,
|
||||
question,
|
||||
scope_digest: scopeDigest,
|
||||
});
|
||||
@@ -214,15 +226,12 @@ export const createResidualUnknown = (
|
||||
question,
|
||||
status:
|
||||
input.contradicting_evidence_ids.length > 0 ? "contradicted" : "open",
|
||||
domain: input.domain,
|
||||
domain,
|
||||
...projectRevisionDetails(input),
|
||||
resolution: null,
|
||||
mutation_evidence_ids: [mutationEvidenceId],
|
||||
} satisfies Omit<ResidualUnknown, "revision_digest">;
|
||||
return residualUnknownSchema.parse({
|
||||
...revision,
|
||||
revision_digest: computeRevisionDigest(revision),
|
||||
});
|
||||
return createRevision(revision);
|
||||
};
|
||||
|
||||
/** Apply a complete, revision-guarded replacement while preserving identity. */
|
||||
@@ -246,10 +255,13 @@ export const updateResidualUnknown = (
|
||||
mutationEvidenceId,
|
||||
]),
|
||||
} satisfies Omit<ResidualUnknown, "revision_digest">;
|
||||
return residualUnknownSchema.parse({
|
||||
...revision,
|
||||
revision_digest: computeRevisionDigest(revision),
|
||||
});
|
||||
return createRevision(revision);
|
||||
};
|
||||
|
||||
/** Validate and normalize owned revision fields before computing their digest. */
|
||||
const createRevision = (value: UnknownRevision): ResidualUnknown => {
|
||||
const revision = generatedRevisionSchema.parse(value);
|
||||
return { ...revision, revision_digest: computeRevisionDigest(revision) };
|
||||
};
|
||||
|
||||
type RevisionDetailsInput = Pick<
|
||||
|
||||
@@ -7,9 +7,9 @@ import type {
|
||||
ManagedMemberInspection,
|
||||
ManagedParseIssue,
|
||||
} from "../domain/managed/managedArtifact.js";
|
||||
import {
|
||||
type ManagedMetadataLayout,
|
||||
type MetadataTableLayout,
|
||||
import type {
|
||||
ManagedMetadataLayout,
|
||||
MetadataTableLayout,
|
||||
} from "./ManagedMetadataLayout.js";
|
||||
import {
|
||||
metadataRowCursor,
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
type ManagedNativeBoundaryInspection,
|
||||
type ManagedParseIssue,
|
||||
} from "../domain/managed/managedArtifact.js";
|
||||
import { type ManagedMetadataLayout } from "./ManagedMetadataLayout.js";
|
||||
import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js";
|
||||
import {
|
||||
metadataRowCursor,
|
||||
metadataCodedToken,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user