chore: merge main into feat/pkg-inventory

Resolve conflicts against main:
- Inventory moved to src/artifacts/inventory (#1032); point the xar and
  cpio reader imports at their new relative paths.
- README.md: take main's simplified English README, which no longer
  lists individual package formats. PKG coverage stays documented in
  the skill reference and docs/testing.md.
- Skill: the references gain the PKG paragraph, so the version moves to
  29 (main already uses 28).
- Regenerate the product catalog, conformance manifest, skill digest and
  package metadata. The MCP tool catalog is now built rather than
  committed (#997), so it is removed here.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kaoru0822
2026-10-08 04:49:54 +00:00
co-authored by Claude Opus 5.5
581 changed files with 29518 additions and 12109 deletions
+60 -42
View File
@@ -3,50 +3,68 @@ import { dirname, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
const output = resolve(root, "build", "fixtures", "rea-ghidra-windows.exe");
const bytes = Buffer.alloc(1024);
const peOffset = 0x80;
const optionalHeader = peOffset + 24;
const sectionTable = optionalHeader + 0xf0;
for (const architecture of ["x86_64", "x86"]) {
const x86 = architecture === "x86";
const output = resolve(
root,
"build",
"fixtures",
x86 ? "rea-ghidra-windows-x86.exe" : "rea-ghidra-windows.exe",
);
const bytes = Buffer.alloc(1024);
const peOffset = 0x80;
const optionalHeader = peOffset + 24;
const optionalHeaderSize = x86 ? 0xe0 : 0xf0;
const sectionTable = optionalHeader + optionalHeaderSize;
bytes.write("MZ", 0, "ascii");
bytes.writeUInt32LE(peOffset, 0x3c);
bytes.write("PE\0\0", peOffset, "binary");
bytes.writeUInt16LE(0x8664, peOffset + 4);
bytes.writeUInt16LE(1, peOffset + 6);
bytes.writeUInt16LE(0xf0, peOffset + 20);
bytes.writeUInt16LE(0x22, peOffset + 22);
bytes.write("MZ", 0, "ascii");
bytes.writeUInt32LE(peOffset, 0x3c);
bytes.write("PE\0\0", peOffset, "binary");
bytes.writeUInt16LE(x86 ? 0x14c : 0x8664, peOffset + 4);
bytes.writeUInt16LE(1, peOffset + 6);
bytes.writeUInt16LE(optionalHeaderSize, peOffset + 20);
bytes.writeUInt16LE(x86 ? 0x102 : 0x22, peOffset + 22);
bytes.writeUInt16LE(0x20b, optionalHeader);
bytes.writeUInt32LE(0x200, optionalHeader + 4);
bytes.writeUInt32LE(0x1000, optionalHeader + 16);
bytes.writeUInt32LE(0x1000, optionalHeader + 20);
bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24);
bytes.writeUInt32LE(0x1000, optionalHeader + 32);
bytes.writeUInt32LE(0x200, optionalHeader + 36);
bytes.writeUInt16LE(6, optionalHeader + 40);
bytes.writeUInt16LE(6, optionalHeader + 48);
bytes.writeUInt32LE(0x2000, optionalHeader + 56);
bytes.writeUInt32LE(0x200, optionalHeader + 60);
bytes.writeUInt16LE(3, optionalHeader + 68);
bytes.writeUInt16LE(0x8160, optionalHeader + 70);
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72);
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80);
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88);
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96);
bytes.writeUInt32LE(16, optionalHeader + 108);
bytes.writeUInt16LE(x86 ? 0x10b : 0x20b, optionalHeader);
bytes.writeUInt32LE(0x200, optionalHeader + 4);
bytes.writeUInt32LE(0x1000, optionalHeader + 16);
bytes.writeUInt32LE(0x1000, optionalHeader + 20);
if (x86) bytes.writeUInt32LE(0x400000, optionalHeader + 28);
else bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24);
bytes.writeUInt32LE(0x1000, optionalHeader + 32);
bytes.writeUInt32LE(0x200, optionalHeader + 36);
bytes.writeUInt16LE(6, optionalHeader + 40);
bytes.writeUInt16LE(6, optionalHeader + 48);
bytes.writeUInt32LE(0x2000, optionalHeader + 56);
bytes.writeUInt32LE(0x200, optionalHeader + 60);
bytes.writeUInt16LE(3, optionalHeader + 68);
bytes.writeUInt16LE(x86 ? 0x8140 : 0x8160, optionalHeader + 70);
if (x86) {
bytes.writeUInt32LE(0x10_0000, optionalHeader + 72);
bytes.writeUInt32LE(0x1000, optionalHeader + 76);
bytes.writeUInt32LE(0x10_0000, optionalHeader + 80);
bytes.writeUInt32LE(0x1000, optionalHeader + 84);
bytes.writeUInt32LE(16, optionalHeader + 92);
} else {
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72);
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80);
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88);
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96);
bytes.writeUInt32LE(16, optionalHeader + 108);
}
bytes.write(".text", sectionTable, "ascii");
bytes.writeUInt32LE(0x16, sectionTable + 8);
bytes.writeUInt32LE(0x1000, sectionTable + 12);
bytes.writeUInt32LE(0x200, sectionTable + 16);
bytes.writeUInt32LE(0x200, sectionTable + 20);
bytes.writeUInt32LE(0x6000_0020, sectionTable + 36);
bytes.write(".text", sectionTable, "ascii");
bytes.writeUInt32LE(0x16, sectionTable + 8);
bytes.writeUInt32LE(0x1000, sectionTable + 12);
bytes.writeUInt32LE(0x200, sectionTable + 16);
bytes.writeUInt32LE(0x200, sectionTable + 20);
bytes.writeUInt32LE(0x6000_0020, sectionTable + 36);
// entry calls the second function, returns zero; second returns 42.
bytes.set([0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], 0x200);
bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210);
// entry calls the second function, returns zero; second returns 42.
bytes.set([0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], 0x200);
bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210);
await mkdir(dirname(output), { recursive: true });
await writeFile(output, bytes);
process.stdout.write(`${output}\n`);
await mkdir(dirname(output), { recursive: true });
await writeFile(output, bytes);
if (!x86) process.stdout.write(`${output}\n`);
}
@@ -0,0 +1,151 @@
import importlib.util
import json
from pathlib import Path
from mitmproxy import connection, ctx, http
from mitmproxy.io import FlowWriter
from mitmproxy.io import tnetstring
from mitmproxy.addons.savehar import SaveHar
from mitmproxy.websocket import WebSocketData, WebSocketMessage
class Generator:
def load(self, loader):
loader.add_option("rea_fixture_root", str, "", "Owned synthetic fixture root")
loader.add_option("rea_adapter_path", str, "", "REA native adapter under verification")
def running(self):
root = Path(ctx.options.rea_fixture_root)
spec = importlib.util.spec_from_file_location("rea_capture_under_test", ctx.options.rea_adapter_path)
adapter = importlib.util.module_from_spec(spec)
spec.loader.exec_module(adapter)
classifications = [
(adapter.CaptureFailure("format", "Malformed retained record.", "/record"), "format"),
(adapter.CaptureFailure("input-limit", "Nesting budget.", "/extension"), "input-limit"),
(MemoryError("payload must not enter diagnostics"), "resource-limit"),
(RuntimeError("payload must not enter diagnostics"), "decoder"),
(OSError("payload must not enter diagnostics"), "decoder"),
]
for error, expected in classifications:
reply = adapter.failure_reply(error)
assert reply["reason"] == expected, reply
assert "payload must not enter diagnostics" not in json.dumps(reply)
for state, expected in [({"metadata": {1: b"bad-key"}}, "format"), ({"metadata": object()}, "decoder")]:
try:
adapter.project_record(state, [])
except adapter.CaptureFailure as error:
assert error.reason == expected, (error.reason, expected)
assert error.pointer == "/metadata", error.pointer
else:
raise AssertionError("Invalid native representation was accepted")
non_utf8 = adapter.project_record({"request": {"headers": [(b"Referer", b"https://byte-user:byte-secret@example.test/path\xff")] }}, [])
assert non_utf8["reported"]["request"]["headers"][0][1] is None, non_utf8
assert non_utf8["binary_fields"][-1]["state"] == "redacted", non_utf8
for value in [" \tHTTPS://ows-user:ows-secret@example.test/path\t ", b" \tHTTPS://ows-user:ows-secret@example.test/path\t "]:
projected = adapter.project_record({"request": {"headers": [(b"Referer", value)]}}, [])
assert projected["reported"]["request"]["headers"][0][1] == " \tHTTPS://example.test/path\t ", projected
assert "ows-secret" not in json.dumps(projected), projected
if isinstance(value, bytes):
assert projected["binary_fields"][-1]["state"] == "redacted", projected
(root / "native-error-classifications.json").write_text(json.dumps({"passed": len(classifications) + 2}))
first = http.HTTPFlow(connection.Client(peername=("127.0.0.1", 1), sockname=("127.0.0.1", 2)), connection.Server(address=("example.test", 80)))
first.id = "original-producer-id"
first.request = http.Request.make("POST", "http://example.test/a?token=ordinary", b"\x00\xffbody", [(b"Authorization", b"Bearer native-transport-secret"), (b"X-Duplicate", b"one"), (b"X-Duplicate", b"two")])
first.request.path = "http://user:password@example.test/a?token=ordinary#fragment"
first.response = http.Response.make(200, b"\x00\xfeanswer", [(b"Set-Cookie", b"session=cookie-secret"), (b"Location", b"https://redirect-user:redirect-password@example.test/b?token=ordinary")])
first.metadata = {"big_integer": 9007199254740993, "nonfinite": float("inf"), "isLosslessNumber": True, "value": "ordinary-extension", "response": {"headers": [["Authorization", "ordinary-extension-credential-name"]]}}
first.websocket = WebSocketData(messages=[WebSocketMessage(2, True, b"\x00\xfdmessage", 123.5)])
first.backup()
first.comment = "caller-selected-fixture"
second = first.copy()
second.id = first.id
second.request.raw_content = None
second.response.raw_content = b""
second.websocket = None
with (root / "flows.mitm").open("wb") as handle:
writer = FlowWriter(handle)
writer.add(first)
writer.add(second)
har_first, har_second = first.copy(), second.copy()
har_first.request.path = "/a?token=ordinary#fragment"
har_second.request.path = "/a?token=ordinary#fragment"
har_first.response.raw_content = b"\x00\xff\x01\xfe"
markers = {"sensitive": "REDACTED", "bracket": "literal[", "overlap": "secret", "ordinary": "unmarked"}
har = SaveHar().make_har([har_first, har_second])
har["log"]["entries"][0]["_markers"] = markers
har["log"]["entries"][0]["_private_properties"] = {"private-property/~": {"number": 123, "text": "private-property"}, "kept": 7}
(root / "producer.har").write_text(json.dumps(har))
har_text = json.dumps(har)
for name, members in [
("equal-number", '"_duplicate": 1, "_duplicate": 1'),
("equal-null", '"_duplicate": null, "_duplicate": null'),
("equal-object", '"_duplicate": {"value": 1}, "_duplicate": {"value": 1}'),
("escaped-key", '"_duplicate": 1, "\\u005fduplicate": 1'),
]:
duplicate = har_text.replace('"cache": {}', '"cache": {}, ' + members, 1)
assert duplicate != har_text
(root / ("duplicate-" + name + ".har")).write_text(duplicate)
duplicate = har_text.replace('"version": "1.2"', '"version": "1.2", "version": "1.2"', 1)
assert duplicate != har_text
(root / "duplicate-core.har").write_text(duplicate)
prototype = json.loads(har_text)
prototype["_extension"] = {"__proto__": {"preserved": 7}, "constructor": "ordinary"}
(root / "prototype.har").write_text(json.dumps(prototype))
prototype_native = first.get_state()
prototype_native["metadata"]["__proto__"] = {"preserved": 7}
with (root / "prototype.mitm").open("wb") as handle:
tnetstring.dump(prototype_native, handle)
invalid_har = json.loads(json.dumps(har))
invalid_har["log"]["entries"][0]["response"]["content"].update({"encoding": "base64", "text": "AR=="})
(root / "invalid-private-parent.har").write_text(json.dumps(invalid_har))
strings = first.get_state()
for state in [strings, strings["backup"]]:
state["request"]["path"] = "https://native-user:string-password@example.test/string-path"
state["request"]["authority"] = "authority-user:authority-password@example.test"
state["request"]["headers"] = [(b"Referer", "https://referer-user:referer-password@example.test/from"), (b"Origin", "//origin-user:origin-password@example.test")]
state["response"]["headers"] = [(b"Location", "https://location-user:location-password@example.test/to")]
state["metadata"]["_markers"] = dict(markers)
state["metadata"]["_private_properties"] = {"private-property/~": {"number": 123, "bytes": b"private-property"}, "kept": 7}
with (root / "string-urls.mitm").open("wb") as handle:
tnetstring.dump(strings, handle)
strings["metadata"]["_private_properties"]["private-property/~"] = {1: b"unsupported-key"}
with (root / "invalid-private-key.mitm").open("wb") as handle:
tnetstring.dump(strings, handle)
extension = "leaf"
for _ in range(66):
extension = {"child": extension}
har["_extension"] = extension
(root / "deep.har").write_text(json.dumps(har))
deep = first.get_state()
deep["_extension"] = extension
with (root / "deep.mitm").open("wb") as handle:
tnetstring.dump(deep, handle)
for name, value in [("invalid-credential-key", {1: b"bad-key"}), ("deep-credential", extension)]:
malformed = first.get_state()
malformed["request"]["headers"] = [(b"Authorization", value)]
with (root / (name + ".mitm")).open("wb") as handle:
tnetstring.dump(malformed, handle)
for prefix, name in [(" \t", "ows"), ("\r\n \v\f\x00\x1f", "controls")]:
url = prefix + "HTTPS://ows-user:ows-secret@example.test/path?token=ordinary#fragment\t "
relative = prefix + "//ows-user:ows-secret@example.test/path?token=ordinary#fragment\t "
whitespace_har = json.loads((root / "producer.har").read_text())
entry = whitespace_har["log"]["entries"][0]
for side, header in [("response", "Location"), ("request", "Referer"), ("request", "Origin")]:
entry[side]["headers"] = [{"name": header, "value": url}] + entry[side]["headers"]
(root / (name + ".har")).write_text(json.dumps(whitespace_har))
whitespace_native = first.get_state()
for state in [whitespace_native, whitespace_native["backup"]]:
state["request"]["headers"] = [(b"Referer", url.encode("utf-8")), (b"Origin", relative)]
state["response"]["headers"] = [(b"Location", url)]
with (root / (name + ".mitm")).open("wb") as handle:
tnetstring.dump(whitespace_native, handle)
non_utf8_native = first.get_state()
for state in [non_utf8_native, non_utf8_native["backup"]]:
state["request"]["path"] = b"https://byte-user:byte-secret@example.test/path\xff"
state["request"]["authority"] = b"byte-user:byte-secret@example.test\xff"
state["request"]["headers"] = [(b"Referer", b" https://byte-user:byte-secret@example.test/from\xff"), (b"Origin", b"//byte-user:byte-secret@example.test\xff")]
state["response"]["headers"] = [(b"Location", b"https://byte-user:byte-secret@example.test/to\xff")]
with (root / "non-utf8-urls.mitm").open("wb") as handle:
tnetstring.dump(non_utf8_native, handle)
(root / "oracle.json").write_text(json.dumps({"records": 2, "request_base64": "AP9ib2R5", "response_base64": "AP5hbnN3ZXI=", "websocket_base64": "AP1tZXNzYWdl", "id": first.id}))
ctx.master.shutdown()
addons = [Generator()]
+3 -2
View File
@@ -17,8 +17,9 @@ for (const argument of arguments_)
throw new Error(`Unknown MCP tool catalog option: ${argument}`);
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
// The checked-in catalog is a portable package artifact. Keep its canonical
// provider metadata independent of the machine running the generator.
// The build-generated catalog is a portable package artifact that is not
// committed to git. Keep its canonical provider metadata independent of the
// machine running the generator. It is regenerated by `npm run build`.
const CATALOG_PLATFORM = "darwin";
const sessionToolNames = new Set([
...TOOL_CONTRACTS.filter(({ kind }) => kind === "session").map(
+65 -7
View File
@@ -24,8 +24,10 @@ SUPPORTED_HOPPER_SHA256 = {
"1339f9e58377442b0c6fcb0dfc3cec20d593cc557408521fad9a00dbc6b8da13",
}
EXPECTED_SCREEN = (1280, 1024)
EXPECTED_DIALOG = (189, 370, 901, 284)
DEMO_CLICK = (305, 632)
DIALOG_TITLES = {"Registration", "Registration — Hopper Disassembler"}
HOPPER_WINDOW_TITLE = "Hopper Disassembler"
DEMO_CLICK_LEFT = 116
DEMO_CLICK_BOTTOM = 22
PRIVATE_DISPLAY_UNAVAILABLE = 70
X11_AUTHORIZATION_FAILED = 71
UNSUPPORTED_HOPPER_BUILD = 72
@@ -197,6 +199,47 @@ def windows(x11: ctypes.CDLL, display: int, root: int) -> list[tuple[int, int, i
return found
def window_title(x11: ctypes.CDLL, display: int, window: int) -> str | None:
title = ctypes.c_char_p()
if not x11.XFetchName(display, window, ctypes.byref(title)) or not title.value:
return None
try:
return title.value.decode("utf-8", "replace")
finally:
x11.XFree(ctypes.cast(title, ctypes.c_void_p))
def window_transient_for(
x11: ctypes.CDLL, display: int, window: int
) -> int | None:
parent = ctypes.c_ulong()
if not x11.XGetTransientForHint(display, window, ctypes.byref(parent)):
return None
return int(parent.value)
def find_demo_dialog(
x11: ctypes.CDLL,
display: int,
observed: list[tuple[int, int, int, int, int]],
) -> tuple[int, int, int, int, int] | None:
for candidate in observed:
window = candidate[0]
if window_title(x11, display, window) not in DIALOG_TITLES:
continue
parent = window_transient_for(x11, display, window)
if parent is None:
continue
if window_title(x11, display, parent) == HOPPER_WINDOW_TITLE:
return candidate
return None
def demo_click(dialog: tuple[int, int, int, int, int]) -> tuple[int, int]:
_window, x, y, _width, height = dialog
return x + DEMO_CLICK_LEFT, y + height - DEMO_CLICK_BOTTOM
def configure_x11(x11: ctypes.CDLL, xtst: ctypes.CDLL) -> None:
window_pointer = ctypes.POINTER(ctypes.c_ulong)
x11.XOpenDisplay.argtypes = [ctypes.c_char_p]
@@ -213,6 +256,18 @@ def configure_x11(x11: ctypes.CDLL, xtst: ctypes.CDLL) -> None:
ctypes.POINTER(XWindowAttributes),
]
x11.XGetWindowAttributes.restype = ctypes.c_int
x11.XFetchName.argtypes = [
ctypes.c_void_p,
ctypes.c_ulong,
ctypes.POINTER(ctypes.c_char_p),
]
x11.XFetchName.restype = ctypes.c_int
x11.XGetTransientForHint.argtypes = [
ctypes.c_void_p,
ctypes.c_ulong,
ctypes.POINTER(ctypes.c_ulong),
]
x11.XGetTransientForHint.restype = ctypes.c_int
x11.XQueryTree.argtypes = [
ctypes.c_void_p,
ctypes.c_ulong,
@@ -645,6 +700,7 @@ def run_launch(args: argparse.Namespace) -> int:
deadline = time.monotonic() + 10
observed: list[tuple[int, int, int, int, int]] = []
dialog: tuple[int, int, int, int, int] | None = None
while time.monotonic() < deadline:
if socket.exists():
x11.XCloseDisplay(display)
@@ -660,7 +716,8 @@ def run_launch(args: argparse.Namespace) -> int:
"owned Hopper process exited",
)
observed = windows(x11, display, root)
if any(window[1:] == EXPECTED_DIALOG for window in observed):
dialog = find_demo_dialog(x11, display, observed)
if dialog is not None:
break
time.sleep(0.1)
else:
@@ -676,8 +733,10 @@ def run_launch(args: argparse.Namespace) -> int:
"expected Hopper demo dialog not found",
)
assert dialog is not None
click_x, click_y = demo_click(dialog)
input_results = (
xtst.XTestFakeMotionEvent(display, -1, DEMO_CLICK[0], DEMO_CLICK[1], 0),
xtst.XTestFakeMotionEvent(display, -1, click_x, click_y, 0),
xtst.XTestFakeButtonEvent(display, 1, True, 0),
xtst.XTestFakeButtonEvent(display, 1, False, 0),
)
@@ -696,9 +755,8 @@ def run_launch(args: argparse.Namespace) -> int:
finally:
if display is not None:
x11.XCloseDisplay(display)
if child is not None and child.poll() is None:
child.terminate()
child.wait()
if child is not None:
stop_process(child)
stop_process(xvfb)
capture.close()
+13
View File
@@ -48,6 +48,8 @@ export const toolFamilyCatalog = (sources) => {
...sources.browserContracts.BROWSER_TOOL_CONTRACTS,
...sources.webRuntimeContracts.WEB_RUNTIME_TOOL_CONTRACTS,
...sources.browserScenarioContracts.BROWSER_SCENARIO_TOOL_CONTRACTS,
...sources.webNetworkCaptureContracts
.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
],
},
{
@@ -170,6 +172,17 @@ export const providerCatalog = (sources) => {
identity: sources.androidProvider.JADX_PROVIDER_IDENTITY,
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
},
{
identity: sources.webNetworkCaptureProvider.HAR_CAPTURE_PROVIDER_IDENTITY,
contracts:
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
},
{
identity:
sources.webNetworkCaptureProvider.MITMPROXY_CAPTURE_PROVIDER_IDENTITY,
contracts:
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
},
{
identity: sources.browserProvider.CDP_BROWSER_PROVIDER_IDENTITY,
contracts: [
+4 -1
View File
@@ -51,6 +51,9 @@ const SOURCE_PATHS = {
"dist/contracts/managed/managedWorkflowToolContracts.js",
browserContracts: "dist/contracts/browserToolContracts.js",
webRuntimeContracts: "dist/contracts/webRuntimeToolContracts.js",
webNetworkCaptureContracts:
"dist/contracts/webNetworkCaptureToolContracts.js",
webNetworkCaptureProvider: "dist/browser/history/CaptureRelease.js",
browserScenarioContracts: "dist/contracts/browserScenarioToolContracts.js",
electronContracts: "dist/contracts/javascript/electronToolContracts.js",
javascriptRuntimeObservationContracts:
@@ -75,7 +78,7 @@ const SOURCE_PATHS = {
evidenceBundle: "dist/domain/evidenceBundle.js",
evidenceCompletion: "dist/domain/evidenceCompletionLedger.js",
completionGeneration: "dist/domain/completionLedgerGeneration.js",
processCapture: "dist/domain/processCapture.js",
processCapture: "dist/domain/process/processCapture.js",
analysisSnapshot: "dist/domain/analysisSnapshot.js",
artifactGraph: "dist/domain/artifactGraph.js",
browserObservation: "dist/domain/browserObservation.js",
+10 -2
View File
@@ -75,6 +75,11 @@ export const documentationFactIssues = async (root, catalog) => {
for (const path of README_PATHS) {
const content = await readFile(join(root, path), "utf8");
requireText(issues, path, content, "MCP-tool_catalog");
if (path === "README.md") {
requireText(issues, path, content, "(docs/installation.md");
requireText(issues, path, content, "(docs/product-catalog.json)");
continue;
}
for (const client of catalog.setup_clients)
requireText(issues, path, content, client.display_name);
try {
@@ -88,6 +93,11 @@ export const documentationFactIssues = async (root, catalog) => {
}
}
const installationPath = "docs/installation.md";
const installation = await readFile(join(root, installationPath), "utf8");
for (const client of catalog.setup_clients)
requireText(issues, installationPath, installation, client.display_name);
const agents = await readFile(join(root, "AGENTS.md"), "utf8");
requireText(issues, "AGENTS.md", agents, "docs/product-catalog.json");
@@ -95,8 +105,6 @@ export const documentationFactIssues = async (root, catalog) => {
const template = await readFile(join(root, templatePath), "utf8");
requireText(issues, templatePath, template, "docs/product-catalog.json");
const english = await readFile(join(root, "README.md"), "utf8");
requireText(issues, "README.md", english, "docs/product-catalog.json");
return issues;
};
+1 -1
View File
@@ -33,7 +33,7 @@ export const ensureGeneratedFile = async ({
return { changed: false };
if (check)
throw new Error(
`${path} is missing or stale. Run \`${generateCommand}\` and commit the result.`,
`${path} is missing or stale. Run \`${generateCommand}\` to regenerate it; commit the result if the file is tracked.`,
);
await writeFile(path, preserveLineEndings(source, existing), "utf8");
return { changed: true };
+1
View File
@@ -6,6 +6,7 @@ export interface ModuleBoundaryViolation {
readonly target: string;
readonly boundary:
| "artifact-acquisition"
| "process-capture"
| "pure-layer"
| "application-composition"
| "provider-construction"
+10
View File
@@ -60,6 +60,14 @@ const failedBoundary = (file, target) => {
if (!file.startsWith("src/") || !target.startsWith("src/")) return undefined;
const layer = file.split("/")[1];
const targetLayer = target.split("/")[1];
if (
file.startsWith("src/process/capture/") &&
(["application", "composition", "server", "cli", "main"].includes(
targetLayer,
) ||
/^src\/(?:cli|main)\./u.test(target))
)
return "process-capture";
if (
file.startsWith("src/application/javascript/") &&
/^src\/artifacts\/(?:Asar|Directory)ArtifactReader\.(?:js|ts)$/u.test(
@@ -70,6 +78,8 @@ const failedBoundary = (file, target) => {
if (
(file.startsWith("src/artifacts/javascript/") ||
file.startsWith("src/artifacts/apple/") ||
file.startsWith("src/artifacts/inventory/") ||
file.startsWith("src/artifacts/extraction/") ||
/^src\/artifacts\/ArtifactHash\.(?:js|ts)$/u.test(file)) &&
(["application", "composition", "server", "cli", "main"].includes(
targetLayer,
+11 -3
View File
@@ -19,7 +19,7 @@ const runtimeFiles = isMcpMode
? ["dist/main.js"]
: isMcpDoctorMode
? ["dist/main.js", "dist/mcpDoctor.js"]
: ["dist/cli.js", "dist/cliOutput.js"];
: ["dist/cli.js", "dist/cliOutput.js", "dist/cli/streamedJsonOutput.js"];
if (!(await compiledRuntimeExists(runtimeFiles))) {
process.stderr.write(
@@ -39,6 +39,8 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
process.exitCode = result.exitCode;
} else {
const { createCli } = await import("../dist/cli.js");
const { createStreamedCliJsonOutput } =
await import("../dist/cli/streamedJsonOutput.js");
const {
renderCliOutputArgumentError,
renderEmptyFilteredCliOutput,
@@ -51,14 +53,20 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
process.exitCode = 1;
} else {
let wroteOutput = false;
await createCli().serve(args, {
const resultOutput = createStreamedCliJsonOutput(args, process.stdout);
await createCli(process.env, resultOutput).serve(args, {
stdout: (output) => {
if (resultOutput?.handled) {
if (resultOutput.failed)
process.stderr.write(sanitizeCliOutput(output));
return;
}
const sanitized = sanitizeCliOutput(output);
if (sanitized.length > 0) wroteOutput = true;
process.stdout.write(sanitized);
},
});
if (!wroteOutput)
if (!wroteOutput && !resultOutput?.handled)
process.stdout.write(renderEmptyFilteredCliOutput(args) ?? "");
}
}
+33 -1
View File
@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { observeNativeUi } from "../dist/native/NativeUiObservation.js";
import { NATIVE_UI_HELPER_MAX_BUFFER } from "../dist/native/NativeUiOutputBudget.js";
if (process.platform !== "darwin")
throw new Error(
@@ -30,7 +31,38 @@ try {
"-o",
childRetrievalTest,
]);
await promisify(execFile)(childRetrievalTest, []);
const childRetrieval = await promisify(execFile)(childRetrievalTest, []);
const [slashRichProducerJson, numericProducerJson] = childRetrieval.stdout
.trimEnd()
.split("\n");
if (
slashRichProducerJson === undefined ||
numericProducerJson === undefined
) {
throw new Error(
"Native UI serializer seam did not emit its producer bytes",
);
}
const slashRichConsumerJson = JSON.stringify(
JSON.parse(slashRichProducerJson),
);
if (slashRichProducerJson !== slashRichConsumerJson) {
throw new Error(
"Swift helper JSON bytes differ from the normalized JavaScript consumer representation",
);
}
const numericConsumerJson = JSON.stringify(JSON.parse(numericProducerJson));
if (
Buffer.byteLength(numericProducerJson) <=
Buffer.byteLength(numericConsumerJson)
) {
throw new Error(
"Foundation numeric fixture no longer exercises larger producer JSON bytes",
);
}
if (Buffer.byteLength(numericProducerJson) > NATIVE_UI_HELPER_MAX_BUFFER) {
throw new Error("Foundation numeric fixture exceeds the raw helper budget");
}
const contents = join(root, "Fixture.app", "Contents");
await mkdir(join(contents, "MacOS"), { recursive: true });
+2 -2
View File
@@ -31,7 +31,7 @@ export async function verifyPackageInstall({
const processCaptureCapabilityUrl = pathToFileURL(
join(
prefix,
"lib/node_modules/rea-agents/dist/application/ProcessCaptureCapability.js",
"lib/node_modules/rea-agents/dist/process/capture/ProcessCaptureCapability.js",
),
).href;
const processCaptureCapability = json(
@@ -77,7 +77,7 @@ export async function verifyPackageInstall({
const noOptionalCapabilityUrl = pathToFileURL(
join(
noOptionalPrefix,
"lib/node_modules/rea-agents/dist/application/ProcessCaptureCapability.js",
"lib/node_modules/rea-agents/dist/process/capture/ProcessCaptureCapability.js",
),
).href;
const noOptionalCapability = json(
+29 -4
View File
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
import { join } from "node:path";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { readFile, rm } from "node:fs/promises";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
@@ -167,6 +168,7 @@ const verifyMcpInlineArtifactEvidence = async (
mcpOptions,
);
assert.notEqual(opened.isError, true, JSON.stringify(opened));
let outputRoot;
try {
const inspected = await client.callTool(
{ name: "inspect_artifact", arguments: {} },
@@ -212,12 +214,35 @@ const verifyMcpInlineArtifactEvidence = async (
evidence,
);
}
} finally {
const closed = await client.callTool(
{ name: "close_binary", arguments: {} },
const extracted = await client.callTool(
{ name: "extract_artifact", arguments: {} },
mcpOptions,
);
assert.notEqual(closed.isError, true, JSON.stringify(closed));
assert.notEqual(extracted.isError, true, JSON.stringify(extracted));
const extraction = parseEvidence(extracted.structuredContent?.evidence);
assert.deepEqual(
extraction.normalized_result,
extracted.structuredContent?.result,
);
const materialized = extracted.structuredContent?.result;
assert.equal(typeof materialized?.output_root, "string");
outputRoot = materialized.output_root;
assert.equal(materialized.containment_verified, true);
assert.equal(
await readFile(join(outputRoot, "app/main.js"), "utf8"),
"main();",
);
} finally {
try {
const closed = await client.callTool(
{ name: "close_binary", arguments: {} },
mcpOptions,
);
assert.notEqual(closed.isError, true, JSON.stringify(closed));
} finally {
if (outputRoot !== undefined)
await rm(outputRoot, { recursive: true, force: true });
}
}
};
+10 -4
View File
@@ -25,10 +25,12 @@ if (process.platform !== "win32" || process.arch !== "x64")
);
const workspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-"));
const exec = promisify(execFile);
let packageRoot = resolve(process.argv[2] ?? ".");
const x86Fixture = process.argv[2] === "--x86";
const packageRootArgument = x86Fixture ? undefined : process.argv[2];
let packageRoot = resolve(packageRootArgument ?? ".");
// The default lane verifies the npm artifact in an isolated prefix. Explicit
// package roots support an already-installed artifact without a second install.
if (process.argv[2] === undefined) {
if (packageRootArgument === undefined) {
try {
const npmEntry =
process.env.npm_execpath ??
@@ -77,7 +79,10 @@ if (process.argv[2] === undefined) {
}
}
const target = resolve(
process.argv[3] ?? "build/fixtures/rea-ghidra-windows.exe",
process.argv[3] ??
(x86Fixture
? "build/fixtures/rea-ghidra-windows-x86.exe"
: "build/fixtures/rea-ghidra-windows.exe"),
);
let TOOL_CONTRACTS, native, token, sha256;
try {
@@ -215,7 +220,8 @@ try {
};
try {
await client.connect(transport);
await call("open_binary", { path: target, provider_id: "ghidra" });
await call("open_binary", { path: target });
await call("binary_overview", {});
assert.equal((await call("list_documents", {})).length, 1);
await call("list_names", {});
const procedures = await call("list_procedures", {});
+28 -3
View File
@@ -63,12 +63,20 @@ if (
);
const root = resolve(fileURLToPath(new URL("..", import.meta.url)));
const targetPath = resolve(root, "build", "fixtures", "rea-ghidra-windows.exe");
const architecture = process.argv[2] === "--x86" ? "x86" : "x86_64";
const targetPath = resolve(
root,
"build",
"fixtures",
architecture === "x86"
? "rea-ghidra-windows-x86.exe"
: "rea-ghidra-windows.exe",
);
const target = await parseBinaryTarget(targetPath);
if (
!target.ok ||
target.value.format !== "pe" ||
target.value.architecture !== "x86_64" ||
target.value.architecture !== architecture ||
target.value.executableRole !== "application" ||
target.value.managed !== false
)
@@ -113,6 +121,9 @@ try {
target.value.sha256,
profile.value.profile.digest,
);
const languagePrefix = architecture === "x86" ? "x86:LE:32:" : "x86:LE:64:";
if (!started.value.target.language_id.startsWith(languagePrefix))
throw new Error("Windows fixture imported with the wrong Ghidra language");
coordinates = client.diagnostics();
assertRuntimeCoordinates(coordinates, target.value.sha256);
@@ -137,6 +148,19 @@ try {
);
if (procedure === undefined)
throw new Error("Windows Ghidra fixture exposed no local procedure");
const calleeAddress = architecture === "x86" ? 0x401010n : 0x140001010n;
const callee = procedures.find(
(item) => BigInt(item.address) === calleeAddress,
);
if (callee === undefined)
throw new Error("Windows fixture exposed no controlled callee");
const pseudoCode = await functionOperation("procedure_pseudo_code", {
procedure: callee.value,
});
if (typeof pseudoCode !== "string" || !/\b(?:42|0x2a)\b/iu.test(pseudoCode))
throw new Error(
"Windows fixture decompilation lost its known return value",
);
await inventory("inspect_native_load_image", {});
const memory = await inventory("read_bytes", {
@@ -210,6 +234,7 @@ try {
report = {
ok: true,
provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION },
language_id: started.value.target.language_id,
target: {
format: target.value.format,
architecture: target.value.architecture,
@@ -228,7 +253,7 @@ try {
limitations: [
"approved-non-sensitive-fixtures-only",
"windows-x64-local-ntfs-only",
"native-x86-64-pe-applications-only",
"native-x86-and-x86-64-pe-applications-only",
"no-gui-or-mutation-authority",
],
};
+994
View File
@@ -0,0 +1,994 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash, randomUUID } from "node:crypto";
import {
access,
copyFile,
chmod,
readFile,
truncate,
writeFile,
} from "node:fs/promises";
import { dirname, isAbsolute, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
import { runOwnedCommand } from "../dist/process/OwnedCommand.js";
import { WEB_NETWORK_CAPTURE_LIMITS } from "../dist/domain/webNetworkCapture.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
const executable = process.env.REA_MITMDUMP_COMMAND;
if (
process.platform !== "linux" ||
executable === undefined ||
!isAbsolute(executable)
)
throw new Error(
"verify:web:network-captures requires Linux and absolute REA_MITMDUMP_COMMAND for caller-supplied mitmdump 12.2.3",
);
await access(executable);
const entrypoint =
process.argv[2] ?? fileURLToPath(new URL("./rea.mjs", import.meta.url));
const run = createVerifierRun();
const runtime = await PrivateRuntimeRoot.create({
prefix: "rea-history-verifier-",
});
const environment = Object.fromEntries(
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
);
const client = new Client({
name: "historical-web-capture-verifier",
version: "1",
});
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env: environment,
stderr: "pipe",
});
const failures = [];
let cases = 0;
try {
await runOwnedCommand(
{
command: executable,
arguments: [
"--no-server",
"-q",
"--set",
`confdir=${join(runtime.path, "config")}`,
"--set",
`rea_fixture_root=${runtime.path}`,
"--set",
`rea_adapter_path=${resolve(dirname(entrypoint), "../bridge/mitmproxy/capture.py")}`,
"-s",
fileURLToPath(
new URL("./fixtures/generate-mitmproxy-capture.py", import.meta.url),
),
],
cwd: runtime.path,
runId: `rea-history-fixture-${randomUUID()}`,
hostEnvironment: environment,
},
{ timeoutMs: 30_000, diagnosticBytes: 1024 * 1024 },
);
assert.deepEqual(
JSON.parse(
await readFile(
join(runtime.path, "native-error-classifications.json"),
"utf8",
),
),
{ passed: 7 },
);
await client.connect(transport);
for (const format of ["har", "mitmproxy"]) {
const path = join(
runtime.path,
format === "har" ? "producer.har" : "flows.mitm",
);
const raw = await readFile(path);
const input = { capture_path: path, format, record_ordinals: [1, 0] };
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, input);
assert.equal(value.total_records, 2);
assert.deepEqual(
value.records.map((record) => record.ordinal),
[1, 0],
);
assert.equal(
value.artifact.sha256,
createHash("sha256").update(raw).digest("hex"),
);
assert.equal(value.runtime_attribution, "unknown");
const first = value.records[1];
if (format === "mitmproxy") {
assert.equal(first.reported.id, "original-producer-id");
assert.equal(value.records[0].reported.id, first.reported.id);
assert.equal(first.location.offset, 0);
assert.equal(
value.records[0].location.offset + value.records[0].location.bytes,
raw.length,
);
assert.equal(
first.reported.request.path,
"http://example.test/a?token=ordinary#fragment",
);
assertBinary(
first,
"/request/content",
Buffer.from([0, 255, ...Buffer.from("body")]),
);
assertBinary(
first,
"/response/content",
Buffer.from([0, 254, ...Buffer.from("answer")]),
);
assertBinary(
first,
"/websocket/messages/0/2",
Buffer.from([0, 253, ...Buffer.from("message")]),
);
assert.equal(value.records[0].reported.request.content, null);
assertBinary(value.records[0], "/response/content", Buffer.alloc(0));
assert.ok(
first.numeric_literals.some(
(number) =>
number.pointer === "/metadata/big_integer" &&
number.literal === "9007199254740993",
),
);
assert.ok(
first.numeric_literals.some(
(number) =>
number.pointer === "/metadata/nonfinite" &&
number.literal === "inf",
),
);
assert.equal(
first.reported.metadata.response.headers[0][1],
"ordinary-extension-credential-name",
);
} else {
assert.equal(value.container.reported.log.creator.name, "mitmproxy");
assert.equal(value.container.reported.log.creator.version, "12.2.3");
assert.equal(value.records[0].reported.request.postData.text, null);
assertBinary(
first,
"/response/content/text",
Buffer.from([0, 255, 1, 254]),
);
}
const text = JSON.stringify(value);
for (const secret of [
"native-transport-secret",
"cookie-secret",
"redirect-password",
"user:password",
])
assert.ok(
!text.includes(secret),
`Credential leaked through ${mode}/${format}`,
);
cases++;
}
}
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, {
capture_path: join(runtime.path, "string-urls.mitm"),
format: "mitmproxy",
});
const first = value.records[0];
for (const state of [first.reported, first.reported.backup]) {
assert.equal(state.request.path, "https://example.test/string-path");
assert.equal(state.request.authority, "example.test");
assert.deepEqual(
state.request.headers.map((field) => field[1]),
["https://example.test/from", "//example.test"],
);
assert.equal(state.response.headers[0][1], "https://example.test/to");
}
assert.ok(!JSON.stringify(value).includes("password"));
assert.ok(
first.redactions.some(
(redaction) =>
redaction.pointer === "/backup/request/authority" &&
redaction.reason === "transport-credential",
),
);
cases++;
}
for (const sensitive_values of [
["REDACTED", "["],
["secret", "REDACTED"],
]) {
for (const format of ["har", "mitmproxy"]) {
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, {
capture_path: join(
runtime.path,
format === "har" ? "producer.har" : "string-urls.mitm",
),
format,
record_ordinals: [0],
sensitive_values,
});
const first = value.records[0];
const markers =
format === "har"
? first.reported._markers
: first.reported.metadata._markers;
assert.equal(markers.sensitive, null);
assert.equal(markers.ordinary, "unmarked");
assert.equal(
sensitive_values.includes("[") ? markers.bracket : markers.overlap,
null,
);
if (format === "mitmproxy")
assert.equal(first.reported.backup.metadata._markers.sensitive, null);
cases++;
}
}
}
for (const name of [
"equal-number",
"equal-null",
"equal-object",
"escaped-key",
"core",
]) {
const path = join(runtime.path, `duplicate-${name}.har`);
const original = await readFile(path);
for (const mode of ["cli", "mcp"]) {
await inspect(
mode,
{ capture_path: path, format: "har" },
"invalid_input",
);
assert.deepEqual(await readFile(path), original);
cases++;
}
}
for (const format of ["har", "mitmproxy"]) {
const path = join(
runtime.path,
format === "har" ? "prototype.har" : "prototype.mitm",
);
const original = await readFile(path);
for (const mode of ["cli", "mcp"]) {
const error = await inspect(
mode,
{ capture_path: path, format },
"unsupported_provider",
);
assert.ok(
JSON.stringify(error).includes("JSON schema boundary cannot preserve"),
);
assert.deepEqual(await readFile(path), original);
cases++;
}
}
for (const format of ["har", "mitmproxy"]) {
const path = join(runtime.path, `malformed-${format}`);
await writeFile(path, format === "har" ? "{invalid-json" : "999999999999:");
for (const mode of ["cli", "mcp"]) {
await inspect(mode, { capture_path: path, format }, "invalid_input");
cases++;
}
}
for (const format of ["har", "mitmproxy"]) {
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, {
capture_path: join(
runtime.path,
format === "har" ? "producer.har" : "string-urls.mitm",
),
format,
sensitive_values: ["private-property"],
});
const record = value.records[0];
const properties =
format === "har"
? record.reported._private_properties
: record.reported.metadata._private_properties;
assert.deepEqual(properties, { kept: 7 });
assert.ok(!JSON.stringify(value).includes("private-property"));
assert.ok(
record.redactions.some(
(item) =>
item.scope === "property-name" &&
item.pointer ===
(format === "har"
? "/_private_properties"
: "/metadata/_private_properties"),
),
);
if (format === "mitmproxy")
assert.deepEqual(record.reported.backup.metadata._private_properties, {
kept: 7,
});
cases++;
}
}
for (const [fixture, prefix] of [
["ows", " \t"],
["controls", "\r\n \v\f\x00\x1f"],
]) {
for (const format of ["har", "mitmproxy"]) {
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, {
capture_path: join(
runtime.path,
`${fixture}.${format === "har" ? "har" : "mitm"}`,
),
format,
});
assert.ok(!JSON.stringify(value).includes("ows-secret"));
const record = value.records[0];
const states =
format === "har"
? [record.reported]
: [record.reported, record.reported.backup];
for (const state of states) {
if (format === "har") {
for (const side of ["request", "response"])
assert.ok(
state[side].headers.some(
(header) =>
header.value ===
`${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `,
),
);
} else {
assert.deepEqual(state.request.headers, [
[
"Referer",
`${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `,
],
[
"Origin",
`${prefix}//example.test/path?token=ordinary#fragment\t `,
],
]);
assert.deepEqual(state.response.headers, [
[
"Location",
`${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `,
],
]);
}
}
if (format === "mitmproxy") {
for (const pointer of [
"/request/headers/0/1",
"/backup/request/headers/0/1",
])
assert.deepEqual(
record.binary_fields.find((field) => field.pointer === pointer),
{
pointer,
representation: "producer-bytes",
state: "redacted",
content_base64: null,
bytes: null,
sha256: null,
},
);
}
cases++;
}
}
}
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, {
capture_path: join(runtime.path, "non-utf8-urls.mitm"),
format: "mitmproxy",
});
const record = value.records[0];
assert.ok(!JSON.stringify(value).includes("byte-secret"));
for (const prefix of ["", "/backup"]) {
for (const suffix of [
"/request/path",
"/request/authority",
"/request/headers/0/1",
"/request/headers/1/1",
"/response/headers/0/1",
]) {
const pointer = prefix + suffix;
assert.deepEqual(
record.binary_fields.find((field) => field.pointer === pointer),
{
pointer,
representation: "producer-bytes",
state: "redacted",
content_base64: null,
bytes: null,
sha256: null,
},
);
}
}
cases++;
}
const nativeCoordinateInput = {
capture_path: join(runtime.path, "flows.mitm"),
format: "mitmproxy",
};
const nativeCoordinates = await inspect("mcp", nativeCoordinateInput);
const originalLocation = nativeCoordinates.records[1].location;
assert.equal(originalLocation.kind, "byte-range");
for (const literal of [
String(originalLocation.offset),
String(originalLocation.bytes),
]) {
for (const mode of ["cli", "mcp"]) {
const excluded = await inspect(mode, {
...nativeCoordinateInput,
sensitive_values: [literal],
});
assert.deepEqual(excluded.records[1].location, {
kind: "unknown",
reason: "explicit-sensitive-value",
});
cases++;
}
}
cases++;
const deniedPath = join(runtime.path, "unreadable.har");
await copyFile(join(runtime.path, "producer.har"), deniedPath);
await chmod(deniedPath, 0o000);
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
const failure = await inspect(
mode,
{ capture_path: deniedPath, format },
"unavailable",
);
assert.equal(failure.code, "access_denied");
assert.equal(failure.details.system_code, "EACCES");
assert.equal(failure.details.path, deniedPath);
cases++;
}
for (const literal of ["log", "entries"]) {
const value = await inspect(mode, {
capture_path: join(runtime.path, "producer.har"),
format: "har",
sensitive_values: [literal],
});
assert.equal(value.container.records_pointer, null);
assert.ok(
value.records.every(
(record) =>
record.location.kind === "unknown" &&
record.location.reason === "explicit-sensitive-value",
),
);
assert.ok(!JSON.stringify(value).includes(literal));
cases++;
}
for (const format of ["har", "mitmproxy"]) {
const value = await inspect(mode, {
capture_path: join(
runtime.path,
format === "har" ? "producer.har" : "string-urls.mitm",
),
format,
sensitive_values: ["~1"],
});
assert.ok(!JSON.stringify(value).includes("~1"));
cases++;
}
}
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
for (const sensitive_values of [
["producer"],
["producer", "REDACTED", "…"],
]) {
await inspect(mode, {
capture_path: join(
runtime.path,
format === "har" ? "producer.har" : "flows.mitm",
),
format,
sensitive_values,
});
cases++;
}
}
}
for (const mode of ["cli", "mcp"]) {
for (const selection of [
{ record_ordinals: [12345], sensitive_values: ["12345"] },
{ record_ordinals: [0, 0], sensitive_values: ["Each"] },
]) {
await inspect(
mode,
{
capture_path: join(runtime.path, "producer.har"),
format: "har",
...selection,
},
"invalid_input",
);
cases++;
}
}
await inspect(
"mcp",
{
capture_path: "relative.har",
format: "har",
sensitive_values: ["absolute"],
},
"invalid_input",
);
cases++;
const privatePath = join(runtime.path, "REDACTED.har");
for (const name of ["mysecret", "/ordinary/secret"]) {
const unknownArgument = await inspect(
"mcp",
{
capture_path: join(runtime.path, "producer.har"),
format: "har",
sensitive_values: ["secret"],
[name]: true,
},
"invalid_input",
);
assert.deepEqual(unknownArgument.details.issues[0].path, []);
cases++;
}
await inspect("mcp", {
capture_path: join(runtime.path, "flows.mitm"),
format: "mitmproxy",
sensitive_values: ["\ud800", "ordinary-scalar"],
});
cases++;
const replacementHar = JSON.parse(
await readFile(join(runtime.path, "producer.har"), "utf8"),
);
replacementHar.log.entries[0].response.content = {
size: 3,
mimeType: "application/octet-stream",
encoding: "base64",
text: "77+9",
};
const replacementHarPath = join(runtime.path, "unicode-scalar.har");
await writeFile(replacementHarPath, JSON.stringify(replacementHar));
for (const literal of ["\ud800", "\udfff", "\ufffd"]) {
const unicode = await inspect("mcp", {
capture_path: replacementHarPath,
format: "har",
sensitive_values: [literal],
});
const body = unicode.records[0].binary_fields.find(
(field) => field.pointer === "/response/content/text",
);
assert.equal(body?.state, literal === "\ufffd" ? "redacted" : "retained");
assert.equal(body.content_base64, literal === "\ufffd" ? null : "77+9");
cases++;
}
const numericHar = (
await readFile(join(runtime.path, "producer.har"), "utf8")
).replace('"cache": {}', '"cache": {}, "_big": 9007199254740993');
const numericPath = join(runtime.path, "numeric.har");
await writeFile(numericPath, numericHar);
assert.ok(numericHar.includes('"_big": 9007199254740993'));
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
for (const literal of ["9007199254740993", "4740993"]) {
const value = await inspect(mode, {
capture_path:
format === "har" ? numericPath : join(runtime.path, "flows.mitm"),
format,
sensitive_values: [literal],
});
assert.ok(!JSON.stringify(value).includes(literal));
const first = value.records[0];
const pointer = format === "har" ? "/_big" : "/metadata/big_integer";
assert.ok(
first.redactions.some(
(redaction) =>
redaction.pointer === pointer &&
redaction.reason === "explicit-sensitive-value",
),
);
assert.ok(
first.numeric_literals.every(
(number) => !number.literal.includes(literal),
),
);
cases++;
}
}
}
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
await inspect(mode, {
capture_path: join(
runtime.path,
format === "har" ? "producer.har" : "flows.mitm",
),
format,
sensitive_values: [format],
});
cases++;
}
}
const encodedHar = JSON.parse(
await readFile(join(runtime.path, "producer.har"), "utf8"),
);
encodedHar.log.entries[0].response.content = {
size: 6,
mimeType: "application/octet-stream",
encoding: "base64",
text: "c2 VjcmV0",
};
const encodedHarPath = join(runtime.path, "encoded-sensitive.har");
await writeFile(encodedHarPath, JSON.stringify(encodedHar));
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
const bytes =
format === "har"
? Buffer.from("secret")
: Buffer.from([0, 255, ...Buffer.from("body")]);
for (const literal of [
bytes.toString("base64"),
bytes.toString("base64").slice(0, 4),
createHash("sha256").update(bytes).digest("hex"),
]) {
const value = await inspect(mode, {
capture_path:
format === "har"
? encodedHarPath
: join(runtime.path, "flows.mitm"),
format,
sensitive_values: [literal],
});
const pointer =
format === "har" ? "/response/content/text" : "/request/content";
const field = value.records[0].binary_fields.find(
(field) => field.pointer === pointer,
);
assert.equal(field?.state, "redacted");
assert.equal(field.content_base64, null);
assert.equal(field.sha256, null);
assert.ok(!JSON.stringify(value).includes(literal));
cases++;
}
}
}
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
const capturePath = join(
runtime.path,
format === "har" ? "producer.har" : "flows.mitm",
);
const digest = createHash("sha256")
.update(await readFile(capturePath))
.digest("hex");
for (const literal of [digest, digest.slice(0, 12)]) {
const value = await inspect(mode, {
capture_path: capturePath,
format,
sensitive_values: [literal],
});
assert.equal(value.artifact.sha256, null);
assert.equal(value.artifact.path, capturePath);
assert.ok(!JSON.stringify(value).includes(literal));
cases++;
}
}
}
const oversizedPath = join(runtime.path, "oversized-capture");
for (const mode of ["cli", "mcp"]) {
for (const literal of ["log", "entries"]) {
const hidden = await inspect(mode, {
capture_path: join(runtime.path, "producer.har"),
format: "har",
sensitive_values: [literal],
});
assert.equal(hidden.total_records, 2);
assert.equal(hidden.container.records_pointer, null);
for (const [ordinal, record] of hidden.records.entries()) {
assert.equal(record.ordinal, ordinal);
assert.deepEqual(record.location, {
kind: "unknown",
reason: "explicit-sensitive-value",
});
assert.equal(record.reported, null);
assert.deepEqual(record.binary_fields, []);
assert.deepEqual(record.numeric_literals, []);
}
cases++;
}
}
await writeFile(oversizedPath, "");
await truncate(oversizedPath, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1);
await copyFile(join(runtime.path, "producer.har"), privatePath);
for (const mode of ["cli", "mcp"]) {
for (const format of ["har", "mitmproxy"]) {
const failure = await inspect(
mode,
{ capture_path: oversizedPath, format },
"invalid_input",
);
assert.equal(failure.details.issues[0].reason, "out_of_range");
assert.deepEqual(failure.details.issues[0].path, ["capture_path"]);
assert.equal(
failure.details.issues[0].expected.maximum_capture_bytes,
WEB_NETWORK_CAPTURE_LIMITS.inputBytes,
);
cases++;
}
for (const format of ["har", "mitmproxy"]) {
const failure = await inspect(
mode,
{
capture_path: join(
runtime.path,
format === "har" ? "deep.har" : "deep.mitm",
),
format,
},
"invalid_input",
);
const issue = failure.details.issues[0];
assert.equal(issue.reason, "out_of_range");
assert.equal(issue.path[0], "capture_path");
assert.match(issue.path[1], /^\/_extension\/child/);
assert.equal(
issue.expected.maximum_capture_nesting,
WEB_NETWORK_CAPTURE_LIMITS.depth,
);
cases++;
}
await inspect(
mode,
{
capture_path: join(runtime.path, "invalid-credential-key.mitm"),
format: "mitmproxy",
},
"invalid_input",
);
cases++;
const credentialDepth = await inspect(
mode,
{
capture_path: join(runtime.path, "deep-credential.mitm"),
format: "mitmproxy",
},
"invalid_input",
);
assert.equal(credentialDepth.details.issues[0].reason, "out_of_range");
assert.match(
credentialDepth.details.issues[0].path[1],
/^\/request\/headers\/0\/1\/child/,
);
cases++;
const value = await inspect(mode, {
capture_path: privatePath,
format: "har",
sensitive_values: ["REDACTED", "["],
});
assert.equal(value.artifact.path, "");
assert.ok(!JSON.stringify(value).includes("REDACTED"));
cases++;
await inspect(
mode,
{
capture_path: join(runtime.path, "invalid-private-parent.har"),
format: "har",
sensitive_values: ["response"],
},
"invalid_input",
);
cases++;
await inspect(
mode,
{
capture_path: join(runtime.path, "private-property-absent.har"),
format: "har",
sensitive_values: ["private-property", "REDACTED"],
},
"invalid_input",
);
cases++;
await inspect(
mode,
{
capture_path: join(runtime.path, "invalid-private-key.mitm"),
format: "mitmproxy",
sensitive_values: ["private-property"],
},
"invalid_input",
);
cases++;
}
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, {
capture_path: join(runtime.path, "flows.mitm"),
format: "mitmproxy",
record_ordinals: [0],
sensitive_values: ["body"],
});
assert.equal(
value.records[0].binary_fields.find(
(field) => field.pointer === "/request/content",
).state,
"redacted",
);
cases++;
}
for (const format of ["har", "mitmproxy"]) {
const path = join(runtime.path, `empty-${format}`);
await writeFile(
path,
format === "har"
? JSON.stringify({
log: {
version: "1.2",
creator: { name: "source-owned-empty-fixture", version: "1" },
entries: [],
},
})
: "",
);
for (const mode of ["cli", "mcp"]) {
const value = await inspect(mode, { capture_path: path, format });
assert.equal(value.total_records, 0);
cases++;
}
}
} catch (cause) {
failures.push(cause);
} finally {
for (const cleanup of [
() => client.close(),
() => transport.close(),
() => runtime.close(),
]) {
try {
await cleanup();
} catch (cause) {
failures.push(cause);
}
}
}
const verifier = await completeVerifierRun(run);
try {
assert.equal(verifier.process_lineage.status, "verified");
assert.deepEqual(verifier.process_lineage.descendants, []);
} catch (cause) {
failures.push(cause);
}
if (failures.length > 0) {
console.error(
JSON.stringify(
{ status: "failed", public_cases: cases, verifier },
null,
2,
),
);
throw new AggregateError(
failures,
"Historical capture verification failed; analysis and cleanup failures are retained.",
);
}
console.log(
JSON.stringify(
{
status: "passed",
public_cases: cases,
upstream: "mitmproxy 12.2.3 FlowWriter + SaveHar",
native_failure_classifications: 7,
offline: true,
verifier,
},
null,
2,
),
);
async function inspect(mode, input, errorCategory) {
if (mode === "mcp") {
const result = await client.callTool({
name: "inspect_web_network_capture",
arguments: input,
});
const value = JSON.parse(mcpTextValue(result));
if (errorCategory !== undefined) {
assert.equal(result.isError, true);
assert.equal(value.error.category, errorCategory);
for (const literal of input.sensitive_values ?? [])
assert.ok(!JSON.stringify(value.error).includes(literal));
return value.error;
}
assert.notEqual(result.isError, true, mcpTextValue(result));
assertSensitiveLimitations(value.evidence, input.sensitive_values ?? []);
return value.result;
}
const args = [
entrypoint,
"inspect-web-network-capture",
input.capture_path,
input.format,
"--json",
...(input.record_ordinals ?? []).flatMap((ordinal) => [
"--record",
String(ordinal),
]),
...(input.sensitive_values ?? []).flatMap((value) => [
"--sensitive-value",
value,
]),
];
let result;
try {
result = await promisify(execFile)(process.execPath, args, {
env: environment,
timeout: 40_000,
maxBuffer: 96 * 1024 * 1024,
});
} catch (cause) {
if (errorCategory === undefined) throw cause;
assert.equal(typeof cause.code, "number");
const value = JSON.parse(cause.stdout);
assert.equal(value.category, errorCategory);
for (const literal of input.sensitive_values ?? [])
assert.ok(!JSON.stringify(value).includes(literal));
return value;
}
assert.equal(errorCategory, undefined, "Expected malformed capture to fail");
const evidence = JSON.parse(result.stdout);
assertSensitiveLimitations(evidence, input.sensitive_values ?? []);
return evidence.normalized_result;
}
function assertSensitiveLimitations(evidence, literals) {
for (const value of Object.values(evidence.parameters))
if (typeof value === "string")
for (const literal of literals)
assert.ok(
!value.includes(literal),
"Declared literal retained in Evidence parameter",
);
const result = evidence.normalized_result;
if (result.artifact.sha256 === null) assert.equal(evidence.subject, null);
else assert.equal(evidence.subject.digest.sha256, result.artifact.sha256);
assert.deepEqual(
evidence.provider,
result.decoder,
"Evidence provider tuple changed",
);
assert.equal(
result.decoder.name,
result.format === "har"
? "REA HAR capture adapter"
: "REA offline native mitmproxy adapter",
);
const texts = [
evidence.limitations,
result.limitations,
...result.records.map((record) => record.limitations),
].flat();
for (const literal of literals)
assert.ok(
texts.every((text) => !text.includes(literal)),
"Declared literal retained in authored limitations",
);
}
function assertBinary(record, pointer, bytes) {
const field = record.binary_fields.find((item) => item.pointer === pointer);
assert.ok(field, `Missing native byte field ${pointer}`);
assert.equal(field.state, "retained");
assert.equal(field.content_base64, bytes.toString("base64"));
assert.equal(field.bytes, bytes.length);
assert.equal(field.sha256, createHash("sha256").update(bytes).digest("hex"));
}
+74
View File
@@ -263,6 +263,80 @@ try {
]);
native.call("runtime_snapshot_cancel", [runtime.handle]);
await assert.rejects(cancelled, /cancelled/u);
assert.equal(await exists(join(runtime.path, "cancelled.bin")), false);
const readbackRuntime = WindowsPrivateRuntime.create(
workspace.replaceAll("\\", "/"),
"readback-",
);
try {
readbackRuntime.writeFile("written.txt", "runtime-write-readback");
assert.equal(
readbackRuntime.readFile("written.txt"),
"runtime-write-readback",
);
const readbackSnapshot = await readbackRuntime.snapshot(
source,
"snapshot.bin",
);
assert.equal(
readbackSnapshot.sha256,
createHash("sha256")
.update(await readFile(source))
.digest("hex"),
);
assert.equal(
readbackRuntime.readFile("snapshot.bin"),
await readFile(source, "utf8"),
);
const writtenPath = join(readbackRuntime.observation.path, "written.txt");
await assert.rejects(writeFile(writtenPath, "replace"));
await assert.rejects(rename(writtenPath, `${writtenPath}.renamed`));
await assert.rejects(rm(writtenPath));
assert.equal(
readbackRuntime.readFile("written.txt"),
"runtime-write-readback",
);
const snapshotPath = join(
readbackRuntime.observation.path,
"snapshot.bin",
);
// Java RandomAccessFile uses read/write sharing without delete sharing.
// Exercise that independent Win32 reader contract without requiring Java.
const readerDigest = execFileSync(
"powershell.exe",
[
"-NoProfile",
"-NonInteractive",
"-Command",
"$ErrorActionPreference = 'Stop'; " +
"$stream = [IO.File]::Open($env:REA_SNAPSHOT_READER_PATH, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite); " +
"$hash = [Security.Cryptography.SHA256]::Create(); " +
"try { [BitConverter]::ToString($hash.ComputeHash($stream)).Replace('-', '').ToLowerInvariant() } " +
"finally { $hash.Dispose(); $stream.Dispose() }",
],
{
encoding: "utf8",
timeout: 15_000,
env: { ...process.env, REA_SNAPSHOT_READER_PATH: snapshotPath },
},
).trim();
assert.equal(readerDigest, readbackSnapshot.sha256);
await assert.rejects(writeFile(snapshotPath, "replace"));
await assert.rejects(rename(snapshotPath, `${snapshotPath}.renamed`));
await assert.rejects(rm(snapshotPath));
assert.equal(
readbackRuntime.readFile("snapshot.bin"),
await readFile(source, "utf8"),
);
report.controls = {
...report.controls,
completedRuntimeFileReadback: true,
completedRuntimeFileMutationDenied: true,
snapshotReadWriteSharingReader: true,
};
} finally {
await readbackRuntime.close();
}
report.controls = {
...report.controls,
protectedDaclReadback: true,
+134
View File
@@ -0,0 +1,134 @@
import assert from "node:assert/strict";
import { constants } from "node:buffer";
import { createHash } from "node:crypto";
import { once } from "node:events";
import { createReadStream, createWriteStream } from "node:fs";
import { mkdtemp, rm, stat, statfs } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { finished } from "node:stream/promises";
import { Cli } from "incur";
import { createStreamedCliJsonOutput } from "../../../dist/cli/streamedJsonOutput.js";
import { createEvidence } from "../../../dist/domain/evidence.js";
// This opt-in format-boundary check writes one temporary file larger than the
// actual engine string limit. Shared leaves keep the analysis input small.
const format = process.argv[2] ?? "json";
assert.ok(format === "json" || format === "jsonl", "Use json or jsonl");
const leaf = "x".repeat(64 * 1024);
const encodedLeaf = JSON.stringify(leaf);
const count = Math.ceil(constants.MAX_STRING_LENGTH / (leaf.length + 3)) + 1;
const volume = await statfs(tmpdir());
assert.ok(
volume.bavail * volume.bsize > count * encodedLeaf.length + 1024 ** 3,
"Large-output verification needs room for one output file and a 1-GiB reserve",
);
const workspace = await mkdtemp(join(tmpdir(), "rea-javascript-output-"));
const path = join(workspace, "result.json");
let destination;
let peakRss = process.memoryUsage().rss;
const sampleRss = () => {
peakRss = Math.max(peakRss, process.memoryUsage().rss);
};
const sampler = setInterval(sampleRss, 25);
sampler.unref();
try {
const evidence = createEvidence(
undefined,
{ id: "large-output-fixture", name: "Large output fixture", version: "1" },
{
operation: "inspect_fixture",
parameters: {},
result: { payload: Array.from({ length: count }, () => leaf) },
},
);
sampleRss();
const metadata = { command: "inspect-output-fixture", duration: "0ms" };
const marker = "__REA_OUTPUT_ENCODING_MARKER__";
const template = {
ok: true,
data: { ...evidence, normalized_result: { payload: [marker] } },
meta: metadata,
};
const skeleton = JSON.stringify(template, null, format === "json" ? 2 : 0);
const encodedMarker = JSON.stringify(marker);
const markerIndex = skeleton.indexOf(encodedMarker);
assert.ok(markerIndex >= 0);
assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker));
const prefix = skeleton.slice(0, markerIndex);
const suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`;
const indentation = prefix.slice(prefix.lastIndexOf("\n") + 1);
const separator = format === "json" ? `,\n${indentation}` : ",";
const expectedHash = createHash("sha256");
expectedHash.update(prefix);
for (let index = 0; index < count; index += 1) {
if (index > 0) expectedHash.update(separator);
expectedHash.update(encodedLeaf);
}
expectedHash.update(suffix);
const expectedBytes =
Buffer.byteLength(prefix) +
count * Buffer.byteLength(encodedLeaf) +
(count - 1) * Buffer.byteLength(separator) +
Buffer.byteLength(suffix);
assert.ok(expectedBytes > constants.MAX_STRING_LENGTH);
destination = createWriteStream(path, { flags: "wx", mode: 0o600 });
const argv = [metadata.command, "--format", format, "--full-output"];
const output = createStreamedCliJsonOutput(argv, destination);
assert.ok(output);
let unexpectedOutputBytes = 0;
let exitCode = 0;
await Cli.create("rea-output-verifier", { sync: false })
.command(metadata.command, {
run: async ({ format: selectedFormat }) => {
assert.equal(
await output.write(evidence, { ...metadata, format: selectedFormat }),
true,
);
},
})
.serve(argv, {
stdout: (text) => {
if (!output.handled) unexpectedOutputBytes += Buffer.byteLength(text);
},
exit: (code) => {
exitCode = code;
},
});
destination.end();
await finished(destination);
assert.equal(exitCode, 0);
assert.equal(output.handled, true);
assert.equal(output.failed, false);
assert.equal(unexpectedOutputBytes, 0);
const actualHash = createHash("sha256");
for await (const chunk of createReadStream(path)) actualHash.update(chunk);
const digest = actualHash.digest("hex");
assert.equal(digest, expectedHash.digest("hex"));
assert.equal((await stat(path)).size, expectedBytes);
sampleRss();
process.stdout.write(
`${JSON.stringify({
json_output_verified: true,
format,
output_bytes: expectedBytes,
engine_string_limit: constants.MAX_STRING_LENGTH,
output_sha256: digest,
evidence_id: evidence.evidence_id,
rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
})}\n`,
);
} finally {
clearInterval(sampler);
if (destination !== undefined && !destination.closed) {
const closed = once(destination, "close");
destination.destroy();
await closed;
}
await rm(workspace, { recursive: true, force: true });
}