chore: merge main into feat/pkg-inventory
Resolve conflicts against main: - Inventory moved to src/artifacts/inventory (#1032); point the xar and cpio reader imports at their new relative paths. - README.md: take main's simplified English README, which no longer lists individual package formats. PKG coverage stays documented in the skill reference and docs/testing.md. - Skill: the references gain the PKG paragraph, so the version moves to 29 (main already uses 28). - Regenerate the product catalog, conformance manifest, skill digest and package metadata. The MCP tool catalog is now built rather than committed (#997), so it is removed here. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -3,50 +3,68 @@ import { dirname, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const output = resolve(root, "build", "fixtures", "rea-ghidra-windows.exe");
|
||||
const bytes = Buffer.alloc(1024);
|
||||
const peOffset = 0x80;
|
||||
const optionalHeader = peOffset + 24;
|
||||
const sectionTable = optionalHeader + 0xf0;
|
||||
for (const architecture of ["x86_64", "x86"]) {
|
||||
const x86 = architecture === "x86";
|
||||
const output = resolve(
|
||||
root,
|
||||
"build",
|
||||
"fixtures",
|
||||
x86 ? "rea-ghidra-windows-x86.exe" : "rea-ghidra-windows.exe",
|
||||
);
|
||||
const bytes = Buffer.alloc(1024);
|
||||
const peOffset = 0x80;
|
||||
const optionalHeader = peOffset + 24;
|
||||
const optionalHeaderSize = x86 ? 0xe0 : 0xf0;
|
||||
const sectionTable = optionalHeader + optionalHeaderSize;
|
||||
|
||||
bytes.write("MZ", 0, "ascii");
|
||||
bytes.writeUInt32LE(peOffset, 0x3c);
|
||||
bytes.write("PE\0\0", peOffset, "binary");
|
||||
bytes.writeUInt16LE(0x8664, peOffset + 4);
|
||||
bytes.writeUInt16LE(1, peOffset + 6);
|
||||
bytes.writeUInt16LE(0xf0, peOffset + 20);
|
||||
bytes.writeUInt16LE(0x22, peOffset + 22);
|
||||
bytes.write("MZ", 0, "ascii");
|
||||
bytes.writeUInt32LE(peOffset, 0x3c);
|
||||
bytes.write("PE\0\0", peOffset, "binary");
|
||||
bytes.writeUInt16LE(x86 ? 0x14c : 0x8664, peOffset + 4);
|
||||
bytes.writeUInt16LE(1, peOffset + 6);
|
||||
bytes.writeUInt16LE(optionalHeaderSize, peOffset + 20);
|
||||
bytes.writeUInt16LE(x86 ? 0x102 : 0x22, peOffset + 22);
|
||||
|
||||
bytes.writeUInt16LE(0x20b, optionalHeader);
|
||||
bytes.writeUInt32LE(0x200, optionalHeader + 4);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 16);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 20);
|
||||
bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 32);
|
||||
bytes.writeUInt32LE(0x200, optionalHeader + 36);
|
||||
bytes.writeUInt16LE(6, optionalHeader + 40);
|
||||
bytes.writeUInt16LE(6, optionalHeader + 48);
|
||||
bytes.writeUInt32LE(0x2000, optionalHeader + 56);
|
||||
bytes.writeUInt32LE(0x200, optionalHeader + 60);
|
||||
bytes.writeUInt16LE(3, optionalHeader + 68);
|
||||
bytes.writeUInt16LE(0x8160, optionalHeader + 70);
|
||||
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72);
|
||||
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80);
|
||||
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88);
|
||||
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96);
|
||||
bytes.writeUInt32LE(16, optionalHeader + 108);
|
||||
bytes.writeUInt16LE(x86 ? 0x10b : 0x20b, optionalHeader);
|
||||
bytes.writeUInt32LE(0x200, optionalHeader + 4);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 16);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 20);
|
||||
if (x86) bytes.writeUInt32LE(0x400000, optionalHeader + 28);
|
||||
else bytes.writeBigUInt64LE(0x1_4000_0000n, optionalHeader + 24);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 32);
|
||||
bytes.writeUInt32LE(0x200, optionalHeader + 36);
|
||||
bytes.writeUInt16LE(6, optionalHeader + 40);
|
||||
bytes.writeUInt16LE(6, optionalHeader + 48);
|
||||
bytes.writeUInt32LE(0x2000, optionalHeader + 56);
|
||||
bytes.writeUInt32LE(0x200, optionalHeader + 60);
|
||||
bytes.writeUInt16LE(3, optionalHeader + 68);
|
||||
bytes.writeUInt16LE(x86 ? 0x8140 : 0x8160, optionalHeader + 70);
|
||||
if (x86) {
|
||||
bytes.writeUInt32LE(0x10_0000, optionalHeader + 72);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 76);
|
||||
bytes.writeUInt32LE(0x10_0000, optionalHeader + 80);
|
||||
bytes.writeUInt32LE(0x1000, optionalHeader + 84);
|
||||
bytes.writeUInt32LE(16, optionalHeader + 92);
|
||||
} else {
|
||||
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 72);
|
||||
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 80);
|
||||
bytes.writeBigUInt64LE(0x10_0000n, optionalHeader + 88);
|
||||
bytes.writeBigUInt64LE(0x1000n, optionalHeader + 96);
|
||||
bytes.writeUInt32LE(16, optionalHeader + 108);
|
||||
}
|
||||
|
||||
bytes.write(".text", sectionTable, "ascii");
|
||||
bytes.writeUInt32LE(0x16, sectionTable + 8);
|
||||
bytes.writeUInt32LE(0x1000, sectionTable + 12);
|
||||
bytes.writeUInt32LE(0x200, sectionTable + 16);
|
||||
bytes.writeUInt32LE(0x200, sectionTable + 20);
|
||||
bytes.writeUInt32LE(0x6000_0020, sectionTable + 36);
|
||||
bytes.write(".text", sectionTable, "ascii");
|
||||
bytes.writeUInt32LE(0x16, sectionTable + 8);
|
||||
bytes.writeUInt32LE(0x1000, sectionTable + 12);
|
||||
bytes.writeUInt32LE(0x200, sectionTable + 16);
|
||||
bytes.writeUInt32LE(0x200, sectionTable + 20);
|
||||
bytes.writeUInt32LE(0x6000_0020, sectionTable + 36);
|
||||
|
||||
// entry calls the second function, returns zero; second returns 42.
|
||||
bytes.set([0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], 0x200);
|
||||
bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210);
|
||||
// entry calls the second function, returns zero; second returns 42.
|
||||
bytes.set([0xe8, 0x0b, 0, 0, 0, 0x31, 0xc0, 0xc3], 0x200);
|
||||
bytes.set([0xb8, 0x2a, 0, 0, 0, 0xc3], 0x210);
|
||||
|
||||
await mkdir(dirname(output), { recursive: true });
|
||||
await writeFile(output, bytes);
|
||||
process.stdout.write(`${output}\n`);
|
||||
await mkdir(dirname(output), { recursive: true });
|
||||
await writeFile(output, bytes);
|
||||
if (!x86) process.stdout.write(`${output}\n`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,151 @@
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from mitmproxy import connection, ctx, http
|
||||
from mitmproxy.io import FlowWriter
|
||||
from mitmproxy.io import tnetstring
|
||||
from mitmproxy.addons.savehar import SaveHar
|
||||
from mitmproxy.websocket import WebSocketData, WebSocketMessage
|
||||
|
||||
class Generator:
|
||||
def load(self, loader):
|
||||
loader.add_option("rea_fixture_root", str, "", "Owned synthetic fixture root")
|
||||
loader.add_option("rea_adapter_path", str, "", "REA native adapter under verification")
|
||||
|
||||
def running(self):
|
||||
root = Path(ctx.options.rea_fixture_root)
|
||||
spec = importlib.util.spec_from_file_location("rea_capture_under_test", ctx.options.rea_adapter_path)
|
||||
adapter = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(adapter)
|
||||
classifications = [
|
||||
(adapter.CaptureFailure("format", "Malformed retained record.", "/record"), "format"),
|
||||
(adapter.CaptureFailure("input-limit", "Nesting budget.", "/extension"), "input-limit"),
|
||||
(MemoryError("payload must not enter diagnostics"), "resource-limit"),
|
||||
(RuntimeError("payload must not enter diagnostics"), "decoder"),
|
||||
(OSError("payload must not enter diagnostics"), "decoder"),
|
||||
]
|
||||
for error, expected in classifications:
|
||||
reply = adapter.failure_reply(error)
|
||||
assert reply["reason"] == expected, reply
|
||||
assert "payload must not enter diagnostics" not in json.dumps(reply)
|
||||
for state, expected in [({"metadata": {1: b"bad-key"}}, "format"), ({"metadata": object()}, "decoder")]:
|
||||
try:
|
||||
adapter.project_record(state, [])
|
||||
except adapter.CaptureFailure as error:
|
||||
assert error.reason == expected, (error.reason, expected)
|
||||
assert error.pointer == "/metadata", error.pointer
|
||||
else:
|
||||
raise AssertionError("Invalid native representation was accepted")
|
||||
non_utf8 = adapter.project_record({"request": {"headers": [(b"Referer", b"https://byte-user:byte-secret@example.test/path\xff")] }}, [])
|
||||
assert non_utf8["reported"]["request"]["headers"][0][1] is None, non_utf8
|
||||
assert non_utf8["binary_fields"][-1]["state"] == "redacted", non_utf8
|
||||
for value in [" \tHTTPS://ows-user:ows-secret@example.test/path\t ", b" \tHTTPS://ows-user:ows-secret@example.test/path\t "]:
|
||||
projected = adapter.project_record({"request": {"headers": [(b"Referer", value)]}}, [])
|
||||
assert projected["reported"]["request"]["headers"][0][1] == " \tHTTPS://example.test/path\t ", projected
|
||||
assert "ows-secret" not in json.dumps(projected), projected
|
||||
if isinstance(value, bytes):
|
||||
assert projected["binary_fields"][-1]["state"] == "redacted", projected
|
||||
(root / "native-error-classifications.json").write_text(json.dumps({"passed": len(classifications) + 2}))
|
||||
first = http.HTTPFlow(connection.Client(peername=("127.0.0.1", 1), sockname=("127.0.0.1", 2)), connection.Server(address=("example.test", 80)))
|
||||
first.id = "original-producer-id"
|
||||
first.request = http.Request.make("POST", "http://example.test/a?token=ordinary", b"\x00\xffbody", [(b"Authorization", b"Bearer native-transport-secret"), (b"X-Duplicate", b"one"), (b"X-Duplicate", b"two")])
|
||||
first.request.path = "http://user:password@example.test/a?token=ordinary#fragment"
|
||||
first.response = http.Response.make(200, b"\x00\xfeanswer", [(b"Set-Cookie", b"session=cookie-secret"), (b"Location", b"https://redirect-user:redirect-password@example.test/b?token=ordinary")])
|
||||
first.metadata = {"big_integer": 9007199254740993, "nonfinite": float("inf"), "isLosslessNumber": True, "value": "ordinary-extension", "response": {"headers": [["Authorization", "ordinary-extension-credential-name"]]}}
|
||||
first.websocket = WebSocketData(messages=[WebSocketMessage(2, True, b"\x00\xfdmessage", 123.5)])
|
||||
first.backup()
|
||||
first.comment = "caller-selected-fixture"
|
||||
second = first.copy()
|
||||
second.id = first.id
|
||||
second.request.raw_content = None
|
||||
second.response.raw_content = b""
|
||||
second.websocket = None
|
||||
with (root / "flows.mitm").open("wb") as handle:
|
||||
writer = FlowWriter(handle)
|
||||
writer.add(first)
|
||||
writer.add(second)
|
||||
har_first, har_second = first.copy(), second.copy()
|
||||
har_first.request.path = "/a?token=ordinary#fragment"
|
||||
har_second.request.path = "/a?token=ordinary#fragment"
|
||||
har_first.response.raw_content = b"\x00\xff\x01\xfe"
|
||||
markers = {"sensitive": "REDACTED", "bracket": "literal[", "overlap": "secret", "ordinary": "unmarked"}
|
||||
har = SaveHar().make_har([har_first, har_second])
|
||||
har["log"]["entries"][0]["_markers"] = markers
|
||||
har["log"]["entries"][0]["_private_properties"] = {"private-property/~": {"number": 123, "text": "private-property"}, "kept": 7}
|
||||
(root / "producer.har").write_text(json.dumps(har))
|
||||
har_text = json.dumps(har)
|
||||
for name, members in [
|
||||
("equal-number", '"_duplicate": 1, "_duplicate": 1'),
|
||||
("equal-null", '"_duplicate": null, "_duplicate": null'),
|
||||
("equal-object", '"_duplicate": {"value": 1}, "_duplicate": {"value": 1}'),
|
||||
("escaped-key", '"_duplicate": 1, "\\u005fduplicate": 1'),
|
||||
]:
|
||||
duplicate = har_text.replace('"cache": {}', '"cache": {}, ' + members, 1)
|
||||
assert duplicate != har_text
|
||||
(root / ("duplicate-" + name + ".har")).write_text(duplicate)
|
||||
duplicate = har_text.replace('"version": "1.2"', '"version": "1.2", "version": "1.2"', 1)
|
||||
assert duplicate != har_text
|
||||
(root / "duplicate-core.har").write_text(duplicate)
|
||||
prototype = json.loads(har_text)
|
||||
prototype["_extension"] = {"__proto__": {"preserved": 7}, "constructor": "ordinary"}
|
||||
(root / "prototype.har").write_text(json.dumps(prototype))
|
||||
prototype_native = first.get_state()
|
||||
prototype_native["metadata"]["__proto__"] = {"preserved": 7}
|
||||
with (root / "prototype.mitm").open("wb") as handle:
|
||||
tnetstring.dump(prototype_native, handle)
|
||||
invalid_har = json.loads(json.dumps(har))
|
||||
invalid_har["log"]["entries"][0]["response"]["content"].update({"encoding": "base64", "text": "AR=="})
|
||||
(root / "invalid-private-parent.har").write_text(json.dumps(invalid_har))
|
||||
strings = first.get_state()
|
||||
for state in [strings, strings["backup"]]:
|
||||
state["request"]["path"] = "https://native-user:string-password@example.test/string-path"
|
||||
state["request"]["authority"] = "authority-user:authority-password@example.test"
|
||||
state["request"]["headers"] = [(b"Referer", "https://referer-user:referer-password@example.test/from"), (b"Origin", "//origin-user:origin-password@example.test")]
|
||||
state["response"]["headers"] = [(b"Location", "https://location-user:location-password@example.test/to")]
|
||||
state["metadata"]["_markers"] = dict(markers)
|
||||
state["metadata"]["_private_properties"] = {"private-property/~": {"number": 123, "bytes": b"private-property"}, "kept": 7}
|
||||
with (root / "string-urls.mitm").open("wb") as handle:
|
||||
tnetstring.dump(strings, handle)
|
||||
strings["metadata"]["_private_properties"]["private-property/~"] = {1: b"unsupported-key"}
|
||||
with (root / "invalid-private-key.mitm").open("wb") as handle:
|
||||
tnetstring.dump(strings, handle)
|
||||
extension = "leaf"
|
||||
for _ in range(66):
|
||||
extension = {"child": extension}
|
||||
har["_extension"] = extension
|
||||
(root / "deep.har").write_text(json.dumps(har))
|
||||
deep = first.get_state()
|
||||
deep["_extension"] = extension
|
||||
with (root / "deep.mitm").open("wb") as handle:
|
||||
tnetstring.dump(deep, handle)
|
||||
for name, value in [("invalid-credential-key", {1: b"bad-key"}), ("deep-credential", extension)]:
|
||||
malformed = first.get_state()
|
||||
malformed["request"]["headers"] = [(b"Authorization", value)]
|
||||
with (root / (name + ".mitm")).open("wb") as handle:
|
||||
tnetstring.dump(malformed, handle)
|
||||
for prefix, name in [(" \t", "ows"), ("\r\n \v\f\x00\x1f", "controls")]:
|
||||
url = prefix + "HTTPS://ows-user:ows-secret@example.test/path?token=ordinary#fragment\t "
|
||||
relative = prefix + "//ows-user:ows-secret@example.test/path?token=ordinary#fragment\t "
|
||||
whitespace_har = json.loads((root / "producer.har").read_text())
|
||||
entry = whitespace_har["log"]["entries"][0]
|
||||
for side, header in [("response", "Location"), ("request", "Referer"), ("request", "Origin")]:
|
||||
entry[side]["headers"] = [{"name": header, "value": url}] + entry[side]["headers"]
|
||||
(root / (name + ".har")).write_text(json.dumps(whitespace_har))
|
||||
whitespace_native = first.get_state()
|
||||
for state in [whitespace_native, whitespace_native["backup"]]:
|
||||
state["request"]["headers"] = [(b"Referer", url.encode("utf-8")), (b"Origin", relative)]
|
||||
state["response"]["headers"] = [(b"Location", url)]
|
||||
with (root / (name + ".mitm")).open("wb") as handle:
|
||||
tnetstring.dump(whitespace_native, handle)
|
||||
non_utf8_native = first.get_state()
|
||||
for state in [non_utf8_native, non_utf8_native["backup"]]:
|
||||
state["request"]["path"] = b"https://byte-user:byte-secret@example.test/path\xff"
|
||||
state["request"]["authority"] = b"byte-user:byte-secret@example.test\xff"
|
||||
state["request"]["headers"] = [(b"Referer", b" https://byte-user:byte-secret@example.test/from\xff"), (b"Origin", b"//byte-user:byte-secret@example.test\xff")]
|
||||
state["response"]["headers"] = [(b"Location", b"https://byte-user:byte-secret@example.test/to\xff")]
|
||||
with (root / "non-utf8-urls.mitm").open("wb") as handle:
|
||||
tnetstring.dump(non_utf8_native, handle)
|
||||
(root / "oracle.json").write_text(json.dumps({"records": 2, "request_base64": "AP9ib2R5", "response_base64": "AP5hbnN3ZXI=", "websocket_base64": "AP1tZXNzYWdl", "id": first.id}))
|
||||
ctx.master.shutdown()
|
||||
|
||||
addons = [Generator()]
|
||||
@@ -17,8 +17,9 @@ for (const argument of arguments_)
|
||||
throw new Error(`Unknown MCP tool catalog option: ${argument}`);
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
// The checked-in catalog is a portable package artifact. Keep its canonical
|
||||
// provider metadata independent of the machine running the generator.
|
||||
// The build-generated catalog is a portable package artifact that is not
|
||||
// committed to git. Keep its canonical provider metadata independent of the
|
||||
// machine running the generator. It is regenerated by `npm run build`.
|
||||
const CATALOG_PLATFORM = "darwin";
|
||||
const sessionToolNames = new Set([
|
||||
...TOOL_CONTRACTS.filter(({ kind }) => kind === "session").map(
|
||||
|
||||
@@ -24,8 +24,10 @@ SUPPORTED_HOPPER_SHA256 = {
|
||||
"1339f9e58377442b0c6fcb0dfc3cec20d593cc557408521fad9a00dbc6b8da13",
|
||||
}
|
||||
EXPECTED_SCREEN = (1280, 1024)
|
||||
EXPECTED_DIALOG = (189, 370, 901, 284)
|
||||
DEMO_CLICK = (305, 632)
|
||||
DIALOG_TITLES = {"Registration", "Registration — Hopper Disassembler"}
|
||||
HOPPER_WINDOW_TITLE = "Hopper Disassembler"
|
||||
DEMO_CLICK_LEFT = 116
|
||||
DEMO_CLICK_BOTTOM = 22
|
||||
PRIVATE_DISPLAY_UNAVAILABLE = 70
|
||||
X11_AUTHORIZATION_FAILED = 71
|
||||
UNSUPPORTED_HOPPER_BUILD = 72
|
||||
@@ -197,6 +199,47 @@ def windows(x11: ctypes.CDLL, display: int, root: int) -> list[tuple[int, int, i
|
||||
return found
|
||||
|
||||
|
||||
def window_title(x11: ctypes.CDLL, display: int, window: int) -> str | None:
|
||||
title = ctypes.c_char_p()
|
||||
if not x11.XFetchName(display, window, ctypes.byref(title)) or not title.value:
|
||||
return None
|
||||
try:
|
||||
return title.value.decode("utf-8", "replace")
|
||||
finally:
|
||||
x11.XFree(ctypes.cast(title, ctypes.c_void_p))
|
||||
|
||||
|
||||
def window_transient_for(
|
||||
x11: ctypes.CDLL, display: int, window: int
|
||||
) -> int | None:
|
||||
parent = ctypes.c_ulong()
|
||||
if not x11.XGetTransientForHint(display, window, ctypes.byref(parent)):
|
||||
return None
|
||||
return int(parent.value)
|
||||
|
||||
|
||||
def find_demo_dialog(
|
||||
x11: ctypes.CDLL,
|
||||
display: int,
|
||||
observed: list[tuple[int, int, int, int, int]],
|
||||
) -> tuple[int, int, int, int, int] | None:
|
||||
for candidate in observed:
|
||||
window = candidate[0]
|
||||
if window_title(x11, display, window) not in DIALOG_TITLES:
|
||||
continue
|
||||
parent = window_transient_for(x11, display, window)
|
||||
if parent is None:
|
||||
continue
|
||||
if window_title(x11, display, parent) == HOPPER_WINDOW_TITLE:
|
||||
return candidate
|
||||
return None
|
||||
|
||||
|
||||
def demo_click(dialog: tuple[int, int, int, int, int]) -> tuple[int, int]:
|
||||
_window, x, y, _width, height = dialog
|
||||
return x + DEMO_CLICK_LEFT, y + height - DEMO_CLICK_BOTTOM
|
||||
|
||||
|
||||
def configure_x11(x11: ctypes.CDLL, xtst: ctypes.CDLL) -> None:
|
||||
window_pointer = ctypes.POINTER(ctypes.c_ulong)
|
||||
x11.XOpenDisplay.argtypes = [ctypes.c_char_p]
|
||||
@@ -213,6 +256,18 @@ def configure_x11(x11: ctypes.CDLL, xtst: ctypes.CDLL) -> None:
|
||||
ctypes.POINTER(XWindowAttributes),
|
||||
]
|
||||
x11.XGetWindowAttributes.restype = ctypes.c_int
|
||||
x11.XFetchName.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_ulong,
|
||||
ctypes.POINTER(ctypes.c_char_p),
|
||||
]
|
||||
x11.XFetchName.restype = ctypes.c_int
|
||||
x11.XGetTransientForHint.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_ulong,
|
||||
ctypes.POINTER(ctypes.c_ulong),
|
||||
]
|
||||
x11.XGetTransientForHint.restype = ctypes.c_int
|
||||
x11.XQueryTree.argtypes = [
|
||||
ctypes.c_void_p,
|
||||
ctypes.c_ulong,
|
||||
@@ -645,6 +700,7 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
|
||||
deadline = time.monotonic() + 10
|
||||
observed: list[tuple[int, int, int, int, int]] = []
|
||||
dialog: tuple[int, int, int, int, int] | None = None
|
||||
while time.monotonic() < deadline:
|
||||
if socket.exists():
|
||||
x11.XCloseDisplay(display)
|
||||
@@ -660,7 +716,8 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
"owned Hopper process exited",
|
||||
)
|
||||
observed = windows(x11, display, root)
|
||||
if any(window[1:] == EXPECTED_DIALOG for window in observed):
|
||||
dialog = find_demo_dialog(x11, display, observed)
|
||||
if dialog is not None:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
@@ -676,8 +733,10 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
"expected Hopper demo dialog not found",
|
||||
)
|
||||
|
||||
assert dialog is not None
|
||||
click_x, click_y = demo_click(dialog)
|
||||
input_results = (
|
||||
xtst.XTestFakeMotionEvent(display, -1, DEMO_CLICK[0], DEMO_CLICK[1], 0),
|
||||
xtst.XTestFakeMotionEvent(display, -1, click_x, click_y, 0),
|
||||
xtst.XTestFakeButtonEvent(display, 1, True, 0),
|
||||
xtst.XTestFakeButtonEvent(display, 1, False, 0),
|
||||
)
|
||||
@@ -696,9 +755,8 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
finally:
|
||||
if display is not None:
|
||||
x11.XCloseDisplay(display)
|
||||
if child is not None and child.poll() is None:
|
||||
child.terminate()
|
||||
child.wait()
|
||||
if child is not None:
|
||||
stop_process(child)
|
||||
stop_process(xvfb)
|
||||
capture.close()
|
||||
|
||||
|
||||
@@ -48,6 +48,8 @@ export const toolFamilyCatalog = (sources) => {
|
||||
...sources.browserContracts.BROWSER_TOOL_CONTRACTS,
|
||||
...sources.webRuntimeContracts.WEB_RUNTIME_TOOL_CONTRACTS,
|
||||
...sources.browserScenarioContracts.BROWSER_SCENARIO_TOOL_CONTRACTS,
|
||||
...sources.webNetworkCaptureContracts
|
||||
.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -170,6 +172,17 @@ export const providerCatalog = (sources) => {
|
||||
identity: sources.androidProvider.JADX_PROVIDER_IDENTITY,
|
||||
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.webNetworkCaptureProvider.HAR_CAPTURE_PROVIDER_IDENTITY,
|
||||
contracts:
|
||||
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity:
|
||||
sources.webNetworkCaptureProvider.MITMPROXY_CAPTURE_PROVIDER_IDENTITY,
|
||||
contracts:
|
||||
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.browserProvider.CDP_BROWSER_PROVIDER_IDENTITY,
|
||||
contracts: [
|
||||
|
||||
@@ -51,6 +51,9 @@ const SOURCE_PATHS = {
|
||||
"dist/contracts/managed/managedWorkflowToolContracts.js",
|
||||
browserContracts: "dist/contracts/browserToolContracts.js",
|
||||
webRuntimeContracts: "dist/contracts/webRuntimeToolContracts.js",
|
||||
webNetworkCaptureContracts:
|
||||
"dist/contracts/webNetworkCaptureToolContracts.js",
|
||||
webNetworkCaptureProvider: "dist/browser/history/CaptureRelease.js",
|
||||
browserScenarioContracts: "dist/contracts/browserScenarioToolContracts.js",
|
||||
electronContracts: "dist/contracts/javascript/electronToolContracts.js",
|
||||
javascriptRuntimeObservationContracts:
|
||||
@@ -75,7 +78,7 @@ const SOURCE_PATHS = {
|
||||
evidenceBundle: "dist/domain/evidenceBundle.js",
|
||||
evidenceCompletion: "dist/domain/evidenceCompletionLedger.js",
|
||||
completionGeneration: "dist/domain/completionLedgerGeneration.js",
|
||||
processCapture: "dist/domain/processCapture.js",
|
||||
processCapture: "dist/domain/process/processCapture.js",
|
||||
analysisSnapshot: "dist/domain/analysisSnapshot.js",
|
||||
artifactGraph: "dist/domain/artifactGraph.js",
|
||||
browserObservation: "dist/domain/browserObservation.js",
|
||||
|
||||
@@ -75,6 +75,11 @@ export const documentationFactIssues = async (root, catalog) => {
|
||||
for (const path of README_PATHS) {
|
||||
const content = await readFile(join(root, path), "utf8");
|
||||
requireText(issues, path, content, "MCP-tool_catalog");
|
||||
if (path === "README.md") {
|
||||
requireText(issues, path, content, "(docs/installation.md");
|
||||
requireText(issues, path, content, "(docs/product-catalog.json)");
|
||||
continue;
|
||||
}
|
||||
for (const client of catalog.setup_clients)
|
||||
requireText(issues, path, content, client.display_name);
|
||||
try {
|
||||
@@ -88,6 +93,11 @@ export const documentationFactIssues = async (root, catalog) => {
|
||||
}
|
||||
}
|
||||
|
||||
const installationPath = "docs/installation.md";
|
||||
const installation = await readFile(join(root, installationPath), "utf8");
|
||||
for (const client of catalog.setup_clients)
|
||||
requireText(issues, installationPath, installation, client.display_name);
|
||||
|
||||
const agents = await readFile(join(root, "AGENTS.md"), "utf8");
|
||||
requireText(issues, "AGENTS.md", agents, "docs/product-catalog.json");
|
||||
|
||||
@@ -95,8 +105,6 @@ export const documentationFactIssues = async (root, catalog) => {
|
||||
const template = await readFile(join(root, templatePath), "utf8");
|
||||
requireText(issues, templatePath, template, "docs/product-catalog.json");
|
||||
|
||||
const english = await readFile(join(root, "README.md"), "utf8");
|
||||
requireText(issues, "README.md", english, "docs/product-catalog.json");
|
||||
return issues;
|
||||
};
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ export const ensureGeneratedFile = async ({
|
||||
return { changed: false };
|
||||
if (check)
|
||||
throw new Error(
|
||||
`${path} is missing or stale. Run \`${generateCommand}\` and commit the result.`,
|
||||
`${path} is missing or stale. Run \`${generateCommand}\` to regenerate it; commit the result if the file is tracked.`,
|
||||
);
|
||||
await writeFile(path, preserveLineEndings(source, existing), "utf8");
|
||||
return { changed: true };
|
||||
|
||||
@@ -6,6 +6,7 @@ export interface ModuleBoundaryViolation {
|
||||
readonly target: string;
|
||||
readonly boundary:
|
||||
| "artifact-acquisition"
|
||||
| "process-capture"
|
||||
| "pure-layer"
|
||||
| "application-composition"
|
||||
| "provider-construction"
|
||||
|
||||
@@ -60,6 +60,14 @@ const failedBoundary = (file, target) => {
|
||||
if (!file.startsWith("src/") || !target.startsWith("src/")) return undefined;
|
||||
const layer = file.split("/")[1];
|
||||
const targetLayer = target.split("/")[1];
|
||||
if (
|
||||
file.startsWith("src/process/capture/") &&
|
||||
(["application", "composition", "server", "cli", "main"].includes(
|
||||
targetLayer,
|
||||
) ||
|
||||
/^src\/(?:cli|main)\./u.test(target))
|
||||
)
|
||||
return "process-capture";
|
||||
if (
|
||||
file.startsWith("src/application/javascript/") &&
|
||||
/^src\/artifacts\/(?:Asar|Directory)ArtifactReader\.(?:js|ts)$/u.test(
|
||||
@@ -70,6 +78,8 @@ const failedBoundary = (file, target) => {
|
||||
if (
|
||||
(file.startsWith("src/artifacts/javascript/") ||
|
||||
file.startsWith("src/artifacts/apple/") ||
|
||||
file.startsWith("src/artifacts/inventory/") ||
|
||||
file.startsWith("src/artifacts/extraction/") ||
|
||||
/^src\/artifacts\/ArtifactHash\.(?:js|ts)$/u.test(file)) &&
|
||||
(["application", "composition", "server", "cli", "main"].includes(
|
||||
targetLayer,
|
||||
|
||||
+11
-3
@@ -19,7 +19,7 @@ const runtimeFiles = isMcpMode
|
||||
? ["dist/main.js"]
|
||||
: isMcpDoctorMode
|
||||
? ["dist/main.js", "dist/mcpDoctor.js"]
|
||||
: ["dist/cli.js", "dist/cliOutput.js"];
|
||||
: ["dist/cli.js", "dist/cliOutput.js", "dist/cli/streamedJsonOutput.js"];
|
||||
|
||||
if (!(await compiledRuntimeExists(runtimeFiles))) {
|
||||
process.stderr.write(
|
||||
@@ -39,6 +39,8 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
|
||||
process.exitCode = result.exitCode;
|
||||
} else {
|
||||
const { createCli } = await import("../dist/cli.js");
|
||||
const { createStreamedCliJsonOutput } =
|
||||
await import("../dist/cli/streamedJsonOutput.js");
|
||||
const {
|
||||
renderCliOutputArgumentError,
|
||||
renderEmptyFilteredCliOutput,
|
||||
@@ -51,14 +53,20 @@ if (!(await compiledRuntimeExists(runtimeFiles))) {
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
let wroteOutput = false;
|
||||
await createCli().serve(args, {
|
||||
const resultOutput = createStreamedCliJsonOutput(args, process.stdout);
|
||||
await createCli(process.env, resultOutput).serve(args, {
|
||||
stdout: (output) => {
|
||||
if (resultOutput?.handled) {
|
||||
if (resultOutput.failed)
|
||||
process.stderr.write(sanitizeCliOutput(output));
|
||||
return;
|
||||
}
|
||||
const sanitized = sanitizeCliOutput(output);
|
||||
if (sanitized.length > 0) wroteOutput = true;
|
||||
process.stdout.write(sanitized);
|
||||
},
|
||||
});
|
||||
if (!wroteOutput)
|
||||
if (!wroteOutput && !resultOutput?.handled)
|
||||
process.stdout.write(renderEmptyFilteredCliOutput(args) ?? "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
|
||||
import { observeNativeUi } from "../dist/native/NativeUiObservation.js";
|
||||
import { NATIVE_UI_HELPER_MAX_BUFFER } from "../dist/native/NativeUiOutputBudget.js";
|
||||
|
||||
if (process.platform !== "darwin")
|
||||
throw new Error(
|
||||
@@ -30,7 +31,38 @@ try {
|
||||
"-o",
|
||||
childRetrievalTest,
|
||||
]);
|
||||
await promisify(execFile)(childRetrievalTest, []);
|
||||
const childRetrieval = await promisify(execFile)(childRetrievalTest, []);
|
||||
const [slashRichProducerJson, numericProducerJson] = childRetrieval.stdout
|
||||
.trimEnd()
|
||||
.split("\n");
|
||||
if (
|
||||
slashRichProducerJson === undefined ||
|
||||
numericProducerJson === undefined
|
||||
) {
|
||||
throw new Error(
|
||||
"Native UI serializer seam did not emit its producer bytes",
|
||||
);
|
||||
}
|
||||
const slashRichConsumerJson = JSON.stringify(
|
||||
JSON.parse(slashRichProducerJson),
|
||||
);
|
||||
if (slashRichProducerJson !== slashRichConsumerJson) {
|
||||
throw new Error(
|
||||
"Swift helper JSON bytes differ from the normalized JavaScript consumer representation",
|
||||
);
|
||||
}
|
||||
const numericConsumerJson = JSON.stringify(JSON.parse(numericProducerJson));
|
||||
if (
|
||||
Buffer.byteLength(numericProducerJson) <=
|
||||
Buffer.byteLength(numericConsumerJson)
|
||||
) {
|
||||
throw new Error(
|
||||
"Foundation numeric fixture no longer exercises larger producer JSON bytes",
|
||||
);
|
||||
}
|
||||
if (Buffer.byteLength(numericProducerJson) > NATIVE_UI_HELPER_MAX_BUFFER) {
|
||||
throw new Error("Foundation numeric fixture exceeds the raw helper budget");
|
||||
}
|
||||
|
||||
const contents = join(root, "Fixture.app", "Contents");
|
||||
await mkdir(join(contents, "MacOS"), { recursive: true });
|
||||
|
||||
@@ -31,7 +31,7 @@ export async function verifyPackageInstall({
|
||||
const processCaptureCapabilityUrl = pathToFileURL(
|
||||
join(
|
||||
prefix,
|
||||
"lib/node_modules/rea-agents/dist/application/ProcessCaptureCapability.js",
|
||||
"lib/node_modules/rea-agents/dist/process/capture/ProcessCaptureCapability.js",
|
||||
),
|
||||
).href;
|
||||
const processCaptureCapability = json(
|
||||
@@ -77,7 +77,7 @@ export async function verifyPackageInstall({
|
||||
const noOptionalCapabilityUrl = pathToFileURL(
|
||||
join(
|
||||
noOptionalPrefix,
|
||||
"lib/node_modules/rea-agents/dist/application/ProcessCaptureCapability.js",
|
||||
"lib/node_modules/rea-agents/dist/process/capture/ProcessCaptureCapability.js",
|
||||
),
|
||||
).href;
|
||||
const noOptionalCapability = json(
|
||||
|
||||
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
||||
import { join } from "node:path";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { readFile, rm } from "node:fs/promises";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
@@ -167,6 +168,7 @@ const verifyMcpInlineArtifactEvidence = async (
|
||||
mcpOptions,
|
||||
);
|
||||
assert.notEqual(opened.isError, true, JSON.stringify(opened));
|
||||
let outputRoot;
|
||||
try {
|
||||
const inspected = await client.callTool(
|
||||
{ name: "inspect_artifact", arguments: {} },
|
||||
@@ -212,12 +214,35 @@ const verifyMcpInlineArtifactEvidence = async (
|
||||
evidence,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
const closed = await client.callTool(
|
||||
{ name: "close_binary", arguments: {} },
|
||||
const extracted = await client.callTool(
|
||||
{ name: "extract_artifact", arguments: {} },
|
||||
mcpOptions,
|
||||
);
|
||||
assert.notEqual(closed.isError, true, JSON.stringify(closed));
|
||||
assert.notEqual(extracted.isError, true, JSON.stringify(extracted));
|
||||
const extraction = parseEvidence(extracted.structuredContent?.evidence);
|
||||
assert.deepEqual(
|
||||
extraction.normalized_result,
|
||||
extracted.structuredContent?.result,
|
||||
);
|
||||
const materialized = extracted.structuredContent?.result;
|
||||
assert.equal(typeof materialized?.output_root, "string");
|
||||
outputRoot = materialized.output_root;
|
||||
assert.equal(materialized.containment_verified, true);
|
||||
assert.equal(
|
||||
await readFile(join(outputRoot, "app/main.js"), "utf8"),
|
||||
"main();",
|
||||
);
|
||||
} finally {
|
||||
try {
|
||||
const closed = await client.callTool(
|
||||
{ name: "close_binary", arguments: {} },
|
||||
mcpOptions,
|
||||
);
|
||||
assert.notEqual(closed.isError, true, JSON.stringify(closed));
|
||||
} finally {
|
||||
if (outputRoot !== undefined)
|
||||
await rm(outputRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -25,10 +25,12 @@ if (process.platform !== "win32" || process.arch !== "x64")
|
||||
);
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-packaged-ghidra-"));
|
||||
const exec = promisify(execFile);
|
||||
let packageRoot = resolve(process.argv[2] ?? ".");
|
||||
const x86Fixture = process.argv[2] === "--x86";
|
||||
const packageRootArgument = x86Fixture ? undefined : process.argv[2];
|
||||
let packageRoot = resolve(packageRootArgument ?? ".");
|
||||
// The default lane verifies the npm artifact in an isolated prefix. Explicit
|
||||
// package roots support an already-installed artifact without a second install.
|
||||
if (process.argv[2] === undefined) {
|
||||
if (packageRootArgument === undefined) {
|
||||
try {
|
||||
const npmEntry =
|
||||
process.env.npm_execpath ??
|
||||
@@ -77,7 +79,10 @@ if (process.argv[2] === undefined) {
|
||||
}
|
||||
}
|
||||
const target = resolve(
|
||||
process.argv[3] ?? "build/fixtures/rea-ghidra-windows.exe",
|
||||
process.argv[3] ??
|
||||
(x86Fixture
|
||||
? "build/fixtures/rea-ghidra-windows-x86.exe"
|
||||
: "build/fixtures/rea-ghidra-windows.exe"),
|
||||
);
|
||||
let TOOL_CONTRACTS, native, token, sha256;
|
||||
try {
|
||||
@@ -215,7 +220,8 @@ try {
|
||||
};
|
||||
try {
|
||||
await client.connect(transport);
|
||||
await call("open_binary", { path: target, provider_id: "ghidra" });
|
||||
await call("open_binary", { path: target });
|
||||
await call("binary_overview", {});
|
||||
assert.equal((await call("list_documents", {})).length, 1);
|
||||
await call("list_names", {});
|
||||
const procedures = await call("list_procedures", {});
|
||||
|
||||
@@ -63,12 +63,20 @@ if (
|
||||
);
|
||||
|
||||
const root = resolve(fileURLToPath(new URL("..", import.meta.url)));
|
||||
const targetPath = resolve(root, "build", "fixtures", "rea-ghidra-windows.exe");
|
||||
const architecture = process.argv[2] === "--x86" ? "x86" : "x86_64";
|
||||
const targetPath = resolve(
|
||||
root,
|
||||
"build",
|
||||
"fixtures",
|
||||
architecture === "x86"
|
||||
? "rea-ghidra-windows-x86.exe"
|
||||
: "rea-ghidra-windows.exe",
|
||||
);
|
||||
const target = await parseBinaryTarget(targetPath);
|
||||
if (
|
||||
!target.ok ||
|
||||
target.value.format !== "pe" ||
|
||||
target.value.architecture !== "x86_64" ||
|
||||
target.value.architecture !== architecture ||
|
||||
target.value.executableRole !== "application" ||
|
||||
target.value.managed !== false
|
||||
)
|
||||
@@ -113,6 +121,9 @@ try {
|
||||
target.value.sha256,
|
||||
profile.value.profile.digest,
|
||||
);
|
||||
const languagePrefix = architecture === "x86" ? "x86:LE:32:" : "x86:LE:64:";
|
||||
if (!started.value.target.language_id.startsWith(languagePrefix))
|
||||
throw new Error("Windows fixture imported with the wrong Ghidra language");
|
||||
coordinates = client.diagnostics();
|
||||
assertRuntimeCoordinates(coordinates, target.value.sha256);
|
||||
|
||||
@@ -137,6 +148,19 @@ try {
|
||||
);
|
||||
if (procedure === undefined)
|
||||
throw new Error("Windows Ghidra fixture exposed no local procedure");
|
||||
const calleeAddress = architecture === "x86" ? 0x401010n : 0x140001010n;
|
||||
const callee = procedures.find(
|
||||
(item) => BigInt(item.address) === calleeAddress,
|
||||
);
|
||||
if (callee === undefined)
|
||||
throw new Error("Windows fixture exposed no controlled callee");
|
||||
const pseudoCode = await functionOperation("procedure_pseudo_code", {
|
||||
procedure: callee.value,
|
||||
});
|
||||
if (typeof pseudoCode !== "string" || !/\b(?:42|0x2a)\b/iu.test(pseudoCode))
|
||||
throw new Error(
|
||||
"Windows fixture decompilation lost its known return value",
|
||||
);
|
||||
|
||||
await inventory("inspect_native_load_image", {});
|
||||
const memory = await inventory("read_bytes", {
|
||||
@@ -210,6 +234,7 @@ try {
|
||||
report = {
|
||||
ok: true,
|
||||
provider: { id: "ghidra", version: SUPPORTED_GHIDRA_VERSION },
|
||||
language_id: started.value.target.language_id,
|
||||
target: {
|
||||
format: target.value.format,
|
||||
architecture: target.value.architecture,
|
||||
@@ -228,7 +253,7 @@ try {
|
||||
limitations: [
|
||||
"approved-non-sensitive-fixtures-only",
|
||||
"windows-x64-local-ntfs-only",
|
||||
"native-x86-64-pe-applications-only",
|
||||
"native-x86-and-x86-64-pe-applications-only",
|
||||
"no-gui-or-mutation-authority",
|
||||
],
|
||||
};
|
||||
|
||||
@@ -0,0 +1,994 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import {
|
||||
access,
|
||||
copyFile,
|
||||
chmod,
|
||||
readFile,
|
||||
truncate,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { dirname, isAbsolute, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { PrivateRuntimeRoot } from "../dist/process/PrivateRuntimeRoot.js";
|
||||
import { runOwnedCommand } from "../dist/process/OwnedCommand.js";
|
||||
import { WEB_NETWORK_CAPTURE_LIMITS } from "../dist/domain/webNetworkCapture.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const executable = process.env.REA_MITMDUMP_COMMAND;
|
||||
if (
|
||||
process.platform !== "linux" ||
|
||||
executable === undefined ||
|
||||
!isAbsolute(executable)
|
||||
)
|
||||
throw new Error(
|
||||
"verify:web:network-captures requires Linux and absolute REA_MITMDUMP_COMMAND for caller-supplied mitmdump 12.2.3",
|
||||
);
|
||||
await access(executable);
|
||||
const entrypoint =
|
||||
process.argv[2] ?? fileURLToPath(new URL("./rea.mjs", import.meta.url));
|
||||
const run = createVerifierRun();
|
||||
const runtime = await PrivateRuntimeRoot.create({
|
||||
prefix: "rea-history-verifier-",
|
||||
});
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
|
||||
);
|
||||
const client = new Client({
|
||||
name: "historical-web-capture-verifier",
|
||||
version: "1",
|
||||
});
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const failures = [];
|
||||
let cases = 0;
|
||||
try {
|
||||
await runOwnedCommand(
|
||||
{
|
||||
command: executable,
|
||||
arguments: [
|
||||
"--no-server",
|
||||
"-q",
|
||||
"--set",
|
||||
`confdir=${join(runtime.path, "config")}`,
|
||||
"--set",
|
||||
`rea_fixture_root=${runtime.path}`,
|
||||
"--set",
|
||||
`rea_adapter_path=${resolve(dirname(entrypoint), "../bridge/mitmproxy/capture.py")}`,
|
||||
"-s",
|
||||
fileURLToPath(
|
||||
new URL("./fixtures/generate-mitmproxy-capture.py", import.meta.url),
|
||||
),
|
||||
],
|
||||
cwd: runtime.path,
|
||||
runId: `rea-history-fixture-${randomUUID()}`,
|
||||
hostEnvironment: environment,
|
||||
},
|
||||
{ timeoutMs: 30_000, diagnosticBytes: 1024 * 1024 },
|
||||
);
|
||||
assert.deepEqual(
|
||||
JSON.parse(
|
||||
await readFile(
|
||||
join(runtime.path, "native-error-classifications.json"),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
{ passed: 7 },
|
||||
);
|
||||
await client.connect(transport);
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const path = join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "flows.mitm",
|
||||
);
|
||||
const raw = await readFile(path);
|
||||
const input = { capture_path: path, format, record_ordinals: [1, 0] };
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, input);
|
||||
assert.equal(value.total_records, 2);
|
||||
assert.deepEqual(
|
||||
value.records.map((record) => record.ordinal),
|
||||
[1, 0],
|
||||
);
|
||||
assert.equal(
|
||||
value.artifact.sha256,
|
||||
createHash("sha256").update(raw).digest("hex"),
|
||||
);
|
||||
assert.equal(value.runtime_attribution, "unknown");
|
||||
const first = value.records[1];
|
||||
if (format === "mitmproxy") {
|
||||
assert.equal(first.reported.id, "original-producer-id");
|
||||
assert.equal(value.records[0].reported.id, first.reported.id);
|
||||
assert.equal(first.location.offset, 0);
|
||||
assert.equal(
|
||||
value.records[0].location.offset + value.records[0].location.bytes,
|
||||
raw.length,
|
||||
);
|
||||
assert.equal(
|
||||
first.reported.request.path,
|
||||
"http://example.test/a?token=ordinary#fragment",
|
||||
);
|
||||
assertBinary(
|
||||
first,
|
||||
"/request/content",
|
||||
Buffer.from([0, 255, ...Buffer.from("body")]),
|
||||
);
|
||||
assertBinary(
|
||||
first,
|
||||
"/response/content",
|
||||
Buffer.from([0, 254, ...Buffer.from("answer")]),
|
||||
);
|
||||
assertBinary(
|
||||
first,
|
||||
"/websocket/messages/0/2",
|
||||
Buffer.from([0, 253, ...Buffer.from("message")]),
|
||||
);
|
||||
assert.equal(value.records[0].reported.request.content, null);
|
||||
assertBinary(value.records[0], "/response/content", Buffer.alloc(0));
|
||||
assert.ok(
|
||||
first.numeric_literals.some(
|
||||
(number) =>
|
||||
number.pointer === "/metadata/big_integer" &&
|
||||
number.literal === "9007199254740993",
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
first.numeric_literals.some(
|
||||
(number) =>
|
||||
number.pointer === "/metadata/nonfinite" &&
|
||||
number.literal === "inf",
|
||||
),
|
||||
);
|
||||
assert.equal(
|
||||
first.reported.metadata.response.headers[0][1],
|
||||
"ordinary-extension-credential-name",
|
||||
);
|
||||
} else {
|
||||
assert.equal(value.container.reported.log.creator.name, "mitmproxy");
|
||||
assert.equal(value.container.reported.log.creator.version, "12.2.3");
|
||||
assert.equal(value.records[0].reported.request.postData.text, null);
|
||||
assertBinary(
|
||||
first,
|
||||
"/response/content/text",
|
||||
Buffer.from([0, 255, 1, 254]),
|
||||
);
|
||||
}
|
||||
const text = JSON.stringify(value);
|
||||
for (const secret of [
|
||||
"native-transport-secret",
|
||||
"cookie-secret",
|
||||
"redirect-password",
|
||||
"user:password",
|
||||
])
|
||||
assert.ok(
|
||||
!text.includes(secret),
|
||||
`Credential leaked through ${mode}/${format}`,
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(runtime.path, "string-urls.mitm"),
|
||||
format: "mitmproxy",
|
||||
});
|
||||
const first = value.records[0];
|
||||
for (const state of [first.reported, first.reported.backup]) {
|
||||
assert.equal(state.request.path, "https://example.test/string-path");
|
||||
assert.equal(state.request.authority, "example.test");
|
||||
assert.deepEqual(
|
||||
state.request.headers.map((field) => field[1]),
|
||||
["https://example.test/from", "//example.test"],
|
||||
);
|
||||
assert.equal(state.response.headers[0][1], "https://example.test/to");
|
||||
}
|
||||
assert.ok(!JSON.stringify(value).includes("password"));
|
||||
assert.ok(
|
||||
first.redactions.some(
|
||||
(redaction) =>
|
||||
redaction.pointer === "/backup/request/authority" &&
|
||||
redaction.reason === "transport-credential",
|
||||
),
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
for (const sensitive_values of [
|
||||
["REDACTED", "["],
|
||||
["secret", "REDACTED"],
|
||||
]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "string-urls.mitm",
|
||||
),
|
||||
format,
|
||||
record_ordinals: [0],
|
||||
sensitive_values,
|
||||
});
|
||||
const first = value.records[0];
|
||||
const markers =
|
||||
format === "har"
|
||||
? first.reported._markers
|
||||
: first.reported.metadata._markers;
|
||||
assert.equal(markers.sensitive, null);
|
||||
assert.equal(markers.ordinary, "unmarked");
|
||||
assert.equal(
|
||||
sensitive_values.includes("[") ? markers.bracket : markers.overlap,
|
||||
null,
|
||||
);
|
||||
if (format === "mitmproxy")
|
||||
assert.equal(first.reported.backup.metadata._markers.sensitive, null);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const name of [
|
||||
"equal-number",
|
||||
"equal-null",
|
||||
"equal-object",
|
||||
"escaped-key",
|
||||
"core",
|
||||
]) {
|
||||
const path = join(runtime.path, `duplicate-${name}.har`);
|
||||
const original = await readFile(path);
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
await inspect(
|
||||
mode,
|
||||
{ capture_path: path, format: "har" },
|
||||
"invalid_input",
|
||||
);
|
||||
assert.deepEqual(await readFile(path), original);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const path = join(
|
||||
runtime.path,
|
||||
format === "har" ? "prototype.har" : "prototype.mitm",
|
||||
);
|
||||
const original = await readFile(path);
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const error = await inspect(
|
||||
mode,
|
||||
{ capture_path: path, format },
|
||||
"unsupported_provider",
|
||||
);
|
||||
assert.ok(
|
||||
JSON.stringify(error).includes("JSON schema boundary cannot preserve"),
|
||||
);
|
||||
assert.deepEqual(await readFile(path), original);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const path = join(runtime.path, `malformed-${format}`);
|
||||
await writeFile(path, format === "har" ? "{invalid-json" : "999999999999:");
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
await inspect(mode, { capture_path: path, format }, "invalid_input");
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "string-urls.mitm",
|
||||
),
|
||||
format,
|
||||
sensitive_values: ["private-property"],
|
||||
});
|
||||
const record = value.records[0];
|
||||
const properties =
|
||||
format === "har"
|
||||
? record.reported._private_properties
|
||||
: record.reported.metadata._private_properties;
|
||||
assert.deepEqual(properties, { kept: 7 });
|
||||
assert.ok(!JSON.stringify(value).includes("private-property"));
|
||||
assert.ok(
|
||||
record.redactions.some(
|
||||
(item) =>
|
||||
item.scope === "property-name" &&
|
||||
item.pointer ===
|
||||
(format === "har"
|
||||
? "/_private_properties"
|
||||
: "/metadata/_private_properties"),
|
||||
),
|
||||
);
|
||||
if (format === "mitmproxy")
|
||||
assert.deepEqual(record.reported.backup.metadata._private_properties, {
|
||||
kept: 7,
|
||||
});
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
for (const [fixture, prefix] of [
|
||||
["ows", " \t"],
|
||||
["controls", "\r\n \v\f\x00\x1f"],
|
||||
]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
`${fixture}.${format === "har" ? "har" : "mitm"}`,
|
||||
),
|
||||
format,
|
||||
});
|
||||
assert.ok(!JSON.stringify(value).includes("ows-secret"));
|
||||
const record = value.records[0];
|
||||
const states =
|
||||
format === "har"
|
||||
? [record.reported]
|
||||
: [record.reported, record.reported.backup];
|
||||
for (const state of states) {
|
||||
if (format === "har") {
|
||||
for (const side of ["request", "response"])
|
||||
assert.ok(
|
||||
state[side].headers.some(
|
||||
(header) =>
|
||||
header.value ===
|
||||
`${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `,
|
||||
),
|
||||
);
|
||||
} else {
|
||||
assert.deepEqual(state.request.headers, [
|
||||
[
|
||||
"Referer",
|
||||
`${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `,
|
||||
],
|
||||
[
|
||||
"Origin",
|
||||
`${prefix}//example.test/path?token=ordinary#fragment\t `,
|
||||
],
|
||||
]);
|
||||
assert.deepEqual(state.response.headers, [
|
||||
[
|
||||
"Location",
|
||||
`${prefix}HTTPS://example.test/path?token=ordinary#fragment\t `,
|
||||
],
|
||||
]);
|
||||
}
|
||||
}
|
||||
if (format === "mitmproxy") {
|
||||
for (const pointer of [
|
||||
"/request/headers/0/1",
|
||||
"/backup/request/headers/0/1",
|
||||
])
|
||||
assert.deepEqual(
|
||||
record.binary_fields.find((field) => field.pointer === pointer),
|
||||
{
|
||||
pointer,
|
||||
representation: "producer-bytes",
|
||||
state: "redacted",
|
||||
content_base64: null,
|
||||
bytes: null,
|
||||
sha256: null,
|
||||
},
|
||||
);
|
||||
}
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(runtime.path, "non-utf8-urls.mitm"),
|
||||
format: "mitmproxy",
|
||||
});
|
||||
const record = value.records[0];
|
||||
assert.ok(!JSON.stringify(value).includes("byte-secret"));
|
||||
for (const prefix of ["", "/backup"]) {
|
||||
for (const suffix of [
|
||||
"/request/path",
|
||||
"/request/authority",
|
||||
"/request/headers/0/1",
|
||||
"/request/headers/1/1",
|
||||
"/response/headers/0/1",
|
||||
]) {
|
||||
const pointer = prefix + suffix;
|
||||
assert.deepEqual(
|
||||
record.binary_fields.find((field) => field.pointer === pointer),
|
||||
{
|
||||
pointer,
|
||||
representation: "producer-bytes",
|
||||
state: "redacted",
|
||||
content_base64: null,
|
||||
bytes: null,
|
||||
sha256: null,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
cases++;
|
||||
}
|
||||
const nativeCoordinateInput = {
|
||||
capture_path: join(runtime.path, "flows.mitm"),
|
||||
format: "mitmproxy",
|
||||
};
|
||||
const nativeCoordinates = await inspect("mcp", nativeCoordinateInput);
|
||||
const originalLocation = nativeCoordinates.records[1].location;
|
||||
assert.equal(originalLocation.kind, "byte-range");
|
||||
for (const literal of [
|
||||
String(originalLocation.offset),
|
||||
String(originalLocation.bytes),
|
||||
]) {
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const excluded = await inspect(mode, {
|
||||
...nativeCoordinateInput,
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
assert.deepEqual(excluded.records[1].location, {
|
||||
kind: "unknown",
|
||||
reason: "explicit-sensitive-value",
|
||||
});
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
cases++;
|
||||
const deniedPath = join(runtime.path, "unreadable.har");
|
||||
await copyFile(join(runtime.path, "producer.har"), deniedPath);
|
||||
await chmod(deniedPath, 0o000);
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const failure = await inspect(
|
||||
mode,
|
||||
{ capture_path: deniedPath, format },
|
||||
"unavailable",
|
||||
);
|
||||
assert.equal(failure.code, "access_denied");
|
||||
assert.equal(failure.details.system_code, "EACCES");
|
||||
assert.equal(failure.details.path, deniedPath);
|
||||
cases++;
|
||||
}
|
||||
for (const literal of ["log", "entries"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(runtime.path, "producer.har"),
|
||||
format: "har",
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
assert.equal(value.container.records_pointer, null);
|
||||
assert.ok(
|
||||
value.records.every(
|
||||
(record) =>
|
||||
record.location.kind === "unknown" &&
|
||||
record.location.reason === "explicit-sensitive-value",
|
||||
),
|
||||
);
|
||||
assert.ok(!JSON.stringify(value).includes(literal));
|
||||
cases++;
|
||||
}
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "string-urls.mitm",
|
||||
),
|
||||
format,
|
||||
sensitive_values: ["~1"],
|
||||
});
|
||||
assert.ok(!JSON.stringify(value).includes("~1"));
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
for (const sensitive_values of [
|
||||
["producer"],
|
||||
["producer", "REDACTED", "…"],
|
||||
]) {
|
||||
await inspect(mode, {
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "flows.mitm",
|
||||
),
|
||||
format,
|
||||
sensitive_values,
|
||||
});
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const selection of [
|
||||
{ record_ordinals: [12345], sensitive_values: ["12345"] },
|
||||
{ record_ordinals: [0, 0], sensitive_values: ["Each"] },
|
||||
]) {
|
||||
await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(runtime.path, "producer.har"),
|
||||
format: "har",
|
||||
...selection,
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
await inspect(
|
||||
"mcp",
|
||||
{
|
||||
capture_path: "relative.har",
|
||||
format: "har",
|
||||
sensitive_values: ["absolute"],
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
cases++;
|
||||
const privatePath = join(runtime.path, "REDACTED.har");
|
||||
for (const name of ["mysecret", "/ordinary/secret"]) {
|
||||
const unknownArgument = await inspect(
|
||||
"mcp",
|
||||
{
|
||||
capture_path: join(runtime.path, "producer.har"),
|
||||
format: "har",
|
||||
sensitive_values: ["secret"],
|
||||
[name]: true,
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
assert.deepEqual(unknownArgument.details.issues[0].path, []);
|
||||
cases++;
|
||||
}
|
||||
await inspect("mcp", {
|
||||
capture_path: join(runtime.path, "flows.mitm"),
|
||||
format: "mitmproxy",
|
||||
sensitive_values: ["\ud800", "ordinary-scalar"],
|
||||
});
|
||||
cases++;
|
||||
const replacementHar = JSON.parse(
|
||||
await readFile(join(runtime.path, "producer.har"), "utf8"),
|
||||
);
|
||||
replacementHar.log.entries[0].response.content = {
|
||||
size: 3,
|
||||
mimeType: "application/octet-stream",
|
||||
encoding: "base64",
|
||||
text: "77+9",
|
||||
};
|
||||
const replacementHarPath = join(runtime.path, "unicode-scalar.har");
|
||||
await writeFile(replacementHarPath, JSON.stringify(replacementHar));
|
||||
for (const literal of ["\ud800", "\udfff", "\ufffd"]) {
|
||||
const unicode = await inspect("mcp", {
|
||||
capture_path: replacementHarPath,
|
||||
format: "har",
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
const body = unicode.records[0].binary_fields.find(
|
||||
(field) => field.pointer === "/response/content/text",
|
||||
);
|
||||
assert.equal(body?.state, literal === "\ufffd" ? "redacted" : "retained");
|
||||
assert.equal(body.content_base64, literal === "\ufffd" ? null : "77+9");
|
||||
cases++;
|
||||
}
|
||||
const numericHar = (
|
||||
await readFile(join(runtime.path, "producer.har"), "utf8")
|
||||
).replace('"cache": {}', '"cache": {}, "_big": 9007199254740993');
|
||||
const numericPath = join(runtime.path, "numeric.har");
|
||||
await writeFile(numericPath, numericHar);
|
||||
assert.ok(numericHar.includes('"_big": 9007199254740993'));
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
for (const literal of ["9007199254740993", "4740993"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path:
|
||||
format === "har" ? numericPath : join(runtime.path, "flows.mitm"),
|
||||
format,
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
assert.ok(!JSON.stringify(value).includes(literal));
|
||||
const first = value.records[0];
|
||||
const pointer = format === "har" ? "/_big" : "/metadata/big_integer";
|
||||
assert.ok(
|
||||
first.redactions.some(
|
||||
(redaction) =>
|
||||
redaction.pointer === pointer &&
|
||||
redaction.reason === "explicit-sensitive-value",
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
first.numeric_literals.every(
|
||||
(number) => !number.literal.includes(literal),
|
||||
),
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
await inspect(mode, {
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "flows.mitm",
|
||||
),
|
||||
format,
|
||||
sensitive_values: [format],
|
||||
});
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
const encodedHar = JSON.parse(
|
||||
await readFile(join(runtime.path, "producer.har"), "utf8"),
|
||||
);
|
||||
encodedHar.log.entries[0].response.content = {
|
||||
size: 6,
|
||||
mimeType: "application/octet-stream",
|
||||
encoding: "base64",
|
||||
text: "c2 VjcmV0",
|
||||
};
|
||||
const encodedHarPath = join(runtime.path, "encoded-sensitive.har");
|
||||
await writeFile(encodedHarPath, JSON.stringify(encodedHar));
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const bytes =
|
||||
format === "har"
|
||||
? Buffer.from("secret")
|
||||
: Buffer.from([0, 255, ...Buffer.from("body")]);
|
||||
for (const literal of [
|
||||
bytes.toString("base64"),
|
||||
bytes.toString("base64").slice(0, 4),
|
||||
createHash("sha256").update(bytes).digest("hex"),
|
||||
]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path:
|
||||
format === "har"
|
||||
? encodedHarPath
|
||||
: join(runtime.path, "flows.mitm"),
|
||||
format,
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
const pointer =
|
||||
format === "har" ? "/response/content/text" : "/request/content";
|
||||
const field = value.records[0].binary_fields.find(
|
||||
(field) => field.pointer === pointer,
|
||||
);
|
||||
assert.equal(field?.state, "redacted");
|
||||
assert.equal(field.content_base64, null);
|
||||
assert.equal(field.sha256, null);
|
||||
assert.ok(!JSON.stringify(value).includes(literal));
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const capturePath = join(
|
||||
runtime.path,
|
||||
format === "har" ? "producer.har" : "flows.mitm",
|
||||
);
|
||||
const digest = createHash("sha256")
|
||||
.update(await readFile(capturePath))
|
||||
.digest("hex");
|
||||
for (const literal of [digest, digest.slice(0, 12)]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: capturePath,
|
||||
format,
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
assert.equal(value.artifact.sha256, null);
|
||||
assert.equal(value.artifact.path, capturePath);
|
||||
assert.ok(!JSON.stringify(value).includes(literal));
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
}
|
||||
const oversizedPath = join(runtime.path, "oversized-capture");
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const literal of ["log", "entries"]) {
|
||||
const hidden = await inspect(mode, {
|
||||
capture_path: join(runtime.path, "producer.har"),
|
||||
format: "har",
|
||||
sensitive_values: [literal],
|
||||
});
|
||||
assert.equal(hidden.total_records, 2);
|
||||
assert.equal(hidden.container.records_pointer, null);
|
||||
for (const [ordinal, record] of hidden.records.entries()) {
|
||||
assert.equal(record.ordinal, ordinal);
|
||||
assert.deepEqual(record.location, {
|
||||
kind: "unknown",
|
||||
reason: "explicit-sensitive-value",
|
||||
});
|
||||
assert.equal(record.reported, null);
|
||||
assert.deepEqual(record.binary_fields, []);
|
||||
assert.deepEqual(record.numeric_literals, []);
|
||||
}
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
await writeFile(oversizedPath, "");
|
||||
await truncate(oversizedPath, WEB_NETWORK_CAPTURE_LIMITS.inputBytes + 1);
|
||||
await copyFile(join(runtime.path, "producer.har"), privatePath);
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const failure = await inspect(
|
||||
mode,
|
||||
{ capture_path: oversizedPath, format },
|
||||
"invalid_input",
|
||||
);
|
||||
assert.equal(failure.details.issues[0].reason, "out_of_range");
|
||||
assert.deepEqual(failure.details.issues[0].path, ["capture_path"]);
|
||||
assert.equal(
|
||||
failure.details.issues[0].expected.maximum_capture_bytes,
|
||||
WEB_NETWORK_CAPTURE_LIMITS.inputBytes,
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const failure = await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(
|
||||
runtime.path,
|
||||
format === "har" ? "deep.har" : "deep.mitm",
|
||||
),
|
||||
format,
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
const issue = failure.details.issues[0];
|
||||
assert.equal(issue.reason, "out_of_range");
|
||||
assert.equal(issue.path[0], "capture_path");
|
||||
assert.match(issue.path[1], /^\/_extension\/child/);
|
||||
assert.equal(
|
||||
issue.expected.maximum_capture_nesting,
|
||||
WEB_NETWORK_CAPTURE_LIMITS.depth,
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(runtime.path, "invalid-credential-key.mitm"),
|
||||
format: "mitmproxy",
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
cases++;
|
||||
const credentialDepth = await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(runtime.path, "deep-credential.mitm"),
|
||||
format: "mitmproxy",
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
assert.equal(credentialDepth.details.issues[0].reason, "out_of_range");
|
||||
assert.match(
|
||||
credentialDepth.details.issues[0].path[1],
|
||||
/^\/request\/headers\/0\/1\/child/,
|
||||
);
|
||||
cases++;
|
||||
const value = await inspect(mode, {
|
||||
capture_path: privatePath,
|
||||
format: "har",
|
||||
sensitive_values: ["REDACTED", "["],
|
||||
});
|
||||
assert.equal(value.artifact.path, "");
|
||||
assert.ok(!JSON.stringify(value).includes("REDACTED"));
|
||||
cases++;
|
||||
await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(runtime.path, "invalid-private-parent.har"),
|
||||
format: "har",
|
||||
sensitive_values: ["response"],
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
cases++;
|
||||
await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(runtime.path, "private-property-absent.har"),
|
||||
format: "har",
|
||||
sensitive_values: ["private-property", "REDACTED"],
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
cases++;
|
||||
await inspect(
|
||||
mode,
|
||||
{
|
||||
capture_path: join(runtime.path, "invalid-private-key.mitm"),
|
||||
format: "mitmproxy",
|
||||
sensitive_values: ["private-property"],
|
||||
},
|
||||
"invalid_input",
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, {
|
||||
capture_path: join(runtime.path, "flows.mitm"),
|
||||
format: "mitmproxy",
|
||||
record_ordinals: [0],
|
||||
sensitive_values: ["body"],
|
||||
});
|
||||
assert.equal(
|
||||
value.records[0].binary_fields.find(
|
||||
(field) => field.pointer === "/request/content",
|
||||
).state,
|
||||
"redacted",
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
for (const format of ["har", "mitmproxy"]) {
|
||||
const path = join(runtime.path, `empty-${format}`);
|
||||
await writeFile(
|
||||
path,
|
||||
format === "har"
|
||||
? JSON.stringify({
|
||||
log: {
|
||||
version: "1.2",
|
||||
creator: { name: "source-owned-empty-fixture", version: "1" },
|
||||
entries: [],
|
||||
},
|
||||
})
|
||||
: "",
|
||||
);
|
||||
for (const mode of ["cli", "mcp"]) {
|
||||
const value = await inspect(mode, { capture_path: path, format });
|
||||
assert.equal(value.total_records, 0);
|
||||
cases++;
|
||||
}
|
||||
}
|
||||
} catch (cause) {
|
||||
failures.push(cause);
|
||||
} finally {
|
||||
for (const cleanup of [
|
||||
() => client.close(),
|
||||
() => transport.close(),
|
||||
() => runtime.close(),
|
||||
]) {
|
||||
try {
|
||||
await cleanup();
|
||||
} catch (cause) {
|
||||
failures.push(cause);
|
||||
}
|
||||
}
|
||||
}
|
||||
const verifier = await completeVerifierRun(run);
|
||||
try {
|
||||
assert.equal(verifier.process_lineage.status, "verified");
|
||||
assert.deepEqual(verifier.process_lineage.descendants, []);
|
||||
} catch (cause) {
|
||||
failures.push(cause);
|
||||
}
|
||||
if (failures.length > 0) {
|
||||
console.error(
|
||||
JSON.stringify(
|
||||
{ status: "failed", public_cases: cases, verifier },
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
throw new AggregateError(
|
||||
failures,
|
||||
"Historical capture verification failed; analysis and cleanup failures are retained.",
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
status: "passed",
|
||||
public_cases: cases,
|
||||
upstream: "mitmproxy 12.2.3 FlowWriter + SaveHar",
|
||||
native_failure_classifications: 7,
|
||||
offline: true,
|
||||
verifier,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
async function inspect(mode, input, errorCategory) {
|
||||
if (mode === "mcp") {
|
||||
const result = await client.callTool({
|
||||
name: "inspect_web_network_capture",
|
||||
arguments: input,
|
||||
});
|
||||
const value = JSON.parse(mcpTextValue(result));
|
||||
if (errorCategory !== undefined) {
|
||||
assert.equal(result.isError, true);
|
||||
assert.equal(value.error.category, errorCategory);
|
||||
for (const literal of input.sensitive_values ?? [])
|
||||
assert.ok(!JSON.stringify(value.error).includes(literal));
|
||||
return value.error;
|
||||
}
|
||||
assert.notEqual(result.isError, true, mcpTextValue(result));
|
||||
assertSensitiveLimitations(value.evidence, input.sensitive_values ?? []);
|
||||
return value.result;
|
||||
}
|
||||
const args = [
|
||||
entrypoint,
|
||||
"inspect-web-network-capture",
|
||||
input.capture_path,
|
||||
input.format,
|
||||
"--json",
|
||||
...(input.record_ordinals ?? []).flatMap((ordinal) => [
|
||||
"--record",
|
||||
String(ordinal),
|
||||
]),
|
||||
...(input.sensitive_values ?? []).flatMap((value) => [
|
||||
"--sensitive-value",
|
||||
value,
|
||||
]),
|
||||
];
|
||||
let result;
|
||||
try {
|
||||
result = await promisify(execFile)(process.execPath, args, {
|
||||
env: environment,
|
||||
timeout: 40_000,
|
||||
maxBuffer: 96 * 1024 * 1024,
|
||||
});
|
||||
} catch (cause) {
|
||||
if (errorCategory === undefined) throw cause;
|
||||
assert.equal(typeof cause.code, "number");
|
||||
const value = JSON.parse(cause.stdout);
|
||||
assert.equal(value.category, errorCategory);
|
||||
for (const literal of input.sensitive_values ?? [])
|
||||
assert.ok(!JSON.stringify(value).includes(literal));
|
||||
return value;
|
||||
}
|
||||
assert.equal(errorCategory, undefined, "Expected malformed capture to fail");
|
||||
const evidence = JSON.parse(result.stdout);
|
||||
assertSensitiveLimitations(evidence, input.sensitive_values ?? []);
|
||||
return evidence.normalized_result;
|
||||
}
|
||||
|
||||
function assertSensitiveLimitations(evidence, literals) {
|
||||
for (const value of Object.values(evidence.parameters))
|
||||
if (typeof value === "string")
|
||||
for (const literal of literals)
|
||||
assert.ok(
|
||||
!value.includes(literal),
|
||||
"Declared literal retained in Evidence parameter",
|
||||
);
|
||||
const result = evidence.normalized_result;
|
||||
if (result.artifact.sha256 === null) assert.equal(evidence.subject, null);
|
||||
else assert.equal(evidence.subject.digest.sha256, result.artifact.sha256);
|
||||
assert.deepEqual(
|
||||
evidence.provider,
|
||||
result.decoder,
|
||||
"Evidence provider tuple changed",
|
||||
);
|
||||
assert.equal(
|
||||
result.decoder.name,
|
||||
result.format === "har"
|
||||
? "REA HAR capture adapter"
|
||||
: "REA offline native mitmproxy adapter",
|
||||
);
|
||||
const texts = [
|
||||
evidence.limitations,
|
||||
result.limitations,
|
||||
...result.records.map((record) => record.limitations),
|
||||
].flat();
|
||||
for (const literal of literals)
|
||||
assert.ok(
|
||||
texts.every((text) => !text.includes(literal)),
|
||||
"Declared literal retained in authored limitations",
|
||||
);
|
||||
}
|
||||
|
||||
function assertBinary(record, pointer, bytes) {
|
||||
const field = record.binary_fields.find((item) => item.pointer === pointer);
|
||||
assert.ok(field, `Missing native byte field ${pointer}`);
|
||||
assert.equal(field.state, "retained");
|
||||
assert.equal(field.content_base64, bytes.toString("base64"));
|
||||
assert.equal(field.bytes, bytes.length);
|
||||
assert.equal(field.sha256, createHash("sha256").update(bytes).digest("hex"));
|
||||
}
|
||||
@@ -263,6 +263,80 @@ try {
|
||||
]);
|
||||
native.call("runtime_snapshot_cancel", [runtime.handle]);
|
||||
await assert.rejects(cancelled, /cancelled/u);
|
||||
assert.equal(await exists(join(runtime.path, "cancelled.bin")), false);
|
||||
const readbackRuntime = WindowsPrivateRuntime.create(
|
||||
workspace.replaceAll("\\", "/"),
|
||||
"readback-",
|
||||
);
|
||||
try {
|
||||
readbackRuntime.writeFile("written.txt", "runtime-write-readback");
|
||||
assert.equal(
|
||||
readbackRuntime.readFile("written.txt"),
|
||||
"runtime-write-readback",
|
||||
);
|
||||
const readbackSnapshot = await readbackRuntime.snapshot(
|
||||
source,
|
||||
"snapshot.bin",
|
||||
);
|
||||
assert.equal(
|
||||
readbackSnapshot.sha256,
|
||||
createHash("sha256")
|
||||
.update(await readFile(source))
|
||||
.digest("hex"),
|
||||
);
|
||||
assert.equal(
|
||||
readbackRuntime.readFile("snapshot.bin"),
|
||||
await readFile(source, "utf8"),
|
||||
);
|
||||
const writtenPath = join(readbackRuntime.observation.path, "written.txt");
|
||||
await assert.rejects(writeFile(writtenPath, "replace"));
|
||||
await assert.rejects(rename(writtenPath, `${writtenPath}.renamed`));
|
||||
await assert.rejects(rm(writtenPath));
|
||||
assert.equal(
|
||||
readbackRuntime.readFile("written.txt"),
|
||||
"runtime-write-readback",
|
||||
);
|
||||
const snapshotPath = join(
|
||||
readbackRuntime.observation.path,
|
||||
"snapshot.bin",
|
||||
);
|
||||
// Java RandomAccessFile uses read/write sharing without delete sharing.
|
||||
// Exercise that independent Win32 reader contract without requiring Java.
|
||||
const readerDigest = execFileSync(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
"$ErrorActionPreference = 'Stop'; " +
|
||||
"$stream = [IO.File]::Open($env:REA_SNAPSHOT_READER_PATH, [IO.FileMode]::Open, [IO.FileAccess]::Read, [IO.FileShare]::ReadWrite); " +
|
||||
"$hash = [Security.Cryptography.SHA256]::Create(); " +
|
||||
"try { [BitConverter]::ToString($hash.ComputeHash($stream)).Replace('-', '').ToLowerInvariant() } " +
|
||||
"finally { $hash.Dispose(); $stream.Dispose() }",
|
||||
],
|
||||
{
|
||||
encoding: "utf8",
|
||||
timeout: 15_000,
|
||||
env: { ...process.env, REA_SNAPSHOT_READER_PATH: snapshotPath },
|
||||
},
|
||||
).trim();
|
||||
assert.equal(readerDigest, readbackSnapshot.sha256);
|
||||
await assert.rejects(writeFile(snapshotPath, "replace"));
|
||||
await assert.rejects(rename(snapshotPath, `${snapshotPath}.renamed`));
|
||||
await assert.rejects(rm(snapshotPath));
|
||||
assert.equal(
|
||||
readbackRuntime.readFile("snapshot.bin"),
|
||||
await readFile(source, "utf8"),
|
||||
);
|
||||
report.controls = {
|
||||
...report.controls,
|
||||
completedRuntimeFileReadback: true,
|
||||
completedRuntimeFileMutationDenied: true,
|
||||
snapshotReadWriteSharingReader: true,
|
||||
};
|
||||
} finally {
|
||||
await readbackRuntime.close();
|
||||
}
|
||||
report.controls = {
|
||||
...report.controls,
|
||||
protectedDaclReadback: true,
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { constants } from "node:buffer";
|
||||
import { createHash } from "node:crypto";
|
||||
import { once } from "node:events";
|
||||
import { createReadStream, createWriteStream } from "node:fs";
|
||||
import { mkdtemp, rm, stat, statfs } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { finished } from "node:stream/promises";
|
||||
|
||||
import { Cli } from "incur";
|
||||
|
||||
import { createStreamedCliJsonOutput } from "../../../dist/cli/streamedJsonOutput.js";
|
||||
import { createEvidence } from "../../../dist/domain/evidence.js";
|
||||
|
||||
// This opt-in format-boundary check writes one temporary file larger than the
|
||||
// actual engine string limit. Shared leaves keep the analysis input small.
|
||||
const format = process.argv[2] ?? "json";
|
||||
assert.ok(format === "json" || format === "jsonl", "Use json or jsonl");
|
||||
const leaf = "x".repeat(64 * 1024);
|
||||
const encodedLeaf = JSON.stringify(leaf);
|
||||
const count = Math.ceil(constants.MAX_STRING_LENGTH / (leaf.length + 3)) + 1;
|
||||
const volume = await statfs(tmpdir());
|
||||
assert.ok(
|
||||
volume.bavail * volume.bsize > count * encodedLeaf.length + 1024 ** 3,
|
||||
"Large-output verification needs room for one output file and a 1-GiB reserve",
|
||||
);
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-javascript-output-"));
|
||||
const path = join(workspace, "result.json");
|
||||
let destination;
|
||||
let peakRss = process.memoryUsage().rss;
|
||||
const sampleRss = () => {
|
||||
peakRss = Math.max(peakRss, process.memoryUsage().rss);
|
||||
};
|
||||
const sampler = setInterval(sampleRss, 25);
|
||||
sampler.unref();
|
||||
|
||||
try {
|
||||
const evidence = createEvidence(
|
||||
undefined,
|
||||
{ id: "large-output-fixture", name: "Large output fixture", version: "1" },
|
||||
{
|
||||
operation: "inspect_fixture",
|
||||
parameters: {},
|
||||
result: { payload: Array.from({ length: count }, () => leaf) },
|
||||
},
|
||||
);
|
||||
sampleRss();
|
||||
const metadata = { command: "inspect-output-fixture", duration: "0ms" };
|
||||
const marker = "__REA_OUTPUT_ENCODING_MARKER__";
|
||||
const template = {
|
||||
ok: true,
|
||||
data: { ...evidence, normalized_result: { payload: [marker] } },
|
||||
meta: metadata,
|
||||
};
|
||||
const skeleton = JSON.stringify(template, null, format === "json" ? 2 : 0);
|
||||
const encodedMarker = JSON.stringify(marker);
|
||||
const markerIndex = skeleton.indexOf(encodedMarker);
|
||||
assert.ok(markerIndex >= 0);
|
||||
assert.equal(markerIndex, skeleton.lastIndexOf(encodedMarker));
|
||||
const prefix = skeleton.slice(0, markerIndex);
|
||||
const suffix = `${skeleton.slice(markerIndex + encodedMarker.length)}\n`;
|
||||
const indentation = prefix.slice(prefix.lastIndexOf("\n") + 1);
|
||||
const separator = format === "json" ? `,\n${indentation}` : ",";
|
||||
const expectedHash = createHash("sha256");
|
||||
expectedHash.update(prefix);
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
if (index > 0) expectedHash.update(separator);
|
||||
expectedHash.update(encodedLeaf);
|
||||
}
|
||||
expectedHash.update(suffix);
|
||||
const expectedBytes =
|
||||
Buffer.byteLength(prefix) +
|
||||
count * Buffer.byteLength(encodedLeaf) +
|
||||
(count - 1) * Buffer.byteLength(separator) +
|
||||
Buffer.byteLength(suffix);
|
||||
assert.ok(expectedBytes > constants.MAX_STRING_LENGTH);
|
||||
|
||||
destination = createWriteStream(path, { flags: "wx", mode: 0o600 });
|
||||
const argv = [metadata.command, "--format", format, "--full-output"];
|
||||
const output = createStreamedCliJsonOutput(argv, destination);
|
||||
assert.ok(output);
|
||||
let unexpectedOutputBytes = 0;
|
||||
let exitCode = 0;
|
||||
await Cli.create("rea-output-verifier", { sync: false })
|
||||
.command(metadata.command, {
|
||||
run: async ({ format: selectedFormat }) => {
|
||||
assert.equal(
|
||||
await output.write(evidence, { ...metadata, format: selectedFormat }),
|
||||
true,
|
||||
);
|
||||
},
|
||||
})
|
||||
.serve(argv, {
|
||||
stdout: (text) => {
|
||||
if (!output.handled) unexpectedOutputBytes += Buffer.byteLength(text);
|
||||
},
|
||||
exit: (code) => {
|
||||
exitCode = code;
|
||||
},
|
||||
});
|
||||
destination.end();
|
||||
await finished(destination);
|
||||
assert.equal(exitCode, 0);
|
||||
assert.equal(output.handled, true);
|
||||
assert.equal(output.failed, false);
|
||||
assert.equal(unexpectedOutputBytes, 0);
|
||||
|
||||
const actualHash = createHash("sha256");
|
||||
for await (const chunk of createReadStream(path)) actualHash.update(chunk);
|
||||
const digest = actualHash.digest("hex");
|
||||
assert.equal(digest, expectedHash.digest("hex"));
|
||||
assert.equal((await stat(path)).size, expectedBytes);
|
||||
sampleRss();
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
json_output_verified: true,
|
||||
format,
|
||||
output_bytes: expectedBytes,
|
||||
engine_string_limit: constants.MAX_STRING_LENGTH,
|
||||
output_sha256: digest,
|
||||
evidence_id: evidence.evidence_id,
|
||||
rss: { method: "node-memoryUsage-rss-sampled", peak_bytes: peakRss },
|
||||
})}\n`,
|
||||
);
|
||||
} finally {
|
||||
clearInterval(sampler);
|
||||
if (destination !== undefined && !destination.closed) {
|
||||
const closed = once(destination, "close");
|
||||
destination.destroy();
|
||||
await closed;
|
||||
}
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
}
|
||||
Reference in New Issue
Block a user