Merge origin/main for artifact admission and evidence fixes

This commit is contained in:
64579525
2026-10-13 14:32:52 +08:00
188 changed files with 8882 additions and 937 deletions
+1
View File
@@ -61,6 +61,7 @@ export const SCOPES = {
"tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts",
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
"tests/boundary/process/ProcessEvidence.host.test.ts",
"tests/acceptance/webModuleCli.test.ts",
+53 -3
View File
@@ -10,6 +10,7 @@ import json
import os
from pathlib import Path
import secrets
import select
import stat
import subprocess
import tempfile
@@ -118,6 +119,7 @@ class XWindowAttributes(ctypes.Structure):
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--probe", action="store_true")
parser.add_argument("--owner-stdin", action="store_true")
parser.add_argument("--hopper")
parser.add_argument("--socket")
parser.add_argument(
@@ -133,6 +135,43 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
class OwnerLifetime:
"""The launcher's private pipe is a lifetime capability, independent of PIDs."""
def __init__(self, enabled: bool) -> None:
self._poll = select.poll() if enabled else None
if self._poll is not None:
# Node/libuv can implement piped stdio with a Unix socket pair.
mode = os.fstat(0).st_mode
if not (stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)):
raise AdapterFailure(
INVALID_LAUNCH_COMMAND,
"invalid_launch_command",
"owner lifetime requires a private stdin pipe",
)
self._poll.register(0, select.POLLIN | select.POLLHUP | select.POLLERR)
def alive(self) -> bool:
if self._poll is None:
return True
for _, events in self._poll.poll(0):
if events & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
return False
if events & select.POLLIN and not os.read(0, 4096):
return False
return True
def wait_for_hopper(child: subprocess.Popen[bytes], owner: OwnerLifetime) -> int:
while owner.alive():
try:
return child.wait(timeout=0.1)
except subprocess.TimeoutExpired:
continue
# run_launch's finally owns cleanup of both direct, unreaped children.
return 0
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as file:
@@ -572,6 +611,7 @@ def start_xvfb(
"-auth",
str(authority),
],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
@@ -627,6 +667,9 @@ def run_launch(args: argparse.Namespace) -> int:
"invalid_launch_command",
"missing Hopper launch coordinates",
)
owner = OwnerLifetime(args.owner_stdin)
if not owner.alive():
return 0
hopper = Path(args.hopper)
socket = Path(args.socket)
try:
@@ -654,6 +697,8 @@ def run_launch(args: argparse.Namespace) -> int:
child: subprocess.Popen[bytes] | None = None
display: int | None = None
try:
if not owner.alive():
return 0
hopper_environment = dict(os.environ)
for name in ("WAYLAND_DISPLAY", "WAYLAND_SOCKET"):
hopper_environment.pop(name, None)
@@ -666,7 +711,10 @@ def run_launch(args: argparse.Namespace) -> int:
"XDG_SESSION_TYPE": "x11",
}
)
child = subprocess.Popen(command, env=hopper_environment)
# Children must not consume or retain REA's lifetime pipe.
child = subprocess.Popen(
command, env=hopper_environment, stdin=subprocess.DEVNULL
)
leader = os.getpid()
os.environ["XAUTHORITY"] = str(authority)
@@ -702,10 +750,12 @@ def run_launch(args: argparse.Namespace) -> int:
observed: list[tuple[int, int, int, int, int]] = []
dialog: tuple[int, int, int, int, int] | None = None
while time.monotonic() < deadline:
if not owner.alive():
return 0
if socket.exists():
x11.XCloseDisplay(display)
display = None
return child.wait()
return wait_for_hopper(child, owner)
if not hopper_is_owned(leader, hopper):
if child.poll() is None:
time.sleep(0.1)
@@ -749,7 +799,7 @@ def run_launch(args: argparse.Namespace) -> int:
x11.XFlush(display)
x11.XCloseDisplay(display)
display = None
return child.wait()
return wait_for_hopper(child, owner)
except AdapterFailure as error:
raise error.with_stderr(capture)
finally:
+9
View File
@@ -52,6 +52,11 @@ export const toolFamilyCatalog = (sources) => {
surface: "android-provider",
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
},
{
id: "pcap",
surface: "pcap-provider",
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
},
{
id: "flutter",
surface: "flutter-provider",
@@ -251,6 +256,10 @@ export const providerCatalog = (sources) => {
identity: sources.androidProvider.JADX_PROVIDER_IDENTITY,
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
},
{
identity: sources.pcapProvider.PCAP_PROVIDER_IDENTITY,
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
},
{
identity: sources.flutterProvider.FLUTTER_PROVIDER_IDENTITY,
contracts: sources.flutterContracts.FLUTTER_TOOL_CONTRACTS,
+2 -2
View File
@@ -68,8 +68,8 @@ const SOURCE_PATHS = {
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
androidProvider: "dist/android/JadxRelease.js",
apktoolContracts: "dist/contracts/apktool/apktoolToolContracts.js",
flutterContracts: "dist/contracts/flutter/flutterToolContracts.js",
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
pcapContracts: "dist/contracts/pcap/pcapToolContracts.js",
pcapProvider: "dist/pcap/PcapProvider.js",
apktoolProvider: "dist/apktool/ApktoolProvider.js",
jebContracts: "dist/contracts/jeb/jebToolContracts.js",
jebProvider: "dist/jeb/JebProvider.js",
+130
View File
@@ -0,0 +1,130 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { setTimeout as delay } from "node:timers/promises";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { verifyNoTokenOwnedProcesses } from "../../dist/process/ProcessOwnership.js";
import {
observeProcessStartIdentity,
signalProcessWithStartIdentity,
} from "../../dist/process/ProcessOwnershipObservation.js";
import { startUnrelatedHopperSentinel } from "./unrelated-hopper-sentinel.mjs";
import {
requireMcpEvidenceResult,
requireMcpLifecycleResult,
} from "./mcp-verifier-results.mjs";
const execFileAsync = promisify(execFile);
/** Prove actual MCP owner exit releases the demo singleton for subsequent CLI/MCP sessions. */
export async function verifyLinuxHopperOwnerExit({
primary,
secondary,
serverCommand,
serverArgs,
environment,
}) {
const sentinel = await startUnrelatedHopperSentinel();
const outcomes = [];
try {
for (const mode of ["graceful", "SIGKILL"]) {
const transport = new StdioClientTransport({
command: serverCommand,
args: serverArgs,
env: environment,
stderr: "pipe",
});
// Drain launcher diagnostics even on the deliberately interrupted owner.
transport.stderr?.resume();
const client = new Client({ name: "hopper-owner-exit", version: "1" });
const options = { timeout: 180_000 };
try {
await client.connect(transport);
const ownerPid = transport.pid;
assert.ok(ownerPid !== null);
const owner = await observeProcessStartIdentity(ownerPid);
assert.equal(owner?.state, "readable");
requireMcpLifecycleResult(
await client.callTool(
{
name: "open_binary",
arguments: { path: primary },
},
options,
),
"open_binary",
);
const procedures = requireMcpEvidenceResult(
await client.callTool(
{ name: "list_procedures", arguments: {} },
options,
),
"list_procedures",
);
assert.ok(procedures.length > 0);
const status = requireMcpLifecycleResult(
await client.callTool(
{ name: "binary_session", arguments: {} },
options,
),
"binary_session",
);
const runId = status.analysis_run?.run_id;
assert.equal(typeof runId, "string");
assert.equal(status.analysis_provider_binding?.provider.id, "hopper");
assert.equal((await verifyNoTokenOwnedProcesses(runId)).cleaned, false);
if (mode === "graceful") await client.close();
else {
assert.equal(
await signalProcessWithStartIdentity(
ownerPid,
owner.identity,
"SIGKILL",
),
"signaled",
);
}
const deadline = Date.now() + 10_000;
let cleanup;
do {
cleanup = await verifyNoTokenOwnedProcesses(runId);
if (cleanup.cleaned) break;
await delay(100);
} while (Date.now() < deadline);
assert.equal(cleanup.cleaned, true, cleanup.reason);
// The SIGKILL case must recover the stale lease for both target identities.
for (const target of [primary, secondary]) {
const result = await execFileAsync(
process.execPath,
[
"scripts/rea.mjs",
"inspect",
target,
"--provider",
"hopper",
"--format",
"json",
],
{ env: environment, timeout: 180_000, maxBuffer: 16 * 1024 * 1024 },
);
assert.ok(JSON.parse(result.stdout).normalized_result);
}
outcomes.push({
mode,
ownedProcessesStopped: true,
sameTargetReopened: true,
differentTargetReopened: true,
});
} finally {
await client.close();
await transport.close();
}
}
} finally {
assert.equal(await sentinel.verifyAndClose(), true);
}
return { outcomes, unrelatedHopperSurvived: true };
}
+6
View File
@@ -16,6 +16,12 @@ const isMcpMode =
const isMcpDoctorMode = args[0] === "mcp" && args[1] === "doctor";
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
// Windows home lookup throws for an empty USERPROFILE. Dependencies resolve
// home during import, before REA's own resolver can apply its account fallback.
if (process.platform === "win32" && process.env.USERPROFILE === "") {
delete process.env.USERPROFILE;
}
// Playwright's registry assumes one of its three desktop host platforms when
// it chooses the browser cache directory. Termux reports `android`, although
// its Node runtime and filesystem are POSIX-compatible. Keep the cache path
+46
View File
@@ -236,6 +236,52 @@ try {
const hci = await inspect({ path: bluetooth, fields: ["bthci_cmd.opcode"] });
assert.deepEqual(hci.packets[0]["bthci_cmd.opcode"], ["0x0c03"]);
assert.deepEqual(await readFile(path), source);
// Exercise the incremental summary workflow against the same owned bytes
// using real installed Wireshark tools; these inputs are format fixtures.
const pcapTool = (await client.listTools()).tools.find(
(t) => t.name === "inspect_pcap_capture",
);
assert.ok(pcapTool?.outputSchema);
const pcapAjv = new Ajv2020({ strict: false, validateFormats: false });
assert.ok(pcapAjv.validateSchema(pcapTool.inputSchema));
assert.ok(pcapAjv.validateSchema(pcapTool.outputSchema));
for (const capturePath of [path, bluetooth]) {
const response = await client.callTool({
name: pcapTool.name,
arguments: { path: capturePath, detail: "summary" },
});
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(
pcapAjv.validate(pcapTool.outputSchema, response.structuredContent),
pcapAjv.errorsText(),
);
const summary = parseEvidence(response.structuredContent).normalized_result;
assert.equal(
summary.summary.target.sha256,
createHash("sha256")
.update(await readFile(capturePath))
.digest("hex"),
);
const source = {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
};
const page = await client.callTool({
name: "inspect_analysis_view",
arguments: {
source,
view: { kind: "pcap", facet: "protocols", limit: 10 },
},
});
assert.notEqual(page.isError, true, JSON.stringify(page));
assert.equal(
parseEvidence(page.structuredContent).normalized_result
.parent_evidence_id,
source.evidence_id,
);
cases++;
}
assert.deepEqual(await readFile(bluetooth), bluetoothSource);
for (const input of [
{ path, fields: ["frame.number"], display_filter: "this is not a filter" },
+131
View File
@@ -0,0 +1,131 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdtemp, readdir, rmdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { hashStableFile } from "../dist/filesystem/StableFileHash.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
const args = process.argv.slice(2);
if (args.includes("--help") || args.includes("-h")) {
console.log(
"Usage: npm run verify:pcap -- [--pcap PATH]\nRequires caller-supplied tshark/capinfos. Without --pcap, only client readiness is verified.",
);
process.exit(0);
}
if (!(args.length === 0 || (args.length === 2 && args[0] === "--pcap")))
throw new Error("Expected --pcap PATH");
const path = args[1] === undefined ? null : resolve(args[1]);
// This parent is shared by independent CLI/MCP owners. A leased Windows
// runtime root would prevent their native admission of its directory handle.
const root = await mkdtemp(join(tmpdir(), "rea-pcap-verifier-"));
const environment = Object.fromEntries(
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
);
Object.assign(environment, {
TMPDIR: root,
TMP: root,
TEMP: root,
});
const entrypoint = resolve("scripts/rea.mjs");
const execute = promisify(execFile);
const client = new Client({ name: "pcap-real-verifier", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env: environment,
stderr: "pipe",
});
transport.stderr?.resume();
try {
await client.connect(transport);
const readiness = await run("inspect_pcap_client", {});
assert(readiness.client.tshark_version && readiness.client.capinfos_version);
if (path === null)
console.log("[unverified] Capture inspection: no --pcap was supplied");
else {
const identity = await hashStableFile(path);
const result = await run("inspect_pcap_capture", { path });
assert.deepEqual(result.target, { path, ...identity });
assert(result.packet_count >= 0);
assert(
result.endpoints.every(
(entry) => entry.packet_count <= result.packet_count,
),
);
assert(
result.protocol_distribution.every(
(entry) => entry.packet_count <= result.packet_count,
),
);
const partial =
result.endpoint_count > result.endpoints.length ||
result.dns_query_count > result.dns_queries.length ||
result.tls_sni_count > result.tls_sni_values.length ||
result.protocol_count > result.protocol_distribution.length ||
result.http_request_uri_count > result.http_request_uris.length ||
result.unattributed_dns_packet_count > 0 ||
result.dns_queries.some(
(entry) =>
entry.response_address_count > entry.response_addresses.length,
);
assert.equal(result.coverage, partial ? "partial" : "complete");
assert.deepEqual(
await hashStableFile(path),
identity,
"Input capture must remain unchanged",
);
console.log(
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
);
}
} finally {
try {
await client.close();
} finally {
await transport.close();
}
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
}
console.log("All asserted pcap CLI/MCP checks passed");
async function run(name, input) {
const contract = toolContract(name);
const { stdout } = await execute(
process.execPath,
[
entrypoint,
name.replaceAll("_", "-"),
...(input.path === undefined ? [] : [input.path]),
"--json",
],
{ env: environment, timeout: 360_000, maxBuffer: 8 * 1024 * 1024 },
);
const cli = contract.outputSchema.parse(JSON.parse(stdout));
const response = await client.callTool(
{ name, arguments: input },
{ timeout: 360_000 },
);
assert.notEqual(response.isError, true, mcpTextValue(response));
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
assert.deepEqual(mcp.normalized_result, cli.normalized_result);
assert.deepEqual(mcp.subject, cli.subject);
if (input.path !== undefined) {
assert.equal(cli.subject.local_path, input.path);
assert.equal(
cli.subject.digest.sha256,
cli.normalized_result.target.sha256,
);
}
console.log(`[ok] ${name}: named output schema and CLI/MCP Evidence parity`);
return cli.normalized_result;
}
+9
View File
@@ -47,6 +47,7 @@ import {
} from "./lib/real-hopper-boundaries.mjs";
import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs";
import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs";
import { verifyLinuxHopperOwnerExit } from "./lib/real-hopper-owner-exit.mjs";
const execFileAsync = promisify(execFile);
const verifierRun = createVerifierRun();
const timeout = 180_000;
@@ -577,6 +578,14 @@ if (sessionsAfter.length > 0) {
}
if (summary === undefined)
throw new Error("Real-Hopper verification did not produce a summary");
if (process.platform === "linux")
summary.ownerExit = await verifyLinuxHopperOwnerExit({
primary: targetA,
secondary: targetB,
serverCommand,
serverArgs,
environment: serverEnvironment,
});
const completedVerifierRun = await completeVerifierRun(verifierRun);
if (
completedVerifierRun.process_lineage.status !== "verified" ||