Merge origin/main for artifact admission and evidence fixes
This commit is contained in:
@@ -61,6 +61,7 @@ export const SCOPES = {
|
||||
"tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts",
|
||||
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
|
||||
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
|
||||
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
|
||||
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
|
||||
"tests/boundary/process/ProcessEvidence.host.test.ts",
|
||||
"tests/acceptance/webModuleCli.test.ts",
|
||||
|
||||
@@ -10,6 +10,7 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import select
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -118,6 +119,7 @@ class XWindowAttributes(ctypes.Structure):
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--probe", action="store_true")
|
||||
parser.add_argument("--owner-stdin", action="store_true")
|
||||
parser.add_argument("--hopper")
|
||||
parser.add_argument("--socket")
|
||||
parser.add_argument(
|
||||
@@ -133,6 +135,43 @@ def parse_args() -> argparse.Namespace:
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
class OwnerLifetime:
|
||||
"""The launcher's private pipe is a lifetime capability, independent of PIDs."""
|
||||
|
||||
def __init__(self, enabled: bool) -> None:
|
||||
self._poll = select.poll() if enabled else None
|
||||
if self._poll is not None:
|
||||
# Node/libuv can implement piped stdio with a Unix socket pair.
|
||||
mode = os.fstat(0).st_mode
|
||||
if not (stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)):
|
||||
raise AdapterFailure(
|
||||
INVALID_LAUNCH_COMMAND,
|
||||
"invalid_launch_command",
|
||||
"owner lifetime requires a private stdin pipe",
|
||||
)
|
||||
self._poll.register(0, select.POLLIN | select.POLLHUP | select.POLLERR)
|
||||
|
||||
def alive(self) -> bool:
|
||||
if self._poll is None:
|
||||
return True
|
||||
for _, events in self._poll.poll(0):
|
||||
if events & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
|
||||
return False
|
||||
if events & select.POLLIN and not os.read(0, 4096):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def wait_for_hopper(child: subprocess.Popen[bytes], owner: OwnerLifetime) -> int:
|
||||
while owner.alive():
|
||||
try:
|
||||
return child.wait(timeout=0.1)
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
# run_launch's finally owns cleanup of both direct, unreaped children.
|
||||
return 0
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
@@ -572,6 +611,7 @@ def start_xvfb(
|
||||
"-auth",
|
||||
str(authority),
|
||||
],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
@@ -627,6 +667,9 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
"invalid_launch_command",
|
||||
"missing Hopper launch coordinates",
|
||||
)
|
||||
owner = OwnerLifetime(args.owner_stdin)
|
||||
if not owner.alive():
|
||||
return 0
|
||||
hopper = Path(args.hopper)
|
||||
socket = Path(args.socket)
|
||||
try:
|
||||
@@ -654,6 +697,8 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
child: subprocess.Popen[bytes] | None = None
|
||||
display: int | None = None
|
||||
try:
|
||||
if not owner.alive():
|
||||
return 0
|
||||
hopper_environment = dict(os.environ)
|
||||
for name in ("WAYLAND_DISPLAY", "WAYLAND_SOCKET"):
|
||||
hopper_environment.pop(name, None)
|
||||
@@ -666,7 +711,10 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
"XDG_SESSION_TYPE": "x11",
|
||||
}
|
||||
)
|
||||
child = subprocess.Popen(command, env=hopper_environment)
|
||||
# Children must not consume or retain REA's lifetime pipe.
|
||||
child = subprocess.Popen(
|
||||
command, env=hopper_environment, stdin=subprocess.DEVNULL
|
||||
)
|
||||
leader = os.getpid()
|
||||
os.environ["XAUTHORITY"] = str(authority)
|
||||
|
||||
@@ -702,10 +750,12 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
observed: list[tuple[int, int, int, int, int]] = []
|
||||
dialog: tuple[int, int, int, int, int] | None = None
|
||||
while time.monotonic() < deadline:
|
||||
if not owner.alive():
|
||||
return 0
|
||||
if socket.exists():
|
||||
x11.XCloseDisplay(display)
|
||||
display = None
|
||||
return child.wait()
|
||||
return wait_for_hopper(child, owner)
|
||||
if not hopper_is_owned(leader, hopper):
|
||||
if child.poll() is None:
|
||||
time.sleep(0.1)
|
||||
@@ -749,7 +799,7 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
x11.XFlush(display)
|
||||
x11.XCloseDisplay(display)
|
||||
display = None
|
||||
return child.wait()
|
||||
return wait_for_hopper(child, owner)
|
||||
except AdapterFailure as error:
|
||||
raise error.with_stderr(capture)
|
||||
finally:
|
||||
|
||||
@@ -52,6 +52,11 @@ export const toolFamilyCatalog = (sources) => {
|
||||
surface: "android-provider",
|
||||
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
id: "pcap",
|
||||
surface: "pcap-provider",
|
||||
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
id: "flutter",
|
||||
surface: "flutter-provider",
|
||||
@@ -251,6 +256,10 @@ export const providerCatalog = (sources) => {
|
||||
identity: sources.androidProvider.JADX_PROVIDER_IDENTITY,
|
||||
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.pcapProvider.PCAP_PROVIDER_IDENTITY,
|
||||
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.flutterProvider.FLUTTER_PROVIDER_IDENTITY,
|
||||
contracts: sources.flutterContracts.FLUTTER_TOOL_CONTRACTS,
|
||||
|
||||
@@ -68,8 +68,8 @@ const SOURCE_PATHS = {
|
||||
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
|
||||
androidProvider: "dist/android/JadxRelease.js",
|
||||
apktoolContracts: "dist/contracts/apktool/apktoolToolContracts.js",
|
||||
flutterContracts: "dist/contracts/flutter/flutterToolContracts.js",
|
||||
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
|
||||
pcapContracts: "dist/contracts/pcap/pcapToolContracts.js",
|
||||
pcapProvider: "dist/pcap/PcapProvider.js",
|
||||
apktoolProvider: "dist/apktool/ApktoolProvider.js",
|
||||
jebContracts: "dist/contracts/jeb/jebToolContracts.js",
|
||||
jebProvider: "dist/jeb/JebProvider.js",
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { verifyNoTokenOwnedProcesses } from "../../dist/process/ProcessOwnership.js";
|
||||
import {
|
||||
observeProcessStartIdentity,
|
||||
signalProcessWithStartIdentity,
|
||||
} from "../../dist/process/ProcessOwnershipObservation.js";
|
||||
import { startUnrelatedHopperSentinel } from "./unrelated-hopper-sentinel.mjs";
|
||||
import {
|
||||
requireMcpEvidenceResult,
|
||||
requireMcpLifecycleResult,
|
||||
} from "./mcp-verifier-results.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Prove actual MCP owner exit releases the demo singleton for subsequent CLI/MCP sessions. */
|
||||
export async function verifyLinuxHopperOwnerExit({
|
||||
primary,
|
||||
secondary,
|
||||
serverCommand,
|
||||
serverArgs,
|
||||
environment,
|
||||
}) {
|
||||
const sentinel = await startUnrelatedHopperSentinel();
|
||||
const outcomes = [];
|
||||
try {
|
||||
for (const mode of ["graceful", "SIGKILL"]) {
|
||||
const transport = new StdioClientTransport({
|
||||
command: serverCommand,
|
||||
args: serverArgs,
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
// Drain launcher diagnostics even on the deliberately interrupted owner.
|
||||
transport.stderr?.resume();
|
||||
const client = new Client({ name: "hopper-owner-exit", version: "1" });
|
||||
const options = { timeout: 180_000 };
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const ownerPid = transport.pid;
|
||||
assert.ok(ownerPid !== null);
|
||||
const owner = await observeProcessStartIdentity(ownerPid);
|
||||
assert.equal(owner?.state, "readable");
|
||||
requireMcpLifecycleResult(
|
||||
await client.callTool(
|
||||
{
|
||||
name: "open_binary",
|
||||
arguments: { path: primary },
|
||||
},
|
||||
options,
|
||||
),
|
||||
"open_binary",
|
||||
);
|
||||
const procedures = requireMcpEvidenceResult(
|
||||
await client.callTool(
|
||||
{ name: "list_procedures", arguments: {} },
|
||||
options,
|
||||
),
|
||||
"list_procedures",
|
||||
);
|
||||
assert.ok(procedures.length > 0);
|
||||
const status = requireMcpLifecycleResult(
|
||||
await client.callTool(
|
||||
{ name: "binary_session", arguments: {} },
|
||||
options,
|
||||
),
|
||||
"binary_session",
|
||||
);
|
||||
const runId = status.analysis_run?.run_id;
|
||||
assert.equal(typeof runId, "string");
|
||||
assert.equal(status.analysis_provider_binding?.provider.id, "hopper");
|
||||
assert.equal((await verifyNoTokenOwnedProcesses(runId)).cleaned, false);
|
||||
if (mode === "graceful") await client.close();
|
||||
else {
|
||||
assert.equal(
|
||||
await signalProcessWithStartIdentity(
|
||||
ownerPid,
|
||||
owner.identity,
|
||||
"SIGKILL",
|
||||
),
|
||||
"signaled",
|
||||
);
|
||||
}
|
||||
const deadline = Date.now() + 10_000;
|
||||
let cleanup;
|
||||
do {
|
||||
cleanup = await verifyNoTokenOwnedProcesses(runId);
|
||||
if (cleanup.cleaned) break;
|
||||
await delay(100);
|
||||
} while (Date.now() < deadline);
|
||||
assert.equal(cleanup.cleaned, true, cleanup.reason);
|
||||
// The SIGKILL case must recover the stale lease for both target identities.
|
||||
for (const target of [primary, secondary]) {
|
||||
const result = await execFileAsync(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/rea.mjs",
|
||||
"inspect",
|
||||
target,
|
||||
"--provider",
|
||||
"hopper",
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
{ env: environment, timeout: 180_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
assert.ok(JSON.parse(result.stdout).normalized_result);
|
||||
}
|
||||
outcomes.push({
|
||||
mode,
|
||||
ownedProcessesStopped: true,
|
||||
sameTargetReopened: true,
|
||||
differentTargetReopened: true,
|
||||
});
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
assert.equal(await sentinel.verifyAndClose(), true);
|
||||
}
|
||||
return { outcomes, unrelatedHopperSurvived: true };
|
||||
}
|
||||
@@ -16,6 +16,12 @@ const isMcpMode =
|
||||
const isMcpDoctorMode = args[0] === "mcp" && args[1] === "doctor";
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
// Windows home lookup throws for an empty USERPROFILE. Dependencies resolve
|
||||
// home during import, before REA's own resolver can apply its account fallback.
|
||||
if (process.platform === "win32" && process.env.USERPROFILE === "") {
|
||||
delete process.env.USERPROFILE;
|
||||
}
|
||||
|
||||
// Playwright's registry assumes one of its three desktop host platforms when
|
||||
// it chooses the browser cache directory. Termux reports `android`, although
|
||||
// its Node runtime and filesystem are POSIX-compatible. Keep the cache path
|
||||
|
||||
@@ -236,6 +236,52 @@ try {
|
||||
const hci = await inspect({ path: bluetooth, fields: ["bthci_cmd.opcode"] });
|
||||
assert.deepEqual(hci.packets[0]["bthci_cmd.opcode"], ["0x0c03"]);
|
||||
assert.deepEqual(await readFile(path), source);
|
||||
// Exercise the incremental summary workflow against the same owned bytes
|
||||
// using real installed Wireshark tools; these inputs are format fixtures.
|
||||
const pcapTool = (await client.listTools()).tools.find(
|
||||
(t) => t.name === "inspect_pcap_capture",
|
||||
);
|
||||
assert.ok(pcapTool?.outputSchema);
|
||||
const pcapAjv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
assert.ok(pcapAjv.validateSchema(pcapTool.inputSchema));
|
||||
assert.ok(pcapAjv.validateSchema(pcapTool.outputSchema));
|
||||
for (const capturePath of [path, bluetooth]) {
|
||||
const response = await client.callTool({
|
||||
name: pcapTool.name,
|
||||
arguments: { path: capturePath, detail: "summary" },
|
||||
});
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(
|
||||
pcapAjv.validate(pcapTool.outputSchema, response.structuredContent),
|
||||
pcapAjv.errorsText(),
|
||||
);
|
||||
const summary = parseEvidence(response.structuredContent).normalized_result;
|
||||
assert.equal(
|
||||
summary.summary.target.sha256,
|
||||
createHash("sha256")
|
||||
.update(await readFile(capturePath))
|
||||
.digest("hex"),
|
||||
);
|
||||
const source = {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: summary.parent_evidence_id,
|
||||
};
|
||||
const page = await client.callTool({
|
||||
name: "inspect_analysis_view",
|
||||
arguments: {
|
||||
source,
|
||||
view: { kind: "pcap", facet: "protocols", limit: 10 },
|
||||
},
|
||||
});
|
||||
assert.notEqual(page.isError, true, JSON.stringify(page));
|
||||
assert.equal(
|
||||
parseEvidence(page.structuredContent).normalized_result
|
||||
.parent_evidence_id,
|
||||
source.evidence_id,
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
|
||||
assert.deepEqual(await readFile(bluetooth), bluetoothSource);
|
||||
for (const input of [
|
||||
{ path, fields: ["frame.number"], display_filter: "this is not a filter" },
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, readdir, rmdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { hashStableFile } from "../dist/filesystem/StableFileHash.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes("--help") || args.includes("-h")) {
|
||||
console.log(
|
||||
"Usage: npm run verify:pcap -- [--pcap PATH]\nRequires caller-supplied tshark/capinfos. Without --pcap, only client readiness is verified.",
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
if (!(args.length === 0 || (args.length === 2 && args[0] === "--pcap")))
|
||||
throw new Error("Expected --pcap PATH");
|
||||
const path = args[1] === undefined ? null : resolve(args[1]);
|
||||
// This parent is shared by independent CLI/MCP owners. A leased Windows
|
||||
// runtime root would prevent their native admission of its directory handle.
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-pcap-verifier-"));
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
|
||||
);
|
||||
Object.assign(environment, {
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
});
|
||||
const entrypoint = resolve("scripts/rea.mjs");
|
||||
const execute = promisify(execFile);
|
||||
const client = new Client({ name: "pcap-real-verifier", version: "1" });
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.resume();
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const readiness = await run("inspect_pcap_client", {});
|
||||
assert(readiness.client.tshark_version && readiness.client.capinfos_version);
|
||||
if (path === null)
|
||||
console.log("[unverified] Capture inspection: no --pcap was supplied");
|
||||
else {
|
||||
const identity = await hashStableFile(path);
|
||||
const result = await run("inspect_pcap_capture", { path });
|
||||
assert.deepEqual(result.target, { path, ...identity });
|
||||
assert(result.packet_count >= 0);
|
||||
assert(
|
||||
result.endpoints.every(
|
||||
(entry) => entry.packet_count <= result.packet_count,
|
||||
),
|
||||
);
|
||||
assert(
|
||||
result.protocol_distribution.every(
|
||||
(entry) => entry.packet_count <= result.packet_count,
|
||||
),
|
||||
);
|
||||
const partial =
|
||||
result.endpoint_count > result.endpoints.length ||
|
||||
result.dns_query_count > result.dns_queries.length ||
|
||||
result.tls_sni_count > result.tls_sni_values.length ||
|
||||
result.protocol_count > result.protocol_distribution.length ||
|
||||
result.http_request_uri_count > result.http_request_uris.length ||
|
||||
result.unattributed_dns_packet_count > 0 ||
|
||||
result.dns_queries.some(
|
||||
(entry) =>
|
||||
entry.response_address_count > entry.response_addresses.length,
|
||||
);
|
||||
assert.equal(result.coverage, partial ? "partial" : "complete");
|
||||
assert.deepEqual(
|
||||
await hashStableFile(path),
|
||||
identity,
|
||||
"Input capture must remain unchanged",
|
||||
);
|
||||
console.log(
|
||||
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
assert.deepEqual(
|
||||
await readdir(root),
|
||||
[],
|
||||
`Owned capture resources remain under ${root}`,
|
||||
);
|
||||
await rmdir(root);
|
||||
}
|
||||
console.log("All asserted pcap CLI/MCP checks passed");
|
||||
|
||||
async function run(name, input) {
|
||||
const contract = toolContract(name);
|
||||
const { stdout } = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
name.replaceAll("_", "-"),
|
||||
...(input.path === undefined ? [] : [input.path]),
|
||||
"--json",
|
||||
],
|
||||
{ env: environment, timeout: 360_000, maxBuffer: 8 * 1024 * 1024 },
|
||||
);
|
||||
const cli = contract.outputSchema.parse(JSON.parse(stdout));
|
||||
const response = await client.callTool(
|
||||
{ name, arguments: input },
|
||||
{ timeout: 360_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
|
||||
assert.deepEqual(mcp.normalized_result, cli.normalized_result);
|
||||
assert.deepEqual(mcp.subject, cli.subject);
|
||||
if (input.path !== undefined) {
|
||||
assert.equal(cli.subject.local_path, input.path);
|
||||
assert.equal(
|
||||
cli.subject.digest.sha256,
|
||||
cli.normalized_result.target.sha256,
|
||||
);
|
||||
}
|
||||
console.log(`[ok] ${name}: named output schema and CLI/MCP Evidence parity`);
|
||||
return cli.normalized_result;
|
||||
}
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
} from "./lib/real-hopper-boundaries.mjs";
|
||||
import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs";
|
||||
import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs";
|
||||
import { verifyLinuxHopperOwnerExit } from "./lib/real-hopper-owner-exit.mjs";
|
||||
const execFileAsync = promisify(execFile);
|
||||
const verifierRun = createVerifierRun();
|
||||
const timeout = 180_000;
|
||||
@@ -577,6 +578,14 @@ if (sessionsAfter.length > 0) {
|
||||
}
|
||||
if (summary === undefined)
|
||||
throw new Error("Real-Hopper verification did not produce a summary");
|
||||
if (process.platform === "linux")
|
||||
summary.ownerExit = await verifyLinuxHopperOwnerExit({
|
||||
primary: targetA,
|
||||
secondary: targetB,
|
||||
serverCommand,
|
||||
serverArgs,
|
||||
environment: serverEnvironment,
|
||||
});
|
||||
const completedVerifierRun = await completeVerifierRun(verifierRun);
|
||||
if (
|
||||
completedVerifierRun.process_lineage.status !== "verified" ||
|
||||
|
||||
Reference in New Issue
Block a user