chore(plugins): sync the captured final review base

This commit is contained in:
morluto
2026-10-13 21:50:53 +08:00
253 changed files with 12485 additions and 1407 deletions
+2
View File
@@ -56,6 +56,8 @@ export const SCOPES = {
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
"tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts",
"src/domain/javascript/javascriptRuntimeStaticCandidates.ts",
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
"tests/boundary/process/ProcessEvidence.host.test.ts",
"tests/acceptance/webModuleCli.test.ts",
+9
View File
@@ -0,0 +1,9 @@
export function assertFeatureTrace(
trace: unknown,
procedure: { readonly address: string },
): void;
export function findMappedNonFunction(
strings: readonly { readonly address: string }[],
resolveAt: (address: string) => Promise<unknown>,
): Promise<string | undefined>;
+52
View File
@@ -0,0 +1,52 @@
import assert from "node:assert/strict";
const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]);
/**
* Assert the `trace_feature` result of the real IDA lane for one selected
* procedure. The query is a substring search, so the trace may also match other
* procedures and strings; each reference is checked against its own matched
* target, and a source outside every function (a data reference) is valid.
*/
export function assertFeatureTrace(trace, procedure) {
assert.ok(
trace.matches.some(
({ type, address }) =>
type === "procedure" && address === procedure.address,
),
"The trace omitted the selected procedure's match.",
);
assert.equal(typeof trace.truncated, "boolean");
const matched = new Set(trace.matches.map(({ address }) => address));
for (const reference of trace.references) {
assert.ok(
matched.has(reference.target_address),
`A reference targets ${reference.target_address}, which is not a match.`,
);
const { containing_procedure: containing } = reference;
if (containing.found)
assert.equal(typeof containing.procedure.address, "string");
else assert.ok(NEGATIVE_REASONS.has(containing.reason));
}
}
/**
* Find a string that IDA maps outside every function. A string inside a
* function chunk (inline data in code) is valid and skipped, so the strings are
* tried in order. Any other answer must be "not in a procedure": IDA lists only
* mapped strings, so an unmapped answer is a containment defect and fails the
* lane. Returns the address, or undefined when every string is inside a function.
*/
export async function findMappedNonFunction(strings, resolveAt) {
for (const { address } of strings) {
const resolved = await resolveAt(address);
if (resolved.found) continue;
assert.equal(
resolved.reason,
"not_in_procedure",
`The listed string at ${address} resolved as ${resolved.reason}.`,
);
return address;
}
return undefined;
}
+30 -17
View File
@@ -8,7 +8,7 @@ import {
realpath,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { join, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import { exec } from "./lib/verify-package-core.mjs";
@@ -18,10 +18,6 @@ import {
toolContract,
} from "../dist/contracts/toolContracts.js";
assert(
process.platform !== "win32",
"The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
);
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
@@ -32,6 +28,13 @@ assert(
"Usage: node scripts/verify-qwen-client.mjs [chat|call]",
);
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
// npm exposes only qwen.cmd on Windows. Resolve a selected JavaScript entry
// before the verifier switches to its disposable workspace.
const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined;
const launch = (args) =>
entryPoint === undefined
? [command, args]
: [process.execPath, [entryPoint, ...args]];
const verifier = createVerifierRun();
const profile = join(lab, "selected profile");
const account = join(lab, "account");
@@ -54,14 +57,18 @@ const baseEnvironment = {
let clientVersion;
try {
clientVersion = (
await exec(command, ["--version"], {
await exec(...launch(["--version"]), {
env: baseEnvironment,
timeout: 30_000,
})
).stdout.trim();
} catch (cause) {
const windowsHint =
process.platform === "win32"
? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js."
: "";
throw new Error(
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`,
);
}
const original = JSON.stringify(
@@ -93,8 +100,8 @@ for (const step of [
assert(
result.plannedActions.every(
(a) =>
a.target.startsWith(profile + "/") ||
a.target.startsWith(sharedSkills + "/"),
a.target.startsWith(profile + sep) ||
a.target.startsWith(sharedSkills + sep),
),
);
await writeFile(
@@ -163,7 +170,14 @@ const fixture = await createOpenAiModelFixture({
skillSeen ||= JSON.stringify(body.messages ?? []).includes(
"reverse-engineer-anything",
);
selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
const messageText = (body.messages ?? [])
.map((message) =>
typeof message.content === "string"
? message.content
: (message.content ?? []).map((part) => part.text ?? "").join("\n"),
)
.join("\n");
selectedSkillLoaded ||= messageText.includes(
join(sharedSkills, "reverse-engineer-anything"),
);
const toolText = JSON.stringify(
@@ -243,9 +257,7 @@ const fixture = await createOpenAiModelFixture({
const invoke = tools.find((t) => t.name === "tool_call");
const skill = tools.find((t) => t.name === "skill");
const shell = tools.find((t) => t.name === "run_shell_command");
const persisted = toolText.match(
/Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
);
const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u);
const call = (name, args) => ({
role: "assistant",
content: null,
@@ -268,13 +280,14 @@ const fixture = await createOpenAiModelFixture({
artifactPath = persisted[1];
assert(
(await realpath(artifactPath)).startsWith(
(await realpath(runtime)) + "/",
(await realpath(runtime)) + sep,
),
);
const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
// Keep paths and JSON quotes out of shell syntax on Windows and POSIX.
const encodedScript = Buffer.from(script).toString("base64");
message = call(shell.name, {
command: `node -e ${quote(script)}`,
command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`,
description:
"Read the complete saved REA Evidence and verify the fixture facts.",
});
@@ -329,7 +342,7 @@ const args = [
let response;
let failure;
try {
response = await exec(command, args, {
response = await exec(...launch(args), {
env,
cwd: workspace,
timeout: 120_000,
+479
View File
@@ -0,0 +1,479 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import {
mkdir,
mkdtemp,
readFile,
readdir,
rm,
symlink,
unlink,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../dist/domain/evidence.js";
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.argv.length !== 2)
throw new Error("Usage: node scripts/verify-real-ghidra-6502.mjs");
if (process.env.GHIDRA_INSTALL_DIR === undefined)
throw new Error(
"Set GHIDRA_INSTALL_DIR to a bring-your-own Ghidra installation.",
);
const exec = promisify(execFile);
const run = createVerifierRun();
const root = fileURLToPath(new URL("..", import.meta.url));
const ca65 = process.env.REA_CA65 ?? "ca65";
const ld65 = process.env.REA_LD65 ?? "ld65";
const toolchain = {};
for (const [name, command] of Object.entries({ ca65, ld65 })) {
const version = await exec(command, ["--version"], { timeout: 10000 });
toolchain[name] = (version.stdout + version.stderr).trim();
}
await mkdir(join(root, "build"), { recursive: true });
const workspace = await mkdtemp(join(root, "build", "rea-6502-proof-"));
const runtime = join(workspace, "runtime");
await mkdir(runtime);
const path = join(workspace, "image.bin");
const object = join(workspace, "image.o");
const entrypoint = join(root, "scripts", "rea.mjs");
const env = {
...process.env,
TMPDIR: runtime,
TMP: runtime,
TEMP: runtime,
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "ghidra",
REA_PROCESS_RUN_ID: run.run_id,
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env,
stderr: "pipe",
});
const client = new Client({ name: "rea-real-6502-proof", version: "1" });
let stderr = "";
transport.stderr?.on("data", (chunk) => {
stderr = (stderr + chunk.toString()).slice(-65536);
});
let report;
try {
await exec(
ca65,
[join(root, "tests/conformance/ghidra/6502.s"), "-o", object],
{ timeout: 10000 },
);
await exec(
ld65,
["-C", join(root, "tests/conformance/ghidra/6502.cfg"), "-o", path, object],
{ timeout: 10000 },
);
const bytes = await readFile(path);
// Independently known opcodes and link addresses from the source fixture.
assert.equal(bytes.toString("hex"), "a907201080c90ad003a92a60a9006018690360");
const sha256 = createHash("sha256").update(bytes).digest("hex");
await client.connect(transport);
const opened = await call("open_binary", {
path,
format: "raw-binary",
architecture: "6502",
entry_point: "0x008001",
provider_id: "ghidra",
});
assert.equal(opened.architecture, "6502");
assert.equal(opened.entry_point, "0x8001");
const status = await call("binary_session");
const image = evidence(await call("inspect_native_load_image"), sha256);
assert.equal(
image.normalized_result.status,
"verified",
JSON.stringify(image.normalized_result),
);
assert.equal(image.normalized_result.architecture, "6502");
assert.equal(image.normalized_result.base_address, "0x8001");
assert.equal(image.normalized_result.language_id, "6502:LE:16:default");
assert.equal(image.normalized_result.compiler_spec_id, "default");
assert.equal(image.normalized_result.file_bytes, bytes.length);
const memory = evidence(
await call("read_bytes", { address: "0x8001", length: bytes.length }),
sha256,
);
assert.equal(memory.normalized_result.bytes_hex, bytes.toString("hex"));
const procedures = evidence(await call("list_procedures"), sha256);
for (const address of ["0x8001", "0x8010"])
assert.ok(
procedures.normalized_result.some(
(procedure) => procedure.address === address,
),
);
const instruction = evidence(
await call("inspect_native_instruction", { address: "0x8003" }),
sha256,
);
assert.equal(instruction.normalized_result.mnemonic.toUpperCase(), "JSR");
assert.ok(
instruction.normalized_result.references.some(
(reference) => reference.call && reference.target_address === "0x8010",
),
);
const pseudo = evidence(
await call("procedure_pseudo_code", { procedure: "0x8010" }),
sha256,
);
assert.match(pseudo.normalized_result, /\+\s*(?:3|0x3|'\\x03')/u);
const filePseudo = evidence(
await call("procedure_pseudo_code", { procedure: "file-offset:0xf" }),
sha256,
);
assert.equal(filePseudo.normalized_result, pseudo.normalized_result);
assert.ok(
filePseudo.locations.some(
(location) => location.kind === "file-offset" && location.offset === 15,
),
);
const rejected = await client.callTool({
name: "open_binary",
arguments: {
path,
format: "raw-binary",
architecture: "6502",
entry_point: "0xffff",
},
});
assert.equal(rejected.isError, true);
assert.match(JSON.stringify(rejected.content), /6502 address width/u);
const preserved = await call("binary_session");
assert.equal(preserved.analysis_run.run_id, status.analysis_run.run_id);
assert.equal(preserved.entry_point, "0x8001");
await call("close_binary");
await client.ping();
const { stdout } = await exec(
process.execPath,
[
entrypoint,
"inspect-native-load-image",
path,
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
"0x8001",
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const cli = parseEvidence(JSON.parse(stdout));
assert.deepEqual(cli.normalized_result, image.normalized_result);
assert.equal(cli.analysis_profile.digest, image.analysis_profile.digest);
assert.equal(cli.subject.digest.sha256, sha256);
const fileCli = await exec(
process.execPath,
[
entrypoint,
"decompile",
path,
"--file-offset=0xf",
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
"0x8001",
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const fileCliEvidence = parseEvidence(JSON.parse(fileCli.stdout));
assert.equal(fileCliEvidence.normalized_result, filePseudo.normalized_result);
assert.deepEqual(fileCliEvidence.locations, filePseudo.locations);
assert.equal(fileCliEvidence.subject.digest.sha256, sha256);
assert.deepEqual(await readFile(path), bytes);
assert.deepEqual(await readdir(runtime), []);
const lowMemory = [];
// BinaryLoader maps file bytes before trying processor-defined blocks. Prove
// that overlapping zero-page/stack definitions never overlay the selected image.
for (const [entry, fullAddressSpace] of [
[0xff, false],
[0, true],
]) {
const lowPath = join(workspace, `low-${entry}.bin`);
await exec(
ld65,
[
"-C",
join(root, "tests/conformance/ghidra/6502.cfg"),
"--start-addr",
String(entry),
"-o",
lowPath,
object,
],
{ timeout: 10000 },
);
const linked = await readFile(lowPath);
const expectedCode = Buffer.from(
"a907200000c90ad003a92a60a9006018690360",
"hex",
);
expectedCode.writeUInt16LE(entry + 15, 3);
assert.deepEqual(linked, expectedCode);
const lowBytes = fullAddressSpace
? Buffer.concat([linked, Buffer.alloc(65536 - linked.length)])
: linked;
await writeFile(lowPath, lowBytes);
const lowSha256 = createHash("sha256").update(lowBytes).digest("hex");
const lowEntry = `0x${entry.toString(16)}`;
const lowOpened = await call("open_binary", {
path: lowPath,
format: "raw-binary",
architecture: "6502",
entry_point: lowEntry,
provider_id: "ghidra",
});
assert.equal(lowOpened.entry_point, lowEntry);
const lowImage = evidence(
await call("inspect_native_load_image"),
lowSha256,
);
assert.equal(
lowImage.normalized_result.status,
"verified",
JSON.stringify(lowImage.normalized_result),
);
assert.equal(lowImage.normalized_result.base_address, lowEntry);
assert.equal(lowImage.normalized_result.file_bytes, lowBytes.length);
const fileMappings =
lowImage.normalized_result.observations.mappings.filter(
(mapping) => mapping.source_file_index !== null,
);
assert.equal(fileMappings.length, 1);
assert.equal(
fileMappings[0].address_space,
lowImage.normalized_result.observations.default_address_space,
);
assert.equal(fileMappings[0].overlay, false);
assert.equal(fileMappings[0].start, lowEntry);
assert.equal(fileMappings[0].length, lowBytes.length);
const lowRead = evidence(
await call("read_bytes", { address: lowEntry, length: linked.length }),
lowSha256,
);
assert.equal(lowRead.normalized_result.bytes_hex, linked.toString("hex"));
const lowCall = evidence(
await call("inspect_native_instruction", {
address: `0x${(entry + 2).toString(16)}`,
}),
lowSha256,
);
assert.equal(lowCall.normalized_result.mnemonic.toUpperCase(), "JSR");
assert.ok(
lowCall.normalized_result.references.some(
(reference) =>
reference.call &&
reference.target_address === `0x${(entry + 15).toString(16)}`,
),
);
await call("close_binary");
const lowCli = await exec(
process.execPath,
[
entrypoint,
"inspect-native-load-image",
lowPath,
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
lowEntry,
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const lowCliEvidence = evidence(JSON.parse(lowCli.stdout), lowSha256);
assert.deepEqual(
lowCliEvidence.normalized_result,
lowImage.normalized_result,
);
assert.equal(
lowCliEvidence.analysis_profile.digest,
lowImage.analysis_profile.digest,
);
assert.deepEqual(await readFile(lowPath), lowBytes);
assert.deepEqual(await readdir(runtime), []);
lowMemory.push({
entry: lowEntry,
bytes: lowBytes.length,
sha256: lowSha256,
});
}
let windowsReparseAdmission = "not-applicable";
if (process.platform === "win32") {
const junction = join(workspace, "linked");
await symlink(workspace, junction, "junction");
try {
const linkedPath = join(junction, "image.bin");
// Reopening the same bytes through a junction must replace the direct session.
await call("open_binary", {
path,
format: "raw-binary",
architecture: "6502",
entry_point: "0x8001",
provider_id: "ghidra",
});
const directImage = evidence(
await call("inspect_native_load_image"),
sha256,
);
assert.equal(directImage.normalized_result.status, "verified");
// Canonical file identity must not bypass admission of the new path.
await call("open_binary", {
path: linkedPath,
format: "raw-binary",
architecture: "6502",
entry_point: "0x8001",
provider_id: "ghidra",
});
// open_binary selects a lazy session; the first analysis admits the snapshot.
const linkedImage = await client.callTool(
{ name: "inspect_native_load_image", arguments: {} },
{ timeout: 240000 },
);
assert.equal(linkedImage.isError, true);
assert.match(
JSON.stringify(linkedImage),
/Reparse path component is unsupported/u,
);
await call("close_binary");
const snapshotPath = join(workspace, "direct-snapshot.json");
const cliArgs = (selectedPath) => [
entrypoint,
"analyze",
selectedPath,
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
"0x8001",
"--provider",
"ghidra",
"--snapshot",
snapshotPath,
"--json",
];
const saved = await exec(process.execPath, cliArgs(path), {
env,
timeout: 240000,
maxBuffer: 16 * 1024 * 1024,
});
assert.equal(
evidence(JSON.parse(saved.stdout), sha256).operation,
"binary_overview",
);
const savedSnapshot = JSON.parse(await readFile(snapshotPath, "utf8"));
assert.ok(
savedSnapshot.workflow_entries.some(
({ operation }) => operation === "binary_overview",
),
);
assert.deepEqual(await readdir(runtime), []);
// Replaying a matching snapshot must still reach native path admission.
await assert.rejects(
exec(process.execPath, cliArgs(linkedPath), {
env,
timeout: 240000,
maxBuffer: 16 * 1024 * 1024,
}),
(cause) => {
assert.match(
cause.stdout + cause.stderr,
/Reparse path component is unsupported/u,
);
return true;
},
);
assert.deepEqual(await readFile(path), bytes);
assert.deepEqual(await readdir(runtime), []);
windowsReparseAdmission = "cli-snapshot-mcp-rejected";
} finally {
await unlink(junction);
}
}
report = {
ok: true,
mocked: false,
lane: "ghidra-6502",
host: process.platform,
toolchain,
provider: image.provider,
target: {
sha256,
bytes: bytes.length,
architecture: "6502",
entry: "0x8001",
},
cli_mcp_load_image_parity: true,
instructions_and_call_reference: true,
decompilation: true,
file_offset_decompilation_cli_mcp: true,
invalid_mapping_preserves_session: true,
low_memory_cli_mcp_parity: lowMemory,
windows_reparse_admission: windowsReparseAdmission,
source_unchanged: true,
runtime_cleanup: "complete",
};
} catch (cause) {
if (stderr) process.stderr.write(stderr);
throw cause;
} finally {
try {
await client.close();
} finally {
await transport.close();
}
}
const verifierRun = await completeVerifierRun(run);
if (process.platform !== "win32") {
assert.equal(verifierRun.process_lineage.status, "verified");
assert.deepEqual(verifierRun.process_lineage.descendants, []);
}
// Keep the workspace for diagnosis unless verification and cleanup both pass.
await rm(workspace, { recursive: true, force: true });
process.stdout.write(
JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n",
);
async function call(name, args = {}) {
const result = await client.callTool(
{ name, arguments: args },
{ timeout: 240000 },
);
assert.notEqual(result.isError, true, JSON.stringify(result));
if (["open_binary", "binary_session", "close_binary"].includes(name))
return requireMcpOperationResult(result, name);
return result.structuredContent;
}
function evidence(value, sha256) {
const result = parseEvidence(value);
assert.equal(result.subject.digest.sha256, sha256);
assert.equal(result.subject.architecture, "6502");
assert.equal(result.subject.format, "raw-binary");
return result;
}
+68
View File
@@ -45,8 +45,13 @@ const transport = new StdioClientTransport({
const source = fileURLToPath(
new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url),
);
const literalSource = fileURLToPath(
new URL("../tests/conformance/swift/string-literals.swift", import.meta.url),
);
const binary = join(root, "swift-inventory");
const literalBinary = join(root, "swift-literals");
const reports = [];
let literalReport;
try {
await execute("swiftc", ["-O", "-g", source, "-o", binary], {
timeout: 60_000,
@@ -54,6 +59,19 @@ try {
await mkdir(join(root, "copied"));
const copied = join(root, "copied", "swift-inventory");
await copyFile(binary, copied);
// Decoding covers AArch64 register pairs, so build that slice on any Mac host.
await execute(
"swiftc",
[
"-O",
"-target",
"arm64-apple-macosx13.0",
literalSource,
"-o",
literalBinary,
],
{ timeout: 60_000 },
);
await client.connect(transport);
for (const [variant, path] of [
["debug", binary],
@@ -132,6 +150,7 @@ try {
await call("close_binary", {});
}
}
literalReport = await verifyStringLiterals();
} finally {
await client.close();
await transport.close();
@@ -143,6 +162,7 @@ console.log(
ok: true,
provider: "ghidra",
fixtures: reports,
swift_string_literals: literalReport,
verifier_run: await completeVerifierRun(run),
},
null,
@@ -159,3 +179,51 @@ async function call(name, arguments_) {
toolContract(name).outputSchema.parse(response.structuredContent);
return requireMcpOperationResult(response, name);
}
// Literal text lives only in register constants, so it must come from the decoded pairs.
async function verifyStringLiterals() {
await call("open_binary", { path: literalBinary, provider_id: "ghidra" });
try {
const procedures = await call("list_procedures", {});
const expected = {
smallLiteral: [["SleepDisabled", "small"]],
largeLiteral: [["The power setting could not be changed.", "large"]],
unicodeLiterals: [
["café", "small"],
["Résumé of the naïve décor, written twice", "large"],
],
emptyAndControlLiterals: [
["", "small"],
["tab\there", "small"],
],
returnedLiteral: [["Holding awake", "small"]],
};
const decoded = {};
for (const [name, literals] of Object.entries(expected)) {
const procedure = procedures.find(({ value }) => value.includes(name));
assert(procedure, `${name} is missing from the procedure inventory`);
const dossier = await call("analyze_function", {
procedure: procedure.address,
});
const observed = dossier.swift_string_literals;
assert(Array.isArray(observed), `${name} recorded no Swift literals`);
assert.deepEqual(dossier.swift_string_literal_omissions, []);
assert.deepEqual(
observed.map(({ value, representation }) => [value, representation]),
literals,
);
for (const literal of observed) {
assert.deepEqual(literal.registers, ["x0", "x1"]);
assert.equal(
literal.address === null,
literal.representation === "small",
`${literal.value} storage does not match its representation`,
);
}
decoded[name] = observed.length;
}
return decoded;
} finally {
await call("close_binary", {});
}
}
+60 -1
View File
@@ -1,4 +1,8 @@
#!/usr/bin/env node
import {
assertFeatureTrace,
findMappedNonFunction,
} from "./lib/ida-feature-trace.mjs";
import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs";
import assert from "node:assert/strict";
@@ -6,7 +10,7 @@ import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { access, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { basename, dirname, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { parseArgs, promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@@ -261,6 +265,61 @@ const mcp = async (cliDossier) => {
break;
}
}
if (config.mode === "headless") {
const documents = observe(await call("list_documents", {}));
assert.equal(documents.length, 1);
assert.equal(basename(documents[0]), basename(target));
assert.ok(
!workspaces.has(dirname(documents[0])),
"list_documents reported the private workspace copy.",
);
const resolveAt = async (address) =>
observe(await call("resolve_containing_procedure", { address }));
const instructionAddresses = dossier.assembly
.map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1])
.filter((address) => address !== undefined);
const queries = [
dossier.procedure.address,
...instructionAddresses.slice(1, 2),
];
for (const queried of queries) {
const containing = await resolveAt(queried);
assert.equal(containing.found, true);
assert.equal(containing.procedure.address, dossier.procedure.address);
assert.equal(containing.procedure.name, dossier.procedure.name);
}
const unmapped = await resolveAt("0xffffffffffffff00");
assert.equal(unmapped.found, false);
assert.equal(unmapped.reason, "outside_segments");
const strings = observe(await call("list_strings", {}));
const mappedString = await findMappedNonFunction(strings, resolveAt);
if (mappedString === undefined)
report.unverified.push(
"Mapped non-function containment: no string of the target is mapped outside every function.",
);
const context = await client.callTool({
name: "inspect_address_context",
arguments: { address: dossier.procedure.address },
});
assert.notEqual(context.isError, true);
const contextResult = context.structuredContent.result;
assert.equal(contextResult.document, documents[0]);
assert.equal(contextResult.procedure.state, "available");
assert.equal(
contextResult.procedure.value.procedure.address,
dossier.procedure.address,
);
const traced = await client.callTool({
name: "trace_feature",
arguments: { query: dossier.procedure.name },
});
assert.notEqual(traced.isError, true);
const trace = traced.structuredContent.normalized_result;
assertFeatureTrace(trace, dossier.procedure);
report.checks.push(
"headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing",
);
}
if (values["expected-string"] === undefined)
report.unverified.push(
"String inventory/search correctness: no independent expected literal supplied.",
+2 -2
View File
@@ -27,9 +27,9 @@ if (
);
const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length;
if (
bytes.toString("ascii", 0, 2) !== "MZ" ||
bytes.toString("latin1", 0, 2) !== "MZ" ||
offset > bytes.length - 26 ||
bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" ||
bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" ||
bytes.readUInt16LE(offset + 4) !== 0x8664 ||
bytes.readUInt16LE(offset + 24) !== 0x20b
)
+298 -204
View File
@@ -1,15 +1,8 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import {
access,
mkdir,
mkdtemp,
readFile,
rm,
symlink,
} from "node:fs/promises";
import * as fs from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import * as nodePath from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
import { parseEvidence } from "../../../dist/domain/evidence.js";
import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js";
const exec = promisify(execFile);
const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const entrypoint = resolve(
const execFileAsync = promisify(execFile);
const { join, resolve, dirname } = nodePath;
const CALL_TIMEOUT_MS = 150_000;
const CONNECT_TIMEOUT_MS = 30_000;
const SEVENZIP_TIMEOUT_MS = 10_000;
const MAX_STDOUT_BYTES = 16 * 1024 * 1024;
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const ENTRYPOINT = resolve(
process.env.REA_FIRMWARE_TEST_ENTRYPOINT ??
join(repository, "scripts/rea.mjs"),
join(REPO_ROOT, "scripts/rea.mjs"),
);
if (process.platform !== "linux")
throw new Error(
"verify:firmware requires Linux; other hosts are not verified",
);
for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
const path = process.env[variable];
if (path === undefined)
throw new Error(
`verify:firmware requires ${variable}; see docs/firmware-analysis.md`,
);
await access(path);
}
try {
await exec("7z", ["i"], { timeout: 10_000 });
} catch (cause) {
throw new Error(
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
{ cause },
);
}
const fixtureRoot = resolve(
const FIXTURE_ROOT = resolve(
process.env.REA_FIRMWARE_FIXTURE_ROOT ??
join(repository, "_reference/firmware-integration/generated"),
join(REPO_ROOT, "_reference/firmware-integration/generated"),
);
await access(join(fixtureRoot, "oracle.json")).catch((cause) => {
throw new Error(
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
{ cause },
);
});
// ---------------------------------------------------------------------------
// Prerequisite checks
// ---------------------------------------------------------------------------
async function checkPrerequisites() {
if (process.platform !== "linux") {
throw new Error(
"verify:firmware requires Linux; other hosts are not verified",
);
}
for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
const value = process.env[name];
if (value === undefined) {
throw new Error(
`verify:firmware requires ${name}; see docs/firmware-analysis.md`,
);
}
await fs.access(value);
}
try {
await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS });
} catch (cause) {
throw new Error(
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
{ cause },
);
}
try {
await fs.access(join(FIXTURE_ROOT, "oracle.json"));
} catch (cause) {
throw new Error(
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
{ cause },
);
}
}
await checkPrerequisites();
// ---------------------------------------------------------------------------
// Shared state
// ---------------------------------------------------------------------------
const oracle = JSON.parse(
await readFile(join(fixtureRoot, "oracle.json"), "utf8"),
await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"),
);
const path = join(fixtureRoot, "firmware.bin");
assert.equal(await hashFirmwareFile(path), oracle.sha256);
const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-"));
const environment = Object.fromEntries(
Object.entries(process.env).filter(([, value]) => value !== undefined),
const firmwarePath = join(FIXTURE_ROOT, "firmware.bin");
assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256);
const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-"));
const baseEnv = Object.fromEntries(
Object.entries(process.env).filter(([, v]) => v !== undefined),
);
const client = new Client({ name: "rea-real-firmware-verifier", version: "1" });
const cli = async (args, env = environment) => {
const response = await exec(
const mcpClient = new Client({
name: "rea-real-firmware-verifier",
version: "1",
});
// ---------------------------------------------------------------------------
// Invocation helpers
// ---------------------------------------------------------------------------
async function runCli(args, env = baseEnv) {
const { stdout } = await execFileAsync(
process.execPath,
[entrypoint, ...args, "--format", "json"],
[ENTRYPOINT, ...args, "--format", "json"],
{
env,
cwd: repository,
timeout: 150_000,
maxBuffer: 16 * 1024 * 1024,
cwd: REPO_ROOT,
timeout: CALL_TIMEOUT_MS,
maxBuffer: MAX_STDOUT_BYTES,
},
);
return parseEvidence(JSON.parse(response.stdout));
};
const call = async (name, args) => {
const response = await client.callTool(
return parseEvidence(JSON.parse(stdout));
}
async function callTool(name, args) {
const result = await mcpClient.callTool(
{ name, arguments: args },
{ timeout: 150_000 },
{ timeout: CALL_TIMEOUT_MS },
);
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(response.structuredContent !== undefined);
return response.structuredContent;
};
const verifyRegions = (evidence) => {
assert.notEqual(result.isError, true, JSON.stringify(result));
assert.ok(result.structuredContent !== undefined);
return result.structuredContent;
}
// ---------------------------------------------------------------------------
// Result verifiers
// ---------------------------------------------------------------------------
function verifyRegions(evidence) {
assert.equal(evidence.subject.digest.sha256, oracle.sha256);
const result = firmwareResultSchemas.inspect_firmware_regions.parse(
const parsed = firmwareResultSchemas.inspect_firmware_regions.parse(
evidence.normalized_result,
);
const gzip = result.regions.find(
(region) => region.signature === "gzip" && region.offset === oracle.offset,
const gzipRegion = parsed.regions.find(
(r) => r.signature === "gzip" && r.offset === oracle.offset,
);
assert.ok(
gzip !== undefined,
gzipRegion !== undefined,
"Binwalk must find the independently generated gzip offset",
);
assert.equal(gzip.reported_size, oracle.length);
assert.equal(gzip.size_basis, "provider_reported_validation_unknown");
return result.regions.map(
({ provider_id: _provider_id, ...region }) => region,
);
};
const verifyFiles = async (evidence) => {
const result = firmwareResultSchemas.extract_firmware.parse(
assert.equal(gzipRegion.reported_size, oracle.length);
assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown");
// Strip provider-specific ids so CLI and MCP results compare equal.
return parsed.regions.map((region) => {
const { provider_id: _ignored, ...rest } = region;
return rest;
});
}
async function verifyFiles(evidence) {
const parsed = firmwareResultSchemas.extract_firmware.parse(
evidence.normalized_result,
);
assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip"));
assert.ok(parsed.chunks.some((c) => c.handler === "gzip"));
for (const [suffix, expected] of Object.entries(oracle.files)) {
const file = result.files.find((candidate) =>
candidate.relative_path.endsWith(`/${suffix}`),
const entry = parsed.files.find((f) =>
f.relative_path.endsWith(`/${suffix}`),
);
assert.ok(file !== undefined, `Missing extracted ${suffix}`);
assert.equal(file.sha256, expected.sha256);
assert.equal(file.size, expected.size);
assert.equal(await hashFirmwareFile(file.path), expected.sha256);
assert.equal(file.original_file_range, null);
assert.equal(file.runtime_address, null);
assert.ok(entry !== undefined, `Missing extracted ${suffix}`);
assert.equal(entry.sha256, expected.sha256);
assert.equal(entry.size, expected.size);
assert.equal(await hashFirmwareFile(entry.path), expected.sha256);
assert.equal(entry.original_file_range, null);
assert.equal(entry.runtime_address, null);
}
return result;
};
try {
const inspected = await cli(["inspect-firmware-regions", path]);
const cliRegions = verifyRegions(inspected);
const extracted = await cli([
"extract-firmware",
path,
join(root, "cli-output"),
return parsed;
}
const toContentIdentity = (parsed) =>
parsed.files.map(({ relative_path, sha256, size }) => ({
relative_path,
sha256,
size,
}));
// ---------------------------------------------------------------------------
// Verification stages
// ---------------------------------------------------------------------------
async function stageCli() {
const regionsEvidence = await runCli([
"inspect-firmware-regions",
firmwarePath,
]);
const cliFiles = await verifyFiles(extracted);
assert.equal(cliFiles.coverage, "partial");
const regions = verifyRegions(regionsEvidence);
const extractEvidence = await runCli([
"extract-firmware",
firmwarePath,
join(workDir, "cli-output"),
]);
const files = await verifyFiles(extractEvidence);
assert.equal(files.coverage, "partial");
assert.ok(
cliFiles.chunks.some(
(chunk) =>
chunk.handler === null &&
chunk.range.offset === 0 &&
chunk.range.length === 64,
files.chunks.some(
(c) =>
c.handler === null && c.range.offset === 0 && c.range.length === 64,
),
);
await client.connect(
new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
cwd: repository,
env: environment,
stderr: "pipe",
}),
{ timeout: 30_000 },
);
const inspectedMcp = await call("inspect_firmware_regions", { path });
const mcpEvidence = parseEvidence(inspectedMcp);
assert.deepEqual(verifyRegions(mcpEvidence), cliRegions);
const extractedMcp = await call("extract_firmware", {
path,
output_directory: join(root, "mcp-output"),
return { regions, files };
}
async function connectMcp() {
const transport = new StdioClientTransport({
command: process.execPath,
args: [ENTRYPOINT, "mcp"],
cwd: REPO_ROOT,
env: baseEnv,
stderr: "pipe",
});
const mcpFiles = await verifyFiles(parseEvidence(extractedMcp));
const contentIdentity = (result) =>
result.files.map(({ relative_path, sha256, size }) => ({
relative_path,
sha256,
size,
}));
assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles));
const selected = await call("extract_firmware", {
path,
output_directory: join(root, "selected-output"),
await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS });
}
async function stageMcp(cli) {
const regionsRaw = await callTool("inspect_firmware_regions", {
path: firmwarePath,
});
assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions);
const extractRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "mcp-output"),
});
const mcpFiles = await verifyFiles(parseEvidence(extractRaw));
assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files));
const rangedRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "selected-output"),
range: { offset: oracle.offset, length: oracle.length },
});
const selectedFiles = await verifyFiles(parseEvidence(selected));
assert.equal(selectedFiles.selection.offset, oracle.offset);
assert.equal(selectedFiles.selection.length, oracle.length);
const depth = await call("extract_firmware", {
path,
output_directory: join(root, "depth-output"),
const ranged = await verifyFiles(parseEvidence(rangedRaw));
assert.equal(ranged.selection.offset, oracle.offset);
assert.equal(ranged.selection.length, oracle.length);
const depthRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "depth-output"),
max_depth: 1,
});
const depthResult = firmwareResultSchemas.extract_firmware.parse(
depth.normalized_result,
const depthParsed = firmwareResultSchemas.extract_firmware.parse(
depthRaw.normalized_result,
);
assert.equal(depthResult.coverage, "partial");
assert.ok(depthResult.depth_limited_paths.length > 0);
assert.equal(depthParsed.coverage, "partial");
assert.ok(depthParsed.depth_limited_paths.length > 0);
console.log(
"PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits",
);
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") {
const ext4 = join(fixtureRoot, "ext4.bin");
await access(ext4);
const response = await call("extract_firmware", {
path: ext4,
output_directory: join(root, "ext4-output"),
});
const result = firmwareResultSchemas.extract_firmware.parse(
response.normalized_result,
return mcpFiles;
}
async function stageExt4() {
const ext4Path = join(FIXTURE_ROOT, "ext4.bin");
await fs.access(ext4Path);
const raw = await callTool("extract_firmware", {
path: ext4Path,
output_directory: join(workDir, "ext4-output"),
});
const parsed = firmwareResultSchemas.extract_firmware.parse(
raw.normalized_result,
);
assert.ok(parsed.chunks.some((c) => c.handler === "extfs"));
assert.equal(parsed.coverage, "complete");
for (const [suffix, expected] of Object.entries(oracle.files)) {
const match = parsed.files.find((f) =>
f.relative_path.endsWith(`/${suffix}`),
);
assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs"));
assert.equal(result.coverage, "complete");
for (const [suffix, expected] of Object.entries(oracle.files))
assert.equal(
result.files.find((file) => file.relative_path.endsWith(`/${suffix}`))
?.sha256,
expected.sha256,
);
console.log("PASS real ext4 extraction with debugfs; no mounts");
// Preserve the process-ownership utility while withholding all extractors.
const withoutExtractors = join(root, "without-extractors");
await access("/bin/ps");
await mkdir(withoutExtractors);
await symlink("/bin/ps", join(withoutExtractors, "ps"));
const missing = await cli(
["extract-firmware", ext4, join(root, "missing-extractor-output")],
{ ...environment, PATH: withoutExtractors },
);
const partial = firmwareResultSchemas.extract_firmware.parse(
missing.normalized_result,
);
assert.equal(partial.coverage, "partial");
assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs"));
console.log(
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
);
assert.equal(match?.sha256, expected.sha256);
}
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") {
if (process.env.GHIDRA_INSTALL_DIR === undefined)
throw new Error(
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
);
const child = mcpFiles.files.find((file) =>
file.relative_path.endsWith("/usr/bin/probe"),
);
assert.ok(child !== undefined);
await call("open_binary", { path: child.path, provider_id: "ghidra" });
const dossier = await call("analyze_function", {
procedure: "firmware_probe",
});
const evidence = parseEvidence(dossier);
assert.equal(
evidence.subject.digest.sha256,
oracle.files["usr/bin/probe"].sha256,
);
const pseudocode = await call("procedure_pseudo_code", {
procedure: "firmware_probe",
});
assert.equal(parseEvidence(pseudocode).provider.id, "ghidra");
assert.ok(
JSON.stringify(evidence.normalized_result).includes("firmware_probe"),
);
await call("close_binary", {});
console.log(
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
console.log("PASS real ext4 extraction with debugfs; no mounts");
// Keep the process-ownership utility available but hide every extractor.
const emptyBin = join(workDir, "without-extractors");
await fs.access("/bin/ps");
await fs.mkdir(emptyBin);
await fs.symlink("/bin/ps", join(emptyBin, "ps"));
const degraded = await runCli(
["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")],
{ ...baseEnv, PATH: emptyBin },
);
const degradedParsed = firmwareResultSchemas.extract_firmware.parse(
degraded.normalized_result,
);
assert.equal(degradedParsed.coverage, "partial");
assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs"));
console.log(
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
);
}
async function stageGhidra(mcpFiles) {
if (process.env.GHIDRA_INSTALL_DIR === undefined) {
throw new Error(
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
);
}
const probe = mcpFiles.files.find((f) =>
f.relative_path.endsWith("/usr/bin/probe"),
);
assert.ok(probe !== undefined);
await callTool("open_binary", { path: probe.path, provider_id: "ghidra" });
const dossierRaw = await callTool("analyze_function", {
procedure: "firmware_probe",
});
const dossier = parseEvidence(dossierRaw);
assert.equal(
await hashFirmwareFile(path),
dossier.subject.digest.sha256,
oracle.files["usr/bin/probe"].sha256,
);
const pseudoRaw = await callTool("procedure_pseudo_code", {
procedure: "firmware_probe",
});
assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra");
assert.ok(
JSON.stringify(dossier.normalized_result).includes("firmware_probe"),
);
await callTool("close_binary", {});
console.log(
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
);
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
try {
const cli = await stageCli();
await connectMcp();
const mcpFiles = await stageMcp(cli);
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4();
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1")
await stageGhidra(mcpFiles);
assert.equal(
await hashFirmwareFile(firmwarePath),
oracle.sha256,
"Original firmware bytes must remain unchanged",
);
} finally {
await client.close();
await rm(root, { recursive: true, force: true });
await mcpClient.close();
await fs.rm(workDir, { recursive: true, force: true });
}