chore(plugins): sync the captured final review base
This commit is contained in:
@@ -56,6 +56,8 @@ export const SCOPES = {
|
||||
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
|
||||
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
|
||||
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
|
||||
"tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts",
|
||||
"src/domain/javascript/javascriptRuntimeStaticCandidates.ts",
|
||||
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
|
||||
"tests/boundary/process/ProcessEvidence.host.test.ts",
|
||||
"tests/acceptance/webModuleCli.test.ts",
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export function assertFeatureTrace(
|
||||
trace: unknown,
|
||||
procedure: { readonly address: string },
|
||||
): void;
|
||||
|
||||
export function findMappedNonFunction(
|
||||
strings: readonly { readonly address: string }[],
|
||||
resolveAt: (address: string) => Promise<unknown>,
|
||||
): Promise<string | undefined>;
|
||||
@@ -0,0 +1,52 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]);
|
||||
|
||||
/**
|
||||
* Assert the `trace_feature` result of the real IDA lane for one selected
|
||||
* procedure. The query is a substring search, so the trace may also match other
|
||||
* procedures and strings; each reference is checked against its own matched
|
||||
* target, and a source outside every function (a data reference) is valid.
|
||||
*/
|
||||
export function assertFeatureTrace(trace, procedure) {
|
||||
assert.ok(
|
||||
trace.matches.some(
|
||||
({ type, address }) =>
|
||||
type === "procedure" && address === procedure.address,
|
||||
),
|
||||
"The trace omitted the selected procedure's match.",
|
||||
);
|
||||
assert.equal(typeof trace.truncated, "boolean");
|
||||
const matched = new Set(trace.matches.map(({ address }) => address));
|
||||
for (const reference of trace.references) {
|
||||
assert.ok(
|
||||
matched.has(reference.target_address),
|
||||
`A reference targets ${reference.target_address}, which is not a match.`,
|
||||
);
|
||||
const { containing_procedure: containing } = reference;
|
||||
if (containing.found)
|
||||
assert.equal(typeof containing.procedure.address, "string");
|
||||
else assert.ok(NEGATIVE_REASONS.has(containing.reason));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a string that IDA maps outside every function. A string inside a
|
||||
* function chunk (inline data in code) is valid and skipped, so the strings are
|
||||
* tried in order. Any other answer must be "not in a procedure": IDA lists only
|
||||
* mapped strings, so an unmapped answer is a containment defect and fails the
|
||||
* lane. Returns the address, or undefined when every string is inside a function.
|
||||
*/
|
||||
export async function findMappedNonFunction(strings, resolveAt) {
|
||||
for (const { address } of strings) {
|
||||
const resolved = await resolveAt(address);
|
||||
if (resolved.found) continue;
|
||||
assert.equal(
|
||||
resolved.reason,
|
||||
"not_in_procedure",
|
||||
`The listed string at ${address} resolved as ${resolved.reason}.`,
|
||||
);
|
||||
return address;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
realpath,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { join, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
@@ -18,10 +18,6 @@ import {
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
|
||||
@@ -32,6 +28,13 @@ assert(
|
||||
"Usage: node scripts/verify-qwen-client.mjs [chat|call]",
|
||||
);
|
||||
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
|
||||
// npm exposes only qwen.cmd on Windows. Resolve a selected JavaScript entry
|
||||
// before the verifier switches to its disposable workspace.
|
||||
const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined;
|
||||
const launch = (args) =>
|
||||
entryPoint === undefined
|
||||
? [command, args]
|
||||
: [process.execPath, [entryPoint, ...args]];
|
||||
const verifier = createVerifierRun();
|
||||
const profile = join(lab, "selected profile");
|
||||
const account = join(lab, "account");
|
||||
@@ -54,14 +57,18 @@ const baseEnvironment = {
|
||||
let clientVersion;
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], {
|
||||
await exec(...launch(["--version"]), {
|
||||
env: baseEnvironment,
|
||||
timeout: 30_000,
|
||||
})
|
||||
).stdout.trim();
|
||||
} catch (cause) {
|
||||
const windowsHint =
|
||||
process.platform === "win32"
|
||||
? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js."
|
||||
: "";
|
||||
throw new Error(
|
||||
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
|
||||
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`,
|
||||
);
|
||||
}
|
||||
const original = JSON.stringify(
|
||||
@@ -93,8 +100,8 @@ for (const step of [
|
||||
assert(
|
||||
result.plannedActions.every(
|
||||
(a) =>
|
||||
a.target.startsWith(profile + "/") ||
|
||||
a.target.startsWith(sharedSkills + "/"),
|
||||
a.target.startsWith(profile + sep) ||
|
||||
a.target.startsWith(sharedSkills + sep),
|
||||
),
|
||||
);
|
||||
await writeFile(
|
||||
@@ -163,7 +170,14 @@ const fixture = await createOpenAiModelFixture({
|
||||
skillSeen ||= JSON.stringify(body.messages ?? []).includes(
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
|
||||
const messageText = (body.messages ?? [])
|
||||
.map((message) =>
|
||||
typeof message.content === "string"
|
||||
? message.content
|
||||
: (message.content ?? []).map((part) => part.text ?? "").join("\n"),
|
||||
)
|
||||
.join("\n");
|
||||
selectedSkillLoaded ||= messageText.includes(
|
||||
join(sharedSkills, "reverse-engineer-anything"),
|
||||
);
|
||||
const toolText = JSON.stringify(
|
||||
@@ -243,9 +257,7 @@ const fixture = await createOpenAiModelFixture({
|
||||
const invoke = tools.find((t) => t.name === "tool_call");
|
||||
const skill = tools.find((t) => t.name === "skill");
|
||||
const shell = tools.find((t) => t.name === "run_shell_command");
|
||||
const persisted = toolText.match(
|
||||
/Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
|
||||
);
|
||||
const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u);
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
@@ -268,13 +280,14 @@ const fixture = await createOpenAiModelFixture({
|
||||
artifactPath = persisted[1];
|
||||
assert(
|
||||
(await realpath(artifactPath)).startsWith(
|
||||
(await realpath(runtime)) + "/",
|
||||
(await realpath(runtime)) + sep,
|
||||
),
|
||||
);
|
||||
const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
|
||||
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
||||
// Keep paths and JSON quotes out of shell syntax on Windows and POSIX.
|
||||
const encodedScript = Buffer.from(script).toString("base64");
|
||||
message = call(shell.name, {
|
||||
command: `node -e ${quote(script)}`,
|
||||
command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`,
|
||||
description:
|
||||
"Read the complete saved REA Evidence and verify the fixture facts.",
|
||||
});
|
||||
@@ -329,7 +342,7 @@ const args = [
|
||||
let response;
|
||||
let failure;
|
||||
try {
|
||||
response = await exec(command, args, {
|
||||
response = await exec(...launch(args), {
|
||||
env,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
symlink,
|
||||
unlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2)
|
||||
throw new Error("Usage: node scripts/verify-real-ghidra-6502.mjs");
|
||||
if (process.env.GHIDRA_INSTALL_DIR === undefined)
|
||||
throw new Error(
|
||||
"Set GHIDRA_INSTALL_DIR to a bring-your-own Ghidra installation.",
|
||||
);
|
||||
const exec = promisify(execFile);
|
||||
const run = createVerifierRun();
|
||||
const root = fileURLToPath(new URL("..", import.meta.url));
|
||||
const ca65 = process.env.REA_CA65 ?? "ca65";
|
||||
const ld65 = process.env.REA_LD65 ?? "ld65";
|
||||
const toolchain = {};
|
||||
for (const [name, command] of Object.entries({ ca65, ld65 })) {
|
||||
const version = await exec(command, ["--version"], { timeout: 10000 });
|
||||
toolchain[name] = (version.stdout + version.stderr).trim();
|
||||
}
|
||||
await mkdir(join(root, "build"), { recursive: true });
|
||||
const workspace = await mkdtemp(join(root, "build", "rea-6502-proof-"));
|
||||
const runtime = join(workspace, "runtime");
|
||||
await mkdir(runtime);
|
||||
const path = join(workspace, "image.bin");
|
||||
const object = join(workspace, "image.o");
|
||||
const entrypoint = join(root, "scripts", "rea.mjs");
|
||||
const env = {
|
||||
...process.env,
|
||||
TMPDIR: runtime,
|
||||
TMP: runtime,
|
||||
TEMP: runtime,
|
||||
REA_LOG_LEVEL: "silent",
|
||||
REA_ANALYSIS_PROVIDER: "ghidra",
|
||||
REA_PROCESS_RUN_ID: run.run_id,
|
||||
};
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({ name: "rea-real-6502-proof", version: "1" });
|
||||
let stderr = "";
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
stderr = (stderr + chunk.toString()).slice(-65536);
|
||||
});
|
||||
let report;
|
||||
try {
|
||||
await exec(
|
||||
ca65,
|
||||
[join(root, "tests/conformance/ghidra/6502.s"), "-o", object],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
await exec(
|
||||
ld65,
|
||||
["-C", join(root, "tests/conformance/ghidra/6502.cfg"), "-o", path, object],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
const bytes = await readFile(path);
|
||||
// Independently known opcodes and link addresses from the source fixture.
|
||||
assert.equal(bytes.toString("hex"), "a907201080c90ad003a92a60a9006018690360");
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
await client.connect(transport);
|
||||
const opened = await call("open_binary", {
|
||||
path,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0x008001",
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
assert.equal(opened.architecture, "6502");
|
||||
assert.equal(opened.entry_point, "0x8001");
|
||||
const status = await call("binary_session");
|
||||
const image = evidence(await call("inspect_native_load_image"), sha256);
|
||||
assert.equal(
|
||||
image.normalized_result.status,
|
||||
"verified",
|
||||
JSON.stringify(image.normalized_result),
|
||||
);
|
||||
assert.equal(image.normalized_result.architecture, "6502");
|
||||
assert.equal(image.normalized_result.base_address, "0x8001");
|
||||
assert.equal(image.normalized_result.language_id, "6502:LE:16:default");
|
||||
assert.equal(image.normalized_result.compiler_spec_id, "default");
|
||||
assert.equal(image.normalized_result.file_bytes, bytes.length);
|
||||
const memory = evidence(
|
||||
await call("read_bytes", { address: "0x8001", length: bytes.length }),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(memory.normalized_result.bytes_hex, bytes.toString("hex"));
|
||||
const procedures = evidence(await call("list_procedures"), sha256);
|
||||
for (const address of ["0x8001", "0x8010"])
|
||||
assert.ok(
|
||||
procedures.normalized_result.some(
|
||||
(procedure) => procedure.address === address,
|
||||
),
|
||||
);
|
||||
const instruction = evidence(
|
||||
await call("inspect_native_instruction", { address: "0x8003" }),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(instruction.normalized_result.mnemonic.toUpperCase(), "JSR");
|
||||
assert.ok(
|
||||
instruction.normalized_result.references.some(
|
||||
(reference) => reference.call && reference.target_address === "0x8010",
|
||||
),
|
||||
);
|
||||
const pseudo = evidence(
|
||||
await call("procedure_pseudo_code", { procedure: "0x8010" }),
|
||||
sha256,
|
||||
);
|
||||
assert.match(pseudo.normalized_result, /\+\s*(?:3|0x3|'\\x03')/u);
|
||||
const filePseudo = evidence(
|
||||
await call("procedure_pseudo_code", { procedure: "file-offset:0xf" }),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(filePseudo.normalized_result, pseudo.normalized_result);
|
||||
assert.ok(
|
||||
filePseudo.locations.some(
|
||||
(location) => location.kind === "file-offset" && location.offset === 15,
|
||||
),
|
||||
);
|
||||
const rejected = await client.callTool({
|
||||
name: "open_binary",
|
||||
arguments: {
|
||||
path,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0xffff",
|
||||
},
|
||||
});
|
||||
assert.equal(rejected.isError, true);
|
||||
assert.match(JSON.stringify(rejected.content), /6502 address width/u);
|
||||
const preserved = await call("binary_session");
|
||||
assert.equal(preserved.analysis_run.run_id, status.analysis_run.run_id);
|
||||
assert.equal(preserved.entry_point, "0x8001");
|
||||
await call("close_binary");
|
||||
await client.ping();
|
||||
const { stdout } = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"inspect-native-load-image",
|
||||
path,
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
"0x8001",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const cli = parseEvidence(JSON.parse(stdout));
|
||||
assert.deepEqual(cli.normalized_result, image.normalized_result);
|
||||
assert.equal(cli.analysis_profile.digest, image.analysis_profile.digest);
|
||||
assert.equal(cli.subject.digest.sha256, sha256);
|
||||
const fileCli = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"decompile",
|
||||
path,
|
||||
"--file-offset=0xf",
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
"0x8001",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const fileCliEvidence = parseEvidence(JSON.parse(fileCli.stdout));
|
||||
assert.equal(fileCliEvidence.normalized_result, filePseudo.normalized_result);
|
||||
assert.deepEqual(fileCliEvidence.locations, filePseudo.locations);
|
||||
assert.equal(fileCliEvidence.subject.digest.sha256, sha256);
|
||||
assert.deepEqual(await readFile(path), bytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
const lowMemory = [];
|
||||
// BinaryLoader maps file bytes before trying processor-defined blocks. Prove
|
||||
// that overlapping zero-page/stack definitions never overlay the selected image.
|
||||
for (const [entry, fullAddressSpace] of [
|
||||
[0xff, false],
|
||||
[0, true],
|
||||
]) {
|
||||
const lowPath = join(workspace, `low-${entry}.bin`);
|
||||
await exec(
|
||||
ld65,
|
||||
[
|
||||
"-C",
|
||||
join(root, "tests/conformance/ghidra/6502.cfg"),
|
||||
"--start-addr",
|
||||
String(entry),
|
||||
"-o",
|
||||
lowPath,
|
||||
object,
|
||||
],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
const linked = await readFile(lowPath);
|
||||
const expectedCode = Buffer.from(
|
||||
"a907200000c90ad003a92a60a9006018690360",
|
||||
"hex",
|
||||
);
|
||||
expectedCode.writeUInt16LE(entry + 15, 3);
|
||||
assert.deepEqual(linked, expectedCode);
|
||||
const lowBytes = fullAddressSpace
|
||||
? Buffer.concat([linked, Buffer.alloc(65536 - linked.length)])
|
||||
: linked;
|
||||
await writeFile(lowPath, lowBytes);
|
||||
const lowSha256 = createHash("sha256").update(lowBytes).digest("hex");
|
||||
const lowEntry = `0x${entry.toString(16)}`;
|
||||
const lowOpened = await call("open_binary", {
|
||||
path: lowPath,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: lowEntry,
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
assert.equal(lowOpened.entry_point, lowEntry);
|
||||
const lowImage = evidence(
|
||||
await call("inspect_native_load_image"),
|
||||
lowSha256,
|
||||
);
|
||||
assert.equal(
|
||||
lowImage.normalized_result.status,
|
||||
"verified",
|
||||
JSON.stringify(lowImage.normalized_result),
|
||||
);
|
||||
assert.equal(lowImage.normalized_result.base_address, lowEntry);
|
||||
assert.equal(lowImage.normalized_result.file_bytes, lowBytes.length);
|
||||
const fileMappings =
|
||||
lowImage.normalized_result.observations.mappings.filter(
|
||||
(mapping) => mapping.source_file_index !== null,
|
||||
);
|
||||
assert.equal(fileMappings.length, 1);
|
||||
assert.equal(
|
||||
fileMappings[0].address_space,
|
||||
lowImage.normalized_result.observations.default_address_space,
|
||||
);
|
||||
assert.equal(fileMappings[0].overlay, false);
|
||||
assert.equal(fileMappings[0].start, lowEntry);
|
||||
assert.equal(fileMappings[0].length, lowBytes.length);
|
||||
const lowRead = evidence(
|
||||
await call("read_bytes", { address: lowEntry, length: linked.length }),
|
||||
lowSha256,
|
||||
);
|
||||
assert.equal(lowRead.normalized_result.bytes_hex, linked.toString("hex"));
|
||||
const lowCall = evidence(
|
||||
await call("inspect_native_instruction", {
|
||||
address: `0x${(entry + 2).toString(16)}`,
|
||||
}),
|
||||
lowSha256,
|
||||
);
|
||||
assert.equal(lowCall.normalized_result.mnemonic.toUpperCase(), "JSR");
|
||||
assert.ok(
|
||||
lowCall.normalized_result.references.some(
|
||||
(reference) =>
|
||||
reference.call &&
|
||||
reference.target_address === `0x${(entry + 15).toString(16)}`,
|
||||
),
|
||||
);
|
||||
await call("close_binary");
|
||||
const lowCli = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"inspect-native-load-image",
|
||||
lowPath,
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
lowEntry,
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const lowCliEvidence = evidence(JSON.parse(lowCli.stdout), lowSha256);
|
||||
assert.deepEqual(
|
||||
lowCliEvidence.normalized_result,
|
||||
lowImage.normalized_result,
|
||||
);
|
||||
assert.equal(
|
||||
lowCliEvidence.analysis_profile.digest,
|
||||
lowImage.analysis_profile.digest,
|
||||
);
|
||||
assert.deepEqual(await readFile(lowPath), lowBytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
lowMemory.push({
|
||||
entry: lowEntry,
|
||||
bytes: lowBytes.length,
|
||||
sha256: lowSha256,
|
||||
});
|
||||
}
|
||||
let windowsReparseAdmission = "not-applicable";
|
||||
if (process.platform === "win32") {
|
||||
const junction = join(workspace, "linked");
|
||||
await symlink(workspace, junction, "junction");
|
||||
try {
|
||||
const linkedPath = join(junction, "image.bin");
|
||||
// Reopening the same bytes through a junction must replace the direct session.
|
||||
await call("open_binary", {
|
||||
path,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0x8001",
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
const directImage = evidence(
|
||||
await call("inspect_native_load_image"),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(directImage.normalized_result.status, "verified");
|
||||
// Canonical file identity must not bypass admission of the new path.
|
||||
await call("open_binary", {
|
||||
path: linkedPath,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0x8001",
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
// open_binary selects a lazy session; the first analysis admits the snapshot.
|
||||
const linkedImage = await client.callTool(
|
||||
{ name: "inspect_native_load_image", arguments: {} },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.equal(linkedImage.isError, true);
|
||||
assert.match(
|
||||
JSON.stringify(linkedImage),
|
||||
/Reparse path component is unsupported/u,
|
||||
);
|
||||
await call("close_binary");
|
||||
const snapshotPath = join(workspace, "direct-snapshot.json");
|
||||
const cliArgs = (selectedPath) => [
|
||||
entrypoint,
|
||||
"analyze",
|
||||
selectedPath,
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
"0x8001",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--snapshot",
|
||||
snapshotPath,
|
||||
"--json",
|
||||
];
|
||||
const saved = await exec(process.execPath, cliArgs(path), {
|
||||
env,
|
||||
timeout: 240000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
assert.equal(
|
||||
evidence(JSON.parse(saved.stdout), sha256).operation,
|
||||
"binary_overview",
|
||||
);
|
||||
const savedSnapshot = JSON.parse(await readFile(snapshotPath, "utf8"));
|
||||
assert.ok(
|
||||
savedSnapshot.workflow_entries.some(
|
||||
({ operation }) => operation === "binary_overview",
|
||||
),
|
||||
);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
// Replaying a matching snapshot must still reach native path admission.
|
||||
await assert.rejects(
|
||||
exec(process.execPath, cliArgs(linkedPath), {
|
||||
env,
|
||||
timeout: 240000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
}),
|
||||
(cause) => {
|
||||
assert.match(
|
||||
cause.stdout + cause.stderr,
|
||||
/Reparse path component is unsupported/u,
|
||||
);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.deepEqual(await readFile(path), bytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
windowsReparseAdmission = "cli-snapshot-mcp-rejected";
|
||||
} finally {
|
||||
await unlink(junction);
|
||||
}
|
||||
}
|
||||
report = {
|
||||
ok: true,
|
||||
mocked: false,
|
||||
lane: "ghidra-6502",
|
||||
host: process.platform,
|
||||
toolchain,
|
||||
provider: image.provider,
|
||||
target: {
|
||||
sha256,
|
||||
bytes: bytes.length,
|
||||
architecture: "6502",
|
||||
entry: "0x8001",
|
||||
},
|
||||
cli_mcp_load_image_parity: true,
|
||||
instructions_and_call_reference: true,
|
||||
decompilation: true,
|
||||
file_offset_decompilation_cli_mcp: true,
|
||||
invalid_mapping_preserves_session: true,
|
||||
low_memory_cli_mcp_parity: lowMemory,
|
||||
windows_reparse_admission: windowsReparseAdmission,
|
||||
source_unchanged: true,
|
||||
runtime_cleanup: "complete",
|
||||
};
|
||||
} catch (cause) {
|
||||
if (stderr) process.stderr.write(stderr);
|
||||
throw cause;
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
const verifierRun = await completeVerifierRun(run);
|
||||
if (process.platform !== "win32") {
|
||||
assert.equal(verifierRun.process_lineage.status, "verified");
|
||||
assert.deepEqual(verifierRun.process_lineage.descendants, []);
|
||||
}
|
||||
// Keep the workspace for diagnosis unless verification and cleanup both pass.
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
process.stdout.write(
|
||||
JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n",
|
||||
);
|
||||
|
||||
async function call(name, args = {}) {
|
||||
const result = await client.callTool(
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
if (["open_binary", "binary_session", "close_binary"].includes(name))
|
||||
return requireMcpOperationResult(result, name);
|
||||
return result.structuredContent;
|
||||
}
|
||||
|
||||
function evidence(value, sha256) {
|
||||
const result = parseEvidence(value);
|
||||
assert.equal(result.subject.digest.sha256, sha256);
|
||||
assert.equal(result.subject.architecture, "6502");
|
||||
assert.equal(result.subject.format, "raw-binary");
|
||||
return result;
|
||||
}
|
||||
@@ -45,8 +45,13 @@ const transport = new StdioClientTransport({
|
||||
const source = fileURLToPath(
|
||||
new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url),
|
||||
);
|
||||
const literalSource = fileURLToPath(
|
||||
new URL("../tests/conformance/swift/string-literals.swift", import.meta.url),
|
||||
);
|
||||
const binary = join(root, "swift-inventory");
|
||||
const literalBinary = join(root, "swift-literals");
|
||||
const reports = [];
|
||||
let literalReport;
|
||||
try {
|
||||
await execute("swiftc", ["-O", "-g", source, "-o", binary], {
|
||||
timeout: 60_000,
|
||||
@@ -54,6 +59,19 @@ try {
|
||||
await mkdir(join(root, "copied"));
|
||||
const copied = join(root, "copied", "swift-inventory");
|
||||
await copyFile(binary, copied);
|
||||
// Decoding covers AArch64 register pairs, so build that slice on any Mac host.
|
||||
await execute(
|
||||
"swiftc",
|
||||
[
|
||||
"-O",
|
||||
"-target",
|
||||
"arm64-apple-macosx13.0",
|
||||
literalSource,
|
||||
"-o",
|
||||
literalBinary,
|
||||
],
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
await client.connect(transport);
|
||||
for (const [variant, path] of [
|
||||
["debug", binary],
|
||||
@@ -132,6 +150,7 @@ try {
|
||||
await call("close_binary", {});
|
||||
}
|
||||
}
|
||||
literalReport = await verifyStringLiterals();
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
@@ -143,6 +162,7 @@ console.log(
|
||||
ok: true,
|
||||
provider: "ghidra",
|
||||
fixtures: reports,
|
||||
swift_string_literals: literalReport,
|
||||
verifier_run: await completeVerifierRun(run),
|
||||
},
|
||||
null,
|
||||
@@ -159,3 +179,51 @@ async function call(name, arguments_) {
|
||||
toolContract(name).outputSchema.parse(response.structuredContent);
|
||||
return requireMcpOperationResult(response, name);
|
||||
}
|
||||
|
||||
// Literal text lives only in register constants, so it must come from the decoded pairs.
|
||||
async function verifyStringLiterals() {
|
||||
await call("open_binary", { path: literalBinary, provider_id: "ghidra" });
|
||||
try {
|
||||
const procedures = await call("list_procedures", {});
|
||||
const expected = {
|
||||
smallLiteral: [["SleepDisabled", "small"]],
|
||||
largeLiteral: [["The power setting could not be changed.", "large"]],
|
||||
unicodeLiterals: [
|
||||
["café", "small"],
|
||||
["Résumé of the naïve décor, written twice", "large"],
|
||||
],
|
||||
emptyAndControlLiterals: [
|
||||
["", "small"],
|
||||
["tab\there", "small"],
|
||||
],
|
||||
returnedLiteral: [["Holding awake", "small"]],
|
||||
};
|
||||
const decoded = {};
|
||||
for (const [name, literals] of Object.entries(expected)) {
|
||||
const procedure = procedures.find(({ value }) => value.includes(name));
|
||||
assert(procedure, `${name} is missing from the procedure inventory`);
|
||||
const dossier = await call("analyze_function", {
|
||||
procedure: procedure.address,
|
||||
});
|
||||
const observed = dossier.swift_string_literals;
|
||||
assert(Array.isArray(observed), `${name} recorded no Swift literals`);
|
||||
assert.deepEqual(dossier.swift_string_literal_omissions, []);
|
||||
assert.deepEqual(
|
||||
observed.map(({ value, representation }) => [value, representation]),
|
||||
literals,
|
||||
);
|
||||
for (const literal of observed) {
|
||||
assert.deepEqual(literal.registers, ["x0", "x1"]);
|
||||
assert.equal(
|
||||
literal.address === null,
|
||||
literal.representation === "small",
|
||||
`${literal.value} storage does not match its representation`,
|
||||
);
|
||||
}
|
||||
decoded[name] = observed.length;
|
||||
}
|
||||
return decoded;
|
||||
} finally {
|
||||
await call("close_binary", {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
#!/usr/bin/env node
|
||||
import {
|
||||
assertFeatureTrace,
|
||||
findMappedNonFunction,
|
||||
} from "./lib/ida-feature-trace.mjs";
|
||||
import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs";
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
@@ -6,7 +10,7 @@ import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { access, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { parseArgs, promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
@@ -261,6 +265,61 @@ const mcp = async (cliDossier) => {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (config.mode === "headless") {
|
||||
const documents = observe(await call("list_documents", {}));
|
||||
assert.equal(documents.length, 1);
|
||||
assert.equal(basename(documents[0]), basename(target));
|
||||
assert.ok(
|
||||
!workspaces.has(dirname(documents[0])),
|
||||
"list_documents reported the private workspace copy.",
|
||||
);
|
||||
const resolveAt = async (address) =>
|
||||
observe(await call("resolve_containing_procedure", { address }));
|
||||
const instructionAddresses = dossier.assembly
|
||||
.map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1])
|
||||
.filter((address) => address !== undefined);
|
||||
const queries = [
|
||||
dossier.procedure.address,
|
||||
...instructionAddresses.slice(1, 2),
|
||||
];
|
||||
for (const queried of queries) {
|
||||
const containing = await resolveAt(queried);
|
||||
assert.equal(containing.found, true);
|
||||
assert.equal(containing.procedure.address, dossier.procedure.address);
|
||||
assert.equal(containing.procedure.name, dossier.procedure.name);
|
||||
}
|
||||
const unmapped = await resolveAt("0xffffffffffffff00");
|
||||
assert.equal(unmapped.found, false);
|
||||
assert.equal(unmapped.reason, "outside_segments");
|
||||
const strings = observe(await call("list_strings", {}));
|
||||
const mappedString = await findMappedNonFunction(strings, resolveAt);
|
||||
if (mappedString === undefined)
|
||||
report.unverified.push(
|
||||
"Mapped non-function containment: no string of the target is mapped outside every function.",
|
||||
);
|
||||
const context = await client.callTool({
|
||||
name: "inspect_address_context",
|
||||
arguments: { address: dossier.procedure.address },
|
||||
});
|
||||
assert.notEqual(context.isError, true);
|
||||
const contextResult = context.structuredContent.result;
|
||||
assert.equal(contextResult.document, documents[0]);
|
||||
assert.equal(contextResult.procedure.state, "available");
|
||||
assert.equal(
|
||||
contextResult.procedure.value.procedure.address,
|
||||
dossier.procedure.address,
|
||||
);
|
||||
const traced = await client.callTool({
|
||||
name: "trace_feature",
|
||||
arguments: { query: dossier.procedure.name },
|
||||
});
|
||||
assert.notEqual(traced.isError, true);
|
||||
const trace = traced.structuredContent.normalized_result;
|
||||
assertFeatureTrace(trace, dossier.procedure);
|
||||
report.checks.push(
|
||||
"headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing",
|
||||
);
|
||||
}
|
||||
if (values["expected-string"] === undefined)
|
||||
report.unverified.push(
|
||||
"String inventory/search correctness: no independent expected literal supplied.",
|
||||
|
||||
@@ -27,9 +27,9 @@ if (
|
||||
);
|
||||
const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length;
|
||||
if (
|
||||
bytes.toString("ascii", 0, 2) !== "MZ" ||
|
||||
bytes.toString("latin1", 0, 2) !== "MZ" ||
|
||||
offset > bytes.length - 26 ||
|
||||
bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" ||
|
||||
bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" ||
|
||||
bytes.readUInt16LE(offset + 4) !== 0x8664 ||
|
||||
bytes.readUInt16LE(offset + 24) !== 0x20b
|
||||
)
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
access,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
symlink,
|
||||
} from "node:fs/promises";
|
||||
import * as fs from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import * as nodePath from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
@@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
|
||||
import { parseEvidence } from "../../../dist/domain/evidence.js";
|
||||
import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const entrypoint = resolve(
|
||||
const execFileAsync = promisify(execFile);
|
||||
const { join, resolve, dirname } = nodePath;
|
||||
|
||||
const CALL_TIMEOUT_MS = 150_000;
|
||||
const CONNECT_TIMEOUT_MS = 30_000;
|
||||
const SEVENZIP_TIMEOUT_MS = 10_000;
|
||||
const MAX_STDOUT_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const ENTRYPOINT = resolve(
|
||||
process.env.REA_FIRMWARE_TEST_ENTRYPOINT ??
|
||||
join(repository, "scripts/rea.mjs"),
|
||||
join(REPO_ROOT, "scripts/rea.mjs"),
|
||||
);
|
||||
if (process.platform !== "linux")
|
||||
throw new Error(
|
||||
"verify:firmware requires Linux; other hosts are not verified",
|
||||
);
|
||||
for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
|
||||
const path = process.env[variable];
|
||||
if (path === undefined)
|
||||
throw new Error(
|
||||
`verify:firmware requires ${variable}; see docs/firmware-analysis.md`,
|
||||
);
|
||||
await access(path);
|
||||
}
|
||||
try {
|
||||
await exec("7z", ["i"], { timeout: 10_000 });
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
const fixtureRoot = resolve(
|
||||
const FIXTURE_ROOT = resolve(
|
||||
process.env.REA_FIRMWARE_FIXTURE_ROOT ??
|
||||
join(repository, "_reference/firmware-integration/generated"),
|
||||
join(REPO_ROOT, "_reference/firmware-integration/generated"),
|
||||
);
|
||||
await access(join(fixtureRoot, "oracle.json")).catch((cause) => {
|
||||
throw new Error(
|
||||
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
|
||||
{ cause },
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prerequisite checks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function checkPrerequisites() {
|
||||
if (process.platform !== "linux") {
|
||||
throw new Error(
|
||||
"verify:firmware requires Linux; other hosts are not verified",
|
||||
);
|
||||
}
|
||||
|
||||
for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
|
||||
const value = process.env[name];
|
||||
if (value === undefined) {
|
||||
throw new Error(
|
||||
`verify:firmware requires ${name}; see docs/firmware-analysis.md`,
|
||||
);
|
||||
}
|
||||
await fs.access(value);
|
||||
}
|
||||
|
||||
try {
|
||||
await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS });
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(join(FIXTURE_ROOT, "oracle.json"));
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await checkPrerequisites();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const oracle = JSON.parse(
|
||||
await readFile(join(fixtureRoot, "oracle.json"), "utf8"),
|
||||
await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"),
|
||||
);
|
||||
const path = join(fixtureRoot, "firmware.bin");
|
||||
assert.equal(await hashFirmwareFile(path), oracle.sha256);
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-"));
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, value]) => value !== undefined),
|
||||
const firmwarePath = join(FIXTURE_ROOT, "firmware.bin");
|
||||
assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256);
|
||||
|
||||
const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-"));
|
||||
const baseEnv = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, v]) => v !== undefined),
|
||||
);
|
||||
const client = new Client({ name: "rea-real-firmware-verifier", version: "1" });
|
||||
const cli = async (args, env = environment) => {
|
||||
const response = await exec(
|
||||
const mcpClient = new Client({
|
||||
name: "rea-real-firmware-verifier",
|
||||
version: "1",
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Invocation helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function runCli(args, env = baseEnv) {
|
||||
const { stdout } = await execFileAsync(
|
||||
process.execPath,
|
||||
[entrypoint, ...args, "--format", "json"],
|
||||
[ENTRYPOINT, ...args, "--format", "json"],
|
||||
{
|
||||
env,
|
||||
cwd: repository,
|
||||
timeout: 150_000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
cwd: REPO_ROOT,
|
||||
timeout: CALL_TIMEOUT_MS,
|
||||
maxBuffer: MAX_STDOUT_BYTES,
|
||||
},
|
||||
);
|
||||
return parseEvidence(JSON.parse(response.stdout));
|
||||
};
|
||||
const call = async (name, args) => {
|
||||
const response = await client.callTool(
|
||||
return parseEvidence(JSON.parse(stdout));
|
||||
}
|
||||
|
||||
async function callTool(name, args) {
|
||||
const result = await mcpClient.callTool(
|
||||
{ name, arguments: args },
|
||||
{ timeout: 150_000 },
|
||||
{ timeout: CALL_TIMEOUT_MS },
|
||||
);
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(response.structuredContent !== undefined);
|
||||
return response.structuredContent;
|
||||
};
|
||||
const verifyRegions = (evidence) => {
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
assert.ok(result.structuredContent !== undefined);
|
||||
return result.structuredContent;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result verifiers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function verifyRegions(evidence) {
|
||||
assert.equal(evidence.subject.digest.sha256, oracle.sha256);
|
||||
const result = firmwareResultSchemas.inspect_firmware_regions.parse(
|
||||
|
||||
const parsed = firmwareResultSchemas.inspect_firmware_regions.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
const gzip = result.regions.find(
|
||||
(region) => region.signature === "gzip" && region.offset === oracle.offset,
|
||||
|
||||
const gzipRegion = parsed.regions.find(
|
||||
(r) => r.signature === "gzip" && r.offset === oracle.offset,
|
||||
);
|
||||
assert.ok(
|
||||
gzip !== undefined,
|
||||
gzipRegion !== undefined,
|
||||
"Binwalk must find the independently generated gzip offset",
|
||||
);
|
||||
assert.equal(gzip.reported_size, oracle.length);
|
||||
assert.equal(gzip.size_basis, "provider_reported_validation_unknown");
|
||||
return result.regions.map(
|
||||
({ provider_id: _provider_id, ...region }) => region,
|
||||
);
|
||||
};
|
||||
const verifyFiles = async (evidence) => {
|
||||
const result = firmwareResultSchemas.extract_firmware.parse(
|
||||
assert.equal(gzipRegion.reported_size, oracle.length);
|
||||
assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown");
|
||||
|
||||
// Strip provider-specific ids so CLI and MCP results compare equal.
|
||||
return parsed.regions.map((region) => {
|
||||
const { provider_id: _ignored, ...rest } = region;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
|
||||
async function verifyFiles(evidence) {
|
||||
const parsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip"));
|
||||
|
||||
assert.ok(parsed.chunks.some((c) => c.handler === "gzip"));
|
||||
|
||||
for (const [suffix, expected] of Object.entries(oracle.files)) {
|
||||
const file = result.files.find((candidate) =>
|
||||
candidate.relative_path.endsWith(`/${suffix}`),
|
||||
const entry = parsed.files.find((f) =>
|
||||
f.relative_path.endsWith(`/${suffix}`),
|
||||
);
|
||||
assert.ok(file !== undefined, `Missing extracted ${suffix}`);
|
||||
assert.equal(file.sha256, expected.sha256);
|
||||
assert.equal(file.size, expected.size);
|
||||
assert.equal(await hashFirmwareFile(file.path), expected.sha256);
|
||||
assert.equal(file.original_file_range, null);
|
||||
assert.equal(file.runtime_address, null);
|
||||
assert.ok(entry !== undefined, `Missing extracted ${suffix}`);
|
||||
assert.equal(entry.sha256, expected.sha256);
|
||||
assert.equal(entry.size, expected.size);
|
||||
assert.equal(await hashFirmwareFile(entry.path), expected.sha256);
|
||||
assert.equal(entry.original_file_range, null);
|
||||
assert.equal(entry.runtime_address, null);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
try {
|
||||
const inspected = await cli(["inspect-firmware-regions", path]);
|
||||
const cliRegions = verifyRegions(inspected);
|
||||
const extracted = await cli([
|
||||
"extract-firmware",
|
||||
path,
|
||||
join(root, "cli-output"),
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const toContentIdentity = (parsed) =>
|
||||
parsed.files.map(({ relative_path, sha256, size }) => ({
|
||||
relative_path,
|
||||
sha256,
|
||||
size,
|
||||
}));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Verification stages
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function stageCli() {
|
||||
const regionsEvidence = await runCli([
|
||||
"inspect-firmware-regions",
|
||||
firmwarePath,
|
||||
]);
|
||||
const cliFiles = await verifyFiles(extracted);
|
||||
assert.equal(cliFiles.coverage, "partial");
|
||||
const regions = verifyRegions(regionsEvidence);
|
||||
|
||||
const extractEvidence = await runCli([
|
||||
"extract-firmware",
|
||||
firmwarePath,
|
||||
join(workDir, "cli-output"),
|
||||
]);
|
||||
const files = await verifyFiles(extractEvidence);
|
||||
|
||||
assert.equal(files.coverage, "partial");
|
||||
assert.ok(
|
||||
cliFiles.chunks.some(
|
||||
(chunk) =>
|
||||
chunk.handler === null &&
|
||||
chunk.range.offset === 0 &&
|
||||
chunk.range.length === 64,
|
||||
files.chunks.some(
|
||||
(c) =>
|
||||
c.handler === null && c.range.offset === 0 && c.range.length === 64,
|
||||
),
|
||||
);
|
||||
await client.connect(
|
||||
new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
cwd: repository,
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
}),
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const inspectedMcp = await call("inspect_firmware_regions", { path });
|
||||
const mcpEvidence = parseEvidence(inspectedMcp);
|
||||
assert.deepEqual(verifyRegions(mcpEvidence), cliRegions);
|
||||
const extractedMcp = await call("extract_firmware", {
|
||||
path,
|
||||
output_directory: join(root, "mcp-output"),
|
||||
|
||||
return { regions, files };
|
||||
}
|
||||
|
||||
async function connectMcp() {
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [ENTRYPOINT, "mcp"],
|
||||
cwd: REPO_ROOT,
|
||||
env: baseEnv,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const mcpFiles = await verifyFiles(parseEvidence(extractedMcp));
|
||||
const contentIdentity = (result) =>
|
||||
result.files.map(({ relative_path, sha256, size }) => ({
|
||||
relative_path,
|
||||
sha256,
|
||||
size,
|
||||
}));
|
||||
assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles));
|
||||
const selected = await call("extract_firmware", {
|
||||
path,
|
||||
output_directory: join(root, "selected-output"),
|
||||
await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
async function stageMcp(cli) {
|
||||
const regionsRaw = await callTool("inspect_firmware_regions", {
|
||||
path: firmwarePath,
|
||||
});
|
||||
assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions);
|
||||
|
||||
const extractRaw = await callTool("extract_firmware", {
|
||||
path: firmwarePath,
|
||||
output_directory: join(workDir, "mcp-output"),
|
||||
});
|
||||
const mcpFiles = await verifyFiles(parseEvidence(extractRaw));
|
||||
assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files));
|
||||
|
||||
const rangedRaw = await callTool("extract_firmware", {
|
||||
path: firmwarePath,
|
||||
output_directory: join(workDir, "selected-output"),
|
||||
range: { offset: oracle.offset, length: oracle.length },
|
||||
});
|
||||
const selectedFiles = await verifyFiles(parseEvidence(selected));
|
||||
assert.equal(selectedFiles.selection.offset, oracle.offset);
|
||||
assert.equal(selectedFiles.selection.length, oracle.length);
|
||||
const depth = await call("extract_firmware", {
|
||||
path,
|
||||
output_directory: join(root, "depth-output"),
|
||||
const ranged = await verifyFiles(parseEvidence(rangedRaw));
|
||||
assert.equal(ranged.selection.offset, oracle.offset);
|
||||
assert.equal(ranged.selection.length, oracle.length);
|
||||
|
||||
const depthRaw = await callTool("extract_firmware", {
|
||||
path: firmwarePath,
|
||||
output_directory: join(workDir, "depth-output"),
|
||||
max_depth: 1,
|
||||
});
|
||||
const depthResult = firmwareResultSchemas.extract_firmware.parse(
|
||||
depth.normalized_result,
|
||||
const depthParsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
depthRaw.normalized_result,
|
||||
);
|
||||
assert.equal(depthResult.coverage, "partial");
|
||||
assert.ok(depthResult.depth_limited_paths.length > 0);
|
||||
assert.equal(depthParsed.coverage, "partial");
|
||||
assert.ok(depthParsed.depth_limited_paths.length > 0);
|
||||
|
||||
console.log(
|
||||
"PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits",
|
||||
);
|
||||
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") {
|
||||
const ext4 = join(fixtureRoot, "ext4.bin");
|
||||
await access(ext4);
|
||||
const response = await call("extract_firmware", {
|
||||
path: ext4,
|
||||
output_directory: join(root, "ext4-output"),
|
||||
});
|
||||
const result = firmwareResultSchemas.extract_firmware.parse(
|
||||
response.normalized_result,
|
||||
|
||||
return mcpFiles;
|
||||
}
|
||||
|
||||
async function stageExt4() {
|
||||
const ext4Path = join(FIXTURE_ROOT, "ext4.bin");
|
||||
await fs.access(ext4Path);
|
||||
|
||||
const raw = await callTool("extract_firmware", {
|
||||
path: ext4Path,
|
||||
output_directory: join(workDir, "ext4-output"),
|
||||
});
|
||||
const parsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
raw.normalized_result,
|
||||
);
|
||||
assert.ok(parsed.chunks.some((c) => c.handler === "extfs"));
|
||||
assert.equal(parsed.coverage, "complete");
|
||||
|
||||
for (const [suffix, expected] of Object.entries(oracle.files)) {
|
||||
const match = parsed.files.find((f) =>
|
||||
f.relative_path.endsWith(`/${suffix}`),
|
||||
);
|
||||
assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs"));
|
||||
assert.equal(result.coverage, "complete");
|
||||
for (const [suffix, expected] of Object.entries(oracle.files))
|
||||
assert.equal(
|
||||
result.files.find((file) => file.relative_path.endsWith(`/${suffix}`))
|
||||
?.sha256,
|
||||
expected.sha256,
|
||||
);
|
||||
console.log("PASS real ext4 extraction with debugfs; no mounts");
|
||||
// Preserve the process-ownership utility while withholding all extractors.
|
||||
const withoutExtractors = join(root, "without-extractors");
|
||||
await access("/bin/ps");
|
||||
await mkdir(withoutExtractors);
|
||||
await symlink("/bin/ps", join(withoutExtractors, "ps"));
|
||||
const missing = await cli(
|
||||
["extract-firmware", ext4, join(root, "missing-extractor-output")],
|
||||
{ ...environment, PATH: withoutExtractors },
|
||||
);
|
||||
const partial = firmwareResultSchemas.extract_firmware.parse(
|
||||
missing.normalized_result,
|
||||
);
|
||||
assert.equal(partial.coverage, "partial");
|
||||
assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs"));
|
||||
console.log(
|
||||
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
|
||||
);
|
||||
assert.equal(match?.sha256, expected.sha256);
|
||||
}
|
||||
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") {
|
||||
if (process.env.GHIDRA_INSTALL_DIR === undefined)
|
||||
throw new Error(
|
||||
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
|
||||
);
|
||||
const child = mcpFiles.files.find((file) =>
|
||||
file.relative_path.endsWith("/usr/bin/probe"),
|
||||
);
|
||||
assert.ok(child !== undefined);
|
||||
await call("open_binary", { path: child.path, provider_id: "ghidra" });
|
||||
const dossier = await call("analyze_function", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
const evidence = parseEvidence(dossier);
|
||||
assert.equal(
|
||||
evidence.subject.digest.sha256,
|
||||
oracle.files["usr/bin/probe"].sha256,
|
||||
);
|
||||
const pseudocode = await call("procedure_pseudo_code", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
assert.equal(parseEvidence(pseudocode).provider.id, "ghidra");
|
||||
assert.ok(
|
||||
JSON.stringify(evidence.normalized_result).includes("firmware_probe"),
|
||||
);
|
||||
await call("close_binary", {});
|
||||
console.log(
|
||||
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
|
||||
console.log("PASS real ext4 extraction with debugfs; no mounts");
|
||||
|
||||
// Keep the process-ownership utility available but hide every extractor.
|
||||
const emptyBin = join(workDir, "without-extractors");
|
||||
await fs.access("/bin/ps");
|
||||
await fs.mkdir(emptyBin);
|
||||
await fs.symlink("/bin/ps", join(emptyBin, "ps"));
|
||||
|
||||
const degraded = await runCli(
|
||||
["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")],
|
||||
{ ...baseEnv, PATH: emptyBin },
|
||||
);
|
||||
const degradedParsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
degraded.normalized_result,
|
||||
);
|
||||
assert.equal(degradedParsed.coverage, "partial");
|
||||
assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs"));
|
||||
console.log(
|
||||
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
|
||||
);
|
||||
}
|
||||
|
||||
async function stageGhidra(mcpFiles) {
|
||||
if (process.env.GHIDRA_INSTALL_DIR === undefined) {
|
||||
throw new Error(
|
||||
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
|
||||
);
|
||||
}
|
||||
|
||||
const probe = mcpFiles.files.find((f) =>
|
||||
f.relative_path.endsWith("/usr/bin/probe"),
|
||||
);
|
||||
assert.ok(probe !== undefined);
|
||||
|
||||
await callTool("open_binary", { path: probe.path, provider_id: "ghidra" });
|
||||
|
||||
const dossierRaw = await callTool("analyze_function", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
const dossier = parseEvidence(dossierRaw);
|
||||
assert.equal(
|
||||
await hashFirmwareFile(path),
|
||||
dossier.subject.digest.sha256,
|
||||
oracle.files["usr/bin/probe"].sha256,
|
||||
);
|
||||
|
||||
const pseudoRaw = await callTool("procedure_pseudo_code", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra");
|
||||
assert.ok(
|
||||
JSON.stringify(dossier.normalized_result).includes("firmware_probe"),
|
||||
);
|
||||
|
||||
await callTool("close_binary", {});
|
||||
console.log(
|
||||
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Entry point
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
try {
|
||||
const cli = await stageCli();
|
||||
await connectMcp();
|
||||
const mcpFiles = await stageMcp(cli);
|
||||
|
||||
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4();
|
||||
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1")
|
||||
await stageGhidra(mcpFiles);
|
||||
|
||||
assert.equal(
|
||||
await hashFirmwareFile(firmwarePath),
|
||||
oracle.sha256,
|
||||
"Original firmware bytes must remain unchanged",
|
||||
);
|
||||
} finally {
|
||||
await client.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
await mcpClient.close();
|
||||
await fs.rm(workDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user