test(hopper): cover real boundary failures and CLI parity

This commit is contained in:
morluto
2026-10-09 02:44:28 +08:00
parent d830cb79b3
commit 64717ad7da
6 changed files with 335 additions and 6 deletions
+53 -2
View File
@@ -7,6 +7,8 @@ import { promisify } from "node:util";
import Ajv from "ajv/dist/2020.js";
import addFormats from "ajv-formats";
import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs";
import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs";
import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs";
/** Exercise real Hopper annotation and malformed-input boundaries over MCP. */
export async function verifyHopperBoundaryContracts(
@@ -47,13 +49,13 @@ export async function verifyHopperBoundaryContracts(
successfulCalls += 1;
return result;
};
const invalid = async (name, args, reason) => {
const invalid = async (name, args, reason, failedOperation = name) => {
const reply = await client.callTool({ name, arguments: args }, options);
assert.equal(reply.isError, true, `${name} accepted invalid input`);
const { error } = JSON.parse(mcpTextValue(reply));
assert.equal(error.code, "invalid_request");
assert.equal(error.category, "invalid_input");
assert.equal(error.details.operation, name);
assert.equal(error.details.operation, failedOperation);
assert.match(error.message, reason);
rejectedCalls += 1;
rejectedOperations.add(name);
@@ -66,6 +68,12 @@ export async function verifyHopperBoundaryContracts(
const foreignDocument = (await call("list_documents")).find(
(name) => name !== activeDocument,
);
const navigationAndText = await verifyHopperNavigationAndText(
call,
invalid,
address,
);
const workflows = await verifyHopperWorkflows(call, invalid, procedure);
try {
if (foreignDocument !== undefined) {
await invalid(
@@ -268,6 +276,8 @@ export async function verifyHopperBoundaryContracts(
rejectedCalls,
rejectedOperations: [...rejectedOperations],
foreignDocumentVerified: foreignDocument !== undefined,
navigationAndText,
workflows,
};
}
@@ -298,6 +308,14 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) {
await copyFile(targets.primary, primary);
await copyFile(targets.secondary, secondary);
await call("open_binary", { path: primary });
const primaryProcedures = await call("list_procedures");
const entry = primaryProcedures.find((item) =>
item.value.endsWith("rea_entry"),
);
assert.ok(entry, "primary fixture omitted its entry procedure");
const expectedFunction = await call("analyze_function", {
procedure: entry.value,
});
const firstDocument = await call("current_document");
await call("open_binary", { path: secondary });
const secondDocument = await call("current_document");
@@ -314,6 +332,10 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) {
length: 8,
document: secondDocument,
});
await call("open_binary", { path: primary });
const query = "REA_C_LEAF __rea_missing_phrase__";
const expectedTrace = await call("trace_feature", { query });
assert.deepEqual(expectedTrace.matches, []);
await call("close_binary");
const dispatcher = new URL("../rea.mjs", import.meta.url).pathname;
@@ -336,6 +358,33 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) {
assert.equal(evidence.provider.id, "hopper");
assert.equal(evidence.subject.local_path, secondary);
assert.deepEqual(evidence.normalized_result, expected);
const traced = await runCli([
dispatcher,
"trace",
primary,
query,
"--provider",
"hopper",
"--format",
"json",
]);
const traceEvidence = JSON.parse(traced.stdout);
assert.equal(traceEvidence.subject.local_path, primary);
assert.deepEqual(traceEvidence.normalized_result, expectedTrace);
const analyzed = await runCli([
dispatcher,
"function",
primary,
entry.value,
"--provider",
"hopper",
"--format",
"json",
]);
assert.deepEqual(
JSON.parse(analyzed.stdout).normalized_result,
expectedFunction,
);
let failure;
try {
await runCli([
@@ -364,6 +413,8 @@ export async function verifyHopperLifecycleAndCli(client, options, targets) {
targetSwitchDisposedDocument: true,
cliByteParity: true,
cliInvalidAddress: true,
cliLiteralTraceParity: true,
cliFunctionDossierParity: true,
callerCancellationRecovered: true,
closedDocumentAbsent: true,
};
+175
View File
@@ -0,0 +1,175 @@
import assert from "node:assert/strict";
/** Verify native cursor readback and reject unrepresentable text before mutations. */
export async function verifyHopperNavigationAndText(call, invalid, address) {
const cursor = await call("current_address");
const procedures = await call("list_procedures");
const other = procedures.find((item) => item.address !== address);
assert.ok(other, "fixture omitted a second procedure for batch validation");
const originalName = await call("address_name", { address });
const originalComment = await call("comment", { address });
const originalInline = await call("inline_comment", { address });
const originalBookmark = (await call("list_bookmarks")).find(
(item) => item.address === address,
);
try {
const instructions = await call("read_function_instructions", {
procedure: address,
});
const info = await call("procedure_info", { procedure: address });
assert.ok(info.locals.length > 0, "fixture omitted real stack locals");
for (const local of info.locals) {
assert.equal(typeof local.name, "string");
assert.match(local.stack_displacement, /^-?\d+$/u);
assert.ok(local.description.includes(local.name));
assert.ok(local.description.includes(local.stack_displacement));
assert.doesNotMatch(local.description, /object at 0x/u);
}
const dossier = await call("analyze_function", { procedure: address });
assert.deepEqual(dossier.procedure.locals, info.locals);
const coordinates = instructions.instructions.map(
(line) => /^0x[0-9a-f]+/u.exec(line)?.[0],
);
assert.equal(coordinates[0], address);
assert.ok(coordinates[1], "fixture omitted a second instruction");
const interior = `0x${(BigInt(address) + 1n).toString(16)}`;
assert.equal(await call("goto_address", { address: interior }), address);
assert.equal(await call("current_address"), address);
assert.equal(
await call("next_address", { address: interior }),
coordinates[1],
);
assert.equal(await call("prev_address", { address: interior }), address);
for (const unmapped of ["0x0", "0xfffffffffffffffe"]) {
for (const operation of ["goto_address", "next_address", "prev_address"])
await invalid(
operation,
{ address: unmapped },
/Address is outside every segment/u,
);
assert.equal(await call("current_address"), address);
}
const segments = await call("list_segments");
const first = segments.reduce((left, right) =>
BigInt(left.start) < BigInt(right.start) ? left : right,
);
const last = segments.reduce((left, right) =>
BigInt(left.end) > BigInt(right.end) ? left : right,
);
await invalid(
"prev_address",
{ address: first.start },
/No previous address exists in mapped memory/u,
);
await invalid(
"next_address",
{ address: `0x${(BigInt(last.end) - 1n).toString(16)}` },
/No adjacent analyzed object exists in mapped memory/u,
);
const comment = "REA representation boundary 测试";
await call("set_comment", { address, comment });
await call("set_inline_comment", { address, comment });
await call("set_bookmark", { address, name: comment });
for (const [text, reason] of [
["before\0after", /contains a NUL character/u],
["before\ud800after", /contains an unpaired Unicode surrogate/u],
]) {
for (const operation of ["procedure_info", "read_bytes"])
await invalid(
operation,
operation === "procedure_info"
? { procedure: text }
: { address: text, length: 1 },
reason,
);
for (const operation of ["set_address_name", "set_bookmark"])
await invalid(operation, { address, name: text }, reason);
for (const operation of ["set_comment", "set_inline_comment"])
await invalid(operation, { address, comment: text }, reason);
await invalid(
"set_addresses_names",
{ names: { [address]: "must_not_apply", [other.address]: text } },
reason,
);
assert.equal(await call("address_name", { address }), originalName);
assert.equal(
await call("address_name", { address: other.address }),
other.value,
);
assert.equal(await call("comment", { address }), comment);
assert.equal(await call("inline_comment", { address }), comment);
assert.equal(
(await call("list_bookmarks")).find((item) => item.address === address)
?.name,
comment,
);
}
await invalid(
"set_address_name",
{ address, name: other.value },
/already assigned to/u,
);
await invalid(
"set_addresses_names",
{ names: { [address]: other.value } },
/already assigned to/u,
);
await invalid(
"set_addresses_names",
{ names: { [address]: "duplicate", [other.address]: "duplicate" } },
/assigns the same name more than once/u,
);
await invalid(
"set_addresses_names",
{ names: { [address]: "first", [address.slice(2)]: "second" } },
/selects the same address more than once/u,
);
assert.equal(await call("address_name", { address }), originalName);
assert.equal(
await call("address_name", { address: other.address }),
other.value,
);
try {
const swapped = await call("set_addresses_names", {
names: { [address]: other.value, [other.address]: originalName },
});
assert.equal(swapped[address], true);
assert.equal(swapped[other.address], true);
assert.equal(await call("address_name", { address }), other.value);
assert.equal(
await call("address_name", { address: other.address }),
originalName,
);
} finally {
await call("set_addresses_names", {
names: { [address]: originalName, [other.address]: other.value },
});
}
return {
cursorReadback: true,
interiorObjectNavigation: true,
unmappedNavigationRejected: true,
nativeTextValidationBeforeMutation: true,
unselectedLabelOwnersPreserved: true,
explicitLabelSwap: true,
observedLocalMetadata: true,
};
} finally {
await call("goto_address", { address: cursor });
await call("set_comment", { address, comment: originalComment ?? "" });
await call("set_inline_comment", {
address,
comment: originalInline ?? "",
});
if (originalBookmark === undefined)
await call("unset_bookmark", { address });
else
await call("set_bookmark", {
address,
...(originalBookmark.name === null
? {}
: { name: originalBookmark.name }),
});
}
}
+86
View File
@@ -0,0 +1,86 @@
import assert from "node:assert/strict";
/** Ground literal matching and graph identity in a real provider's function chain. */
export async function verifyHopperWorkflows(call, invalid, procedure) {
const address = procedure.address;
const originalName = await call("address_name", { address });
const callees = await call("procedure_callees", { procedure: address });
assert.ok(callees.length > 0, "fixture omitted a reachable call target");
const goal = callees[0];
const goalName = await call("address_name", { address: goal });
assert.equal(typeof goalName, "string");
const canonicalPath = await call("trace_call_path", { start: address, goal });
assert.equal(canonicalPath.goal_status, "reached");
assert.deepEqual(
await call("trace_call_path", { start: originalName, goal: goalName }),
canonicalPath,
);
assert.deepEqual(
await call("trace_call_path", {
start: address.toUpperCase(),
goal: `0x${(BigInt(goal) + 1n).toString(16)}`,
}),
canonicalPath,
);
const graph = await call("get_call_graph", { address });
assert.deepEqual(
await call("get_call_graph", { address: originalName }),
graph,
);
assert.deepEqual(
await call("get_call_graph", { address: address.toUpperCase() }),
graph,
);
const procedures = await call("list_procedures");
const cycle = procedures.filter((item) =>
/rea_cycle_[ab]$/u.test(item.value),
);
assert.equal(cycle.length, 2, "fixture omitted its source-owned call cycle");
const cycleGraph = await call("get_call_graph", { address: cycle[0].value });
const cycleNodes = Object.values(cycleGraph).flat();
assert.equal(cycleNodes.length, 2);
assert.deepEqual(
cycleNodes.map((node) => node.address).sort(),
cycle.map((item) => item.address).sort(),
);
for (const node of cycleNodes) {
assert.equal(node.status, "ok");
assert.deepEqual(
node.calls,
cycle
.filter((item) => item.address !== node.address)
.map((item) => item.address),
);
}
await invalid(
"trace_call_path",
{ start: "__rea_missing_procedure__", goal: "__rea_missing_procedure__" },
/Unknown Hopper address or name/u,
"procedure_address",
);
const missingPhrase = await call("trace_feature", {
query: "REA_C_LEAF __rea_missing_phrase__",
});
assert.deepEqual(missingPhrase.matches, []);
assert.deepEqual(missingPhrase.references, []);
const phrase = "REA boundary phrase";
try {
await call("set_address_name", { address, name: phrase });
const traced = await call("trace_feature", { query: phrase });
assert.deepEqual(traced.matches, [
{ type: "procedure", address, value: phrase },
]);
const spaced = await call("trace_feature", { query: ` ${phrase} ` });
assert.deepEqual(spaced.matches, []);
assert.deepEqual(spaced.references, []);
} finally {
await call("set_address_name", { address, name: originalName });
}
return {
canonicalGraphIdentities: true,
sourceOwnedCallCycle: true,
symbolAndInteriorGoalParity: true,
invalidSelfPathRejected: true,
completeLiteralQueryPreserved: true,
};
}
+1 -1
View File
@@ -250,7 +250,7 @@ let stderrBytes = 0;
const stderrChunks = [];
transport.stderr?.on("data", (chunk) => {
stderrBytes += chunk.length;
if (stderrBytes <= 16_384) stderrChunks.push(chunk.toString("utf8"));
stderrChunks.push(chunk.toString("utf8"));
});
const client = new Client({ name: "real-hopper-verifier", version: "1.0.0" });
const unrelatedHopper = await startUnrelatedHopperSentinel();