test(hopper): add semantic runtime conformance

This commit is contained in:
morluto
2026-07-18 04:50:23 +00:00
parent e6cfce3902
commit da1da3bfb8
17 changed files with 1251 additions and 71 deletions
+2
View File
@@ -6,6 +6,7 @@ import { promisify } from "node:util";
import {
generateLargeFixture,
HOPPER_C_ORACLE,
LARGE_FIXTURE_COUNT,
sha256,
sourceDigest,
@@ -47,6 +48,7 @@ await build(
expectations: {
symbols: ["_rea_entry", "_rea_branch", "_rea_leaf", "_rea_c_global"],
strings: ["REA_C_ENTRY", "REA_C_LEAF"],
hopperOracle: HOPPER_C_ORACLE,
},
},
);
+9
View File
@@ -1,4 +1,13 @@
export const LARGE_FIXTURE_COUNT: number;
export const HOPPER_C_ORACLE: Readonly<{
mainProcedure: "main";
entryProcedure: "rea_entry";
branchProcedure: "rea_branch";
leafProcedure: "rea_leaf";
entryString: "REA_C_ENTRY";
leafString: "REA_C_LEAF";
globalName: "rea_c_global";
}>;
export function sha256(value: string | Uint8Array): string;
export function generateLargeFixture(count?: number): string;
export function sourceDigest(
+10
View File
@@ -2,6 +2,16 @@ import { createHash } from "node:crypto";
export const LARGE_FIXTURE_COUNT = 1_205;
export const HOPPER_C_ORACLE = Object.freeze({
mainProcedure: "main",
entryProcedure: "rea_entry",
branchProcedure: "rea_branch",
leafProcedure: "rea_leaf",
entryString: "REA_C_ENTRY",
leafString: "REA_C_LEAF",
globalName: "rea_c_global",
});
export const sha256 = (value) =>
createHash("sha256").update(value).digest("hex");
+34
View File
@@ -0,0 +1,34 @@
export interface RealHopperFixtureOracle {
readonly mainProcedure: string;
readonly entryProcedure: string;
readonly branchProcedure: string;
readonly leafProcedure: string;
readonly entryString: string;
readonly leafString: string;
readonly globalName: string;
}
export interface RealHopperFixtureTarget {
readonly path: string;
readonly sha256: string;
}
export interface RealHopperLargeFixtureOracle {
readonly symbolPrefix: string;
readonly symbolCount: number;
readonly stringPrefix: string;
readonly stringCount: number;
}
export interface RealHopperFixtureTargets {
readonly manifestPath: string;
readonly primary: RealHopperFixtureTarget;
readonly secondary: RealHopperFixtureTarget;
readonly large: RealHopperFixtureTarget;
readonly oracle: RealHopperFixtureOracle;
readonly largeOracle: RealHopperLargeFixtureOracle;
}
export function loadRealHopperFixtureTargets(
manifestPath: string,
): Promise<RealHopperFixtureTargets>;
+113
View File
@@ -0,0 +1,113 @@
import { createHash } from "node:crypto";
import { readFile, realpath } from "node:fs/promises";
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
/** Bind the real-Hopper semantic verifier to source-owned manifest artifacts. */
export async function loadRealHopperFixtureTargets(manifestPath) {
const canonicalManifest = await realpath(manifestPath);
const manifest = JSON.parse(await readFile(canonicalManifest, "utf8"));
if (manifest?.schemaVersion !== 1 || !Array.isArray(manifest.fixtures))
throw new Error("Invalid conformance fixture manifest");
const primary = fixtureEntry(manifest.fixtures, "c");
const secondary = fixtureEntry(manifest.fixtures, "version-v2");
const large = fixtureEntry(manifest.fixtures, "large");
const oracle = requireCOracle(primary.expectations?.hopperOracle);
const largeOracle = requireLargeOracle(large.expectations);
const root = dirname(canonicalManifest);
const [primaryArtifact, secondaryArtifact, largeArtifact] = await Promise.all(
[
bindFixtureArtifact(root, primary),
bindFixtureArtifact(root, secondary),
bindFixtureArtifact(root, large),
],
);
requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]);
return {
manifestPath: canonicalManifest,
primary: primaryArtifact,
secondary: secondaryArtifact,
large: largeArtifact,
oracle,
largeOracle,
};
}
function requireCOracle(oracle) {
if (
oracle === null ||
typeof oracle !== "object" ||
typeof oracle.mainProcedure !== "string" ||
typeof oracle.entryProcedure !== "string" ||
typeof oracle.branchProcedure !== "string" ||
typeof oracle.leafProcedure !== "string" ||
typeof oracle.entryString !== "string" ||
typeof oracle.leafString !== "string" ||
typeof oracle.globalName !== "string" ||
[
oracle.mainProcedure,
oracle.entryProcedure,
oracle.branchProcedure,
oracle.leafProcedure,
oracle.entryString,
oracle.leafString,
oracle.globalName,
].some((value) => value.length === 0)
)
throw new Error("C fixture omitted its Hopper semantic oracle");
return oracle;
}
function requireLargeOracle(largeOracle) {
if (
largeOracle === null ||
typeof largeOracle !== "object" ||
typeof largeOracle.symbolPrefix !== "string" ||
largeOracle.symbolPrefix.length === 0 ||
typeof largeOracle.stringPrefix !== "string" ||
largeOracle.stringPrefix.length === 0 ||
!Number.isInteger(largeOracle.symbolCount) ||
largeOracle.symbolCount !== largeOracle.stringCount ||
largeOracle.symbolCount <= 0
)
throw new Error("Large fixture omitted its exact pagination oracle");
return largeOracle;
}
function requireDistinctArtifacts(artifacts) {
if (
new Set(artifacts.map(({ path }) => path)).size !== artifacts.length ||
new Set(artifacts.map(({ sha256 }) => sha256)).size !== artifacts.length
)
throw new Error("Hopper conformance fixtures must be distinct artifacts");
}
function fixtureEntry(fixtures, name) {
const matches = fixtures.filter((fixture) => fixture?.name === name);
if (matches.length !== 1) throw new Error(`Missing fixture ${name}`);
return matches[0];
}
async function bindFixtureArtifact(root, fixture) {
if (
typeof fixture.artifact !== "string" ||
typeof fixture.artifactSha256 !== "string" ||
!/^[0-9a-f]{64}$/u.test(fixture.artifactSha256)
)
throw new Error(`Invalid artifact declaration for ${fixture.name}`);
const path = await realpath(resolve(root, fixture.artifact));
const relation = relative(root, path);
if (
relation === ".." ||
relation.startsWith(`..${sep}`) ||
isAbsolute(relation)
)
throw new Error(`Fixture ${fixture.name} escapes its manifest root`);
const actual = sha256(await readFile(path));
if (actual !== fixture.artifactSha256)
throw new Error(
`Fixture ${fixture.name} digest did not match its manifest`,
);
return { path, sha256: actual };
}
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+33
View File
@@ -0,0 +1,33 @@
interface ToolClient {
callTool(request: unknown, options: unknown): Promise<unknown>;
}
interface PaginationSummary {
readonly count: number;
readonly pages: number;
}
export function verifyLargeFixturePagination(input: {
readonly client: ToolClient;
readonly options: unknown;
readonly normalizedResult: (value: unknown, operation: string) => unknown;
readonly expectedCount: number;
readonly symbolPrefix: string;
readonly stringPrefix: string;
}): Promise<{
readonly procedures: PaginationSummary;
readonly strings: PaginationSummary;
}>;
export function openAndVerifyLargeFixture(input: {
readonly client: ToolClient;
readonly options: unknown;
readonly normalizedResult: (value: unknown, operation: string) => unknown;
readonly path: string;
readonly expectedCount: number;
readonly symbolPrefix: string;
readonly stringPrefix: string;
}): Promise<{
readonly procedures: PaginationSummary;
readonly strings: PaginationSummary;
}>;
+138
View File
@@ -0,0 +1,138 @@
const PAGE_LIMIT = 100;
/** Open the large fixture and exhaustively prove its public search pages. */
export async function openAndVerifyLargeFixture({
client,
options,
normalizedResult,
path,
expectedCount,
symbolPrefix,
stringPrefix,
}) {
const opened = await client.callTool(
{ name: "open_binary", arguments: { path } },
options,
);
if (opened.isError === true)
throw new Error("Hopper rejected the large fixture");
return verifyLargeFixturePagination({
client,
options,
normalizedResult,
expectedCount,
symbolPrefix,
stringPrefix,
});
}
/** Exhaustively prove the source-owned large fixture through public search pages. */
export async function verifyLargeFixturePagination({
client,
options,
normalizedResult,
expectedCount,
symbolPrefix,
stringPrefix,
}) {
const procedures = await collectExactPages({
client,
options,
normalizedResult,
tool: "search_procedures",
pattern: symbolPrefix.replace(/^_+/u, ""),
expectedPrefix: symbolPrefix.replace(/^_+/u, ""),
expectedCount,
normalize: (value) => value.replace(/^_+/u, ""),
});
const strings = await collectExactPages({
client,
options,
normalizedResult,
tool: "search_strings",
pattern: stringPrefix,
expectedPrefix: stringPrefix,
expectedCount,
normalize: (value) => value,
});
return { procedures, strings };
}
async function collectExactPages({
client,
options,
normalizedResult,
tool,
pattern,
expectedPrefix,
expectedCount,
normalize,
}) {
const seenAddresses = new Set();
const seenValues = new Set();
let offset = 0;
let pages = 0;
while (true) {
const page = normalizedResult(
await client.callTool(
{
name: tool,
arguments: {
pattern,
mode: "literal",
case_sensitive: true,
offset,
limit: PAGE_LIMIT,
},
},
options,
),
`${tool} offset ${offset}`,
);
validatePage(page, { offset, expectedCount });
for (const item of page.items) {
const value = normalize(item.value);
if (item.value_truncated || !/^0x[0-9a-f]+$/iu.test(item.address))
throw new Error(`${tool} returned truncated or malformed evidence`);
if (seenAddresses.has(item.address) || seenValues.has(value))
throw new Error(`${tool} returned duplicate evidence`);
seenAddresses.add(item.address);
seenValues.add(value);
}
pages += 1;
if (page.next_offset === null) break;
offset = page.next_offset;
}
const expectedValues = Array.from(
{ length: expectedCount },
(_, index) => `${expectedPrefix}${String(index).padStart(4, "0")}`,
);
if (
seenValues.size !== expectedCount ||
expectedValues.some((value) => !seenValues.has(value))
)
throw new Error(`${tool} did not exhaustively prove the fixture values`);
return { count: seenValues.size, pages };
}
function validatePage(page, { offset, expectedCount }) {
if (
page === null ||
typeof page !== "object" ||
!Array.isArray(page.items) ||
page.offset !== offset ||
page.limit !== PAGE_LIMIT ||
page.total !== expectedCount ||
page.items.length !== Math.min(PAGE_LIMIT, expectedCount - offset)
)
throw new Error("Hopper returned an inconsistent fixture search page");
const expectedNext =
offset + page.items.length < expectedCount
? offset + page.items.length
: null;
if (
page.next_offset !== expectedNext ||
page.has_more !== (expectedNext !== null)
)
throw new Error("Hopper returned inconsistent pagination metadata");
}
+34
View File
@@ -0,0 +1,34 @@
export interface FixtureProcedure {
readonly address: string;
readonly name: string;
}
interface ToolClient {
callTool(
request: { readonly name: string; readonly arguments: unknown },
options: unknown,
): Promise<unknown>;
}
export function resolveFixtureProcedure(
client: ToolClient,
options: unknown,
expectedName: string,
normalizedResult: (value: unknown, operation: string) => unknown,
): Promise<FixtureProcedure>;
export function requireSafeDiagnostics(chunks: readonly string[]): number;
export function requireCurrentDocument(
client: ToolClient,
options: unknown,
documents: readonly string[],
normalizedResult: (value: unknown, operation: string) => unknown,
): Promise<string>;
export function analyzeFixtureProcedure(
client: ToolClient,
options: unknown,
procedure: FixtureProcedure,
normalizedResult: (value: unknown, operation: string) => unknown,
): Promise<unknown>;
+194
View File
@@ -0,0 +1,194 @@
import { requireFunctionDossierOracle } from "../../dist/application/RealHopperAssertions.js";
/** Verify named positive semantics from the source-owned C fixture. */
export async function verifyRealHopperFixture({
client,
options,
document,
oracle,
normalizedResult,
}) {
const main = await resolveFixtureProcedure(
client,
options,
oracle.mainProcedure,
normalizedResult,
);
const entry = await resolveFixtureProcedure(
client,
options,
oracle.entryProcedure,
normalizedResult,
);
const branch = await resolveFixtureProcedure(
client,
options,
oracle.branchProcedure,
normalizedResult,
);
const leaf = await resolveFixtureProcedure(
client,
options,
oracle.leafProcedure,
normalizedResult,
);
const comment = "REA real-Hopper conformance verifier";
const commentSet = normalizedResult(
await client.callTool(
{
name: "set_comment",
arguments: { address: entry.address, comment, document },
},
options,
),
"set_comment fixture marker",
);
if (commentSet !== true)
throw new Error("Hopper did not read back the fixture comment marker");
const entryDossier = requireFunctionDossierOracle(
await analyzeFixtureProcedure(client, options, entry, normalizedResult),
{
procedure_address: entry.address,
callee_address: branch.address,
caller_address: main.address,
referenced_string: oracle.entryString,
comment,
require_assembly: true,
},
);
const branchDossier = requireFunctionDossierOracle(
await analyzeFixtureProcedure(client, options, branch, normalizedResult),
{
procedure_address: branch.address,
callee_address: leaf.address,
caller_address: entry.address,
require_cfg_successor: true,
require_assembly: true,
},
);
const leafDossier = requireFunctionDossierOracle(
await analyzeFixtureProcedure(client, options, leaf, normalizedResult),
{
procedure_address: leaf.address,
caller_address: branch.address,
referenced_string: oracle.leafString,
referenced_name: oracle.globalName,
require_assembly: true,
},
);
return {
procedures: { main, entry, branch, leaf },
entryComments: entryDossier.comments.returned,
entryStrings: entryDossier.referenced_strings.returned,
entryCallees: entryDossier.callees.returned,
branchCallees: branchDossier.callees.returned,
branchBlocks: branchDossier.basic_blocks.returned,
leafStrings: leafDossier.referenced_strings.returned,
leafNames: leafDossier.referenced_names.returned,
};
}
/** Reject malformed, unsafe, or error-level diagnostics from the MCP runtime. */
export function requireSafeDiagnostics(chunks) {
const lines = chunks
.join("")
.split("\n")
.map((line) => line.trim())
.filter((line) => line.length > 0);
for (const line of lines) {
let diagnostic;
try {
diagnostic = JSON.parse(line);
} catch {
throw new Error(`The MCP runtime emitted malformed stderr: ${line}`);
}
if (
diagnostic === null ||
typeof diagnostic !== "object" ||
diagnostic.application !== "rea" ||
typeof diagnostic.level !== "number" ||
diagnostic.level >= 40
)
throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`);
}
return lines.length;
}
/** Bind mutations to the active document rather than list ordering. */
export async function requireCurrentDocument(
client,
options,
documents,
normalizedResult,
) {
const current = normalizedResult(
await client.callTool({ name: "current_document", arguments: {} }, options),
"current_document",
);
if (
typeof current !== "string" ||
current.length === 0 ||
!documents.includes(current)
)
throw new Error("current_document was not present in list_documents");
return current;
}
export async function resolveFixtureProcedure(
client,
options,
expectedName,
normalizedResult,
) {
const page = normalizedResult(
await client.callTool(
{
name: "search_procedures",
arguments: {
pattern: expectedName,
mode: "literal",
case_sensitive: true,
limit: 100,
},
},
options,
),
`search_procedures ${expectedName}`,
);
const matches = Array.isArray(page?.items)
? page.items.filter(
(item) =>
typeof item?.value === "string" &&
item.value_truncated === false &&
item.value.replace(/^_+/u, "") === expectedName,
)
: [];
if (
matches.length !== 1 ||
typeof matches[0]?.address !== "string" ||
!/^0x[0-9a-f]+$/iu.test(matches[0].address)
)
throw new Error(
`Expected exactly one Hopper procedure named ${expectedName}`,
);
return { address: matches[0].address, name: matches[0].value };
}
/** Request the assembly evidence required by the fixture oracle. */
export const analyzeFixtureProcedure = async (
client,
options,
procedure,
normalizedResult,
) =>
normalizedResult(
await client.callTool(
{
name: "analyze_function",
arguments: { procedure: procedure.address, include_assembly: true },
},
options,
),
`analyze_function ${procedure.name}`,
);
+49 -59
View File
@@ -1,7 +1,5 @@
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { readdir, realpath } from "node:fs/promises";
import { readdir } from "node:fs/promises";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@@ -10,17 +8,21 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
import {
firstProcedureAddress,
requireDistinctTargetHashes,
requireAddressArray,
requireFunctionDossier,
requirePseudocode,
} from "../dist/application/RealHopperAssertions.js";
import { HOPPER_PROVIDER_IDENTITY } from "../dist/hopper/HopperProvider.js";
import { REA_WORKFLOW_PROVIDER } from "../dist/application/InvestigationProviders.js";
import { loadRealHopperFixtureTargets } from "./lib/real-hopper-fixture.mjs";
import {
requireCurrentDocument,
requireSafeDiagnostics,
verifyRealHopperFixture,
} from "./lib/real-hopper-semantic.mjs";
import { openAndVerifyLargeFixture } from "./lib/real-hopper-pagination.mjs";
const execFileAsync = promisify(execFile);
const timeout = 180_000;
const parseServerArgs = (encoded) => {
const parsed = JSON.parse(encoded);
if (
@@ -34,7 +36,6 @@ const parseServerArgs = (encoded) => {
const sessionsBefore = new Set(
(await readdir("/tmp")).filter((name) => name.startsWith("rea-")),
);
const textValue = (result) => {
const text = result.content?.find((item) => item.type === "text")?.text;
if (typeof text !== "string") throw new Error("Tool result omitted text");
@@ -285,49 +286,15 @@ const verifyCurrentTarget = async (client, options) => {
};
};
const requireSafeDiagnostics = (chunks) => {
const lines = chunks
.join("")
.split("\n")
.map((line) => line.trim())
.filter((line) => line.length > 0);
for (const line of lines) {
let diagnostic;
try {
diagnostic = JSON.parse(line);
} catch {
throw new Error(`The MCP runtime emitted malformed stderr: ${line}`);
}
if (
diagnostic === null ||
typeof diagnostic !== "object" ||
diagnostic.application !== "rea" ||
typeof diagnostic.level !== "number" ||
diagnostic.level >= 40
) {
throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`);
}
}
return lines.length;
};
const targetPath = process.env.HOPPER_TARGET_PATH;
const secondTargetPath = process.env.HOPPER_SECOND_TARGET_PATH;
if (!targetPath || !secondTargetPath)
throw new Error(
"HOPPER_TARGET_PATH and HOPPER_SECOND_TARGET_PATH are required and must name distinct binaries",
);
const [targetA, targetB] = await Promise.all([
realpath(targetPath),
realpath(secondTargetPath),
]);
if (targetA === targetB)
throw new Error("Real-Hopper verification requires two distinct targets");
const [targetHashA, targetHashB] = await Promise.all([
sha256File(targetA),
sha256File(targetB),
]);
requireDistinctTargetHashes(targetHashA, targetHashB);
const fixtureManifestPath = process.env.REA_HOPPER_CONFORMANCE_MANIFEST_PATH;
if (!fixtureManifestPath)
throw new Error("REA_HOPPER_CONFORMANCE_MANIFEST_PATH is required");
const fixtureTargets = await loadRealHopperFixtureTargets(fixtureManifestPath);
const targetA = fixtureTargets.primary.path;
const targetB = fixtureTargets.secondary.path;
const largeTarget = fixtureTargets.large.path;
const targetHashA = fixtureTargets.primary.sha256;
const targetHashB = fixtureTargets.secondary.sha256;
const serverEnvironment = { ...process.env };
serverEnvironment.REA_ANALYSIS_PROVIDER = "auto";
delete serverEnvironment.HOPPER_TARGET_PATH;
@@ -409,13 +376,30 @@ try {
requireSuccessfulTool(segments, "list_segments"),
);
const firstDocuments = requireSuccessfulTool(documents, "list_documents");
if (!Array.isArray(firstDocuments) || firstDocuments.length === 0)
if (
!Array.isArray(firstDocuments) ||
firstDocuments.length === 0 ||
typeof firstDocuments[0] !== "string"
)
throw new Error("list_documents returned no real Hopper document");
const currentDocument = await requireCurrentDocument(
client,
options,
firstDocuments,
requireSuccessfulTool,
);
const firstOverview = requireOverview(
requireSuccessfulTool(overview, "binary_overview"),
"binary_overview",
);
const firstAnalysis = await verifyCurrentTarget(client, options);
const fixtureAnalysis = await verifyRealHopperFixture({
client,
options,
document: currentDocument,
oracle: fixtureTargets.oracle,
normalizedResult: requireSuccessfulTool,
});
const switched = await client.callTool(
{ name: "open_binary", arguments: { path: targetB } },
options,
@@ -439,6 +423,15 @@ try {
"binary_overview after target switch",
);
const secondAnalysis = await verifyCurrentTarget(client, options);
const largePagination = await openAndVerifyLargeFixture({
client,
options,
normalizedResult: requireSuccessfulTool,
path: largeTarget,
expectedCount: fixtureTargets.largeOracle.symbolCount,
symbolPrefix: fixtureTargets.largeOracle.symbolPrefix,
stringPrefix: fixtureTargets.largeOracle.stringPrefix,
});
const processList = await execFileAsync("ps", ["-ax", "-o", "command="]);
const bundledMcpRunning = processList.stdout
@@ -468,13 +461,16 @@ try {
overview: firstOverview,
segmentCount: firstOverview.segment_count,
analyses: [firstAnalysis, secondAnalysis],
fixtureAnalysis,
largePagination,
fixtureManifest: fixtureTargets.manifestPath,
bundledMcpRunning,
stderrBytes,
diagnosticCount,
dynamicSession: true,
providerBinding,
targets: [targetA, targetB],
targetHashes: [targetHashA, targetHashB],
targets: [targetA, targetB, largeTarget],
targetHashes: [targetHashA, targetHashB, fixtureTargets.large.sha256],
switched: true,
secondOverview: verifiedSecondOverview,
};
@@ -520,9 +516,3 @@ await new Promise((resolve, reject) => {
},
);
});
async function sha256File(path) {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk);
return hash.digest("hex");
}