test(hopper): add semantic runtime conformance

This commit is contained in:
morluto
2026-07-18 04:50:23 +00:00
parent e6cfce3902
commit da1da3bfb8
17 changed files with 1251 additions and 71 deletions
+9
View File
@@ -1,4 +1,13 @@
export const LARGE_FIXTURE_COUNT: number;
export const HOPPER_C_ORACLE: Readonly<{
mainProcedure: "main";
entryProcedure: "rea_entry";
branchProcedure: "rea_branch";
leafProcedure: "rea_leaf";
entryString: "REA_C_ENTRY";
leafString: "REA_C_LEAF";
globalName: "rea_c_global";
}>;
export function sha256(value: string | Uint8Array): string;
export function generateLargeFixture(count?: number): string;
export function sourceDigest(
+10
View File
@@ -2,6 +2,16 @@ import { createHash } from "node:crypto";
export const LARGE_FIXTURE_COUNT = 1_205;
export const HOPPER_C_ORACLE = Object.freeze({
mainProcedure: "main",
entryProcedure: "rea_entry",
branchProcedure: "rea_branch",
leafProcedure: "rea_leaf",
entryString: "REA_C_ENTRY",
leafString: "REA_C_LEAF",
globalName: "rea_c_global",
});
export const sha256 = (value) =>
createHash("sha256").update(value).digest("hex");
+34
View File
@@ -0,0 +1,34 @@
export interface RealHopperFixtureOracle {
readonly mainProcedure: string;
readonly entryProcedure: string;
readonly branchProcedure: string;
readonly leafProcedure: string;
readonly entryString: string;
readonly leafString: string;
readonly globalName: string;
}
export interface RealHopperFixtureTarget {
readonly path: string;
readonly sha256: string;
}
export interface RealHopperLargeFixtureOracle {
readonly symbolPrefix: string;
readonly symbolCount: number;
readonly stringPrefix: string;
readonly stringCount: number;
}
export interface RealHopperFixtureTargets {
readonly manifestPath: string;
readonly primary: RealHopperFixtureTarget;
readonly secondary: RealHopperFixtureTarget;
readonly large: RealHopperFixtureTarget;
readonly oracle: RealHopperFixtureOracle;
readonly largeOracle: RealHopperLargeFixtureOracle;
}
export function loadRealHopperFixtureTargets(
manifestPath: string,
): Promise<RealHopperFixtureTargets>;
+113
View File
@@ -0,0 +1,113 @@
import { createHash } from "node:crypto";
import { readFile, realpath } from "node:fs/promises";
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
/** Bind the real-Hopper semantic verifier to source-owned manifest artifacts. */
export async function loadRealHopperFixtureTargets(manifestPath) {
const canonicalManifest = await realpath(manifestPath);
const manifest = JSON.parse(await readFile(canonicalManifest, "utf8"));
if (manifest?.schemaVersion !== 1 || !Array.isArray(manifest.fixtures))
throw new Error("Invalid conformance fixture manifest");
const primary = fixtureEntry(manifest.fixtures, "c");
const secondary = fixtureEntry(manifest.fixtures, "version-v2");
const large = fixtureEntry(manifest.fixtures, "large");
const oracle = requireCOracle(primary.expectations?.hopperOracle);
const largeOracle = requireLargeOracle(large.expectations);
const root = dirname(canonicalManifest);
const [primaryArtifact, secondaryArtifact, largeArtifact] = await Promise.all(
[
bindFixtureArtifact(root, primary),
bindFixtureArtifact(root, secondary),
bindFixtureArtifact(root, large),
],
);
requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]);
return {
manifestPath: canonicalManifest,
primary: primaryArtifact,
secondary: secondaryArtifact,
large: largeArtifact,
oracle,
largeOracle,
};
}
function requireCOracle(oracle) {
if (
oracle === null ||
typeof oracle !== "object" ||
typeof oracle.mainProcedure !== "string" ||
typeof oracle.entryProcedure !== "string" ||
typeof oracle.branchProcedure !== "string" ||
typeof oracle.leafProcedure !== "string" ||
typeof oracle.entryString !== "string" ||
typeof oracle.leafString !== "string" ||
typeof oracle.globalName !== "string" ||
[
oracle.mainProcedure,
oracle.entryProcedure,
oracle.branchProcedure,
oracle.leafProcedure,
oracle.entryString,
oracle.leafString,
oracle.globalName,
].some((value) => value.length === 0)
)
throw new Error("C fixture omitted its Hopper semantic oracle");
return oracle;
}
function requireLargeOracle(largeOracle) {
if (
largeOracle === null ||
typeof largeOracle !== "object" ||
typeof largeOracle.symbolPrefix !== "string" ||
largeOracle.symbolPrefix.length === 0 ||
typeof largeOracle.stringPrefix !== "string" ||
largeOracle.stringPrefix.length === 0 ||
!Number.isInteger(largeOracle.symbolCount) ||
largeOracle.symbolCount !== largeOracle.stringCount ||
largeOracle.symbolCount <= 0
)
throw new Error("Large fixture omitted its exact pagination oracle");
return largeOracle;
}
function requireDistinctArtifacts(artifacts) {
if (
new Set(artifacts.map(({ path }) => path)).size !== artifacts.length ||
new Set(artifacts.map(({ sha256 }) => sha256)).size !== artifacts.length
)
throw new Error("Hopper conformance fixtures must be distinct artifacts");
}
function fixtureEntry(fixtures, name) {
const matches = fixtures.filter((fixture) => fixture?.name === name);
if (matches.length !== 1) throw new Error(`Missing fixture ${name}`);
return matches[0];
}
async function bindFixtureArtifact(root, fixture) {
if (
typeof fixture.artifact !== "string" ||
typeof fixture.artifactSha256 !== "string" ||
!/^[0-9a-f]{64}$/u.test(fixture.artifactSha256)
)
throw new Error(`Invalid artifact declaration for ${fixture.name}`);
const path = await realpath(resolve(root, fixture.artifact));
const relation = relative(root, path);
if (
relation === ".." ||
relation.startsWith(`..${sep}`) ||
isAbsolute(relation)
)
throw new Error(`Fixture ${fixture.name} escapes its manifest root`);
const actual = sha256(await readFile(path));
if (actual !== fixture.artifactSha256)
throw new Error(
`Fixture ${fixture.name} digest did not match its manifest`,
);
return { path, sha256: actual };
}
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+33
View File
@@ -0,0 +1,33 @@
interface ToolClient {
callTool(request: unknown, options: unknown): Promise<unknown>;
}
interface PaginationSummary {
readonly count: number;
readonly pages: number;
}
export function verifyLargeFixturePagination(input: {
readonly client: ToolClient;
readonly options: unknown;
readonly normalizedResult: (value: unknown, operation: string) => unknown;
readonly expectedCount: number;
readonly symbolPrefix: string;
readonly stringPrefix: string;
}): Promise<{
readonly procedures: PaginationSummary;
readonly strings: PaginationSummary;
}>;
export function openAndVerifyLargeFixture(input: {
readonly client: ToolClient;
readonly options: unknown;
readonly normalizedResult: (value: unknown, operation: string) => unknown;
readonly path: string;
readonly expectedCount: number;
readonly symbolPrefix: string;
readonly stringPrefix: string;
}): Promise<{
readonly procedures: PaginationSummary;
readonly strings: PaginationSummary;
}>;
+138
View File
@@ -0,0 +1,138 @@
const PAGE_LIMIT = 100;
/** Open the large fixture and exhaustively prove its public search pages. */
export async function openAndVerifyLargeFixture({
client,
options,
normalizedResult,
path,
expectedCount,
symbolPrefix,
stringPrefix,
}) {
const opened = await client.callTool(
{ name: "open_binary", arguments: { path } },
options,
);
if (opened.isError === true)
throw new Error("Hopper rejected the large fixture");
return verifyLargeFixturePagination({
client,
options,
normalizedResult,
expectedCount,
symbolPrefix,
stringPrefix,
});
}
/** Exhaustively prove the source-owned large fixture through public search pages. */
export async function verifyLargeFixturePagination({
client,
options,
normalizedResult,
expectedCount,
symbolPrefix,
stringPrefix,
}) {
const procedures = await collectExactPages({
client,
options,
normalizedResult,
tool: "search_procedures",
pattern: symbolPrefix.replace(/^_+/u, ""),
expectedPrefix: symbolPrefix.replace(/^_+/u, ""),
expectedCount,
normalize: (value) => value.replace(/^_+/u, ""),
});
const strings = await collectExactPages({
client,
options,
normalizedResult,
tool: "search_strings",
pattern: stringPrefix,
expectedPrefix: stringPrefix,
expectedCount,
normalize: (value) => value,
});
return { procedures, strings };
}
async function collectExactPages({
client,
options,
normalizedResult,
tool,
pattern,
expectedPrefix,
expectedCount,
normalize,
}) {
const seenAddresses = new Set();
const seenValues = new Set();
let offset = 0;
let pages = 0;
while (true) {
const page = normalizedResult(
await client.callTool(
{
name: tool,
arguments: {
pattern,
mode: "literal",
case_sensitive: true,
offset,
limit: PAGE_LIMIT,
},
},
options,
),
`${tool} offset ${offset}`,
);
validatePage(page, { offset, expectedCount });
for (const item of page.items) {
const value = normalize(item.value);
if (item.value_truncated || !/^0x[0-9a-f]+$/iu.test(item.address))
throw new Error(`${tool} returned truncated or malformed evidence`);
if (seenAddresses.has(item.address) || seenValues.has(value))
throw new Error(`${tool} returned duplicate evidence`);
seenAddresses.add(item.address);
seenValues.add(value);
}
pages += 1;
if (page.next_offset === null) break;
offset = page.next_offset;
}
const expectedValues = Array.from(
{ length: expectedCount },
(_, index) => `${expectedPrefix}${String(index).padStart(4, "0")}`,
);
if (
seenValues.size !== expectedCount ||
expectedValues.some((value) => !seenValues.has(value))
)
throw new Error(`${tool} did not exhaustively prove the fixture values`);
return { count: seenValues.size, pages };
}
function validatePage(page, { offset, expectedCount }) {
if (
page === null ||
typeof page !== "object" ||
!Array.isArray(page.items) ||
page.offset !== offset ||
page.limit !== PAGE_LIMIT ||
page.total !== expectedCount ||
page.items.length !== Math.min(PAGE_LIMIT, expectedCount - offset)
)
throw new Error("Hopper returned an inconsistent fixture search page");
const expectedNext =
offset + page.items.length < expectedCount
? offset + page.items.length
: null;
if (
page.next_offset !== expectedNext ||
page.has_more !== (expectedNext !== null)
)
throw new Error("Hopper returned inconsistent pagination metadata");
}
+34
View File
@@ -0,0 +1,34 @@
export interface FixtureProcedure {
readonly address: string;
readonly name: string;
}
interface ToolClient {
callTool(
request: { readonly name: string; readonly arguments: unknown },
options: unknown,
): Promise<unknown>;
}
export function resolveFixtureProcedure(
client: ToolClient,
options: unknown,
expectedName: string,
normalizedResult: (value: unknown, operation: string) => unknown,
): Promise<FixtureProcedure>;
export function requireSafeDiagnostics(chunks: readonly string[]): number;
export function requireCurrentDocument(
client: ToolClient,
options: unknown,
documents: readonly string[],
normalizedResult: (value: unknown, operation: string) => unknown,
): Promise<string>;
export function analyzeFixtureProcedure(
client: ToolClient,
options: unknown,
procedure: FixtureProcedure,
normalizedResult: (value: unknown, operation: string) => unknown,
): Promise<unknown>;
+194
View File
@@ -0,0 +1,194 @@
import { requireFunctionDossierOracle } from "../../dist/application/RealHopperAssertions.js";
/** Verify named positive semantics from the source-owned C fixture. */
export async function verifyRealHopperFixture({
client,
options,
document,
oracle,
normalizedResult,
}) {
const main = await resolveFixtureProcedure(
client,
options,
oracle.mainProcedure,
normalizedResult,
);
const entry = await resolveFixtureProcedure(
client,
options,
oracle.entryProcedure,
normalizedResult,
);
const branch = await resolveFixtureProcedure(
client,
options,
oracle.branchProcedure,
normalizedResult,
);
const leaf = await resolveFixtureProcedure(
client,
options,
oracle.leafProcedure,
normalizedResult,
);
const comment = "REA real-Hopper conformance verifier";
const commentSet = normalizedResult(
await client.callTool(
{
name: "set_comment",
arguments: { address: entry.address, comment, document },
},
options,
),
"set_comment fixture marker",
);
if (commentSet !== true)
throw new Error("Hopper did not read back the fixture comment marker");
const entryDossier = requireFunctionDossierOracle(
await analyzeFixtureProcedure(client, options, entry, normalizedResult),
{
procedure_address: entry.address,
callee_address: branch.address,
caller_address: main.address,
referenced_string: oracle.entryString,
comment,
require_assembly: true,
},
);
const branchDossier = requireFunctionDossierOracle(
await analyzeFixtureProcedure(client, options, branch, normalizedResult),
{
procedure_address: branch.address,
callee_address: leaf.address,
caller_address: entry.address,
require_cfg_successor: true,
require_assembly: true,
},
);
const leafDossier = requireFunctionDossierOracle(
await analyzeFixtureProcedure(client, options, leaf, normalizedResult),
{
procedure_address: leaf.address,
caller_address: branch.address,
referenced_string: oracle.leafString,
referenced_name: oracle.globalName,
require_assembly: true,
},
);
return {
procedures: { main, entry, branch, leaf },
entryComments: entryDossier.comments.returned,
entryStrings: entryDossier.referenced_strings.returned,
entryCallees: entryDossier.callees.returned,
branchCallees: branchDossier.callees.returned,
branchBlocks: branchDossier.basic_blocks.returned,
leafStrings: leafDossier.referenced_strings.returned,
leafNames: leafDossier.referenced_names.returned,
};
}
/** Reject malformed, unsafe, or error-level diagnostics from the MCP runtime. */
export function requireSafeDiagnostics(chunks) {
const lines = chunks
.join("")
.split("\n")
.map((line) => line.trim())
.filter((line) => line.length > 0);
for (const line of lines) {
let diagnostic;
try {
diagnostic = JSON.parse(line);
} catch {
throw new Error(`The MCP runtime emitted malformed stderr: ${line}`);
}
if (
diagnostic === null ||
typeof diagnostic !== "object" ||
diagnostic.application !== "rea" ||
typeof diagnostic.level !== "number" ||
diagnostic.level >= 40
)
throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`);
}
return lines.length;
}
/** Bind mutations to the active document rather than list ordering. */
export async function requireCurrentDocument(
client,
options,
documents,
normalizedResult,
) {
const current = normalizedResult(
await client.callTool({ name: "current_document", arguments: {} }, options),
"current_document",
);
if (
typeof current !== "string" ||
current.length === 0 ||
!documents.includes(current)
)
throw new Error("current_document was not present in list_documents");
return current;
}
export async function resolveFixtureProcedure(
client,
options,
expectedName,
normalizedResult,
) {
const page = normalizedResult(
await client.callTool(
{
name: "search_procedures",
arguments: {
pattern: expectedName,
mode: "literal",
case_sensitive: true,
limit: 100,
},
},
options,
),
`search_procedures ${expectedName}`,
);
const matches = Array.isArray(page?.items)
? page.items.filter(
(item) =>
typeof item?.value === "string" &&
item.value_truncated === false &&
item.value.replace(/^_+/u, "") === expectedName,
)
: [];
if (
matches.length !== 1 ||
typeof matches[0]?.address !== "string" ||
!/^0x[0-9a-f]+$/iu.test(matches[0].address)
)
throw new Error(
`Expected exactly one Hopper procedure named ${expectedName}`,
);
return { address: matches[0].address, name: matches[0].value };
}
/** Request the assembly evidence required by the fixture oracle. */
export const analyzeFixtureProcedure = async (
client,
options,
procedure,
normalizedResult,
) =>
normalizedResult(
await client.callTool(
{
name: "analyze_function",
arguments: { procedure: procedure.address, include_assembly: true },
},
options,
),
`analyze_function ${procedure.name}`,
);