test(hopper): add semantic runtime conformance
This commit is contained in:
@@ -1,4 +1,13 @@
|
||||
export const LARGE_FIXTURE_COUNT: number;
|
||||
export const HOPPER_C_ORACLE: Readonly<{
|
||||
mainProcedure: "main";
|
||||
entryProcedure: "rea_entry";
|
||||
branchProcedure: "rea_branch";
|
||||
leafProcedure: "rea_leaf";
|
||||
entryString: "REA_C_ENTRY";
|
||||
leafString: "REA_C_LEAF";
|
||||
globalName: "rea_c_global";
|
||||
}>;
|
||||
export function sha256(value: string | Uint8Array): string;
|
||||
export function generateLargeFixture(count?: number): string;
|
||||
export function sourceDigest(
|
||||
|
||||
@@ -2,6 +2,16 @@ import { createHash } from "node:crypto";
|
||||
|
||||
export const LARGE_FIXTURE_COUNT = 1_205;
|
||||
|
||||
export const HOPPER_C_ORACLE = Object.freeze({
|
||||
mainProcedure: "main",
|
||||
entryProcedure: "rea_entry",
|
||||
branchProcedure: "rea_branch",
|
||||
leafProcedure: "rea_leaf",
|
||||
entryString: "REA_C_ENTRY",
|
||||
leafString: "REA_C_LEAF",
|
||||
globalName: "rea_c_global",
|
||||
});
|
||||
|
||||
export const sha256 = (value) =>
|
||||
createHash("sha256").update(value).digest("hex");
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
export interface RealHopperFixtureOracle {
|
||||
readonly mainProcedure: string;
|
||||
readonly entryProcedure: string;
|
||||
readonly branchProcedure: string;
|
||||
readonly leafProcedure: string;
|
||||
readonly entryString: string;
|
||||
readonly leafString: string;
|
||||
readonly globalName: string;
|
||||
}
|
||||
|
||||
export interface RealHopperFixtureTarget {
|
||||
readonly path: string;
|
||||
readonly sha256: string;
|
||||
}
|
||||
|
||||
export interface RealHopperLargeFixtureOracle {
|
||||
readonly symbolPrefix: string;
|
||||
readonly symbolCount: number;
|
||||
readonly stringPrefix: string;
|
||||
readonly stringCount: number;
|
||||
}
|
||||
|
||||
export interface RealHopperFixtureTargets {
|
||||
readonly manifestPath: string;
|
||||
readonly primary: RealHopperFixtureTarget;
|
||||
readonly secondary: RealHopperFixtureTarget;
|
||||
readonly large: RealHopperFixtureTarget;
|
||||
readonly oracle: RealHopperFixtureOracle;
|
||||
readonly largeOracle: RealHopperLargeFixtureOracle;
|
||||
}
|
||||
|
||||
export function loadRealHopperFixtureTargets(
|
||||
manifestPath: string,
|
||||
): Promise<RealHopperFixtureTargets>;
|
||||
@@ -0,0 +1,113 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, realpath } from "node:fs/promises";
|
||||
import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
|
||||
|
||||
/** Bind the real-Hopper semantic verifier to source-owned manifest artifacts. */
|
||||
export async function loadRealHopperFixtureTargets(manifestPath) {
|
||||
const canonicalManifest = await realpath(manifestPath);
|
||||
const manifest = JSON.parse(await readFile(canonicalManifest, "utf8"));
|
||||
if (manifest?.schemaVersion !== 1 || !Array.isArray(manifest.fixtures))
|
||||
throw new Error("Invalid conformance fixture manifest");
|
||||
const primary = fixtureEntry(manifest.fixtures, "c");
|
||||
const secondary = fixtureEntry(manifest.fixtures, "version-v2");
|
||||
const large = fixtureEntry(manifest.fixtures, "large");
|
||||
const oracle = requireCOracle(primary.expectations?.hopperOracle);
|
||||
const largeOracle = requireLargeOracle(large.expectations);
|
||||
const root = dirname(canonicalManifest);
|
||||
const [primaryArtifact, secondaryArtifact, largeArtifact] = await Promise.all(
|
||||
[
|
||||
bindFixtureArtifact(root, primary),
|
||||
bindFixtureArtifact(root, secondary),
|
||||
bindFixtureArtifact(root, large),
|
||||
],
|
||||
);
|
||||
requireDistinctArtifacts([primaryArtifact, secondaryArtifact, largeArtifact]);
|
||||
return {
|
||||
manifestPath: canonicalManifest,
|
||||
primary: primaryArtifact,
|
||||
secondary: secondaryArtifact,
|
||||
large: largeArtifact,
|
||||
oracle,
|
||||
largeOracle,
|
||||
};
|
||||
}
|
||||
|
||||
function requireCOracle(oracle) {
|
||||
if (
|
||||
oracle === null ||
|
||||
typeof oracle !== "object" ||
|
||||
typeof oracle.mainProcedure !== "string" ||
|
||||
typeof oracle.entryProcedure !== "string" ||
|
||||
typeof oracle.branchProcedure !== "string" ||
|
||||
typeof oracle.leafProcedure !== "string" ||
|
||||
typeof oracle.entryString !== "string" ||
|
||||
typeof oracle.leafString !== "string" ||
|
||||
typeof oracle.globalName !== "string" ||
|
||||
[
|
||||
oracle.mainProcedure,
|
||||
oracle.entryProcedure,
|
||||
oracle.branchProcedure,
|
||||
oracle.leafProcedure,
|
||||
oracle.entryString,
|
||||
oracle.leafString,
|
||||
oracle.globalName,
|
||||
].some((value) => value.length === 0)
|
||||
)
|
||||
throw new Error("C fixture omitted its Hopper semantic oracle");
|
||||
return oracle;
|
||||
}
|
||||
|
||||
function requireLargeOracle(largeOracle) {
|
||||
if (
|
||||
largeOracle === null ||
|
||||
typeof largeOracle !== "object" ||
|
||||
typeof largeOracle.symbolPrefix !== "string" ||
|
||||
largeOracle.symbolPrefix.length === 0 ||
|
||||
typeof largeOracle.stringPrefix !== "string" ||
|
||||
largeOracle.stringPrefix.length === 0 ||
|
||||
!Number.isInteger(largeOracle.symbolCount) ||
|
||||
largeOracle.symbolCount !== largeOracle.stringCount ||
|
||||
largeOracle.symbolCount <= 0
|
||||
)
|
||||
throw new Error("Large fixture omitted its exact pagination oracle");
|
||||
return largeOracle;
|
||||
}
|
||||
|
||||
function requireDistinctArtifacts(artifacts) {
|
||||
if (
|
||||
new Set(artifacts.map(({ path }) => path)).size !== artifacts.length ||
|
||||
new Set(artifacts.map(({ sha256 }) => sha256)).size !== artifacts.length
|
||||
)
|
||||
throw new Error("Hopper conformance fixtures must be distinct artifacts");
|
||||
}
|
||||
|
||||
function fixtureEntry(fixtures, name) {
|
||||
const matches = fixtures.filter((fixture) => fixture?.name === name);
|
||||
if (matches.length !== 1) throw new Error(`Missing fixture ${name}`);
|
||||
return matches[0];
|
||||
}
|
||||
|
||||
async function bindFixtureArtifact(root, fixture) {
|
||||
if (
|
||||
typeof fixture.artifact !== "string" ||
|
||||
typeof fixture.artifactSha256 !== "string" ||
|
||||
!/^[0-9a-f]{64}$/u.test(fixture.artifactSha256)
|
||||
)
|
||||
throw new Error(`Invalid artifact declaration for ${fixture.name}`);
|
||||
const path = await realpath(resolve(root, fixture.artifact));
|
||||
const relation = relative(root, path);
|
||||
if (
|
||||
relation === ".." ||
|
||||
relation.startsWith(`..${sep}`) ||
|
||||
isAbsolute(relation)
|
||||
)
|
||||
throw new Error(`Fixture ${fixture.name} escapes its manifest root`);
|
||||
const actual = sha256(await readFile(path));
|
||||
if (actual !== fixture.artifactSha256)
|
||||
throw new Error(
|
||||
`Fixture ${fixture.name} digest did not match its manifest`,
|
||||
);
|
||||
return { path, sha256: actual };
|
||||
}
|
||||
|
||||
const sha256 = (value) => createHash("sha256").update(value).digest("hex");
|
||||
@@ -0,0 +1,33 @@
|
||||
interface ToolClient {
|
||||
callTool(request: unknown, options: unknown): Promise<unknown>;
|
||||
}
|
||||
|
||||
interface PaginationSummary {
|
||||
readonly count: number;
|
||||
readonly pages: number;
|
||||
}
|
||||
|
||||
export function verifyLargeFixturePagination(input: {
|
||||
readonly client: ToolClient;
|
||||
readonly options: unknown;
|
||||
readonly normalizedResult: (value: unknown, operation: string) => unknown;
|
||||
readonly expectedCount: number;
|
||||
readonly symbolPrefix: string;
|
||||
readonly stringPrefix: string;
|
||||
}): Promise<{
|
||||
readonly procedures: PaginationSummary;
|
||||
readonly strings: PaginationSummary;
|
||||
}>;
|
||||
|
||||
export function openAndVerifyLargeFixture(input: {
|
||||
readonly client: ToolClient;
|
||||
readonly options: unknown;
|
||||
readonly normalizedResult: (value: unknown, operation: string) => unknown;
|
||||
readonly path: string;
|
||||
readonly expectedCount: number;
|
||||
readonly symbolPrefix: string;
|
||||
readonly stringPrefix: string;
|
||||
}): Promise<{
|
||||
readonly procedures: PaginationSummary;
|
||||
readonly strings: PaginationSummary;
|
||||
}>;
|
||||
@@ -0,0 +1,138 @@
|
||||
const PAGE_LIMIT = 100;
|
||||
|
||||
/** Open the large fixture and exhaustively prove its public search pages. */
|
||||
export async function openAndVerifyLargeFixture({
|
||||
client,
|
||||
options,
|
||||
normalizedResult,
|
||||
path,
|
||||
expectedCount,
|
||||
symbolPrefix,
|
||||
stringPrefix,
|
||||
}) {
|
||||
const opened = await client.callTool(
|
||||
{ name: "open_binary", arguments: { path } },
|
||||
options,
|
||||
);
|
||||
if (opened.isError === true)
|
||||
throw new Error("Hopper rejected the large fixture");
|
||||
return verifyLargeFixturePagination({
|
||||
client,
|
||||
options,
|
||||
normalizedResult,
|
||||
expectedCount,
|
||||
symbolPrefix,
|
||||
stringPrefix,
|
||||
});
|
||||
}
|
||||
|
||||
/** Exhaustively prove the source-owned large fixture through public search pages. */
|
||||
export async function verifyLargeFixturePagination({
|
||||
client,
|
||||
options,
|
||||
normalizedResult,
|
||||
expectedCount,
|
||||
symbolPrefix,
|
||||
stringPrefix,
|
||||
}) {
|
||||
const procedures = await collectExactPages({
|
||||
client,
|
||||
options,
|
||||
normalizedResult,
|
||||
tool: "search_procedures",
|
||||
pattern: symbolPrefix.replace(/^_+/u, ""),
|
||||
expectedPrefix: symbolPrefix.replace(/^_+/u, ""),
|
||||
expectedCount,
|
||||
normalize: (value) => value.replace(/^_+/u, ""),
|
||||
});
|
||||
const strings = await collectExactPages({
|
||||
client,
|
||||
options,
|
||||
normalizedResult,
|
||||
tool: "search_strings",
|
||||
pattern: stringPrefix,
|
||||
expectedPrefix: stringPrefix,
|
||||
expectedCount,
|
||||
normalize: (value) => value,
|
||||
});
|
||||
return { procedures, strings };
|
||||
}
|
||||
|
||||
async function collectExactPages({
|
||||
client,
|
||||
options,
|
||||
normalizedResult,
|
||||
tool,
|
||||
pattern,
|
||||
expectedPrefix,
|
||||
expectedCount,
|
||||
normalize,
|
||||
}) {
|
||||
const seenAddresses = new Set();
|
||||
const seenValues = new Set();
|
||||
let offset = 0;
|
||||
let pages = 0;
|
||||
while (true) {
|
||||
const page = normalizedResult(
|
||||
await client.callTool(
|
||||
{
|
||||
name: tool,
|
||||
arguments: {
|
||||
pattern,
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
offset,
|
||||
limit: PAGE_LIMIT,
|
||||
},
|
||||
},
|
||||
options,
|
||||
),
|
||||
`${tool} offset ${offset}`,
|
||||
);
|
||||
validatePage(page, { offset, expectedCount });
|
||||
for (const item of page.items) {
|
||||
const value = normalize(item.value);
|
||||
if (item.value_truncated || !/^0x[0-9a-f]+$/iu.test(item.address))
|
||||
throw new Error(`${tool} returned truncated or malformed evidence`);
|
||||
if (seenAddresses.has(item.address) || seenValues.has(value))
|
||||
throw new Error(`${tool} returned duplicate evidence`);
|
||||
seenAddresses.add(item.address);
|
||||
seenValues.add(value);
|
||||
}
|
||||
pages += 1;
|
||||
if (page.next_offset === null) break;
|
||||
offset = page.next_offset;
|
||||
}
|
||||
const expectedValues = Array.from(
|
||||
{ length: expectedCount },
|
||||
(_, index) => `${expectedPrefix}${String(index).padStart(4, "0")}`,
|
||||
);
|
||||
if (
|
||||
seenValues.size !== expectedCount ||
|
||||
expectedValues.some((value) => !seenValues.has(value))
|
||||
)
|
||||
throw new Error(`${tool} did not exhaustively prove the fixture values`);
|
||||
return { count: seenValues.size, pages };
|
||||
}
|
||||
|
||||
function validatePage(page, { offset, expectedCount }) {
|
||||
if (
|
||||
page === null ||
|
||||
typeof page !== "object" ||
|
||||
!Array.isArray(page.items) ||
|
||||
page.offset !== offset ||
|
||||
page.limit !== PAGE_LIMIT ||
|
||||
page.total !== expectedCount ||
|
||||
page.items.length !== Math.min(PAGE_LIMIT, expectedCount - offset)
|
||||
)
|
||||
throw new Error("Hopper returned an inconsistent fixture search page");
|
||||
const expectedNext =
|
||||
offset + page.items.length < expectedCount
|
||||
? offset + page.items.length
|
||||
: null;
|
||||
if (
|
||||
page.next_offset !== expectedNext ||
|
||||
page.has_more !== (expectedNext !== null)
|
||||
)
|
||||
throw new Error("Hopper returned inconsistent pagination metadata");
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
export interface FixtureProcedure {
|
||||
readonly address: string;
|
||||
readonly name: string;
|
||||
}
|
||||
|
||||
interface ToolClient {
|
||||
callTool(
|
||||
request: { readonly name: string; readonly arguments: unknown },
|
||||
options: unknown,
|
||||
): Promise<unknown>;
|
||||
}
|
||||
|
||||
export function resolveFixtureProcedure(
|
||||
client: ToolClient,
|
||||
options: unknown,
|
||||
expectedName: string,
|
||||
normalizedResult: (value: unknown, operation: string) => unknown,
|
||||
): Promise<FixtureProcedure>;
|
||||
|
||||
export function requireSafeDiagnostics(chunks: readonly string[]): number;
|
||||
|
||||
export function requireCurrentDocument(
|
||||
client: ToolClient,
|
||||
options: unknown,
|
||||
documents: readonly string[],
|
||||
normalizedResult: (value: unknown, operation: string) => unknown,
|
||||
): Promise<string>;
|
||||
|
||||
export function analyzeFixtureProcedure(
|
||||
client: ToolClient,
|
||||
options: unknown,
|
||||
procedure: FixtureProcedure,
|
||||
normalizedResult: (value: unknown, operation: string) => unknown,
|
||||
): Promise<unknown>;
|
||||
@@ -0,0 +1,194 @@
|
||||
import { requireFunctionDossierOracle } from "../../dist/application/RealHopperAssertions.js";
|
||||
|
||||
/** Verify named positive semantics from the source-owned C fixture. */
|
||||
export async function verifyRealHopperFixture({
|
||||
client,
|
||||
options,
|
||||
document,
|
||||
oracle,
|
||||
normalizedResult,
|
||||
}) {
|
||||
const main = await resolveFixtureProcedure(
|
||||
client,
|
||||
options,
|
||||
oracle.mainProcedure,
|
||||
normalizedResult,
|
||||
);
|
||||
const entry = await resolveFixtureProcedure(
|
||||
client,
|
||||
options,
|
||||
oracle.entryProcedure,
|
||||
normalizedResult,
|
||||
);
|
||||
const branch = await resolveFixtureProcedure(
|
||||
client,
|
||||
options,
|
||||
oracle.branchProcedure,
|
||||
normalizedResult,
|
||||
);
|
||||
const leaf = await resolveFixtureProcedure(
|
||||
client,
|
||||
options,
|
||||
oracle.leafProcedure,
|
||||
normalizedResult,
|
||||
);
|
||||
const comment = "REA real-Hopper conformance verifier";
|
||||
const commentSet = normalizedResult(
|
||||
await client.callTool(
|
||||
{
|
||||
name: "set_comment",
|
||||
arguments: { address: entry.address, comment, document },
|
||||
},
|
||||
options,
|
||||
),
|
||||
"set_comment fixture marker",
|
||||
);
|
||||
if (commentSet !== true)
|
||||
throw new Error("Hopper did not read back the fixture comment marker");
|
||||
|
||||
const entryDossier = requireFunctionDossierOracle(
|
||||
await analyzeFixtureProcedure(client, options, entry, normalizedResult),
|
||||
{
|
||||
procedure_address: entry.address,
|
||||
callee_address: branch.address,
|
||||
caller_address: main.address,
|
||||
referenced_string: oracle.entryString,
|
||||
comment,
|
||||
require_assembly: true,
|
||||
},
|
||||
);
|
||||
const branchDossier = requireFunctionDossierOracle(
|
||||
await analyzeFixtureProcedure(client, options, branch, normalizedResult),
|
||||
{
|
||||
procedure_address: branch.address,
|
||||
callee_address: leaf.address,
|
||||
caller_address: entry.address,
|
||||
require_cfg_successor: true,
|
||||
require_assembly: true,
|
||||
},
|
||||
);
|
||||
const leafDossier = requireFunctionDossierOracle(
|
||||
await analyzeFixtureProcedure(client, options, leaf, normalizedResult),
|
||||
{
|
||||
procedure_address: leaf.address,
|
||||
caller_address: branch.address,
|
||||
referenced_string: oracle.leafString,
|
||||
referenced_name: oracle.globalName,
|
||||
require_assembly: true,
|
||||
},
|
||||
);
|
||||
return {
|
||||
procedures: { main, entry, branch, leaf },
|
||||
entryComments: entryDossier.comments.returned,
|
||||
entryStrings: entryDossier.referenced_strings.returned,
|
||||
entryCallees: entryDossier.callees.returned,
|
||||
branchCallees: branchDossier.callees.returned,
|
||||
branchBlocks: branchDossier.basic_blocks.returned,
|
||||
leafStrings: leafDossier.referenced_strings.returned,
|
||||
leafNames: leafDossier.referenced_names.returned,
|
||||
};
|
||||
}
|
||||
|
||||
/** Reject malformed, unsafe, or error-level diagnostics from the MCP runtime. */
|
||||
export function requireSafeDiagnostics(chunks) {
|
||||
const lines = chunks
|
||||
.join("")
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter((line) => line.length > 0);
|
||||
for (const line of lines) {
|
||||
let diagnostic;
|
||||
try {
|
||||
diagnostic = JSON.parse(line);
|
||||
} catch {
|
||||
throw new Error(`The MCP runtime emitted malformed stderr: ${line}`);
|
||||
}
|
||||
if (
|
||||
diagnostic === null ||
|
||||
typeof diagnostic !== "object" ||
|
||||
diagnostic.application !== "rea" ||
|
||||
typeof diagnostic.level !== "number" ||
|
||||
diagnostic.level >= 40
|
||||
)
|
||||
throw new Error(`The MCP runtime emitted unsafe stderr: ${line}`);
|
||||
}
|
||||
return lines.length;
|
||||
}
|
||||
|
||||
/** Bind mutations to the active document rather than list ordering. */
|
||||
export async function requireCurrentDocument(
|
||||
client,
|
||||
options,
|
||||
documents,
|
||||
normalizedResult,
|
||||
) {
|
||||
const current = normalizedResult(
|
||||
await client.callTool({ name: "current_document", arguments: {} }, options),
|
||||
"current_document",
|
||||
);
|
||||
if (
|
||||
typeof current !== "string" ||
|
||||
current.length === 0 ||
|
||||
!documents.includes(current)
|
||||
)
|
||||
throw new Error("current_document was not present in list_documents");
|
||||
return current;
|
||||
}
|
||||
|
||||
export async function resolveFixtureProcedure(
|
||||
client,
|
||||
options,
|
||||
expectedName,
|
||||
normalizedResult,
|
||||
) {
|
||||
const page = normalizedResult(
|
||||
await client.callTool(
|
||||
{
|
||||
name: "search_procedures",
|
||||
arguments: {
|
||||
pattern: expectedName,
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
limit: 100,
|
||||
},
|
||||
},
|
||||
options,
|
||||
),
|
||||
`search_procedures ${expectedName}`,
|
||||
);
|
||||
const matches = Array.isArray(page?.items)
|
||||
? page.items.filter(
|
||||
(item) =>
|
||||
typeof item?.value === "string" &&
|
||||
item.value_truncated === false &&
|
||||
item.value.replace(/^_+/u, "") === expectedName,
|
||||
)
|
||||
: [];
|
||||
if (
|
||||
matches.length !== 1 ||
|
||||
typeof matches[0]?.address !== "string" ||
|
||||
!/^0x[0-9a-f]+$/iu.test(matches[0].address)
|
||||
)
|
||||
throw new Error(
|
||||
`Expected exactly one Hopper procedure named ${expectedName}`,
|
||||
);
|
||||
return { address: matches[0].address, name: matches[0].value };
|
||||
}
|
||||
|
||||
/** Request the assembly evidence required by the fixture oracle. */
|
||||
export const analyzeFixtureProcedure = async (
|
||||
client,
|
||||
options,
|
||||
procedure,
|
||||
normalizedResult,
|
||||
) =>
|
||||
normalizedResult(
|
||||
await client.callTool(
|
||||
{
|
||||
name: "analyze_function",
|
||||
arguments: { procedure: procedure.address, include_assembly: true },
|
||||
},
|
||||
options,
|
||||
),
|
||||
`analyze_function ${procedure.name}`,
|
||||
);
|
||||
Reference in New Issue
Block a user