feat(native): add inspection primitives, dispatch traces and approved UI observation (#500)

* fix(hopper): normalize string-backed addresses and update verification

* feat(native): add inspection primitives, dispatch traces and approved UI observation

* test: prioritize real artifact E2E and captured golden regressions

* fix(process): queue observed output before reactive rendering actions

* test: prune redundant cases and strengthen boundary assertions

* fix(cli): route native value tracing through shared workflows

* test: replace synthetic success with real workflows and captured data

* fix(catalog): separate portable metadata from native host availability
This commit is contained in:
morluto
2026-10-04 16:02:36 +08:00
committed by GitHub
parent 0356f7677f
commit 4fbc501121
205 changed files with 8136 additions and 1816 deletions
@@ -7,6 +7,21 @@ export async function startBrowserVerifierSite() {
let port = 0;
let sessionGeneration = 0;
const server = createServer((request, response) => {
if (request.url === "/screenshot-noise") {
response.setHeader("content-type", "text/html");
response.end(`<!doctype html><html><body style="margin:0"><canvas width="2048" height="1536"></canvas><script>
const context = document.querySelector('canvas').getContext('2d');
const image = context.createImageData(2048, 1536);
let state = 0x6d2b79f5;
for (let index = 0; index < image.data.length; index++) {
state ^= state << 13; state ^= state >>> 17; state ^= state << 5;
image.data[index] = index % 4 === 3 ? 255 : state & 255;
}
context.putImageData(image, 0, 0);
document.body.dataset.ready = 'true';
</script></body></html>`);
return;
}
if (request.url === "/app.js.map") {
response.setHeader("content-type", "application/source-map+json");
response.end(sourceMap());
+1 -1
View File
@@ -78,7 +78,7 @@ async function sdkToolCatalog() {
}
}
const auxiliaryProviders = [
new ArtifactProvider(false, false),
new ArtifactProvider(false, false, CATALOG_PLATFORM),
new NativeMacOSProvider(undefined, CATALOG_PLATFORM),
new ManagedStaticProvider(),
].map((provider) => ({
+64
View File
@@ -0,0 +1,64 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
const environment = () => ({
PATH: process.env.PATH ?? "/usr/bin:/bin",
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "auto",
// Artifact workflows have no deep-provider prerequisite or GUI authority.
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
});
/** Execute the compiled CLI with production providers and no test adapters. */
export async function artifactCli(command, target, arguments_ = []) {
const { stdout } = await promisify(execFile)(
process.execPath,
[
fileURLToPath(new URL("../rea.mjs", import.meta.url)),
command,
target,
...arguments_,
"--json",
],
{ env: environment(), timeout: 60000, maxBuffer: 16 * 1024 * 1024 },
);
const evidence = JSON.parse(stdout);
assert.equal(evidence.error, undefined, stdout);
assert.ok(evidence.normalized_result, "CLI omitted normalized result");
return evidence.normalized_result;
}
/** Run a real stdio MCP subprocess, owning and closing its transport. */
export async function withArtifactMcp(target, verify) {
const transport = new StdioClientTransport({
command: process.execPath,
args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
env: environment(),
stderr: "pipe",
});
const client = new Client({ name: "artifact-real-e2e", version: "1" });
try {
await client.connect(transport);
const opened = await client.callTool({
name: "open_binary",
arguments: { path: target },
});
assert.notEqual(opened.isError, true, JSON.stringify(opened));
await verify(client);
} finally {
await client.close();
await transport.close();
}
}
/** Require a successful structured MCP tool result rather than its text rendering. */
export async function artifactMcpResult(client, name, arguments_ = {}) {
const result = await client.callTool({ name, arguments: arguments_ });
assert.notEqual(result.isError, true, JSON.stringify(result));
assert.ok(result.structuredContent?.result, "MCP omitted structured result");
return result.structuredContent.result;
}
+115
View File
@@ -0,0 +1,115 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { chromium } from "playwright-core";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { webScreenshotSchema } from "../../dist/domain/webScreenshot.js";
import { comparePngScreenshots } from "../../dist/browser/PngVisualDiff.js";
/** Verify a real, large PNG through production CLI and stdio MCP boundaries. */
export async function verifyLargeScreenshotE2e(endpoint, origin) {
const browser = await chromium.connectOverCDP(endpoint);
const context = await browser.newContext({
viewport: { width: 2048, height: 1536 },
deviceScaleFactor: 1,
});
const entrypoint = fileURLToPath(new URL("../rea.mjs", import.meta.url));
const env = {
PATH: process.env.PATH ?? "/usr/bin:/bin",
REA_LOG_LEVEL: "silent",
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
REA_BROWSER_OBSERVE_ENABLED: "true",
REA_BROWSER_CDP_ENDPOINTS_JSON: JSON.stringify([endpoint]),
REA_BROWSER_ALLOWED_ORIGINS_JSON: JSON.stringify([origin]),
};
let client;
let transport;
try {
const page = await context.newPage();
await page.goto(`${origin}/screenshot-noise`);
await page.waitForFunction(() => document.body.dataset.ready === "true");
const targets = await (await fetch(`${endpoint}/json/list`)).json();
const target = targets.find((value) => value.url === page.url());
assert.ok(
target,
"Owned screenshot page is absent from actual CDP discovery",
);
const { stdout } = await promisify(execFile)(
process.execPath,
[entrypoint, "capture-web-screenshot", endpoint, target.id, "--json"],
{ env, timeout: 60000, maxBuffer: 72 * 1024 * 1024 },
);
const cliEvidence = parseEvidence(JSON.parse(stdout));
assert.equal(cliEvidence.operation, "capture_web_screenshot");
const screenshot = webScreenshotSchema.parse(cliEvidence.normalized_result);
assert.ok(
screenshot.artifact.bytes > 8 * 1024 * 1024,
"Real PNG did not cross the former screenshot byte ceiling",
);
const bytes = Buffer.from(screenshot.artifact.data_base64, "base64");
assert.equal(bytes.byteLength, screenshot.artifact.bytes);
assert.deepEqual(
bytes.subarray(0, 8),
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
);
assert.equal(bytes.readUInt32BE(16), 2048);
assert.equal(bytes.readUInt32BE(20), 1536);
transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env,
stderr: "pipe",
// The SDK defaults to a 10 MiB receive buffer; inline PNG output is larger.
maxBufferSize: 64 * 1024 * 1024,
});
client = new Client({ name: "browser-screenshot-real-e2e", version: "1" });
await client.connect(transport);
const captured = await client.callTool(
{
name: "capture_web_screenshot",
arguments: {
cdp_endpoint: endpoint,
allowed_origins: [origin],
target_id: target.id,
},
},
{ timeout: 60000 },
);
assert.notEqual(captured.isError, true, JSON.stringify(captured));
const mcpEvidence = parseEvidence({
...captured.structuredContent?.evidence,
normalized_result: captured.structuredContent?.result,
});
const mcpScreenshot = webScreenshotSchema.parse(
mcpEvidence.normalized_result,
);
assert.deepEqual(mcpScreenshot.artifact, screenshot.artifact);
const compared = comparePngScreenshots({
before: screenshot.artifact,
after: mcpScreenshot.artifact,
channel_threshold: 0,
});
assert.equal(compared.status, "identical");
assert.equal(compared.changed_pixels, 0);
return {
mocked: false,
cli: true,
stdio_mcp: true,
png_bytes: bytes.byteLength,
};
} finally {
try {
if (client !== undefined) await client.close();
} finally {
try {
if (transport !== undefined) await transport.close();
} finally {
await context.close();
await browser.close();
}
}
}
}
+183
View File
@@ -0,0 +1,183 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { analysisProfileSchema } from "../../dist/domain/analysisProfile.js";
import { nativeValueTraceSchema } from "../../dist/domain/nativeValueTrace.js";
const entrypoint = fileURLToPath(new URL("../rea.mjs", import.meta.url));
const environment = () => ({
PATH: process.env.PATH ?? "/usr/bin:/bin",
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "ghidra",
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
...(process.env.GHIDRA_INSTALL_DIR === undefined
? {}
: { GHIDRA_INSTALL_DIR: process.env.GHIDRA_INSTALL_DIR }),
...(process.env.JAVA_HOME === undefined
? {}
: { JAVA_HOME: process.env.JAVA_HOME }),
});
const verifyGraph = (input, target, procedure, globalAddress) => {
const graph = nativeValueTraceSchema.parse(input);
assert.equal(graph.target_sha256, target.sha256);
assert.equal(graph.seed, procedure);
const nodes = new Map(graph.nodes.map((node) => [node.id, node]));
for (const kind of ["argument-binding", "parameter-use", "return-binding"]) {
const edges = graph.edges.filter((edge) => edge.kind === kind);
assert.ok(edges.length > 0, `Real native dependency graph lacks ${kind}`);
for (const edge of edges) {
const source = nodes.get(edge.source);
const destination = nodes.get(edge.target);
assert.ok(source, `Missing source node ${edge.source}`);
assert.ok(destination, `Missing target node ${edge.target}`);
assert.equal(edge.status, "derived");
if (kind === "argument-binding") {
assert.match(source.operation?.opcode ?? "", /^CALL(?:IND)?$/u);
assert.equal(destination.kind, "parameter");
assert.equal(edge.input_index, destination.parameter.ordinal + 1);
} else if (kind === "parameter-use") {
assert.equal(source.kind, "parameter");
assert.ok(destination.operation.inputs[edge.input_index]);
} else {
assert.equal(source.operation?.opcode, "RETURN");
assert.match(destination.operation?.opcode ?? "", /^CALL(?:IND)?$/u);
assert.equal(edge.input_index, 1);
}
}
}
assert.ok(
graph.nodes.some((node) =>
node.operation?.inputs.some((value) => value.location === globalAddress),
),
"Source-owned global operand is absent from the real dependency graph",
);
for (const node of graph.nodes)
assert.match(node.evidence_id, /^ev_[a-f0-9]{64}$/u);
for (const procedure of graph.procedures) {
assert.equal(procedure.provider, "ghidra");
assert.match(procedure.analysis_profile_digest, /^[a-f0-9]{64}$/u);
}
return graph;
};
/** Verify real Ghidra dependency composition through production CLI and stdio MCP. */
export async function verifyNativeValueE2e(target, procedure, globalAddress) {
const { stdout } = await promisify(execFile)(
process.execPath,
[
entrypoint,
"trace-native-values",
target.path,
procedure,
"--provider",
"ghidra",
"--max-depth",
"2",
"--max-functions",
"8",
"--limit",
"5000",
"--json",
],
{ env: environment(), timeout: 240000, maxBuffer: 72 * 1024 * 1024 },
);
const evidence = parseEvidence(JSON.parse(stdout));
assert.equal(evidence.operation, "trace_native_values");
assert.equal(evidence.provider.id, "rea-workflow");
const workflowProfile = analysisProfileSchema.parse(
evidence.analysis_profile,
);
assert.equal(workflowProfile.provider.id, "rea-workflow");
const upstreamProfile = analysisProfileSchema.parse(
workflowProfile.parameters.upstream_analysis_profile,
);
assert.equal(upstreamProfile.provider.id, "ghidra");
const cliGraph = verifyGraph(
evidence.normalized_result,
target,
procedure,
globalAddress,
);
for (const procedure of cliGraph.procedures)
assert.equal(procedure.analysis_profile_digest, upstreamProfile.digest);
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env: environment(),
stderr: "pipe",
});
const client = new Client({ name: "native-values-real-e2e", version: "1" });
try {
await client.connect(transport);
const opened = await client.callTool(
{
name: "open_binary",
arguments: { path: target.path, provider: "ghidra" },
},
{ timeout: 180000 },
);
assert.notEqual(opened.isError, true, JSON.stringify(opened));
const inventory = await client.callTool({
name: "binary_session",
arguments: {},
});
assert.notEqual(inventory.isError, true, JSON.stringify(inventory));
assert.ok(
inventory.structuredContent?.result?.tool_availability.some(
(tool) => tool.name === "trace_native_values" && tool.available,
),
"Native value tracing is unavailable in real provider discovery",
);
const result = await client.callTool(
{
name: "trace_native_values",
arguments: { procedure, max_depth: 2, max_functions: 8, limit: 5000 },
},
{ timeout: 180000 },
);
assert.notEqual(result.isError, true, JSON.stringify(result));
const mcpEvidence = parseEvidence({
...result.structuredContent?.evidence,
normalized_result: result.structuredContent?.result,
});
assert.equal(mcpEvidence.operation, "trace_native_values");
assert.equal(mcpEvidence.provider.id, "rea-workflow");
assert.deepEqual(mcpEvidence.analysis_profile, workflowProfile);
const mcpGraph = verifyGraph(
result.structuredContent?.result,
target,
procedure,
globalAddress,
);
assert.deepEqual(mcpEvidence.normalized_result, mcpGraph);
assert.deepEqual(mcpGraph, cliGraph);
const closed = await client.callTool({
name: "close_binary",
arguments: {},
});
assert.notEqual(closed.isError, true, JSON.stringify(closed));
const status = await client.callTool({
name: "binary_session",
arguments: {},
});
assert.equal(status.structuredContent?.result?.open, false);
} finally {
await client.close();
await transport.close();
}
return {
mocked: false,
cli: true,
stdio_mcp: true,
nodes: cliGraph.total_nodes,
edges: cliGraph.total_edges,
decompilations: cliGraph.decompilations,
unknowns: cliGraph.unknowns.length,
};
}
+8 -13
View File
@@ -31,23 +31,21 @@ export const verifyHopperFunctionBasics = async (
arguments: {
procedure,
direction: "outgoing",
limit: 10,
max_instructions: 100,
},
},
options,
),
"procedure_references",
);
if (!Array.isArray(references?.references?.items)) {
throw new Error("procedure_references returned an invalid bounded result");
if (!Array.isArray(references?.references)) {
throw new Error("procedure_references returned an invalid result");
}
const instructions = requireMcpResult(
await client.callTool(
{
name: "read_function_instructions",
arguments: { procedure, limit: 2 },
arguments: { procedure },
},
options,
),
@@ -55,17 +53,14 @@ export const verifyHopperFunctionBasics = async (
);
if (
instructions?.procedure?.address !== procedure ||
!Array.isArray(instructions.instructions?.items) ||
instructions.instructions.items.length === 0 ||
instructions.instructions.items.length > 2
!Array.isArray(instructions.instructions) ||
instructions.instructions.length === 0
) {
throw new Error(
"read_function_instructions returned an invalid bounded window",
);
throw new Error("read_function_instructions returned an invalid result");
}
return {
outgoingReferenceCount: references.references.items.length,
instructionWindowCount: instructions.instructions.items.length,
outgoingReferenceCount: references.references.length,
instructionWindowCount: instructions.instructions.length,
};
};
+133
View File
@@ -0,0 +1,133 @@
import { execFile } from "node:child_process";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { inspectAppleDispatchMetadata } from "../dist/native/AppleDispatchMetadata.js";
if (process.platform !== "darwin")
throw new Error(
"Apple dispatch verification requires macOS with a host Objective-C compiler and Foundation SDK",
);
const exec = promisify(execFile);
const root = await mkdtemp(join(tmpdir(), "rea-dispatch-"));
try {
const targetPath = join(root, "fixture");
await exec("/usr/bin/xcrun", [
"clang",
"-O2",
"-framework",
"Foundation",
"-Wl,-no_fixup_chains",
fileURLToPath(
new URL("../tests/conformance/native/dispatch.m", import.meta.url),
),
"-o",
targetPath,
]);
const results = [];
for (const variant of ["symbols", "stripped"]) {
if (variant === "stripped")
await exec("/usr/bin/strip", ["-x", targetPath]);
const target = await parseBinaryTarget(targetPath);
if (!target.ok) throw target.error;
const observation = await inspectAppleDispatchMetadata(target.value, 5000);
const metadata = observation.result;
if (
!metadata.objc_dispatch_implementations.some(
(item) =>
item.class_name === "ReaDispatchFixture" &&
item.selector === "performAction:" &&
item.implementation_address !== null,
) ||
!metadata.objc_dispatch_implementations.some(
(item) =>
item.selector === "fixtureVersion" && item.method_type === "class",
) ||
!metadata.objc_ivars.some(
(item) => item.name === "state" && item.offset !== null,
) ||
!metadata.objc_protocol_records.some(
(protocol) =>
protocol.name === "ReaDispatchProtocol" &&
protocol.methods.some(
(method) => method.selector === "performAction:",
) &&
protocol.optional_methods.some(
(method) => method.selector === "optionalFixtureValue",
),
)
)
throw new Error(
`Apple metadata fixture drifted: ${JSON.stringify(observation)}`,
);
results.push({
variant,
target_sha256: target.value.sha256,
implementations: metadata.objc_dispatch_implementations.length,
ivars: metadata.objc_ivars.length,
coverage: metadata.coverage,
});
}
const swiftPath = join(root, "libReaWitnessFixture.dylib");
await exec("/usr/bin/xcrun", [
"swiftc",
"-emit-library",
"-O",
"-Xlinker",
"-no_fixup_chains",
fileURLToPath(
new URL("../tests/conformance/native/dispatch.swift", import.meta.url),
),
"-o",
swiftPath,
]);
for (const variant of ["swift-symbols", "swift-stripped"]) {
if (variant === "swift-stripped")
await exec("/usr/bin/strip", ["-x", swiftPath]);
const target = await parseBinaryTarget(swiftPath);
if (!target.ok) throw target.error;
const observation = await inspectAppleDispatchMetadata(target.value, 5000);
const metadata = observation.result;
if (
!metadata.swift_dispatch_slots.some(
(slot) =>
slot.table_kind === "class_vtable" &&
slot.owner === "ReaVtableFixture" &&
slot.decode.status === "decoded",
)
)
throw new Error(
`Swift class vtable fixture unavailable: ${JSON.stringify(metadata.coverage)}`,
);
if (
!metadata.swift_conformances.some(
(item) =>
item.type_name === "ReaWitnessFixture" &&
item.protocol_name === "ReaWitnessFixtureProtocol",
) ||
!metadata.swift_dispatch_slots.some(
(item) =>
item.owner === "ReaWitnessFixture: ReaWitnessFixtureProtocol" &&
item.implementation_address !== null,
)
)
throw new Error(
`Swift metadata fixture drifted: ${JSON.stringify(observation)}`,
);
results.push({
variant,
target_sha256: target.value.sha256,
conformances: metadata.swift_conformances.length,
slots: metadata.swift_dispatch_slots.length,
coverage: metadata.coverage,
});
}
process.stdout.write(
`${JSON.stringify({ ok: true, target_executed: false, fixtures: results })}\n`,
);
} finally {
await rm(root, { recursive: true, force: true });
}
+103
View File
@@ -0,0 +1,103 @@
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { execFile } from "node:child_process";
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import {
artifactCli,
artifactMcpResult,
withArtifactMcp,
} from "./lib/artifact-e2e.mjs";
if (process.platform !== "darwin")
throw new Error(
"Asset catalog E2E requires macOS, Xcode actool and assetutil",
);
const exec = promisify(execFile);
await exec("/usr/bin/xcrun", ["--find", "actool"]);
await exec("/usr/bin/xcrun", ["--find", "clang"]);
await exec("/usr/bin/assetutil", ["--version"]);
const root = await mkdtemp(join(tmpdir(), "rea-asset-e2e-"));
try {
const app = join(root, "Fixture.app");
const resources = join(app, "Contents", "Resources");
await mkdir(resources, { recursive: true });
await mkdir(join(app, "Contents", "MacOS"), { recursive: true });
await exec("/usr/bin/xcrun", [
"clang",
fileURLToPath(
new URL(
"../tests/conformance/interface-builder/fixture.c",
import.meta.url,
),
),
"-o",
join(app, "Contents", "MacOS", "Fixture"),
]);
await writeFile(
join(app, "Contents", "Info.plist"),
'<?xml version="1.0"?><plist version="1.0"><dict><key>CFBundleExecutable</key><string>Fixture</string><key>CFBundleIdentifier</key><string>dev.rea.asset-fixture</string></dict></plist>',
);
await exec("/usr/bin/xcrun", [
"actool",
fileURLToPath(
new URL("../tests/conformance/native/Assets.xcassets", import.meta.url),
),
"--compile",
resources,
"--platform",
"macosx",
"--minimum-deployment-target",
"12.0",
]);
const raw = JSON.parse(
(
await exec("/usr/bin/assetutil", [
"--info",
join(resources, "Assets.car"),
])
).stdout,
);
const sha256 = createHash("sha256")
.update(await readFile(join(resources, "Assets.car")))
.digest("hex");
const cli = await artifactCli("inspect-asset-catalog", app);
assert.deepEqual(
cli.records.map(({ metadata }) => metadata),
raw,
);
assert.equal(cli.catalogs[0].sha256, sha256);
assert.ok(
cli.records.some(
({ asset_name, metadata }) =>
asset_name === "FixtureColor" && metadata.AssetType === "Color",
),
);
await withArtifactMcp(app, async (client) => {
assert.deepEqual(
await artifactMcpResult(client, "inspect_asset_catalog"),
cli,
);
const first = await artifactMcpResult(client, "inspect_asset_catalog", {
limit: 1,
});
assert.equal(first.next_offset, 1);
const next = await artifactMcpResult(client, "inspect_asset_catalog", {
offset: first.next_offset,
});
assert.deepEqual([...first.records, ...next.records], cli.records);
const invalid = await client.callTool({
name: "inspect_asset_catalog",
arguments: { offset: -1 },
});
assert.equal(invalid.isError, true);
});
process.stdout.write(
`${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, records: raw.length, pagination: true, malformed_input_rejected: true })}\n`,
);
} finally {
await rm(root, { recursive: true, force: true });
}
+125
View File
@@ -0,0 +1,125 @@
import { execFile } from "node:child_process";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import {
artifactCli,
artifactMcpResult,
withArtifactMcp,
} from "./lib/artifact-e2e.mjs";
if (process.platform !== "darwin")
throw new Error(
"Real Foundation keyed archive verification requires macOS and Xcode Swift tools",
);
const root = await mkdtemp(join(tmpdir(), "rea-keyed-fixture-"));
try {
const archive = join(root, "model.plist");
await promisify(execFile)("/usr/bin/xcrun", [
"swift",
"-module-cache-path",
join(root, "modules"),
fileURLToPath(
new URL(
"../tests/conformance/native/keyed-archive.swift",
import.meta.url,
),
),
archive,
]);
const graph = await artifactCli("inspect-keyed-archive", archive);
const digest = createHash("sha256")
.update(await readFile(archive))
.digest("hex");
assert.equal(graph.archive_sha256, digest);
assert.equal(graph.target_sha256, digest);
await withArtifactMcp(archive, async (client) => {
assert.deepEqual(
await artifactMcpResult(client, "inspect_keyed_archive"),
graph,
);
const page = await artifactMcpResult(client, "inspect_keyed_archive", {
offset: 1,
limit: 1,
});
assert.deepEqual(page.objects, [graph.objects[1]]);
assert.equal(page.next_offset, 2);
const invalid = await client.callTool({
name: "inspect_keyed_archive",
arguments: { path: "../other.plist" },
});
assert.equal(invalid.isError, true);
});
const xml = join(root, "model.xml.plist");
await promisify(execFile)("/usr/bin/plutil", [
"-convert",
"xml1",
"-o",
xml,
archive,
]);
const xmlGraph = await artifactCli("inspect-keyed-archive", xml);
const expected = JSON.parse(
await readFile(
new URL(
"../tests/fixtures/golden/keyed-archive/graph.json",
import.meta.url,
),
"utf8",
),
);
assert.deepEqual(
Object.fromEntries(
Object.entries(xmlGraph).filter(
([key]) =>
!["archive_path", "archive_sha256", "target_sha256"].includes(key),
),
),
expected,
);
assert.deepEqual(
graph.references.map(({ source, path, target, status }) => ({
source,
path,
target,
status,
})),
xmlGraph.references.map(({ source, path, target, status }) => ({
source,
path,
target,
status,
})),
);
if (
graph?.archive_format !== "binary-plist" ||
!graph.references.some(
(link) =>
link.source !== null &&
link.source === link.target &&
link.status === "resolved",
) ||
!graph.objects.some((item) => item.class_name?.includes("ReaArchiveRecord"))
)
throw new Error(
`Real Foundation archive graph drifted: ${JSON.stringify(graph)}`,
);
const record = graph.objects.find((item) =>
item.class_name?.includes("ReaArchiveRecord"),
);
if (
graph.references.filter(
(link) => link.target === record.id && link.source !== record.id,
).length < 2
)
throw new Error("Real shared object identity was not preserved");
process.stdout.write(
`${JSON.stringify({ ok: true, mocked: false, cli: true, stdio_mcp: true, xml_golden: true, format: graph.archive_format, objects: graph.total_objects, references: graph.total_references, shared_identity: true, cyclic_identity: true, target_classes_instantiated_by_reader: false })}\n`,
);
} finally {
await rm(root, { recursive: true, force: true });
}
+166
View File
@@ -0,0 +1,166 @@
import { execFile, spawn } from "node:child_process";
import { mkdtemp, mkdir, writeFile, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { observeNativeUi } from "../dist/native/NativeUiObservation.js";
if (process.platform !== "darwin")
throw new Error(
"Native UI verification requires macOS, an interactive desktop and Xcode command-line tools",
);
const root = await mkdtemp(join(tmpdir(), "rea-ui-fixture-"));
let child;
let fixturePid;
try {
const contents = join(root, "Fixture.app", "Contents");
await mkdir(join(contents, "MacOS"), { recursive: true });
await writeFile(
join(contents, "Info.plist"),
`<?xml version="1.0"?><plist version="1.0"><dict><key>CFBundleExecutable</key><string>fixture</string><key>CFBundleIdentifier</key><string>io.rea.source-owned-ui-fixture</string><key>CFBundleName</key><string>REA UI Fixture</string><key>CFBundlePackageType</key><string>APPL</string></dict></plist>`,
);
const executable = join(contents, "MacOS", "fixture");
await promisify(execFile)("/usr/bin/xcrun", [
"swiftc",
"-module-cache-path",
join(root, "modules"),
fileURLToPath(
new URL("../tests/conformance/native/ui.swift", import.meta.url),
),
"-o",
executable,
]);
const coordinates = join(root, "scope.json");
await writeFile(coordinates, "");
child = spawn(
"/usr/bin/open",
["-W", "-n", "-g", "--stdout", coordinates, join(root, "Fixture.app")],
{ stdio: "ignore" },
);
let scope;
const deadline = Date.now() + 20000;
while (Date.now() < deadline) {
const output = await readFile(coordinates, "utf8");
if (output.includes("\n")) {
scope = JSON.parse(output.split("\n")[0]);
break;
}
await new Promise((resolve) => setTimeout(resolve, 100));
}
if (scope === undefined)
throw new Error(
"Source-owned UI fixture failed to expose one window within 20 seconds",
);
fixturePid = scope.pid;
const target = await parseBinaryTarget(executable);
if (!target.ok) throw target.error;
const observation = await observeNativeUi(target.value, "observe_native_ui", {
...scope,
observation_approved: true,
});
if (
!observation.ok &&
!/accessibility-denied|screen-recording-denied/u.test(
observation.error.message,
)
)
throw observation.error;
if (!observation.ok && !process.argv.includes("--allow-permission-denial"))
throw new Error(
`Native UI E2E requires successful capture; permissions prevented coverage: ${observation.error.message}. Use --allow-permission-denial only for the separate permission diagnostic lane.`,
);
let scenarioStatus = "not-run-permission-denied";
if (observation.ok) {
const button = observation.value.initial.nodes.find(
(node) => node.title === "Increment REA fixture",
);
if (button === undefined)
throw new Error("Source-owned fixture button is absent from AX capture");
const scenario = await observeNativeUi(
target.value,
"capture_native_ui_scenario",
{
...scope,
observation_approved: true,
actions_approved: true,
restore: "leave-as-is",
screenshot: false,
steps: [
{ kind: "click", path: button.path },
{ kind: "wait", milliseconds: 200 },
],
},
);
if (!scenario.ok) throw scenario.error;
if (
scenario.value.steps.some((step) => step.outcome !== "completed") ||
!scenario.value.steps
.at(-1)
?.after?.nodes.some((node) => node.title === "REA fixture incremented")
)
throw new Error(
`Source-owned AX scenario failed: ${JSON.stringify(scenario.value.steps.map(({ outcome, reason }) => ({ outcome, reason })))}`,
);
scenarioStatus = "selected-AX-button-action-observed";
const cli = await promisify(execFile)(
process.execPath,
[
fileURLToPath(new URL("./rea.mjs", import.meta.url)),
"capture-native-ui-scenario",
executable,
"--pid",
String(scope.pid),
"--window-id",
String(scope.window_id),
"--observation-approved",
"--actions-approved",
"--restore",
"leave-as-is",
"--steps",
JSON.stringify([
{ kind: "click", path: button.path },
{ kind: "wait", milliseconds: 200 },
]),
"--json",
],
{ maxBuffer: 32 * 1024 * 1024, timeout: 180000 },
);
const evidence = JSON.parse(cli.stdout);
if (
evidence.normalized_result?.steps?.some(
(step) => step.outcome !== "completed",
) ||
evidence.normalized_result?.steps?.length !== 2
)
throw new Error(
`Native UI CLI did not preserve the approved scenario: ${JSON.stringify(evidence.error ?? evidence.normalized_result?.steps?.map(({ outcome, reason }) => ({ outcome, reason })))}`,
);
scenarioStatus = "selected-AX-button-action-and-CLI-parity-observed";
}
const mismatch = await observeNativeUi(
{ ...target.value, sha256: "0".repeat(64) },
"observe_native_ui",
{ ...scope, observation_approved: true },
);
if (mismatch.ok || !mismatch.error.message.includes("target-mismatch"))
throw new Error("Real helper did not reject changed target bytes");
process.stdout.write(
`${JSON.stringify({ ok: true, verification_status: observation.ok ? "passed" : "permission-boundary-only", positive_e2e: observation.ok, observation: observation.ok ? "captured-selected-fixture-window" : "os-permission-denial-verified", scenario: scenarioStatus, permission_failure: observation.ok ? null : observation.error.message, mismatch_rejected: true, target_owned: true })}\n`,
);
} finally {
if (fixturePid !== undefined) {
try {
process.kill(fixturePid, "SIGTERM");
} catch (error) {
if (error.code !== "ESRCH") throw error;
}
}
if (child !== undefined && child.exitCode === null) {
const exited = new Promise((resolve) => child.once("exit", resolve));
child.kill("SIGTERM");
await exited;
}
await rm(root, { recursive: true, force: true });
}
+3
View File
@@ -33,6 +33,7 @@ import {
scenarioProfiles,
} from "./lib/browser-scenario-verifier.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
const SECRET_VALUES = [
"network-secret-value",
@@ -297,6 +298,7 @@ try {
)
throw new Error(`Browser scenario retained sensitive value: ${secret}`);
const largeScreenshot = await verifyLargeScreenshotE2e(endpoint, site.origin);
process.stdout.write(
`${JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
@@ -314,6 +316,7 @@ try {
sessionEvents: session.value.timeline.length,
pageScopedTransport: true,
screenshotBytes: screenshot.value.artifact.bytes,
largeScreenshot,
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
+4 -1
View File
@@ -249,7 +249,10 @@ export async function assertSession(
session.read_only !== true ||
session.analysis_complete !== true ||
session.analysis_timed_out !== false ||
session.capabilities.join(",") !== GHIDRA_SESSION_CAPABILITIES.join(",") ||
session.capabilities.length !== GHIDRA_SESSION_CAPABILITIES.length ||
GHIDRA_SESSION_CAPABILITIES.some(
(name) => !session.capabilities.includes(name),
) ||
!/^0x[0-9a-f]+$/u.test(session.target.image_base) ||
session.target.default_address_space.length === 0
)
+138
View File
@@ -6,6 +6,7 @@ import {
parseGhidraInventoryInput,
parseGhidraInventoryResult,
} from "../dist/ghidra/GhidraInventoryValues.js";
import { verifyNativeValueE2e } from "./lib/native-value-e2e.mjs";
import {
assertDenseSwitchDossier,
@@ -84,6 +85,39 @@ export async function verifyDebugFunctionOperations(
entryXrefs,
] = await collectEntryFunctionData(client, entry);
assertLocalProcedureInfo(info, entry.address);
const instruction = await functionCall(client, "inspect_native_instruction", {
document: null,
address: entry.address,
});
if (
instruction.status !== "decoded" ||
instruction.length < 1 ||
instruction.bytes.length !== instruction.length * 2 ||
instruction.operands.length === 0
)
throw new Error("Structured native instruction facts drifted");
const directSite = outgoing.references.find(
(reference) => reference.kind.call && !reference.kind.computed,
);
if (directSite === undefined)
throw new Error("Direct call fixture site unavailable");
const direct = await functionCall(client, "resolve_native_call_targets", {
document: null,
address: directSite.source_address,
});
if (
direct.status !== "direct" ||
!direct.targets.some(
(target) => target.address === directSite.target_address,
)
)
throw new Error("Direct call target resolution drifted");
const missingType = await functionCall(client, "inspect_native_data_type", {
document: null,
type: "/REA_MISSING_TYPE",
});
if (missingType.status !== "unavailable")
throw new Error("Missing native type was not explicit");
assertProviderText(assembly, pseudocode, entry.address);
if (
instructionWindow.procedure?.address !== entry.address ||
@@ -475,3 +509,107 @@ async function findCrossBranchAddress(client, variant, callees) {
}
throw new Error(`${variant} did not recover the multi-block callee`);
}
export async function verifyNativeTypeLayout(client, typeNames) {
const layoutSymbol = typeNames.find((item) =>
matchesSymbol(item.value, "rea_ghidra_inventory_layout"),
);
if (layoutSymbol === undefined)
throw new Error("Native layout fixture symbol unavailable");
const layout = await functionCall(client, "inspect_native_data_type", {
document: null,
address: layoutSymbol.address,
});
if (
layout.kind !== "struct" ||
layout.fields.length !== 3 ||
layout.size_bytes < 12
)
throw new Error(
`Native struct layout unavailable: ${JSON.stringify(layout)}`,
);
const payloadField = layout.fields.find((field) => field.name === "payload");
const stateField = layout.fields.find((field) => field.name === "state");
if (payloadField === undefined || stateField === undefined)
throw new Error(
`Nested layout field identities unavailable: ${JSON.stringify(layout)}`,
);
const payload = await functionCall(client, "inspect_native_data_type", {
document: null,
type: payloadField.type_id,
});
const state = await functionCall(client, "inspect_native_data_type", {
document: null,
type: stateField.type_id,
});
if (
payload.kind !== "union" ||
payload.fields.length !== 2 ||
state.kind !== "enum" ||
!state.members.some(
(member) => member.name === "REA_FIXTURE_READY" && member.value === "3",
)
)
throw new Error("Nested union or enum layout drifted");
return { layout, payload, state };
}
export async function verifyRelativeSwitch(client, procedures, entrySize) {
// Relocatable loaders may choose an assembler-generated alias as the function name.
const names = await inventoryCall(client, "list_names", {
document: null,
address: null,
});
const procedure = requireProcedure(names, "rea_relative_switch");
const dossier = await functionCall(client, "analyze_function", {
procedure: procedure.address,
});
const table = dossier.native_api?.jump_tables.find((item) =>
item.data_sources.some(
(source) =>
source.entry_size_bytes === entrySize && source.entry_count === 4,
),
);
if (table === undefined)
throw new Error(
`Exact ${entrySize}-byte relative table unavailable: ${JSON.stringify(dossier.native_api)}`,
);
for (let value = 0; value < 4; value++) {
const mapping = table.mappings.find((item) => item.case_value === value);
if (mapping === undefined || mapping.confidence !== "high")
throw new Error("Relative switch case mapping was not verified");
const instruction = await functionCall(
client,
"inspect_native_instruction",
{ address: mapping.target_address },
);
if (
instruction.status !== "decoded" ||
!instruction.operands.some((operand) =>
operand.components.some(
(component) =>
component.kind === "immediate" &&
component.value === `0x${(100 + value).toString(16)}`,
),
)
)
throw new Error(
`Relative switch target does not implement source case ${value}: ${JSON.stringify(instruction)}`,
);
}
return {
entry_size: entrySize,
dispatch_address: table.dispatch_address,
mappings: table.mappings.length,
};
}
export async function verifyNativeValueTrace(client, procedures, target) {
const entry = requireProcedure(procedures, "rea_ghidra_inventory_entry");
const names = await inventoryCall(client, "list_names", {
document: null,
address: null,
});
const global = requireProcedure(names, "rea_ghidra_inventory_global");
return verifyNativeValueE2e(target, entry.address, global.address);
}
+74 -12
View File
@@ -34,6 +34,11 @@ import {
} from "./verify-real-ghidra-inventory.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import {
verifyNativeTypeLayout,
verifyRelativeSwitch,
verifyNativeValueTrace,
} from "./verify-real-ghidra-function.mjs";
const exec = promisify(execFile);
const verifierRun = createVerifierRun();
const installDir = process.env.GHIDRA_INSTALL_DIR;
@@ -71,6 +76,7 @@ const crossFormatSourcePath = fileURLToPath(
new URL("../tests/conformance/ghidra/cross-format.c", import.meta.url),
);
const debugPath = join(fixtureRoot, "rea-ghidra-inventory-debug");
const layoutPath = join(fixtureRoot, "rea-ghidra-layout.o");
const strippedPath = join(fixtureRoot, "rea-ghidra-inventory-stripped");
const arm64ElfPath = join(fixtureRoot, "rea-ghidra-cross-arm64");
const peObjectPath = join(fixtureRoot, "rea-ghidra-cross-x86_64.obj");
@@ -113,12 +119,49 @@ try {
const common = ["-O0", "-g", "-fno-inline"];
if (expectedNativeTarget.format === "elf") common.push("-fno-pie", "-no-pie");
await exec(compiler, [...common, sourcePath, "-o", debugPath]);
await exec(compiler, [
"-O0",
"-g",
"-gdwarf-4",
"-c",
sourcePath,
"-o",
layoutPath,
]);
const strippedFlags = [...common, "-s"];
if (expectedNativeTarget.format === "mach-o")
strippedFlags.push("-fvisibility=hidden");
await exec(compiler, [...strippedFlags, sourcePath, "-o", strippedPath]);
const crossTargets = [];
if (aarch64JumpTableOnly) {
const relativeSource = fileURLToPath(
new URL("../tests/conformance/ghidra/relative-switch.S", import.meta.url),
);
for (const entrySize of [1, 2]) {
for (const format of [
"elf",
...(expectedNativeTarget.architecture === "arm64" ? ["mach-o"] : []),
]) {
const targetPath = join(
fixtureRoot,
`relative-${entrySize}-${format}.o`,
);
await exec(clang, [
...(format === "elf" ? ["--target=aarch64-linux-gnu"] : []),
`-DREA_ENTRY_BYTES=${entrySize}`,
"-c",
relativeSource,
"-o",
targetPath,
]);
crossTargets.push([
targetPath,
`relative-${entrySize}`,
{ format, architecture: "arm64" },
]);
}
}
await exec(clang, [
"--target=aarch64-linux-gnu",
"-O2",
@@ -197,6 +240,11 @@ try {
const debug = await verifyTarget(debugPath, "debug", expectedNativeTarget);
assertDebugFixture(debug);
const layout = await verifyTarget(
layoutPath,
"type-layout",
expectedNativeTarget,
);
const stripped = await verifyTarget(
strippedPath,
"stripped",
@@ -206,7 +254,8 @@ try {
const crossResults = [];
for (const [targetPath, variant, expectedTarget] of crossTargets) {
const result = await verifyTarget(targetPath, variant, expectedTarget);
if (variant !== "aarch64-jump-table") assertCrossFixture(result);
if (variant !== "aarch64-jump-table" && !variant.startsWith("relative-"))
assertCrossFixture(result);
crossResults.push(result);
}
await assertMalformedFixture(malformedPath);
@@ -228,11 +277,12 @@ try {
crossFormat || aarch64JumpTableOnly
? [sourcePath, crossFormatSourcePath]
: [sourcePath],
fixtures: [debug, stripped, ...crossResults].map(summary),
fixtures: [debug, stripped, layout, ...crossResults].map(summary),
malformed_target: "rejected-before-provider-start",
native_api_cli: aarch64JumpTableOnly
? (crossResults[0]?.native_api_cli ?? null)
: debug.native_api_cli,
native_value_e2e: debug.native_values,
custom_target: custom === null ? null : summary(custom),
cleanup: "complete",
})}\n`,
@@ -417,16 +467,27 @@ async function verifyTarget(targetPath, variant, expectedTarget = null) {
: "rea_ghidra_inventory_dense_switch",
)
: null;
const probes =
variant === "custom" || variant === "aarch64-jump-table"
? null
: await verifyInventoryOperations({
client,
variant,
procedures,
names,
strings,
});
const probes = variant.startsWith("relative-")
? await verifyRelativeSwitch(
client,
procedures,
Number(variant.slice(-1)),
)
: variant === "type-layout"
? await verifyNativeTypeLayout(client, names)
: variant === "custom" || variant === "aarch64-jump-table"
? null
: await verifyInventoryOperations({
client,
variant,
procedures,
names,
strings,
});
const nativeValues =
variant === "debug"
? await verifyNativeValueTrace(client, procedures, parsedTarget.value)
: null;
return {
variant,
target: parsedTarget.value,
@@ -439,6 +500,7 @@ async function verifyTarget(targetPath, variant, expectedTarget = null) {
strings,
probes,
native_api_cli: nativeApiCli,
native_values: nativeValues,
};
} finally {
await client.close();
+1
View File
@@ -309,6 +309,7 @@ try {
{ name: "binary_overview", arguments: {} },
options,
);
requireSuccessfulTool(overview, "binary_overview");
requireEvidenceProvider(
documents,
"list_documents",