chore: reconcile captured main with PR #2801

This commit is contained in:
morluto
2026-10-13 21:43:32 +08:00
808 changed files with 74776 additions and 5305 deletions
+76
View File
@@ -0,0 +1,76 @@
import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
const repository = process.env.CI_REPOSITORY;
const branch = process.env.CI_DEFAULT_BRANCH;
const head = process.env.HEAD_SHA;
const shaPattern = /^[0-9a-f]{40}$/i;
function ancestor(base) {
try {
execFileSync("git", ["merge-base", "--is-ancestor", base, head], {
stdio: "ignore",
});
return true;
} catch {
return false;
}
}
function write(base, reason) {
const output = { base, reason };
if (process.env.GITHUB_OUTPUT) {
appendFileSync(
process.env.GITHUB_OUTPUT,
`base=${base}\nreason=${reason.replace(/[\r\n]/g, " ")}\n`,
);
}
console.log(JSON.stringify(output));
}
if (!repository || !branch || !head || !shaPattern.test(head)) {
write("", "Repository, default branch, or checked-out head is invalid");
} else {
try {
const response = execFileSync(
"gh",
[
"api",
"--method",
"GET",
`repos/${repository}/actions/workflows/ci.yml/runs`,
"-f",
`branch=${branch}`,
"-f",
"event=push",
"-f",
"status=success",
"-F",
"per_page=100",
],
{ encoding: "utf8", timeout: 15_000, maxBuffer: 4 * 1024 * 1024 },
);
const runs = JSON.parse(response).workflow_runs;
if (!Array.isArray(runs))
throw new Error("GitHub returned no workflow run list");
const base = runs.find(
(run) =>
run?.event === "push" &&
run?.conclusion === "success" &&
run?.head_branch === branch &&
run?.head_repository?.full_name === repository &&
typeof run?.head_sha === "string" &&
shaPattern.test(run.head_sha) &&
run.head_sha.toLowerCase() !== head.toLowerCase() &&
ancestor(run.head_sha),
)?.head_sha;
if (base) write(base, "");
else
write(
"",
"No trusted successful CI push baseline is an ancestor of this head",
);
} catch {
write("", "Could not read a successful CI push baseline from GitHub");
}
}
+268
View File
@@ -0,0 +1,268 @@
import { execFile } from "node:child_process";
import { appendFile, mkdir, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { collectJobTimings } from "./job-timings.mjs";
const execFileAsync = promisify(execFile);
const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings");
const MAX_BUFFER = 16 * 1024 * 1024;
const API_TIMEOUT_MS = 30_000;
const WORKFLOWS = [
{ id: "ci.yml", name: "CI" },
{ id: "release.yml", name: "Release" },
];
const LOOKBACK_MS = 24 * 60 * 60 * 1_000;
const MAX_RUNS_PER_WORKFLOW = 100;
const TIME_BUDGET_MS = 240_000;
/** Collect a bounded recent sample of completed CI and Release attempts. */
export async function collectJobTimingsBatch({
repository,
outputDirectory = OUTPUT_DIRECTORY,
observedAt = new Date().toISOString(),
env = process.env,
now = Date.now,
collect = collectJobTimings,
} = {}) {
const identity = validateRepository(repository);
const started = now();
const cutoff = new Date(started - LOOKBACK_MS).toISOString();
const manifest = {
schemaVersion: 1,
observedAt,
repository: identity,
selection: {
workflows: WORKFLOWS,
status: "completed",
createdAfter: cutoff,
lookbackHours: 24,
overlap: "Each hourly run re-reads the preceding 24 hours.",
pagination:
"Run listings use the first 100; jobs are paginated within the remaining time and 16 MiB response limits.",
maxRunsPerWorkflow: MAX_RUNS_PER_WORKFLOW,
timeBudgetMs: TIME_BUDGET_MS,
exhaustive: false,
},
coverage: {
workflowsQueried: [],
runsListed: 0,
selectedRuns: 0,
collectedRuns: 0,
},
truncation: [],
errors: [],
runs: [],
elapsedMs: 0,
};
const budgetExpired = () => now() - started >= TIME_BUDGET_MS;
const selected = [];
for (const workflow of WORKFLOWS) {
if (budgetExpired()) {
manifest.truncation.push("Time budget reached while listing workflows.");
break;
}
try {
const response = await apiJson(
`repos/${identity}/actions/workflows/${workflow.id}/runs?status=completed&created=%3E%3D${encodeURIComponent(cutoff)}&per_page=${String(MAX_RUNS_PER_WORKFLOW)}&page=1`,
env,
started,
now,
);
if (!Array.isArray(response.workflow_runs))
throw new Error("run listing must contain a workflow_runs array");
manifest.coverage.workflowsQueried.push(workflow.name);
manifest.coverage.runsListed += response.workflow_runs.length;
if (response.workflow_runs.length >= MAX_RUNS_PER_WORKFLOW)
manifest.truncation.push(
`${workflow.name} run listing reached the 100-run first-page limit.`,
);
for (const run of response.workflow_runs.slice(
0,
MAX_RUNS_PER_WORKFLOW,
)) {
if (
run.status !== "completed" ||
!Number.isSafeInteger(run.id) ||
!Number.isSafeInteger(run.run_attempt)
) {
manifest.errors.push({
scope: `workflow:${workflow.name}`,
runId: run.id ?? null,
message:
"Run record missing completed status, numeric id, or numeric latest attempt.",
});
continue;
}
if (run.created_at && Date.parse(run.created_at) < Date.parse(cutoff))
continue;
selected.push({
id: run.id,
runAttempt: run.run_attempt,
workflow: workflow.name,
conclusion: run.conclusion ?? null,
createdAt: run.created_at ?? null,
});
}
} catch (cause) {
manifest.errors.push({
scope: `workflow:${workflow.name}`,
message: errorMessage(cause),
});
}
}
selected.sort(
(left, right) =>
(Date.parse(right.createdAt ?? "") || 0) -
(Date.parse(left.createdAt ?? "") || 0),
);
manifest.coverage.selectedRuns = selected.length;
for (const run of selected) {
if (budgetExpired()) {
manifest.truncation.push(
"Time budget reached before all selected runs were collected.",
);
break;
}
const runRecord = {
...run,
status: "pending",
rawPath: null,
reportPath: null,
error: null,
};
manifest.runs.push(runRecord);
const runDirectory = join(
outputDirectory,
"runs",
`${String(run.id)}-${String(run.runAttempt)}`,
);
try {
const result = await collect({
repository: identity,
runId: String(run.id),
runAttempt: String(run.runAttempt),
outputDirectory: runDirectory,
env,
timeoutMs: Math.max(
1,
Math.min(API_TIMEOUT_MS, TIME_BUDGET_MS - (now() - started)),
),
});
runRecord.status = "collected";
runRecord.rawPath = result.rawPath;
runRecord.reportPath = result.reportPath;
manifest.coverage.collectedRuns += 1;
} catch (cause) {
runRecord.status = "error";
runRecord.error = errorMessage(cause);
manifest.errors.push({
scope: "run",
runId: run.id,
runAttempt: run.runAttempt,
message: runRecord.error,
});
}
}
if (budgetExpired() && manifest.runs.length < manifest.coverage.selectedRuns)
manifest.truncation.push(
"Time budget reached; selected runs remain unattempted.",
);
return await writeManifest(outputDirectory, manifest, started, now, env);
}
function validateRepository(repository) {
if (
typeof repository !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
)
throw new Error(
"CI_REPOSITORY must be an owner/name repository identifier",
);
return repository;
}
async function apiJson(endpoint, env, started, now) {
const timeout = Math.max(
1,
Math.min(API_TIMEOUT_MS, TIME_BUDGET_MS - (now() - started)),
);
const { stdout } = await execFileAsync("gh", ["api", endpoint], {
env,
encoding: "utf8",
maxBuffer: MAX_BUFFER,
timeout,
});
try {
return JSON.parse(stdout);
} catch (cause) {
throw new Error("GitHub Actions API returned malformed JSON", { cause });
}
}
async function writeManifest(outputDirectory, manifest, started, now, env) {
manifest.elapsedMs = Math.max(0, now() - started);
if (
manifest.elapsedMs >= TIME_BUDGET_MS &&
!manifest.truncation.some((item) => item.includes("Time budget"))
)
manifest.truncation.push("Time budget reached.");
await mkdir(outputDirectory, { recursive: true });
const manifestPath = join(outputDirectory, "batch-manifest.json");
await writeFile(
manifestPath,
`${JSON.stringify(manifest, null, 2)}\n`,
"utf8",
);
if (env.GITHUB_STEP_SUMMARY)
await appendBatchSummary(env.GITHUB_STEP_SUMMARY, manifest, manifestPath);
return { manifestPath, manifest };
}
async function appendBatchSummary(summaryPath, manifest, manifestPath) {
const collected = manifest.runs.filter(
({ status }) => status === "collected",
).length;
const failed = manifest.runs.filter(
({ status }) => status === "error",
).length;
const lines = [
"### Batched CI timing snapshot",
"",
`Observed ${manifest.observedAt}; collected ${String(collected)} of ${String(manifest.coverage.selectedRuns)} selected recent completed runs (${String(failed)} run errors).`,
`Coverage uses the first ${String(MAX_RUNS_PER_WORKFLOW)} runs per workflow created in the preceding 24 hours and a ${String(TIME_BUDGET_MS / 1_000)} second total budget. This sample is bounded and is not exhaustive.`,
`Manifest: \`${manifestPath}\`.`,
...manifest.truncation.map((message) => `- Truncated: ${message}`),
...manifest.errors.map(
({ scope, message }) => `- Error (${scope}): ${message}`,
),
"",
];
await appendFile(summaryPath, lines.join("\n"), "utf8");
}
function errorMessage(cause) {
return cause instanceof Error ? cause.message : String(cause);
}
if (
process.argv[1] !== undefined &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
try {
const result = await collectJobTimingsBatch({
repository: process.env.CI_REPOSITORY,
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY,
});
process.stdout.write(
`Wrote batch manifest for ${String(result.manifest.coverage.collectedRuns)} collected runs to ${result.manifestPath}\n`,
);
} catch (cause) {
process.stderr.write(`CI timing batch failed: ${errorMessage(cause)}\n`);
process.exitCode = 1;
}
}
+354
View File
@@ -0,0 +1,354 @@
import { execFile } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
const execFileAsync = promisify(execFile);
const MAX_BUFFER = 16 * 1024 * 1024;
const TIMEOUT_MS = 30_000;
const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings");
const STEP_COSTS = [
{ key: "npmCi", pattern: /\bnpm ci\b/iu },
{ key: "artifactUpload", pattern: /upload.*artifact/iu },
{ key: "artifactDownload", pattern: /download.*artifact/iu },
];
/** Fetch and summarize one workflow attempt without turning telemetry into a gate. */
export async function collectJobTimings({
repository,
runId,
runAttempt,
outputDirectory = OUTPUT_DIRECTORY,
summaryPath,
observedAt = new Date().toISOString(),
env = process.env,
timeoutMs = TIMEOUT_MS,
}) {
const identity = validateIdentity({ repository, runId, runAttempt });
const endpoint = `repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100`;
const { stdout } = await execFileAsync(
"gh",
["api", "--paginate", "--slurp", endpoint],
{ env, encoding: "utf8", maxBuffer: MAX_BUFFER, timeout: timeoutMs },
);
const rawPath = join(outputDirectory, "jobs.json");
const reportPath = join(outputDirectory, "report.json");
await mkdir(outputDirectory, { recursive: true });
await writeFile(rawPath, stdout, "utf8");
const pages = parsePages(stdout);
const jobs = pages.flatMap(({ jobs: pageJobs }, index) =>
validateJobs(pageJobs, index),
);
const report = normalizeReport({ identity, jobs, observedAt });
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
if (summaryPath !== undefined)
await appendSummary(summaryPath, report, rawPath, reportPath);
return { rawPath, reportPath, report };
}
function validateIdentity({ repository, runId, runAttempt }) {
if (
typeof repository !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
)
throw new Error(
"CI_REPOSITORY must be an owner/name repository identifier",
);
return {
repository,
runId: parsePositiveInteger(runId, "CI_TIMING_RUN_ID"),
runAttempt: parsePositiveInteger(runAttempt, "CI_TIMING_RUN_ATTEMPT"),
};
}
function parsePositiveInteger(value, name) {
if (typeof value === "string" && /^[1-9]\d*$/u.test(value)) {
const parsed = Number(value);
if (Number.isSafeInteger(parsed)) return parsed;
}
throw new Error(`${name} must be a positive safe integer`);
}
function parsePages(stdout) {
let pages;
try {
pages = JSON.parse(stdout);
} catch (cause) {
throw new Error("GitHub jobs API returned malformed JSON", { cause });
}
if (!Array.isArray(pages) || pages.length === 0)
throw new Error("GitHub jobs API slurp must contain a nonempty page array");
for (const page of pages)
if (
typeof page !== "object" ||
page === null ||
Array.isArray(page) ||
!Array.isArray(page.jobs)
)
throw new Error("GitHub jobs API page must contain a jobs array");
return pages;
}
function validateJobs(jobs, pageIndex) {
return jobs.map((job, jobIndex) => {
if (
typeof job !== "object" ||
job === null ||
Array.isArray(job) ||
typeof job.name !== "string" ||
job.name.length === 0 ||
typeof job.status !== "string"
)
throw new Error(
`GitHub jobs API page ${String(pageIndex + 1)} job ${String(jobIndex + 1)} is malformed`,
);
return job;
});
}
function normalizeReport({ identity, jobs, observedAt }) {
const normalizedJobs = jobs.map(normalizeJob);
const queues = normalizedJobs
.map(({ queue }) => queue.seconds)
.filter((value) => value !== null);
const completedExecutions = normalizedJobs
.filter(
({ status, conclusion }) =>
status === "completed" && conclusion !== "skipped",
)
.map(({ execution }) => execution.seconds)
.filter((value) => value !== null);
const completedDurationSumSeconds = completedExecutions.reduce(
(sum, seconds) => sum + seconds,
0,
);
const completedExecutionUnknown = normalizedJobs.filter(
({ status, conclusion, execution }) =>
status === "completed" &&
conclusion !== "skipped" &&
execution.seconds === null,
).length;
const skipped = normalizedJobs.filter(
({ conclusion }) => conclusion === "skipped",
).length;
const costs = Object.fromEntries(STEP_COSTS.map(({ key }) => [key, []]));
for (const job of normalizedJobs)
for (const step of job.steps)
if (step.costKey !== null)
costs[step.costKey].push({ job: job.name, ...step });
return {
schemaVersion: 1,
run: {
...identity,
observedAt,
snapshot:
normalizedJobs.length > 0 &&
normalizedJobs.every(({ status }) => status === "completed")
? "completed job status snapshot; unknown timestamps remain unmeasured"
: "partial; job statuses may still be queued or in progress",
},
counts: {
jobs: normalizedJobs.length,
completed: normalizedJobs.filter(({ status }) => status === "completed")
.length,
skipped,
queueUnknown: normalizedJobs.filter(({ queue }) => queue.seconds === null)
.length,
executionUnknown: normalizedJobs.filter(
({ execution }) => execution.seconds === null,
).length,
completedExecutionUnknown,
},
aggregates: {
queueSeconds: percentileSummary(queues),
completedExecutionSeconds: percentileSummary(completedExecutions),
summedCompletedJobElapsedSeconds: {
value:
completedExecutions.length === 0
? null
: round(completedDurationSumSeconds),
knownCount: completedExecutions.length,
unknownCount: completedExecutionUnknown,
label:
"sum of known completed job elapsed durations; incomplete when timestamps are unknown; not CPU time, billing, or workflow wall time",
},
},
jobs: normalizedJobs,
stepCostInterpretation:
"Matched install/artifact steps by step name; these are not total artifact transfer costs.",
stepCosts: costs,
};
}
function normalizeJob(job) {
const skipped = job.conclusion === "skipped";
const hasStarted = ["in_progress", "completed"].includes(job.status);
const queue = skipped
? unknownDuration("job was skipped")
: !hasStarted
? unknownDuration("job start is not established by its status")
: duration(job.created_at, job.started_at);
const execution = skipped
? unknownDuration("job was skipped")
: job.status === "completed"
? duration(job.started_at, job.completed_at)
: unknownDuration("job is not completed");
const steps = Array.isArray(job.steps) ? job.steps.map(normalizeStep) : [];
return {
id: Number.isSafeInteger(job.id) && job.id > 0 ? job.id : null,
url: stringOrNull(job.html_url),
name: job.name,
workflowName: stringOrNull(job.workflow_name),
status: job.status,
conclusion: stringOrNull(job.conclusion),
createdAt: stringOrNull(job.created_at),
startedAt: stringOrNull(job.started_at),
completedAt: stringOrNull(job.completed_at),
queue,
execution,
steps,
};
}
function normalizeStep(step) {
if (
typeof step !== "object" ||
step === null ||
Array.isArray(step) ||
typeof step.name !== "string"
)
throw new Error("GitHub jobs API contains a malformed job step");
const cost = STEP_COSTS.find(({ pattern }) => pattern.test(step.name));
return {
name: step.name,
costKey: cost?.key ?? null,
duration: duration(step.started_at, step.completed_at),
};
}
function duration(startValue, endValue) {
if (typeof startValue !== "string" || typeof endValue !== "string")
return unknownDuration("timestamp unavailable");
const start = Date.parse(startValue);
const end = Date.parse(endValue);
if (!Number.isFinite(start) || !Number.isFinite(end))
return unknownDuration("invalid timestamp");
const milliseconds = end - start;
if (!Number.isFinite(milliseconds) || milliseconds < 0)
return unknownDuration("timestamps out of order");
return { seconds: round(milliseconds / 1_000), reason: null };
}
function unknownDuration(reason) {
return { seconds: null, reason };
}
function percentileSummary(values) {
return {
count: values.length,
p50: nearestRank(values, 0.5),
p95: nearestRank(values, 0.95),
unit: "seconds",
};
}
function nearestRank(values, percentile) {
if (values.length === 0) return null;
const sorted = [...values].sort((left, right) => left - right);
return round(sorted[Math.ceil(percentile * sorted.length) - 1]);
}
function round(value) {
return Math.round(value * 1_000) / 1_000;
}
function stringOrNull(value) {
return typeof value === "string" ? value : null;
}
async function appendSummary(summaryPath, report, rawPath, reportPath) {
await mkdir(dirname(summaryPath), { recursive: true });
const rows = [
["Queue", report.aggregates.queueSeconds],
["Completed execution", report.aggregates.completedExecutionSeconds],
]
.map(
([label, item]) =>
`| ${label} | ${display(item.p50)} | ${display(item.p95)} | ${String(item.count)} |`,
)
.join("\n");
const jobRows = report.jobs
.map(
(job) =>
`| ${escapeCell(job.name)} | ${job.status} | ${job.conclusion ?? "unknown"} | ${display(job.queue.seconds)} | ${job.queue.seconds === null ? job.queue.reason : ""} | ${display(job.execution.seconds)} | ${job.execution.seconds === null ? job.execution.reason : ""} |`,
)
.join("\n");
const costRows = STEP_COSTS.map(({ key }) => {
const samples = report.stepCosts[key];
const values = samples
.map(({ duration: item }) => item.seconds)
.filter((value) => value !== null);
return `| ${key} | ${String(samples.length)} | ${display(nearestRank(values, 0.5))} | ${samples.filter(({ duration: item }) => item.seconds === null).length} |`;
}).join("\n");
const text = [
"### CI job timing snapshot",
"",
`Observed at ${report.run.observedAt} for run ${String(report.run.runId)}, attempt ${String(report.run.runAttempt)}. This is a point-in-time snapshot of fetched jobs. Queue starts at job creation and excludes preceding dependency waiting. Missing, queued, in-progress, invalid, or out-of-order timestamps are reported as unknown. Elapsed durations are not CPU time or billing data.`,
"",
`Fetched ${String(report.counts.jobs)} jobs; ${String(report.counts.completed)} are completed (${String(report.counts.skipped)} skipped); ${String(report.counts.queueUnknown)} queue durations and ${String(report.counts.executionUnknown)} execution durations (${String(report.counts.completedExecutionUnknown)} for non-skipped completed jobs) are unknown.`,
"",
"| Metric | p50 (s) | p95 (s) | Samples |",
"| --- | ---: | ---: | ---: |",
rows,
"",
"| Job | Status | Conclusion | Queue (s) | Queue detail | Execution (s) | Execution detail |",
"| --- | --- | --- | ---: | --- | ---: | --- |",
jobRows,
"",
`Sum of known completed job elapsed durations: **${display(report.aggregates.summedCompletedJobElapsedSeconds.value)} s** across ${String(report.aggregates.summedCompletedJobElapsedSeconds.knownCount)} known and ${String(report.aggregates.summedCompletedJobElapsedSeconds.unknownCount)} unknown jobs; this is not CPU time, billing, or workflow wall time.`,
"",
"Matched install/artifact steps by name; these are not total artifact transfer costs.",
"",
"| Matched step group | Samples | p50 (s) | Unknown |",
"| --- | ---: | ---: | ---: |",
costRows,
"",
`Reports: \`${rawPath}\`, \`${reportPath}\`.`,
"",
].join("\n");
await writeFile(summaryPath, text, { encoding: "utf8", flag: "a" });
}
function display(value) {
return value === null ? "unknown" : String(value);
}
function escapeCell(value) {
return value.replaceAll("|", "\\|").replaceAll("\n", " ");
}
if (
process.argv[1] !== undefined &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
try {
const result = await collectJobTimings({
repository: process.env.CI_REPOSITORY,
runId: process.env.CI_TIMING_RUN_ID,
runAttempt: process.env.CI_TIMING_RUN_ATTEMPT,
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY,
summaryPath: process.env.GITHUB_STEP_SUMMARY,
});
process.stdout.write(
`Wrote ${result.report.jobs.length} job timing records to ${result.reportPath}\n`,
);
} catch (cause) {
process.stderr.write(
`CI timing telemetry failed: ${cause instanceof Error ? cause.message : String(cause)}\n`,
);
process.exitCode = 1;
}
}
+151 -106
View File
@@ -1,30 +1,20 @@
import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { matchesGlob } from "node:path";
import { MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
import { planTestSelection } from "./test-selection.mjs";
const flags = [
...new Set([
"code",
"runtime",
"package",
"apple",
"fixtures",
"readiness",
"docs",
"workflows",
...Object.keys(SCOPES),
]),
];
// Test files select the native-host lanes that run them explicitly. Provider
// scopes remain implementation-owned, so test-only edits stay off those lanes.
const NATIVE_TEST_OWNERS = ["apple", "package"];
// Test files select the host/runtime lanes that execute their focused suites.
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
// Actionlint validates workflow syntax only. The semantic suite asserts these
// workflows' release, publication and self-hosted runner safeguards.
const GUARDED_WORKFLOWS = new Set([
".github/workflows/pr-title.yml",
".github/workflows/ci-full-pr.yml",
".github/workflows/release.yml",
".github/workflows/real-ghidra-windows.yml",
".github/workflows/sqlite-runtime.yml",
".github/workflows/ci-timings.yml",
]);
// Sources compiled by build-conformance-fixtures.mjs. The readiness journey
// analyzes the built C fixture as well as its own applications.
@@ -36,6 +26,15 @@ const FIXTURE_SOURCES = [
"tests/conformance/versions/**",
];
const READINESS_SOURCES = ["tests/conformance/readiness/**"];
// These reporters affect CI output only; they do not select a host or provider
// verification lane when changed on their own.
const PORTABLE_CI_REPORTERS = new Set([
"scripts/ci/job-timings.mjs",
"scripts/ci/job-timings-batch.mjs",
"scripts/ci/test-report.mjs",
"scripts/ci/test-timing-reporter.mjs",
"scripts/ci/test-history.mjs",
]);
const matchesAny = (file, patterns) =>
patterns.some((pattern) => matchesGlob(file, pattern));
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
@@ -63,6 +62,10 @@ function classifyPackage(base, head) {
]),
].filter((key) => beforeScripts?.[key] !== afterScripts?.[key]);
for (const script of changed) {
if (script === "test:sqlite" || script === "test:sqlite:run") {
selected.sqlite = true;
continue;
}
if (
/^(test(?::|$)|check(?::|$)|lint(?::|$)|format(?::|$)|knip$|jscpd$)/.test(
script,
@@ -108,101 +111,137 @@ function classifyPackage(base, head) {
}
const event = process.env.GITHUB_EVENT_NAME;
let files = [];
if (
event === "push" ||
if (!["pull_request", "push", "schedule", "workflow_dispatch"].includes(event))
throw new Error(`Unsupported CI event: ${event}`);
const requestedFull =
event === "schedule" ||
event === "workflow_dispatch" ||
process.env.CI_FULL === "true"
) {
process.env.CI_FULL === "true";
const tier =
event === "schedule" &&
process.env.CI_SKIP_FULL === "true" &&
process.env.CI_FULL !== "true"
? "reused"
: requestedFull
? "full"
: "fast";
let files = [];
if (tier === "reused") {
reasons.push(
"Recent exact-SHA full qualification reused; no fresh provider observation",
);
} else if (tier === "full") {
full("Full baseline requested by event or ci:full label");
} else if (event === "pull_request") {
const base = git("merge-base", process.env.BASE_SHA, process.env.HEAD_SHA);
} else if (event === "pull_request" || event === "push") {
const head = process.env.HEAD_SHA;
// Disable rename detection so both the removed and added paths participate.
files = git("diff", "--name-only", "--no-renames", "-z", base, head)
.split("\0")
.filter(Boolean);
for (const file of files) {
if (
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
file,
) ||
file.startsWith("docs/") ||
file.startsWith(".github/ISSUE_TEMPLATE/")
) {
selected.docs = true;
selected.website ||= matchesAny(file, SCOPES.website);
continue;
}
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
selected.code = true;
for (const owner of NATIVE_TEST_OWNERS)
selected[owner] ||= matchesAny(file, SCOPES[owner]);
continue;
}
if (matchesAny(file, MANUAL_VERIFIERS)) {
selected.code = true;
continue;
}
const owners = Object.entries(SCOPES)
.filter(([, patterns]) =>
patterns.some((pattern) => matchesGlob(file, pattern)),
)
.map(([owner]) => owner);
for (const owner of owners) selected[owner] = true;
if (
file.startsWith(".github/workflows/") ||
file === ".github/actionlint.yaml"
) {
selected.workflows = true;
if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
if (file !== ".github/workflows/ci.yml") continue;
}
// The Notes example is also the source-owned agent evaluation fixture.
if (file.startsWith("website/public/examples/notes-electron/"))
selected.code = true;
if (owners.length === 1 && owners[0] === "website") continue;
if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
if (file === "package.json") {
classifyPackage(base, head);
continue;
}
selected.code = true;
if (matchesAny(file, FIXTURE_SOURCES)) {
selected.fixtures = true;
selected.readiness = true;
}
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
if (file.startsWith("tests/")) continue;
if (
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
file,
)
) {
full(`Shared boundary: ${file}`);
} else if (owners.length) {
if (file.startsWith("src/") || file.startsWith("bridge/"))
selected.runtime = true;
} else if (/^(src|bridge)\//.test(file)) {
// Unowned runtime sources keep the Linux x64 installed-package baseline;
// ci:full, main pushes and manual dispatch retain every native host.
selected.runtime = true;
reasons.push(`Unowned runtime source: ${file}`);
} else if (file.startsWith(".agents/skills/")) {
selected.runtime = true;
selected.docs = true;
} else {
full(`Unclassified input: ${file}`);
let base;
if (event === "pull_request") {
base = git("merge-base", process.env.BASE_SHA, head);
} else {
const candidate = process.env.BASE_SHA;
const baselineReason = process.env.BASELINE_REASON;
try {
if (!candidate || !/^[0-9a-f]{40}$/i.test(candidate))
throw new Error(
baselineReason || "No valid successful push baseline was found",
);
execFileSync("git", ["merge-base", "--is-ancestor", candidate, head], {
stdio: "ignore",
});
base = candidate;
} catch {
full(
baselineReason ||
"No valid successful push baseline ancestor; selecting all lanes",
);
}
}
} else throw new Error(`Unsupported CI event: ${event}`);
if (base) {
// Disable rename detection so both the removed and added paths participate.
files = git("diff", "--name-only", "--no-renames", "-z", base, head)
.split("\0")
.filter(Boolean);
for (const file of files) {
if (
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
file,
) ||
file.startsWith("docs/") ||
file.startsWith(".github/ISSUE_TEMPLATE/")
) {
selected.docs = true;
selected.website ||= matchesAny(file, SCOPES.website);
continue;
}
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
selected.code = true;
for (const owner of HOST_TEST_OWNERS)
selected[owner] ||= matchesAny(file, SCOPES[owner]);
continue;
}
if (matchesAny(file, MANUAL_VERIFIERS)) {
selected.code = true;
continue;
}
const owners = Object.entries(SCOPES)
.filter(([, patterns]) =>
patterns.some((pattern) => matchesGlob(file, pattern)),
)
.map(([owner]) => owner);
for (const owner of owners) selected[owner] = true;
if (
file.startsWith(".github/workflows/") ||
file === ".github/actionlint.yaml"
) {
selected.workflows = true;
if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
if (file !== ".github/workflows/ci.yml") continue;
}
// The Notes example is also the source-owned agent evaluation fixture.
if (file.startsWith("website/public/examples/notes-electron/"))
selected.code = true;
if (owners.length === 1 && owners[0] === "website") continue;
if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
if (file === "package.json") {
classifyPackage(base, head);
continue;
}
selected.code = true;
if (matchesAny(file, FIXTURE_SOURCES)) {
selected.fixtures = true;
selected.readiness = true;
}
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
if (file.startsWith("tests/")) continue;
if (PORTABLE_CI_REPORTERS.has(file)) continue;
if (
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
file,
)
) {
full(`Shared boundary: ${file}`);
} else if (owners.length) {
if (file.startsWith("src/") || file.startsWith("bridge/"))
selected.runtime = true;
} else if (/^(src|bridge)\//.test(file)) {
// Unowned runtime sources keep the Linux x64 installed-package baseline;
// full qualification retains every native host.
selected.runtime = true;
reasons.push(`Unowned runtime source: ${file}`);
} else if (file.startsWith(".agents/skills/")) {
selected.runtime = true;
selected.docs = true;
} else {
full(`Unclassified input: ${file}`);
}
}
}
}
selected.runtime ||= selected.package;
const linux = { os: "ubuntu-latest", platform: "linux", architecture: "x64" };
// macOS x64 is the slowest package host. Pull requests run it only with ci:full;
// main pushes, schedules and manual dispatch always include it.
// macOS x64 is the slowest package host. Fast qualification omits it.
const intel =
event !== "pull_request" || process.env.CI_FULL === "true"
tier === "full"
? [{ os: "macos-15-intel", platform: "darwin", architecture: "x64" }]
: [];
const matrix = {
@@ -215,17 +254,23 @@ const matrix = {
]
: [linux],
};
const plan = { ...selected, matrix, files, reasons };
const testSelection = planTestSelection(files, event, tier);
const plan = { ...selected, tier, matrix, files, reasons, testSelection };
if (process.env.GITHUB_OUTPUT)
appendFileSync(
process.env.GITHUB_OUTPUT,
Object.entries({ ...selected, matrix: JSON.stringify(matrix) })
Object.entries({
...selected,
tier,
matrix: JSON.stringify(matrix),
test_selection: JSON.stringify(testSelection),
})
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
);
if (process.env.GITHUB_STEP_SUMMARY)
appendFileSync(
process.env.GITHUB_STEP_SUMMARY,
`### Selected CI lanes\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
);
console.log(JSON.stringify(plan));
+176
View File
@@ -0,0 +1,176 @@
import { execFileSync } from "node:child_process";
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
const ARTIFACT_NAME = "ci-full-qualification-v1";
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
const positiveId = (value) =>
(typeof value === "string" || typeof value === "number") &&
/^[1-9]\d*$/.test(String(value)) &&
Number.isSafeInteger(Number(value));
const commitSha = (value) =>
typeof value === "string" && /^[0-9a-f]{40}$/.test(value);
/** Record only after the required gate has verified every full-tier obligation. */
export function recordQualification(directory) {
const sourceSha = process.env.GITHUB_SHA;
const runId = process.env.GITHUB_RUN_ID;
const runAttempt = process.env.GITHUB_RUN_ATTEMPT;
const repository = process.env.GITHUB_REPOSITORY;
if (
!commitSha(sourceSha) ||
!positiveId(runId) ||
!positiveId(runAttempt) ||
!repository
)
throw new Error(
"Full qualification receipt requires source and workflow identity",
);
const checkoutSha = execFileSync("git", ["rev-parse", "HEAD"], {
encoding: "utf8",
}).trim();
if (checkoutSha !== sourceSha)
throw new Error("Qualification checkout differs from workflow source");
mkdirSync(directory, { recursive: true });
const path = resolve(directory, "receipt.json");
writeFileSync(
path,
`${JSON.stringify(
{
schemaVersion: 1,
tier: "full",
sourceSha,
repository,
runId: Number(runId),
runAttempt: Number(runAttempt),
qualifiedAt: new Date().toISOString(),
},
null,
2,
)}\n`,
);
if (process.env.GITHUB_OUTPUT)
appendFileSync(
process.env.GITHUB_OUTPUT,
`receipt_name=${ARTIFACT_NAME}\nreceipt_path=${path}\n`,
);
}
function admit() {
if (process.env.CI_FORCE_FULL === "true")
return {
run_full: true,
reason: "Forced daily runner and dependency refresh",
};
const repository = process.env.CI_REPOSITORY;
const repositoryId = Number(process.env.CI_REPOSITORY_ID);
const branch = process.env.CI_DEFAULT_BRANCH;
const head = process.env.HEAD_SHA;
if (!repository || !positiveId(repositoryId) || !branch || !commitSha(head))
return {
run_full: true,
reason: "Qualification lookup identity is incomplete",
};
const now = Date.now();
const deadline = now + 20_000;
const api = (...arguments_) => {
const remaining = deadline - Date.now();
if (remaining <= 0)
throw new Error("Qualification lookup budget exhausted");
return JSON.parse(
execFileSync("gh", ["api", "--method", "GET", ...arguments_], {
encoding: "utf8",
timeout: Math.min(15_000, remaining),
maxBuffer: 8 * 1024 * 1024,
stdio: ["ignore", "pipe", "pipe"],
}),
);
};
try {
const { artifacts } = api(
`repos/${repository}/actions/artifacts`,
"-f",
`name=${ARTIFACT_NAME}`,
"-F",
"per_page=100",
);
if (!Array.isArray(artifacts))
throw new Error("Missing qualification artifact inventory");
const candidates = artifacts.filter((artifact) => {
const lineage = artifact?.workflow_run;
const age = now - Date.parse(artifact?.created_at);
return (
artifact?.name === ARTIFACT_NAME &&
artifact?.expired === false &&
typeof artifact?.created_at === "string" &&
Number.isFinite(age) &&
age >= 0 &&
age < MAX_AGE_MS &&
positiveId(lineage?.id) &&
String(lineage.id) !== process.env.GITHUB_RUN_ID &&
lineage?.repository_id === repositoryId &&
lineage?.head_repository_id === repositoryId &&
lineage?.head_branch === branch &&
lineage?.head_sha === head
);
});
// Artifact metadata narrows the search before run lookups. Never scan an
// unbounded history or treat a missing receipt as a successful qualification.
for (const artifact of candidates.slice(0, 3)) {
const run = api(
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
);
// An old run can upload a new artifact when only its gate is rerun.
// Bound the original run age as well so retries cannot renew stale proof.
const runAge = Date.now() - Date.parse(run.created_at);
if (
typeof run.created_at === "string" &&
Number.isFinite(runAge) &&
runAge >= 0 &&
runAge < MAX_AGE_MS &&
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.path === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_sha === head &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
) {
return {
run_full: false,
run_id: run.id,
reason: `Reusing full qualification from run ${run.id}; no fresh provider observation`,
};
}
}
return {
run_full: true,
reason: "No trusted full qualification for this SHA within 24 hours",
};
} catch {
return {
run_full: true,
reason: "Qualification lookup unavailable; running full validation",
};
}
}
if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) {
const decision = admit();
if (process.env.GITHUB_OUTPUT)
appendFileSync(
process.env.GITHUB_OUTPUT,
Object.entries(decision)
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
);
if (process.env.GITHUB_STEP_SUMMARY)
appendFileSync(
process.env.GITHUB_STEP_SUMMARY,
`### Scheduled qualification\n\n${decision.reason}.\n`,
);
console.log(JSON.stringify(decision));
}
+40 -6
View File
@@ -1,18 +1,46 @@
import { CI_FLAGS } from "./scopes.mjs";
import { recordQualification } from "./qualification.mjs";
const jobs = JSON.parse(process.env.CI_JOBS);
const outputs = jobs.changes.outputs;
const required = ["changes", "static", "test"];
const outputs = jobs?.changes?.outputs;
if (!["fast", "full", "reused"].includes(outputs?.tier))
throw new Error(
"CI classification requires a valid tier: fast, full, or reused",
);
const invalidFlags = CI_FLAGS.filter(
(flag) => outputs[flag] !== "true" && outputs[flag] !== "false",
);
if (invalidFlags.length)
throw new Error(
`CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`,
);
const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true");
const reused = outputs.tier === "reused";
if (
reused &&
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
selectedScopes.length !== 0)
)
throw new Error(
"Reused qualification requires a scheduled admission receipt and no selected lanes",
);
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
const required = reused ? ["changes"] : ["changes", "static", "test"];
const lanes = {
code: ["build", "test-shard", "sqlite-runtime"],
code: ["build", "test-shard"],
runtime: ["windows-native-build", "package-e2e"],
package: ["windows-curated"],
sqlite: ["sqlite-runtime"],
apple: ["apple-artifacts"],
fixtures: ["conformance-fixtures"],
readiness: ["reconstruction-readiness"],
docs: ["docs"],
workflows: ["static"],
};
for (const [scope, value] of Object.entries(outputs)) {
if (value === "true") required.push(...(lanes[scope] ?? [scope]));
}
for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope]));
const failed = [...new Set(required)].filter(
(name) => jobs[name]?.result !== "success",
);
@@ -23,3 +51,9 @@ for (const [name, job] of Object.entries(jobs)) {
if (failed.length)
throw new Error(`CI lanes did not pass: ${failed.join(", ")}`);
console.log("Every selected CI lane passed");
if (reused)
console.log(
`Qualification reused from run ${outputs.qualification_run_id}; no fresh provider observation`,
);
if (outputs.tier === "full" && process.env.CI_QUALIFICATION_DIRECTORY)
recordQualification(process.env.CI_QUALIFICATION_DIRECTORY);
+67
View File
@@ -55,6 +55,9 @@ export const SCOPES = {
"tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts",
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
"tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts",
"src/domain/javascript/javascriptRuntimeStaticCandidates.ts",
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
"tests/boundary/process/ProcessEvidence.host.test.ts",
"tests/acceptance/webModuleCli.test.ts",
@@ -84,6 +87,55 @@ export const SCOPES = {
"tests/boundary/filesystem/javascriptArtifactContentIdentity.test.ts",
"tests/boundary/process/sourceMapUncHosts.test.ts",
],
// Native SQLite runtime compatibility for the producer, public adapters,
// fixtures, shared lifecycle, and the package script that defines its suite.
sqlite: [
"src/sqlite/**",
"src/domain/sqlite/**",
"src/contracts/sqlite/**",
"src/application/sqlite/**",
"src/composition/sqlite.ts",
"src/cli/sqliteCommands.ts",
"src/server/registerSqliteTools.ts",
"src/application/DirectAnalysisStatus*.ts",
"src/application/CapabilityInventory*.ts",
"src/application/AnalysisProvider*.ts",
"src/contracts/toolContracts.ts",
"src/contracts/toolContractTypes.ts",
"src/contracts/toolEffects*.ts",
"src/contracts/toolOutputSchemaPrimitives.ts",
"src/domain/analysisError*.ts",
"src/domain/evidence*.ts",
"src/domain/inputIssueProjection*.ts",
"src/domain/jsonSerialization*.ts",
"src/domain/jsonValue*.ts",
"src/domain/localPath*.ts",
"src/domain/providerAdapterError*.ts",
"src/domain/providerCleanupError*.ts",
"src/domain/result*.ts",
"src/process/OwnedCommand*.ts",
"src/process/ProviderProcess*.ts",
"src/process/PrivateRuntimeRoot*.ts",
"src/artifacts/ArtifactReader*.ts",
"src/artifacts/ArtifactResourceScope*.ts",
"src/artifacts/ArtifactPaths*.ts",
"src/artifacts/readStableArtifact*.ts",
"src/filesystem/OwnedFileHandle*.ts",
"src/server/EvidenceMcpServer*.ts",
"src/server/analysisAdmission*.ts",
"src/server/contractToolHandler*.ts",
"src/server/toolRegistrationOptions*.ts",
"src/server/createServer*.ts",
"src/cli.ts",
"src/cliCommandNames.ts",
"src/cliLogging.ts",
"src/cli/commandCancellation.ts",
"tests/boundary/sqlite/**",
"tests/fixtures/sqlite/**",
"tests/acceptance/sqliteDatabaseCli.test.ts",
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
".github/workflows/sqlite-runtime.yml",
],
browser: [
".github/workflows/real-browser.yml",
"scripts/verify-real-browser.mjs",
@@ -299,6 +351,7 @@ export const SCOPES = {
nativeaot: [
"bridge/nativeaot/**",
"scripts/generate-nativeaot-fixtures.mjs",
"scripts/lib/nativeaot-build.*",
"tests/fixtures/nativeaot/**",
".github/workflows/nativeaot-fixtures.yml",
],
@@ -311,6 +364,20 @@ export const SCOPES = {
],
};
export const CI_FLAGS = [
...new Set([
"code",
"runtime",
"package",
"apple",
"fixtures",
"readiness",
"docs",
"workflows",
...Object.keys(SCOPES),
]),
];
// Verifier entrypoints that only manual, licensed, or self-hosted workflows
// run. No selected CI lane executes them, so their edits need only the static
// checks and the source-test suite. Shared helpers under scripts/lib remain
+114
View File
@@ -0,0 +1,114 @@
import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
const ARTIFACT_NAME = "test-timings";
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
const MAX_CANDIDATES = 5;
const positiveId = (value) =>
(typeof value === "string" || typeof value === "number") &&
/^[1-9]\d*$/.test(String(value)) &&
Number.isSafeInteger(Number(value));
function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
if (
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository ?? "") ||
!branch ||
!positiveId(repositoryId)
)
return "";
const deadline = Date.now() + 20_000;
const api = (...arguments_) => {
const remaining = deadline - Date.now();
if (remaining <= 0)
throw new Error("Timing history lookup budget exhausted");
return JSON.parse(
execFileSync("gh", ["api", "--method", "GET", ...arguments_], {
encoding: "utf8",
timeout: Math.min(10_000, remaining),
maxBuffer: 4 * 1024 * 1024,
stdio: ["ignore", "pipe", "pipe"],
}),
);
};
const { artifacts } = api(
`repos/${repository}/actions/artifacts`,
"-f",
`name=${ARTIFACT_NAME}`,
"-F",
"per_page=100",
);
if (!Array.isArray(artifacts))
throw new Error("Missing timing artifact inventory");
const candidates = artifacts
.filter((artifact) => {
const lineage = artifact?.workflow_run;
const age = now - Date.parse(artifact?.created_at);
return (
artifact?.name === ARTIFACT_NAME &&
artifact?.expired === false &&
typeof artifact?.created_at === "string" &&
Number.isFinite(age) &&
age >= 0 &&
age < MAX_AGE_MS &&
positiveId(lineage?.id) &&
positiveId(lineage?.repository_id) &&
positiveId(lineage?.head_repository_id) &&
lineage.repository_id === lineage.head_repository_id &&
lineage.repository_id === Number(repositoryId) &&
lineage.head_branch === branch
);
})
.sort(
(left, right) =>
Date.parse(right.created_at) - Date.parse(left.created_at),
)
.slice(0, MAX_CANDIDATES);
for (const artifact of candidates) {
const run = api(
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
);
const runAge = now - Date.parse(run?.created_at);
if (
typeof run?.created_at === "string" &&
Number.isFinite(runAge) &&
runAge >= 0 &&
runAge < MAX_AGE_MS &&
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.path === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
)
return String(run.id);
}
return "";
}
function write(runId) {
if (process.env.GITHUB_OUTPUT)
appendFileSync(process.env.GITHUB_OUTPUT, `run-id=${runId}\n`);
console.log(JSON.stringify({ run_id: runId }));
}
if (
process.argv[1] &&
fileURLToPath(import.meta.url) === resolve(process.argv[1])
) {
let runId = "";
try {
runId = findTimingRun({
repository: process.env.CI_REPOSITORY,
branch: process.env.CI_DEFAULT_BRANCH,
repositoryId: process.env.CI_REPOSITORY_ID,
});
} catch {
// History improves shard ordering but must never block CI.
}
write(runId);
}
+82 -17
View File
@@ -1,28 +1,36 @@
import { execFileSync } from "node:child_process";
import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { z } from "zod";
import { compareTestSelection } from "./test-selection.mjs";
const root = process.cwd();
const historyDirectory = join(root, ".cache/ci-test-history");
const historyPath = join(historyDirectory, "test-results.json");
const selectionDirectory = join(root, ".cache/ci-test-selection");
const selectionPath = join(selectionDirectory, "report.json");
// Never let a failed or partial rerun leave a previous successful history file.
await rm(historyPath, { force: true });
await rm(selectionPath, { force: true });
const report = z
.object({
version: z.literal(1),
success: z.boolean(),
numFailedTests: z.number().int().nonnegative(),
testResults: z.array(
reason: z.string().optional(),
unhandledErrors: z.number().int().nonnegative().optional(),
files: z.array(
z.object({
name: z.string().min(1),
startTime: z.number().finite(),
endTime: z.number().finite(),
durationMs: z.number().finite().nonnegative(),
state: z.string().optional(),
}),
),
})
.parse(JSON.parse(await readFile("test-results.json", "utf8")));
.parse(JSON.parse(await readFile("test-timings.json", "utf8")));
const files = report.testResults.map(({ name, startTime, endTime }) => {
const files = report.files.map(({ name, durationMs, state }) => {
const path = relative(root, resolve(root, name));
if (
path === "" ||
@@ -31,32 +39,85 @@ const files = report.testResults.map(({ name, startTime, endTime }) => {
isAbsolute(path)
)
throw new Error(`Test report path is outside the checkout: ${name}`);
const duration = endTime - startTime;
if (!Number.isFinite(duration) || duration < 0)
throw new Error(`Test report has an invalid duration: ${name}`);
return { name: path.split(sep).join("/"), startTime: 0, endTime: duration };
return { name: path.split(sep).join("/"), durationMs, state };
});
const complete = process.env.SHARD_RESULT === "success";
const passed =
complete &&
process.env.MERGE_RESULT === "success" &&
report.success &&
report.numFailedTests === 0;
complete && process.env.MERGE_RESULT === "success" && report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
const expected =
verifyInventory || process.env.CI_TEST_SELECTION
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
})
.split("\0")
.filter((path) => path.endsWith(".test.ts"))
: [];
let selection;
if (process.env.CI_TEST_SELECTION) {
const comparison = compareTestSelection(
JSON.parse(process.env.CI_TEST_SELECTION),
expected,
files,
);
selection = {
...comparison,
complete:
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
["success", "failure"].includes(process.env.SHARD_RESULT) &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
runId: process.env.GITHUB_RUN_ID ?? null,
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
};
await mkdir(selectionDirectory, { recursive: true });
await writeFile(selectionPath, JSON.stringify(selection));
}
if (passed && verifyInventory) {
const actual = files.map(({ name }) => name);
const expectedSet = new Set(expected);
const actualSet = new Set(actual);
const missing = expected.filter((path) => !actualSet.has(path));
const unexpected = actual.filter((path) => !expectedSet.has(path));
if (missing.length || unexpected.length || actualSet.size !== actual.length)
throw new Error(
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
);
}
const summary = [
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
"",
...(selection
? [
"### Observational PR test selection",
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"",
]
: []),
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
passed
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
"",
"Durations include tests and hooks, worker preparation, environment, imports, and setup. They are scheduling weights, not summed workflow wall time.",
"",
"| Slowest file | Duration (ms) |",
"| --- | ---: |",
...files
.toSorted((left, right) => right.endTime - left.endTime)
.toSorted((left, right) => right.durationMs - left.durationMs)
.slice(0, 20)
.map(
({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`,
({ name, durationMs }) =>
`| ${name.replaceAll("|", "\\|")} | ${durationMs} |`,
),
"",
].join("\n");
@@ -68,6 +129,10 @@ if (passed) {
await mkdir(historyDirectory, { recursive: true });
await writeFile(
historyPath,
JSON.stringify({ success: true, numFailedTests: 0, testResults: files }),
JSON.stringify({
version: 1,
success: true,
files: files.map(({ name, durationMs }) => ({ name, durationMs })),
}),
);
}
+35
View File
@@ -0,0 +1,35 @@
export interface TestSelectionPlan {
readonly version: 1;
readonly mode: "full" | "candidate";
readonly owners: readonly string[];
readonly changedTests: readonly string[];
readonly changedPaths: readonly string[];
readonly reason: string;
}
export function planTestSelection(
files: readonly string[],
event: string,
tier: string,
): TestSelectionPlan;
export function compareTestSelection(
plan: unknown,
inventory: readonly string[],
results: readonly {
readonly name: string;
readonly durationMs: number;
readonly state?: string;
}[],
): {
readonly version: 1;
readonly observational: true;
readonly plan: TestSelectionPlan;
readonly selected: readonly string[];
readonly omitted: readonly string[];
readonly missing: readonly string[];
readonly unexpected: readonly string[];
readonly duplicateFiles: number;
readonly unobserved: readonly string[];
readonly missedFailures: readonly string[];
readonly complete: boolean;
readonly omittedDurationMs: number;
};
+151
View File
@@ -0,0 +1,151 @@
import { matchesGlob } from "node:path";
// Candidate ownership only: these leaf adapters and their producer tests can be
// compared with full-suite outcomes. This table never changes test execution.
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
const OWNERS = {
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
apktool: ["src/apktool/**"],
dotnet: ["src/dotnet/**"],
evm: ["src/evm/**", "tests/boundary/evm/**"],
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
flutter: ["src/flutter/**"],
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
jeb: ["src/jeb/**"],
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
wasm: ["src/wasm/**"],
};
const ownersOf = (path) =>
Object.entries(OWNERS)
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
.map(([owner]) => owner);
const validPath = (path) =>
typeof path === "string" &&
/^(src|tests)\//.test(path) &&
!/[\\\0\r\n]/.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part));
const full = (reason, changedPaths = []) => ({
version: 1,
mode: "full",
owners: [],
changedTests: [],
changedPaths,
reason,
});
/** Plan an observational PR subset from the classifier's existing Git diff. */
export function planTestSelection(files, event, tier) {
if (event !== "pull_request" || tier !== "fast")
return full("Only ordinary PRs are candidates", files);
if (!files.length) return full("No changed paths available");
const owners = new Set();
const changedTests = [];
for (const path of files) {
if (!validPath(path))
return full(`Shared or unknown input: ${path}`, files);
const matches = ownersOf(path);
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
for (const owner of matches) owners.add(owner);
if (path.endsWith(".test.ts")) changedTests.push(path);
}
return {
version: 1,
mode: "candidate",
owners: [...owners].sort(),
changedTests: [...new Set(changedTests)].sort(),
changedPaths: files,
reason:
"Observe affected leaf owners; retain integration and unowned tests",
};
}
/** Compare a proposal with the full run; missing observations never prove parity. */
export function compareTestSelection(plan, inventory, results) {
if (
plan?.version !== 1 ||
!["full", "candidate"].includes(plan.mode) ||
!Array.isArray(plan.owners) ||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
!Array.isArray(plan.changedTests) ||
!Array.isArray(plan.changedPaths) ||
plan.changedPaths.some((path) => typeof path !== "string") ||
plan.changedTests.some(
(path) => !validPath(path) || !path.endsWith(".test.ts"),
) ||
typeof plan.reason !== "string" ||
(plan.mode === "candidate" && !plan.owners.length) ||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
)
throw new Error("Invalid observational test selection plan");
const expected = new Set(inventory);
if (!expected.size || expected.size !== inventory.length)
throw new Error("Test selection requires a nonempty unique inventory");
const ownersByFile = new Map(
plan.mode === "candidate"
? inventory.map((path) => [path, ownersOf(path)])
: [],
);
// Removed/renamed tests have no current executable identity. Keep all tests.
const removed = plan.changedTests.filter((path) => !expected.has(path));
const missingOwners = plan.owners.filter(
(owner) =>
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
);
const effective = removed.length
? full(
`Changed tests absent from current inventory: ${removed.join(", ")}`,
plan.changedPaths,
)
: missingOwners.length
? full(
`Owners without executable tests: ${missingOwners.join(", ")}`,
plan.changedPaths,
)
: plan;
const selected = [];
const omitted = [];
for (const path of inventory.toSorted()) {
const owners = ownersByFile.get(path) ?? [];
const keep =
effective.mode === "full" ||
!owners.length ||
effective.changedTests.includes(path) ||
owners.some((owner) => effective.owners.includes(owner));
(keep ? selected : omitted).push(path);
}
const actual = new Set(results.map((file) => file.name));
const missing = inventory.filter((path) => !actual.has(path));
const unexpected = [...actual].filter((path) => !expected.has(path));
const duplicateFiles = results.length - actual.size;
const unobserved = results
.filter((file) => !["passed", "failed"].includes(file.state))
.map((file) => file.name);
const omittedSet = new Set(omitted);
const missedFailures = results
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
.map((file) => file.name);
return {
version: 1,
observational: true,
plan: effective,
selected,
omitted,
missing,
unexpected,
duplicateFiles,
unobserved,
missedFailures,
complete:
!missing.length &&
!unexpected.length &&
!duplicateFiles &&
!unobserved.length,
// Phase totals are scheduling weights, not an estimate of wall-time savings.
omittedDurationMs: results
.filter((file) => omittedSet.has(file.name))
.reduce((sum, file) => sum + file.durationMs, 0),
};
}
+43
View File
@@ -0,0 +1,43 @@
import { writeFile } from "node:fs/promises";
/** @implements {import("vitest/node").Reporter} */
export default class TestTimingReporter {
/**
* @param {ReadonlyArray<import("vitest/node").TestModule>} modules
* @param {ReadonlyArray<import("vitest/node").SerializedError>} errors
* @param {import("vitest/node").TestRunEndReason} reason
*/
async onTestRunEnd(modules, errors, reason) {
const files = modules.map((module) => {
const diagnostic = module.diagnostic();
// These disjoint phases match Vitest's per-file wall-time accounting.
// Import timings already include transform waits; do not add them again.
const phases = {
execution: diagnostic.duration,
preparation: diagnostic.prepareDuration,
environment: diagnostic.environmentSetupDuration,
imports: diagnostic.collectDuration,
setup: diagnostic.setupDuration,
};
return {
name: module.moduleId,
state: module.state(),
durationMs: Object.values(phases).reduce(
(total, value) => total + value,
0,
),
phases,
};
});
await writeFile(
"test-timings.json",
JSON.stringify({
version: 1,
success: reason === "passed" && errors.length === 0,
reason,
unhandledErrors: errors.length,
files,
}),
);
}
}
+235
View File
@@ -0,0 +1,235 @@
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { execFile } from "node:child_process";
import { copyFile, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
const MANIFEST_NAME = "manifest.json";
const TARBALL_NAME = "package.tgz";
const MANIFEST_KEYS = [
"schemaVersion",
"sourceSha",
"runId",
"runAttempt",
"platform",
"architecture",
"packageVersion",
"nodeVersion",
"sha256",
].sort();
const execFileAsync = promisify(execFile);
/** Normalize workflow identity and host facts into the manifest-owned fields. */
export function createVerifiedPackageMetadata({
sourceSha,
runId,
runAttempt,
platform,
architecture,
packageVersion,
nodeVersion,
}) {
if (!/^[a-f0-9]{40}$/u.test(sourceSha))
throw new Error("Source SHA must be a full commit SHA");
return {
sourceSha,
runId: parsePositiveInteger(runId, "GITHUB_RUN_ID"),
runAttempt: parsePositiveInteger(runAttempt, "GITHUB_RUN_ATTEMPT"),
platform,
architecture,
packageVersion,
nodeVersion,
};
}
/** Build the single supported manifest representation for a verified tarball. */
function createVerifiedPackageManifest(metadata) {
const manifest = { schemaVersion: 1, ...metadata };
validateManifestShape(manifest);
return manifest;
}
/** Write a manifest after the caller has completed qualification and cleanup. */
export async function writeVerifiedPackageManifest(directory, metadata) {
const manifest = createVerifiedPackageManifest(metadata);
await writeFile(
join(directory, MANIFEST_NAME),
`${JSON.stringify(manifest, null, 2)}\n`,
{ encoding: "utf8" },
);
return manifest;
}
/** Copy the qualified archive under its fixed name and prepare its manifest. */
export async function retainVerifiedPackage({
sourceTarball,
directory,
metadata,
}) {
const tarball = join(directory, TARBALL_NAME);
await copyFile(sourceTarball, tarball);
const sha256 = await sha256File(tarball);
return {
directory,
metadata: createVerifiedPackageManifest({ ...metadata, sha256 }),
};
}
/** Validate a retained package against the current checkout and workflow attempt. */
async function validateVerifiedPackage({
directory,
sourceSha,
runId,
runAttempt,
packageVersion,
nodeVersion,
}) {
if (!/^[a-f0-9]{40}$/u.test(sourceSha))
throw new Error("Expected source SHA must be a full commit SHA");
if (!Number.isSafeInteger(runId) || runId <= 0)
throw new Error("Expected workflow run ID must be a positive integer");
if (!Number.isSafeInteger(runAttempt) || runAttempt <= 0)
throw new Error("Expected workflow run attempt must be a positive integer");
const manifestPath = join(directory, MANIFEST_NAME);
let manifest;
try {
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
} catch (cause) {
throw new Error(`Could not read verified package manifest: ${cause}`, {
cause,
});
}
validateManifestShape(manifest);
assertEqual(manifest.sourceSha, sourceSha, "source SHA");
assertEqual(manifest.runId, runId, "workflow run ID");
// A successful producer may belong to an earlier attempt when only failed
// publisher jobs are rerun. The workflow still requires same-run qualification.
if (manifest.runAttempt > runAttempt)
throw new Error("Verified package workflow run attempt is from the future");
assertEqual(manifest.platform, "linux", "verifier platform");
assertEqual(manifest.architecture, "x64", "verifier architecture");
assertEqual(manifest.packageVersion, packageVersion, "package version");
assertEqual(manifest.nodeVersion, nodeVersion, "Node.js version");
// The artifact format deliberately fixes the name; no manifest value can
// redirect validation to another path.
const tarball = resolve(directory, TARBALL_NAME);
const digest = await sha256File(tarball);
assertEqual(manifest.sha256, digest, "tarball SHA-256");
return tarball;
}
function validateManifestShape(manifest) {
if (
typeof manifest !== "object" ||
manifest === null ||
Array.isArray(manifest)
)
throw new Error("Verified package manifest must be a JSON object");
const keys = Object.keys(manifest).sort();
if (
keys.length !== MANIFEST_KEYS.length ||
keys.some((key, index) => key !== MANIFEST_KEYS[index])
)
throw new Error("Verified package manifest has missing or unknown fields");
if (manifest.schemaVersion !== 1)
throw new Error("Unsupported verified package manifest schema version");
if (
typeof manifest.sourceSha !== "string" ||
!/^[a-f0-9]{40}$/u.test(manifest.sourceSha)
)
throw new Error("Verified package manifest has an invalid source SHA");
if (!Number.isSafeInteger(manifest.runId) || manifest.runId <= 0)
throw new Error("Verified package manifest has an invalid workflow run ID");
if (!Number.isSafeInteger(manifest.runAttempt) || manifest.runAttempt <= 0)
throw new Error(
"Verified package manifest has an invalid workflow run attempt",
);
for (const key of [
"platform",
"architecture",
"packageVersion",
"nodeVersion",
])
if (typeof manifest[key] !== "string" || manifest[key].length === 0)
throw new Error(`Verified package manifest has an invalid ${key}`);
if (
typeof manifest.sha256 !== "string" ||
!/^[a-f0-9]{64}$/u.test(manifest.sha256)
)
throw new Error("Verified package manifest has an invalid tarball SHA-256");
}
function assertEqual(actual, expected, label) {
if (actual !== expected)
throw new Error(`Verified package ${label} mismatch`);
}
async function sha256File(path) {
const hash = createHash("sha256");
for await (const chunk of createReadStream(path)) hash.update(chunk);
return hash.digest("hex");
}
async function validateFromCli(directory) {
const sourceSha = process.env.GITHUB_SHA;
const runId = parsePositiveInteger(
process.env.GITHUB_RUN_ID,
"GITHUB_RUN_ID",
);
const runAttempt = parsePositiveInteger(
process.env.GITHUB_RUN_ATTEMPT,
"GITHUB_RUN_ATTEMPT",
);
if (sourceSha === undefined || !/^[a-f0-9]{40}$/u.test(sourceSha))
throw new Error("GITHUB_SHA must be a full commit SHA");
const checkoutSha = (
await execFileAsync("git", ["rev-parse", "HEAD"])
).stdout.trim();
assertEqual(checkoutSha, sourceSha, "checked out source SHA");
const packageJson = JSON.parse(await readFile("package.json", "utf8"));
const tarball = await validateVerifiedPackage({
directory,
sourceSha,
runId,
runAttempt,
packageVersion: packageJson.version,
nodeVersion: process.version,
});
if (process.env.GITHUB_OUTPUT === undefined)
throw new Error(
"GITHUB_OUTPUT is required to publish the validated tarball path",
);
await writeFile(process.env.GITHUB_OUTPUT, `tarball=${tarball}\n`, {
encoding: "utf8",
flag: "a",
});
process.stdout.write(`${tarball}\n`);
}
function parsePositiveInteger(value, name) {
if (value === undefined || !/^[1-9]\d*$/u.test(value))
throw new Error(`${name} must be a positive integer`);
const parsed = Number(value);
if (!Number.isSafeInteger(parsed))
throw new Error(`${name} must be a safe positive integer`);
return parsed;
}
const invokedPath = process.argv[1];
if (
invokedPath !== undefined &&
resolve(invokedPath) === fileURLToPath(import.meta.url)
) {
const [command, directory, ...extra] = process.argv.slice(2);
if (command !== "validate" || directory === undefined || extra.length > 0) {
process.stderr.write(
"Usage: node scripts/ci/verified-package.mjs validate <artifact-directory>\n",
);
process.exitCode = 2;
} else {
await validateFromCli(directory);
}
}
@@ -45,6 +45,20 @@ class Generator:
assert "ows-secret" not in json.dumps(projected), projected
if isinstance(value, bytes):
assert projected["binary_fields"][-1]["state"] == "redacted", projected
body = b"A" * 12345
state = {"response": {"content": body}}
control = adapter.project_record(state, [])
field = control["binary_fields"][0]
assert field["state"] == "retained" and field["bytes"] == len(body), control
for literal in ["12345", "2345"]:
# Only the generated byte count matches this exclusion.
assert literal not in control["reported"]["response"]["content"]
assert literal not in field["content_base64"] and literal not in field["sha256"]
projected = adapter.project_record(state, [literal])
assert projected["reported"]["response"]["content"] is None, projected
assert projected["binary_fields"] == [{"pointer": "/response/content", "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}], projected
assert {"pointer": "/response/content", "reason": "explicit-sensitive-value"} in projected["redactions"], projected
assert literal not in json.dumps(projected), projected
(root / "native-error-classifications.json").write_text(json.dumps({"passed": len(classifications) + 2}))
first = http.HTTPFlow(connection.Client(peername=("127.0.0.1", 1), sockname=("127.0.0.1", 2)), connection.Server(address=("example.test", 80)))
first.id = "original-producer-id"
+12 -17
View File
@@ -4,9 +4,14 @@ import { createHash } from "node:crypto";
import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import {
buildNativeAotFixture,
nativeAotCommandError,
supportsNativeAotFixtureHost,
} from "./lib/nativeaot-build.mjs";
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
if (process.arch !== "x64" || !["linux", "win32"].includes(process.platform))
if (!supportsNativeAotFixtureHost(process.platform, process.arch))
throw new Error(
"NativeAOT fixture lane requires Linux or Windows x64; it does not cross-compile NativeAOT.",
);
@@ -40,9 +45,7 @@ const run = (command, args) => {
maxBuffer: 8 * 1024 * 1024,
});
if (result.error || result.status !== 0)
throw new Error(
`NativeAOT fixture lane requires/action failed ${command}: ${result.error?.message ?? ""}\n${result.stdout ?? ""}${result.stderr ?? ""}`,
);
throw nativeAotCommandError(command, args[0], result);
return result.stdout;
};
const dotnet = process.env.REA_NATIVEAOT_DOTNET ?? "dotnet";
@@ -57,20 +60,12 @@ if (process.platform === "linux") {
run("clang", ["--version"]);
}
const map = join(output, "NativeAotFixture.map");
run(dotnet, [
"publish",
join(root, "tests/fixtures/nativeaot/NativeAotFixture.csproj"),
"-r",
await buildNativeAotFixture(run, {
dotnet,
project: join(root, "tests/fixtures/nativeaot/NativeAotFixture.csproj"),
runtime,
"-c",
"Release",
"-o",
join(output, "symbols"),
"--maxcpucount:1",
`-p:BaseIntermediateOutputPath=${join(output, "obj")}/`,
`-p:BaseOutputPath=${join(output, "bin")}/`,
`-p:NativeAotMapPath=${map}`,
]);
output,
});
const name =
process.platform === "win32" ? "NativeAotFixture.exe" : "NativeAotFixture";
const binary = join(output, "symbols", name);
+120
View File
@@ -0,0 +1,120 @@
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { ensureGeneratedFile } from "./lib/generated-file.mjs";
const arguments_ = new Set(process.argv.slice(2));
for (const argument of arguments_)
if (argument !== "--check")
throw new Error(`Unknown inventory option: ${argument}`);
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
const lock = JSON.parse(
await readFile(join(root, "package-lock.json"), "utf8"),
);
const entries = Object.entries(lock.packages)
.filter(([path]) => path !== "")
.map(([path, record]) => ({
path,
name: record.name ?? path.slice(path.lastIndexOf("node_modules/") + 13),
version: record.version,
license: record.license ?? "UNDECLARED",
resolved: record.resolved,
integrity: record.integrity,
dev: record.dev === true,
optional: record.optional === true,
}))
.sort((left, right) =>
left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
);
for (const entry of entries)
for (const field of ["name", "version", "resolved", "integrity"])
if (typeof entry[field] !== "string" || entry[field].length === 0)
throw new Error(`Missing locked ${field}: ${entry.path}`);
const runtime = entries.filter(({ dev }) => !dev);
const runtimeFingerprint = createHash("sha256")
.update(JSON.stringify(runtime))
.digest("hex");
const source = [
"REA npm dependency inventory",
"Prepared from the root package-lock.json; refresh this inventory when locked dependencies change.",
"License identifiers are upstream lockfile declarations, not a legal assessment.",
"Runtime includes transitive and optional platform packages; development includes build, test and website tooling.",
`Locked package entries: ${entries.length}; runtime: ${runtime.length}.`,
"Full runtime copyright/license/NOTICE texts: npm-runtime-notices.txt.",
"",
...entries.flatMap((entry) => [
`${entry.name}@${entry.version} | ${entry.dev ? "development" : "runtime"}${entry.optional ? ", optional" : ""} | ${entry.license}`,
` Lockfile path: ${entry.path}`,
` Package: https://www.npmjs.com/package/${entry.name}/v/${entry.version}`,
` Archive: ${entry.resolved}`,
` Integrity: ${entry.integrity}`,
]),
"",
].join("\n");
await ensureGeneratedFile({
path: join(root, "licenses/npm-dependencies.txt"),
source,
check: arguments_.has("--check"),
generateCommand: "npm run licenses:generate",
});
if (arguments_.has("--check")) {
const notices = await readFile(
join(root, "licenses/npm-runtime-notices.txt"),
"utf8",
);
const headings = [...notices.matchAll(/^===== (.+) =====$/gmu)];
const sections = new Map();
for (const [index, heading] of headings.entries()) {
if (sections.has(heading[1]))
throw new Error(`Duplicate runtime notice: ${heading[1]}`);
sections.set(
heading[1],
notices.slice(
heading.index + heading[0].length + 1,
headings[index + 1]?.index,
),
);
}
for (const entry of runtime) {
const identity = `${entry.name}@${entry.version}`;
const section = sections.get(identity);
if (section === undefined)
throw new Error(`Missing runtime notice: ${identity}`);
const metadata = [
`Declared license: ${entry.license}`,
`Archive: ${entry.resolved}`,
`Integrity: ${entry.integrity}`,
"",
].join("\n");
if (!section.startsWith(metadata))
throw new Error(`Runtime notice metadata differs for ${identity}`);
}
if (
!notices
.slice(0, headings[0]?.index)
.includes(
`Runtime dependency fingerprint (SHA-256): ${runtimeFingerprint}\n`,
)
)
throw new Error(
`Runtime notice fingerprint differs; refresh verified notices for ${runtimeFingerprint}`,
);
const references = JSON.parse(
await readFile(join(root, "licenses/upstream-sources.json"), "utf8"),
);
for (const reference of references) {
const bytes = await readFile(join(root, reference.file));
if (createHash("sha256").update(bytes).digest("hex") !== reference.sha256)
throw new Error(`Preserved notice digest differs: ${reference.file}`);
}
}
console.log(
JSON.stringify({
locked_packages: entries.length,
runtime_packages: runtime.length,
runtime_dependency_fingerprint: runtimeFingerprint,
}),
);
+53 -3
View File
@@ -10,6 +10,7 @@ import json
import os
from pathlib import Path
import secrets
import select
import stat
import subprocess
import tempfile
@@ -118,6 +119,7 @@ class XWindowAttributes(ctypes.Structure):
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser()
parser.add_argument("--probe", action="store_true")
parser.add_argument("--owner-stdin", action="store_true")
parser.add_argument("--hopper")
parser.add_argument("--socket")
parser.add_argument(
@@ -133,6 +135,43 @@ def parse_args() -> argparse.Namespace:
return parser.parse_args()
class OwnerLifetime:
"""The launcher's private pipe is a lifetime capability, independent of PIDs."""
def __init__(self, enabled: bool) -> None:
self._poll = select.poll() if enabled else None
if self._poll is not None:
# Node/libuv can implement piped stdio with a Unix socket pair.
mode = os.fstat(0).st_mode
if not (stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)):
raise AdapterFailure(
INVALID_LAUNCH_COMMAND,
"invalid_launch_command",
"owner lifetime requires a private stdin pipe",
)
self._poll.register(0, select.POLLIN | select.POLLHUP | select.POLLERR)
def alive(self) -> bool:
if self._poll is None:
return True
for _, events in self._poll.poll(0):
if events & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
return False
if events & select.POLLIN and not os.read(0, 4096):
return False
return True
def wait_for_hopper(child: subprocess.Popen[bytes], owner: OwnerLifetime) -> int:
while owner.alive():
try:
return child.wait(timeout=0.1)
except subprocess.TimeoutExpired:
continue
# run_launch's finally owns cleanup of both direct, unreaped children.
return 0
def digest(path: Path) -> str:
value = hashlib.sha256()
with path.open("rb") as file:
@@ -572,6 +611,7 @@ def start_xvfb(
"-auth",
str(authority),
],
stdin=subprocess.DEVNULL,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
@@ -627,6 +667,9 @@ def run_launch(args: argparse.Namespace) -> int:
"invalid_launch_command",
"missing Hopper launch coordinates",
)
owner = OwnerLifetime(args.owner_stdin)
if not owner.alive():
return 0
hopper = Path(args.hopper)
socket = Path(args.socket)
try:
@@ -654,6 +697,8 @@ def run_launch(args: argparse.Namespace) -> int:
child: subprocess.Popen[bytes] | None = None
display: int | None = None
try:
if not owner.alive():
return 0
hopper_environment = dict(os.environ)
for name in ("WAYLAND_DISPLAY", "WAYLAND_SOCKET"):
hopper_environment.pop(name, None)
@@ -666,7 +711,10 @@ def run_launch(args: argparse.Namespace) -> int:
"XDG_SESSION_TYPE": "x11",
}
)
child = subprocess.Popen(command, env=hopper_environment)
# Children must not consume or retain REA's lifetime pipe.
child = subprocess.Popen(
command, env=hopper_environment, stdin=subprocess.DEVNULL
)
leader = os.getpid()
os.environ["XAUTHORITY"] = str(authority)
@@ -702,10 +750,12 @@ def run_launch(args: argparse.Namespace) -> int:
observed: list[tuple[int, int, int, int, int]] = []
dialog: tuple[int, int, int, int, int] | None = None
while time.monotonic() < deadline:
if not owner.alive():
return 0
if socket.exists():
x11.XCloseDisplay(display)
display = None
return child.wait()
return wait_for_hopper(child, owner)
if not hopper_is_owned(leader, hopper):
if child.poll() is None:
time.sleep(0.1)
@@ -749,7 +799,7 @@ def run_launch(args: argparse.Namespace) -> int:
x11.XFlush(display)
x11.XCloseDisplay(display)
display = None
return child.wait()
return wait_for_hopper(child, owner)
except AdapterFailure as error:
raise error.with_stderr(capture)
finally:
+14
View File
@@ -52,6 +52,11 @@ export const toolFamilyCatalog = (sources) => {
surface: "android-provider",
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
},
{
id: "pcap",
surface: "pcap-provider",
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
},
{
id: "flutter",
surface: "flutter-provider",
@@ -251,6 +256,10 @@ export const providerCatalog = (sources) => {
identity: sources.androidProvider.JADX_PROVIDER_IDENTITY,
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
},
{
identity: sources.pcapProvider.PCAP_PROVIDER_IDENTITY,
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
},
{
identity: sources.flutterProvider.FLUTTER_PROVIDER_IDENTITY,
contracts: sources.flutterContracts.FLUTTER_TOOL_CONTRACTS,
@@ -272,6 +281,11 @@ export const providerCatalog = (sources) => {
contracts:
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
},
{
identity: sources.webNetworkCaptureProvider.SAZ_CAPTURE_PROVIDER_IDENTITY,
contracts:
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
},
{
identity:
sources.webNetworkCaptureProvider.MITMPROXY_CAPTURE_PROVIDER_IDENTITY,
+2 -2
View File
@@ -68,8 +68,8 @@ const SOURCE_PATHS = {
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
androidProvider: "dist/android/JadxRelease.js",
apktoolContracts: "dist/contracts/apktool/apktoolToolContracts.js",
flutterContracts: "dist/contracts/flutter/flutterToolContracts.js",
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
pcapContracts: "dist/contracts/pcap/pcapToolContracts.js",
pcapProvider: "dist/pcap/PcapProvider.js",
apktoolProvider: "dist/apktool/ApktoolProvider.js",
jebContracts: "dist/contracts/jeb/jebToolContracts.js",
jebProvider: "dist/jeb/JebProvider.js",
@@ -0,0 +1,3 @@
export function assertCleanLinuxHopperRuntime(
launcherPaths?: readonly string[],
): Promise<void>;
@@ -0,0 +1,64 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { readdir, stat } from "node:fs/promises";
const execFileAsync = promisify(execFile);
/**
* Observe leftover demo processes on a dedicated clean Linux runner, including
* groups reparented after their launcher exits. This is not ownership proof and
* never authorizes signaling; unrelated runtime processes also block this lane.
*/
export const assertCleanLinuxHopperRuntime = async (
launcherPaths = [
"/opt/hopper/bin/Hopper",
...(process.env.HOPPER_LAUNCHER_PATH === undefined
? []
: [process.env.HOPPER_LAUNCHER_PATH]),
],
) => {
const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]);
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout))
throw new Error("Clean Linux runner retained a Hopper demo process");
const launchers = [];
for (const path of launcherPaths) {
try {
launchers.push(await stat(path, { bigint: true }));
} catch (error) {
if (error?.code !== "ENOENT") throw error;
}
}
if (launchers.length === 0) return;
// /proc exposes the executed file even when argv[0] names a removed symlink.
// Compare file identity, not a display path or a process name.
let inspectionError;
for (const entry of await readdir("/proc", { withFileTypes: true })) {
if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue;
let executable;
try {
executable = await stat(`/proc/${entry.name}/exe`, { bigint: true });
} catch (error) {
if (error?.code === "ENOENT" || error?.code === "ESRCH") continue;
// Other users' executables may be unreadable. The setup lane launches
// providers as the caller; unreadable caller-owned processes fail closed.
if (error?.code === "EACCES") {
const owner = await stat(`/proc/${entry.name}`).catch((cause) => {
if (cause?.code === "ENOENT") return undefined;
throw cause;
});
if (owner === undefined || owner.uid !== process.getuid()) continue;
}
// Finish the inventory so an unrelated restricted process cannot mask a
// positively identified leak. Unknown caller processes still fail closed.
inspectionError ??= error;
continue;
}
if (
launchers.some(
({ dev, ino }) => dev === executable.dev && ino === executable.ino,
)
)
throw new Error("Clean Linux runner retained a Hopper demo process");
}
if (inspectionError !== undefined) throw inspectionError;
};
+9
View File
@@ -0,0 +1,9 @@
export function assertFeatureTrace(
trace: unknown,
procedure: { readonly address: string },
): void;
export function findMappedNonFunction(
strings: readonly { readonly address: string }[],
resolveAt: (address: string) => Promise<unknown>,
): Promise<string | undefined>;
+52
View File
@@ -0,0 +1,52 @@
import assert from "node:assert/strict";
const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]);
/**
* Assert the `trace_feature` result of the real IDA lane for one selected
* procedure. The query is a substring search, so the trace may also match other
* procedures and strings; each reference is checked against its own matched
* target, and a source outside every function (a data reference) is valid.
*/
export function assertFeatureTrace(trace, procedure) {
assert.ok(
trace.matches.some(
({ type, address }) =>
type === "procedure" && address === procedure.address,
),
"The trace omitted the selected procedure's match.",
);
assert.equal(typeof trace.truncated, "boolean");
const matched = new Set(trace.matches.map(({ address }) => address));
for (const reference of trace.references) {
assert.ok(
matched.has(reference.target_address),
`A reference targets ${reference.target_address}, which is not a match.`,
);
const { containing_procedure: containing } = reference;
if (containing.found)
assert.equal(typeof containing.procedure.address, "string");
else assert.ok(NEGATIVE_REASONS.has(containing.reason));
}
}
/**
* Find a string that IDA maps outside every function. A string inside a
* function chunk (inline data in code) is valid and skipped, so the strings are
* tried in order. Any other answer must be "not in a procedure": IDA lists only
* mapped strings, so an unmapped answer is a containment defect and fails the
* lane. Returns the address, or undefined when every string is inside a function.
*/
export async function findMappedNonFunction(strings, resolveAt) {
for (const { address } of strings) {
const resolved = await resolveAt(address);
if (resolved.found) continue;
assert.equal(
resolved.reason,
"not_in_procedure",
`The listed string at ${address} resolved as ${resolved.reason}.`,
);
return address;
}
return undefined;
}
+25
View File
@@ -0,0 +1,25 @@
export function supportsNativeAotFixtureHost(
platform: string,
architecture: string,
): boolean;
export function buildNativeAotFixture(
run: (command: string, args: string[]) => unknown,
options: {
dotnet: string;
project: string;
runtime: string;
output: string;
},
): Promise<void>;
export function nativeAotCommandError(
command: string,
operation: string,
result: {
error?: { message: string };
status: number | null;
stdout?: string | null;
stderr?: string | null;
},
): Error;
+60
View File
@@ -0,0 +1,60 @@
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
/** NativeAOT fixtures use native toolchains, not cross-compilation. */
export function supportsNativeAotFixtureHost(platform, architecture) {
return architecture === "x64" && ["linux", "win32"].includes(platform);
}
/** Keep network recovery separate from compilation and preserve restore failures. */
export async function buildNativeAotFixture(run, options) {
const { dotnet, project, runtime, output } = options;
const common = [
project,
"-r",
runtime,
"--maxcpucount:1",
"-p:Configuration=Release",
`-p:BaseIntermediateOutputPath=${join(output, "obj")}/`,
`-p:BaseOutputPath=${join(output, "bin")}/`,
`-p:NativeAotMapPath=${join(output, "NativeAotFixture.map")}`,
];
for (let attempt = 1; attempt <= 2; attempt++) {
console.error(`NativeAOT dependency restore: attempt ${attempt}/2`);
try {
run(dotnet, ["restore", ...common, "--verbosity", "normal"]);
break;
} catch (cause) {
if (attempt === 2)
throw new Error(
"NativeAOT dependency restore failed after two attempts",
{
cause,
},
);
// Child diagnostics can contain authenticated feed URLs; keep retry logs static.
console.error(
"NativeAOT dependency restore failed; retrying in 1 second",
);
await delay(1_000);
}
}
console.error("NativeAOT compilation (dependencies already restored)");
run(dotnet, [
"publish",
...common,
"--no-restore",
"-o",
join(output, "symbols"),
]);
}
/** Sanitize child diagnostics before any retry, cause chain, or uncaught output. */
export function nativeAotCommandError(command, operation, result) {
const diagnostic = `NativeAOT fixture lane requires/action failed ${command} ${operation}: ${result.error?.message ?? `exit ${result.status}`}\n${result.stdout ?? ""}${result.stderr ?? ""}`;
// Only URL authority userinfo is authentication; preserve host, path, query,
// and unrelated diagnostic values. Never retain the raw result as a cause.
return new Error(
diagnostic.replace(/\b(https?:\/\/)[^\\\s/?#]*@/giu, "$1[redacted]@"),
);
}
+264
View File
@@ -0,0 +1,264 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import {
mkdir,
mkdtemp,
readFile,
readdir,
rm,
writeFile,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../../dist/domain/evidence.js";
import {
requireMcpOperationResult,
requireMcpToolError,
mcpTextValue,
} from "./mcp-verifier-results.mjs";
/** Exercise file coordinates through public CLI/MCP using the source-owned PE fixture. */
export async function verifyGhidraFileOffsets(
targetPath,
entrypoint,
procedure,
) {
const original = await readFile(targetPath);
const workspace = await mkdtemp(join(tmpdir(), "rea-file-offsets-"));
const runtime = join(workspace, "runtime");
await mkdir(runtime);
const env = {
...process.env,
TEMP: runtime,
TMP: runtime,
TMPDIR: runtime,
REA_ANALYSIS_PROVIDER: "ghidra",
REA_LOG_LEVEL: "silent",
};
const client = new Client({ name: "rea-file-offsets", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env,
stderr: "pipe",
});
let connected = false;
let complete = false;
let primaryFailure;
const call = async (name, arguments_ = {}) => {
const result = await client.callTool(
{ name, arguments: arguments_ },
{ timeout: 240_000 },
);
const projected = requireMcpOperationResult(result, name);
return ["open_binary", "binary_session", "close_binary"].includes(name)
? projected
: (result.structuredContent ?? JSON.parse(mcpTextValue(result)));
};
try {
await client.connect(transport);
connected = true;
await call("open_binary", { path: targetPath, provider_id: "ghidra" });
const session = await call("binary_session");
const mapping = parseEvidence(
await call("address_to_file_offset", { address: procedure }),
);
const offset = mapping.normalized_result.file_offset;
const coordinate = `0x${offset.toString(16)}`;
const expected = parseEvidence(
await call("procedure_pseudo_code", { procedure }),
);
const explicit = parseEvidence(
await call("procedure_pseudo_code", {
procedure: `file-offset:${coordinate}`,
}),
);
const automatic = parseEvidence(
await call("procedure_pseudo_code", { procedure: coordinate }),
);
for (const actual of [explicit, automatic]) {
assert.equal(actual.normalized_result, expected.normalized_result);
assert.equal(actual.raw_result.selection.file_offset, offset);
assert.equal(actual.raw_result.selection.mapped_address, procedure);
assert.ok(
actual.locations.some(
(location) =>
location.kind === "file-offset" && location.offset === offset,
),
);
}
const invalid = await client.callTool({
name: "procedure_pseudo_code",
arguments: { procedure: "file-offset:0x20000000000000" },
});
assert.equal(requireMcpToolError(invalid).code, "invalid_request");
const unmapped = await client.callTool({
name: "procedure_pseudo_code",
arguments: { procedure: "file-offset:0x100000" },
});
assert.ok(
JSON.stringify(requireMcpToolError(unmapped)).includes(
"no loaded source mapping",
),
);
assert.equal(
(await call("binary_session")).analysis_run.run_id,
session.analysis_run.run_id,
);
const image = parseEvidence(await call("inspect_native_load_image"));
const headerMapping = image.normalized_result.observations.mappings.find(
(item) => item.initialized && item.file_offset === 0,
);
assert.ok(
headerMapping,
"Fixture header must have an observed source mapping",
);
const headerOffset = parseEvidence(
await call("address_to_file_offset", { address: headerMapping.start }),
);
assert.equal(headerOffset.normalized_result.file_offset, 0);
for (const selector of ["file-offset:0x0", "0x0"]) {
const missing = requireMcpToolError(
await client.callTool({
name: "procedure_pseudo_code",
arguments: { procedure: selector },
}),
);
assertMissingMappedProcedure(missing, headerMapping.start);
}
assert.equal(
(await call("binary_session")).analysis_run.run_id,
session.analysis_run.run_id,
);
await call("close_binary");
const cli = JSON.parse(
(
await promisify(execFile)(
process.execPath,
[
entrypoint,
"decompile",
targetPath,
`--file-offset=${coordinate}`,
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
)
).stdout,
);
assert.equal(cli.normalized_result, explicit.normalized_result);
assert.deepEqual(cli.locations, explicit.locations);
const missingCli = await promisify(execFile)(
process.execPath,
[
entrypoint,
"decompile",
targetPath,
"--file-offset=0x0",
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
).then(
() => {
throw new Error(
"Mapped header bytes must not decompile as a procedure",
);
},
(cause) => JSON.parse(cause.stdout),
);
assertMissingMappedProcedure(missingCli, headerMapping.start);
const alias = Buffer.from(original);
const pe = alias.readUInt32LE(0x3c);
if (alias.readUInt16LE(pe + 6) >= 2) {
const secondSection = pe + 24 + alias.readUInt16LE(pe + 20) + 40;
// Both sections now carry the same original bytes at distinct virtual addresses.
alias.writeUInt32LE(0x200, secondSection + 20);
const ambiguousPath = join(workspace, "ambiguous.exe");
await writeFile(ambiguousPath, alias);
await call("open_binary", { path: ambiguousPath, provider_id: "ghidra" });
const ambiguous = await client.callTool(
{
name: "procedure_pseudo_code",
arguments: { procedure: `file-offset:${coordinate}` },
},
{ timeout: 240_000 },
);
assert.ok(
JSON.stringify(requireMcpToolError(ambiguous)).includes(
"multiple loaded mappings",
),
);
await call("close_binary");
}
assert.deepEqual(await readFile(targetPath), original);
await client.close();
connected = false;
assert.deepEqual(await readdir(runtime), []);
complete = true;
return {
cli_mcp_parity: true,
explicit_and_numeric_offsets: true,
invalid_and_unmapped_offsets: true,
mapped_offset_without_procedure: true,
ambiguous_mappings:
alias.readUInt16LE(pe + 6) >= 2
? "verified"
: "unavailable-single-section",
source_unchanged: true,
cleanup: "complete",
};
} catch (cause) {
primaryFailure = cause;
throw cause;
} finally {
const cleanupFailures = [];
if (connected) {
try {
await call("close_binary");
} catch (cause) {
cleanupFailures.push(cause);
}
}
try {
await client.close();
} catch (cause) {
cleanupFailures.push(cause);
}
if (cleanupFailures.length > 0) {
process.stderr.write(
`Retained file-offset verifier workspace: ${workspace}\n`,
);
throw new AggregateError(
primaryFailure === undefined
? cleanupFailures
: [primaryFailure, ...cleanupFailures],
"File-offset verifier cleanup failed",
);
}
if (complete) await rm(workspace, { recursive: true, force: true });
else
process.stderr.write(
`Retained file-offset verifier workspace: ${workspace}\n`,
);
}
}
function assertMissingMappedProcedure(error, mappedAddress) {
assert.equal(error.code, "invalid_request");
const message = error.details?.issues?.[0]?.message;
assert.equal(typeof message, "string");
assert.ok(
message.includes("File offset 0x0 maps to analysis address"),
message,
);
assert.ok(message.includes(mappedAddress), message);
assert.ok(message.includes("no procedure starts at or contains"), message);
assert.ok(!message.includes("--file-offset"), message);
}
+130
View File
@@ -0,0 +1,130 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { setTimeout as delay } from "node:timers/promises";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { verifyNoTokenOwnedProcesses } from "../../dist/process/ProcessOwnership.js";
import {
observeProcessStartIdentity,
signalProcessWithStartIdentity,
} from "../../dist/process/ProcessOwnershipObservation.js";
import { startUnrelatedHopperSentinel } from "./unrelated-hopper-sentinel.mjs";
import {
requireMcpEvidenceResult,
requireMcpLifecycleResult,
} from "./mcp-verifier-results.mjs";
const execFileAsync = promisify(execFile);
/** Prove actual MCP owner exit releases the demo singleton for subsequent CLI/MCP sessions. */
export async function verifyLinuxHopperOwnerExit({
primary,
secondary,
serverCommand,
serverArgs,
environment,
}) {
const sentinel = await startUnrelatedHopperSentinel();
const outcomes = [];
try {
for (const mode of ["graceful", "SIGKILL"]) {
const transport = new StdioClientTransport({
command: serverCommand,
args: serverArgs,
env: environment,
stderr: "pipe",
});
// Drain launcher diagnostics even on the deliberately interrupted owner.
transport.stderr?.resume();
const client = new Client({ name: "hopper-owner-exit", version: "1" });
const options = { timeout: 180_000 };
try {
await client.connect(transport);
const ownerPid = transport.pid;
assert.ok(ownerPid !== null);
const owner = await observeProcessStartIdentity(ownerPid);
assert.equal(owner?.state, "readable");
requireMcpLifecycleResult(
await client.callTool(
{
name: "open_binary",
arguments: { path: primary },
},
options,
),
"open_binary",
);
const procedures = requireMcpEvidenceResult(
await client.callTool(
{ name: "list_procedures", arguments: {} },
options,
),
"list_procedures",
);
assert.ok(procedures.length > 0);
const status = requireMcpLifecycleResult(
await client.callTool(
{ name: "binary_session", arguments: {} },
options,
),
"binary_session",
);
const runId = status.analysis_run?.run_id;
assert.equal(typeof runId, "string");
assert.equal(status.analysis_provider_binding?.provider.id, "hopper");
assert.equal((await verifyNoTokenOwnedProcesses(runId)).cleaned, false);
if (mode === "graceful") await client.close();
else {
assert.equal(
await signalProcessWithStartIdentity(
ownerPid,
owner.identity,
"SIGKILL",
),
"signaled",
);
}
const deadline = Date.now() + 10_000;
let cleanup;
do {
cleanup = await verifyNoTokenOwnedProcesses(runId);
if (cleanup.cleaned) break;
await delay(100);
} while (Date.now() < deadline);
assert.equal(cleanup.cleaned, true, cleanup.reason);
// The SIGKILL case must recover the stale lease for both target identities.
for (const target of [primary, secondary]) {
const result = await execFileAsync(
process.execPath,
[
"scripts/rea.mjs",
"inspect",
target,
"--provider",
"hopper",
"--format",
"json",
],
{ env: environment, timeout: 180_000, maxBuffer: 16 * 1024 * 1024 },
);
assert.ok(JSON.parse(result.stdout).normalized_result);
}
outcomes.push({
mode,
ownedProcessesStopped: true,
sameTargetReopened: true,
differentTargetReopened: true,
});
} finally {
await client.close();
await transport.close();
}
}
} finally {
assert.equal(await sentinel.verifyAndClose(), true);
}
return { outcomes, unrelatedHopperSurvived: true };
}
+30
View File
@@ -0,0 +1,30 @@
type VerifierCall = (
name: string,
arguments_?: Record<string, unknown>,
) => Promise<unknown>;
interface StringExpectation {
readonly value: string;
readonly encoding: string;
readonly symbol?: string;
readonly recognitionOptional?: boolean;
}
/** Verify source bytes, observed decoding, address filters and literal searches. */
export function verifyHopperStringObjects(
call: VerifierCall,
expectations: readonly StringExpectation[],
inventory?: readonly z.output<typeof analysisStringSchema>[] | null,
): Promise<{
readonly sourceBytesVerified: true;
readonly recognition: readonly {
readonly value: string;
readonly address: string;
readonly state: "observed" | "not_observed";
readonly decodedObject: "verified" | "unverified";
}[];
readonly displayEscapesExcludedFromMatching: true;
}>;
import type { z } from "zod";
import type { analysisStringSchema } from "../../src/domain/hopperValues.js";
+132 -28
View File
@@ -1,13 +1,16 @@
import assert from "node:assert/strict";
import { mcpTextValue } from "./mcp-verifier-results.mjs";
const fixtureSymbolPrefix = process.platform === "darwin" ? "_" : "";
/** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */
export async function verifyHopperSearch(call, invalid, procedure) {
const address = procedure.address;
const original = await call("address_name", { address });
let stringCoverage;
try {
await verifySymbolNames(call, invalid, address);
await verifySearchModes(call, address);
stringCoverage = await verifySearchModes(call, address);
} finally {
await call("set_address_name", { address, name: original });
}
@@ -26,7 +29,7 @@ export async function verifyHopperSearch(call, invalid, procedure) {
symbolUtf16LimitValidatedBeforeMutation: true,
regexAndCaseModes: true,
annotationCacheInvalidation: true,
escapedUtf8AndByteStrings: true,
stringCoverage,
};
}
@@ -103,11 +106,26 @@ async function verifySearchModes(call, address) {
[{ address, value: "REA_Search_Upper" }],
);
const strings = await call("list_strings");
const long = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle";
const longItem = strings.find((item) =>
item.value.startsWith("REA_LONG_LITERAL_"),
const linux = process.platform === "linux";
const fullLiteral = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle";
// Recognition is a provider/loader capability, not an adapter guarantee.
// Always retain the original size in raw-byte coverage; Linux additionally
// requires an extent observed to produce native fragmented string objects.
const fullBytes = Buffer.from(fullLiteral + "\0");
const raw = await call("read_bytes", {
address: `${fixtureSymbolPrefix}rea_long_literal`,
length: fullBytes.length,
});
assert.equal(raw.complete, true);
assert.equal(raw.bytes_hex, fullBytes.toString("hex"));
const long = linux
? "REA_FRAGMENTED_LITERAL_" + "x".repeat(512) + "fragment_tail"
: fullLiteral;
const longItem = strings.find(
(item) => long.startsWith(item.value) && item.value.startsWith("REA_"),
);
assert.ok(longItem, "fixture omitted its real long literal");
if (!linux) assert.equal(longItem.address, raw.address);
assert.ok(long.startsWith(longItem.value));
assert.ok(longItem.provider_value.endsWith("…"));
assert.equal(longItem.string.termination, "missing");
@@ -158,7 +176,7 @@ async function verifySearchModes(call, address) {
}
assert.deepEqual(
await call("search_strings", {
pattern: "needle",
pattern: linux ? "fragment_tail" : "needle",
mode: "literal",
case_sensitive: true,
}),
@@ -176,36 +194,122 @@ async function verifySearchModes(call, address) {
}),
expected,
);
await verifyHopperStringObjects(call, [
{ value: "REA_UTF8_é_😀", encoding: "utf-8" },
{ value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" },
{ value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" },
{ value: "REA_LITERAL_…", encoding: "utf-8" },
{ value: "REA_LATIN1_ÿ", encoding: "latin-1" },
]);
const objects = await verifyHopperStringObjects(
call,
[
{
value: "REA_UTF8_é_😀",
encoding: "utf-8",
symbol: `${fixtureSymbolPrefix}rea_utf8_literal`,
recognitionOptional: linux,
},
{ value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" },
{ value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" },
{
value: "REA_LITERAL_…",
encoding: "utf-8",
symbol: `${fixtureSymbolPrefix}rea_ellipsis_literal`,
recognitionOptional: linux,
},
{
value: "REA_LATIN1_ÿ",
encoding: "latin-1",
symbol: `${fixtureSymbolPrefix}rea_latin1_literal`,
recognitionOptional: linux,
},
],
strings,
);
return {
fullLiteralSourceBytes: fullBytes.length,
fragmentedLiteralSourceBytes: Buffer.byteLength(long),
fullLiteralRecognition: {
address: raw.address,
state: strings.some((item) => item.address === raw.address)
? "observed"
: "not_observed",
// Linux validates the original through raw bytes; its required decoded
// fragment/dossier checks exercise the separate supported-size literal.
decodedObject: linux ? "unverified" : "verified",
},
...objects,
};
}
/** Compare native string identities and values with the source fixture's bytes. */
export async function verifyHopperStringObjects(call, expectations) {
const strings = await call("list_strings");
for (const { value, encoding } of expectations) {
const item = strings.find((candidate) => candidate.value === value);
assert.ok(
item,
`Hopper omitted the decoded string ${JSON.stringify(value)}`,
);
assert.equal(item.string.encoding, encoding);
assert.equal(item.string.encoding_status, "inferred");
assert.equal(item.string.termination, "present_or_not_required");
export async function verifyHopperStringObjects(
call,
expectations,
inventory = null,
) {
// Reuse only the caller's inventory for this unchanged document, never live
// state from an earlier session. Address-filter and search checks remain real.
const strings = inventory ?? (await call("list_strings"));
const recognition = [];
for (const {
value,
encoding,
symbol,
recognitionOptional = false,
} of expectations) {
const expectedBytes = Buffer.from(
value + "\0",
encoding.replaceAll("-", ""),
);
assert.equal(item.string.byte_length, expectedBytes.length);
const bytes = await call("read_bytes", {
let sourceBytes;
if (symbol !== undefined) {
const raw = await call("read_bytes", {
address: symbol,
length: expectedBytes.length,
});
sourceBytes = raw;
assert.equal(raw.complete, true);
assert.equal(raw.bytes_hex, expectedBytes.toString("hex"));
}
const item = strings.find((candidate) =>
sourceBytes === undefined
? candidate.value === value
: candidate.address === sourceBytes.address,
);
if (item === undefined && recognitionOptional) {
assert.ok(
symbol,
"optional recognition requires independent source-byte coverage",
);
recognition.push({
value,
address: sourceBytes.address,
state: "not_observed",
decodedObject: "unverified",
});
continue;
}
assert.ok(
item,
`Hopper omitted the decoded string ${JSON.stringify(value)}`,
);
assert.equal(
item.value,
value,
"native string decoding changed the source value",
);
recognition.push({
value,
address: item.address,
length: expectedBytes.length,
state: "observed",
decodedObject: "verified",
});
assert.equal(item.string.encoding, encoding);
assert.equal(item.string.encoding_status, "inferred");
assert.equal(item.string.termination, "present_or_not_required");
assert.equal(item.string.byte_length, expectedBytes.length);
const bytes =
sourceBytes ??
(await call("read_bytes", {
address: item.address,
length: expectedBytes.length,
}));
assert.equal(bytes.complete, true);
assert.equal(bytes.bytes_hex, expectedBytes.toString("hex"));
assert.deepEqual(await call("list_strings", { address: item.address }), [
item,
@@ -225,7 +329,7 @@ export async function verifyHopperStringObjects(call, expectations) {
);
return {
sourceBytesVerified: true,
inferredEncodingReported: true,
recognition,
displayEscapesExcludedFromMatching: true,
};
}
+6
View File
@@ -16,6 +16,12 @@ const isMcpMode =
const isMcpDoctorMode = args[0] === "mcp" && args[1] === "doctor";
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
// Windows home lookup throws for an empty USERPROFILE. Dependencies resolve
// home during import, before REA's own resolver can apply its account fallback.
if (process.platform === "win32" && process.env.USERPROFILE === "") {
delete process.env.USERPROFILE;
}
// Playwright's registry assumes one of its three desktop host platforms when
// it chooses the browser cache directory. Termux reports `android`, although
// its Node runtime and filesystem are POSIX-compatible. Keep the cache path
+30 -24
View File
@@ -12,6 +12,7 @@ import { tmpdir } from "node:os";
import { promisify } from "node:util";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { assertCleanLinuxHopperRuntime } from "./lib/clean-linux-hopper-runtime.mjs";
const execFileAsync = promisify(execFile);
const verifierRun = createVerifierRun();
@@ -19,12 +20,12 @@ const command = process.env.REA_VERIFY_SERVER_COMMAND;
if (process.platform !== "linux" || process.getuid?.() !== 0)
throw new Error("Linux setup verification requires a root Linux runner");
if (!command) throw new Error("REA_VERIFY_SERVER_COMMAND is required");
await assertCleanLinuxHopperRuntime();
try {
await access("/opt/hopper/bin/Hopper");
throw new Error("Linux setup verification requires a clean runner");
} catch (error) {
if (error instanceof Error && error.message.includes("clean runner"))
throw error;
if (error?.code !== "ENOENT") throw error;
}
const runJson = async (args) => {
@@ -35,14 +36,25 @@ const runJson = async (args) => {
return JSON.parse(stdout);
};
const plan = await runJson(["setup", "--json"]);
if (plan.status !== "needs_confirmation")
throw new Error("setup did not produce a read-only confirmation plan");
const plan = await runJson([
"setup",
"--install-hopper",
"--dry-run",
"--json",
]);
if (
plan.status !== "planned" ||
!plan.plannedActions.some(({ kind }) => kind === "install_hopper") ||
plan.appliedActions.length !== 0
)
throw new Error("setup did not produce a read-only Hopper installation plan");
await access("/opt/hopper/bin/Hopper").then(
() => {
throw new Error("setup plan mutated the clean runner");
},
() => undefined,
(error) => {
if (error?.code !== "ENOENT") throw error;
},
);
const installed = await runJson([
@@ -58,7 +70,7 @@ if (
throw new Error(
`approved setup did not become ready: ${JSON.stringify(installed)}`,
);
const doctor = await runJson(["doctor", "--json"]);
const doctor = await runJson(["doctor", "--provider", "hopper", "--json"]);
if (doctor.healthy !== true)
throw new Error(`doctor is unhealthy: ${JSON.stringify(doctor)}`);
@@ -67,8 +79,6 @@ await execFileAsync(command, ["analyze", "/usr/bin/true"], {
timeout: 180_000,
maxBuffer: 10 * 1024 * 1024,
});
process.env.HOPPER_TARGET_PATH = "/usr/bin/true";
process.env.HOPPER_SECOND_TARGET_PATH = "/usr/bin/false";
process.env.REA_VERIFY_SERVER_ARGS_JSON = '["mcp"]';
await import("./verify-real-hopper-linux.mjs");
@@ -95,28 +105,24 @@ try {
await rm(modifiedDirectory, { recursive: true, force: true });
}
const repeated = await runJson([
"setup",
"--yes",
"--install-hopper",
"--json",
]);
const repeated = await runJson(["setup", "--yes", "--json"]);
if (
repeated.status !== "ready" ||
repeated.appliedActions.includes("installed_hopper")
repeated.plannedActions.length !== 0 ||
repeated.appliedActions.length !== 0
)
throw new Error("a second setup was not idempotent");
const { stdout: processes } = await execFileAsync("ps", [
"-ax",
"-o",
"command=",
]);
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(processes))
throw new Error("Linux setup verification leaked a Hopper demo session");
await assertCleanLinuxHopperRuntime();
const completedVerifierRun = await completeVerifierRun(verifierRun);
if (
completedVerifierRun.process_lineage.status !== "verified" ||
completedVerifierRun.process_lineage.descendants.length !== 0
)
throw new Error("Linux setup verification did not prove owned child cleanup");
console.log(
JSON.stringify({
verifier_run: await completeVerifierRun(verifierRun),
verifier_run: completedVerifierRun,
status: "verified",
setup: true,
cli: true,
+45 -1
View File
@@ -1,6 +1,8 @@
import { mkdtemp, rm } from "node:fs/promises";
import { mkdir, mkdtemp, rm, unlink } from "node:fs/promises";
import { execFile } from "node:child_process";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { promisify } from "node:util";
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
import { PRODUCT_IDENTITY } from "../dist/identity.js";
@@ -21,12 +23,19 @@ import { verifyRepositorySkill } from "./verify-repository-skill.mjs";
import { verifyManaged } from "./verify-package-managed.mjs";
import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import {
createVerifiedPackageMetadata,
retainVerifiedPackage,
writeVerifiedPackageManifest,
} from "./ci/verified-package.mjs";
import {
measureMcpStartup,
profileMcpModuleLoading,
} from "./lib/mcp-startup-probe.mjs";
const root = process.cwd();
const execFileAsync = promisify(execFile);
const retentionDirectory = process.env.REA_VERIFIED_PACKAGE_DIRECTORY;
const verifierRun = createVerifierRun();
const workspace = await mkdtemp(join(tmpdir(), "rea-package-"));
const evidenceRoot = join(workspace, "evidence");
@@ -37,6 +46,8 @@ let mcpModuleLoading;
let update;
let repositorySkill;
let report;
let retainedPackage;
let retentionMetadata;
const runStage = async (name, action) => {
const started = performance.now();
@@ -49,6 +60,27 @@ const runStage = async (name, action) => {
};
try {
if (retentionDirectory !== undefined && retentionDirectory !== "") {
const sourceSha = (
await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root })
).stdout.trim();
retentionMetadata = createVerifiedPackageMetadata({
sourceSha,
runId: process.env.GITHUB_RUN_ID,
runAttempt: process.env.GITHUB_RUN_ATTEMPT,
platform: process.platform,
architecture: process.arch,
packageVersion: PRODUCT_IDENTITY.packageVersion,
nodeVersion: process.version,
});
await mkdir(retentionDirectory, { recursive: true });
await unlink(join(retentionDirectory, "manifest.json")).catch((cause) => {
if (
!(cause instanceof Error && "code" in cause && cause.code === "ENOENT")
)
throw cause;
});
}
repositorySkill = await runStage("repository skill", () =>
verifyRepositorySkill({ root, workspace }),
);
@@ -215,6 +247,13 @@ try {
mcpStartup,
mcpModuleLoading,
};
if (retentionMetadata !== undefined) {
retainedPackage = await retainVerifiedPackage({
sourceTarball: tarball,
directory: retentionDirectory,
metadata: retentionMetadata,
});
}
} finally {
await runStage("workspace cleanup", () =>
rm(workspace, { recursive: true, force: true }),
@@ -223,6 +262,11 @@ try {
const verifierLineage = await runStage("verifier lineage", () =>
completeVerifierRun(verifierRun),
);
if (retainedPackage !== undefined)
await writeVerifiedPackageManifest(
retainedPackage.directory,
retainedPackage.metadata,
);
process.stdout.write(
`${JSON.stringify({ verifier_run: verifierLineage, ...report })}\n`,
);
+196 -3
View File
@@ -1,8 +1,9 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { estimateTokenCount } from "tokenx";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile, writeFile, readdir } from "node:fs/promises";
import { readFile, writeFile, readdir, mkdir, rm } from "node:fs/promises";
import { isAbsolute, join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
@@ -91,6 +92,20 @@ const source = pcap(1, [
// LINKTYPE_BLUETOOTH_HCI_H4 = 187; H4 command packet, HCI Reset opcode 0x0c03, no parameters.
const bluetoothSource = pcap(187, [Buffer.from("01030c00", "hex")]);
let cases = 0;
const discoveryMeasurements = [];
const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8"))
.split("\n")
.find((line) => line.startsWith("Max address space "))
?.trim()
.match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u);
assert.ok(inheritedAddressSpace);
const expectedAddressSpace = Math.min(
1073741824,
...inheritedAddressSpace
.slice(1)
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
try {
await writeFile(path, source);
await writeFile(bluetooth, bluetoothSource);
@@ -121,14 +136,16 @@ try {
.digest("hex"),
);
assert.equal(report.hardware_acquisition, "not-performed");
assert.equal(report.limits.address_space_bytes, 1073741824);
assert.equal(report.limits.address_space_bytes, expectedAddressSpace);
assert.equal(report.tool.configured_path, process.env.REA_TSHARK_COMMAND);
assert.equal(
report.resource_limiter.configured_path,
process.env.REA_TSHARK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit",
);
assert.ok(
report.resource_limiter.arguments.includes("--as=1073741824:1073741824"),
report.resource_limiter.arguments.includes(
`--as=${expectedAddressSpace}:${expectedAddressSpace}`,
),
);
assert.equal(report.configuration, "private-empty-personal-profile");
const flags = Object.entries(input)
@@ -236,6 +253,52 @@ try {
const hci = await inspect({ path: bluetooth, fields: ["bthci_cmd.opcode"] });
assert.deepEqual(hci.packets[0]["bthci_cmd.opcode"], ["0x0c03"]);
assert.deepEqual(await readFile(path), source);
// Exercise the incremental summary workflow against the same owned bytes
// using real installed Wireshark tools; these inputs are format fixtures.
const pcapTool = (await client.listTools()).tools.find(
(t) => t.name === "inspect_pcap_capture",
);
assert.ok(pcapTool?.outputSchema);
const pcapAjv = new Ajv2020({ strict: false, validateFormats: false });
assert.ok(pcapAjv.validateSchema(pcapTool.inputSchema));
assert.ok(pcapAjv.validateSchema(pcapTool.outputSchema));
for (const capturePath of [path, bluetooth]) {
const response = await client.callTool({
name: pcapTool.name,
arguments: { path: capturePath, detail: "summary" },
});
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(
pcapAjv.validate(pcapTool.outputSchema, response.structuredContent),
pcapAjv.errorsText(),
);
const summary = parseEvidence(response.structuredContent).normalized_result;
assert.equal(
summary.summary.target.sha256,
createHash("sha256")
.update(await readFile(capturePath))
.digest("hex"),
);
const source = {
kind: "retained-evidence",
evidence_id: summary.parent_evidence_id,
};
const page = await client.callTool({
name: "inspect_analysis_view",
arguments: {
source,
view: { kind: "pcap", facet: "protocols", limit: 10 },
},
});
assert.notEqual(page.isError, true, JSON.stringify(page));
assert.equal(
parseEvidence(page.structuredContent).normalized_result
.parent_evidence_id,
source.evidence_id,
);
cases++;
}
assert.deepEqual(await readFile(bluetooth), bluetoothSource);
for (const input of [
{ path, fields: ["frame.number"], display_filter: "this is not a filter" },
@@ -267,6 +330,133 @@ try {
true,
);
cases++;
const discovery = (await client.listTools()).tools.find(
({ name }) => name === "inspect_packet_fields",
);
assert.ok(discovery?.outputSchema);
assert.ok(ajv.validateSchema(discovery.inputSchema));
assert.ok(ajv.validateSchema(discovery.outputSchema));
const nativeProfile = join(root.path, "native-profile");
await mkdir(nativeProfile);
const native = (
await execute(process.env.REA_TSHARK_COMMAND, ["-G", "fields"], {
env: { ...environment, WIRESHARK_CONFIG_DIR: nativeProfile },
timeout: 30000,
maxBuffer: 32 * 1024 * 1024,
})
).stdout;
await rm(nativeProfile, { recursive: true });
const nativeLines = native.split(/\r?\n/);
if (nativeLines.at(-1) === "") nativeLines.pop();
for (const input of [
{ query: "usb" },
{ limit: 1 },
{ offset: 1, limit: 2 },
{ kind: "fields", protocol: "usb", query: "idVendor" },
{ kind: "fields", protocol: "udp", limit: null },
{ kind: "fields", query: "rea_no_such_field" },
]) {
const started = performance.now();
const response = await client.callTool({
name: discovery.name,
arguments: input,
});
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(
ajv.validate(discovery.outputSchema, response.structuredContent),
JSON.stringify(ajv.errors),
);
const evidence = parseEvidence(response.structuredContent);
const report = evidence.normalized_result;
const kind = input.kind ?? "protocols";
const query = (input.query ?? "").toLowerCase();
const expectedMatches = nativeLines.flatMap((line, index) => {
const columns = line.split("\t");
if ((kind === "protocols") !== (columns[0] === "P")) return [];
if (
input.protocol !== undefined &&
(columns[0] === "P" ? columns[2] : columns[4]) !== input.protocol
)
return [];
if (
![
columns[1],
columns[2],
...(columns[0] === "F" ? [columns[7]] : []),
].some((value) => value.toLowerCase().includes(query))
)
return [];
return [{ sourceLine: index + 1, columns }];
});
const offset = input.offset ?? 0;
const limit = input.limit === null ? undefined : (input.limit ?? 50);
const expectedPage = expectedMatches.slice(
offset,
limit === undefined ? undefined : offset + limit,
);
assert.deepEqual(
report.records.map((record) => record.source_line),
expectedPage.map((record) => record.sourceLine),
);
assert.deepEqual(report.output_window, {
examined: nativeLines.filter((line) =>
line.startsWith(kind === "protocols" ? "P\t" : "F\t"),
).length,
matching: expectedMatches.length,
returned: expectedPage.length,
has_more:
Math.max(0, expectedMatches.length - offset) > expectedPage.length,
});
discoveryMeasurements.push({
input,
elapsed_ms: +(performance.now() - started).toFixed(2),
response_bytes: Buffer.byteLength(JSON.stringify(response)),
estimated_tokens_one_tool_request_response: estimateTokenCount(
JSON.stringify(discovery) +
JSON.stringify(input) +
JSON.stringify(response),
),
});
for (const record of report.records) {
const columns = nativeLines[record.source_line - 1].split("\t");
assert.equal(record.name, columns[1]);
assert.equal(record.abbreviation, columns[2]);
assert.equal(record.field_type, columns[0] === "F" ? columns[3] : null);
assert.equal(
record.parent_protocol,
columns[0] === "F" ? columns[4] : null,
);
assert.equal(record.description, columns[0] === "F" ? columns[7] : null);
assert.equal(record.display_base, columns[0] === "F" ? columns[5] : null);
assert.equal(record.bitmask, columns[0] === "F" ? columns[6] : null);
}
if (input.query === "idVendor")
assert.ok(
report.records.some(
(row) =>
row.abbreviation === "usb.idVendor" &&
row.field_type === "FT_UINT16",
),
);
if (input.query === "usb")
assert.ok(report.records.some((row) => row.abbreviation === "usb"));
if (input.query === "rea_no_such_field")
assert.equal(report.output_window.matching, 0);
const flags = Object.entries(input).flatMap(([key, value]) =>
value === null ? ["--all"] : [`--${key}`, String(value)],
);
const cli = JSON.parse(
(
await execute(
process.execPath,
[entrypoint, "inspect-packet-fields", ...flags, "--format", "json"],
{ env: environment, timeout: 60000, maxBuffer: 16 * 1024 * 1024 },
)
).stdout,
);
assert.deepEqual(cli.normalized_result, report);
cases++;
}
await client.close();
assert.deepEqual((await readdir(root.path)).sort(), [
"bluetooth.pcap",
@@ -283,6 +473,9 @@ console.log(
{
status: "passed",
cases,
discovery_measurements: discoveryMeasurements,
measurement_scope:
"Warm MCP server, fresh TShark processes; tokenx estimates include one advertised tool/request/response, not billed usage. Model-driven selection, CPU and peak process-family memory are unmeasured; no cache/speedup claim.",
host: `${process.platform}/${process.arch}`,
scope:
"real TShark CLI/MCP; synthetic Ethernet/IPv4/UDP/DNS and Bluetooth HCI PCAP; no hardware acquisition",
+131
View File
@@ -0,0 +1,131 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdtemp, readdir, rmdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { toolContract } from "../dist/contracts/toolContracts.js";
import { hashStableFile } from "../dist/filesystem/StableFileHash.js";
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
const args = process.argv.slice(2);
if (args.includes("--help") || args.includes("-h")) {
console.log(
"Usage: npm run verify:pcap -- [--pcap PATH]\nRequires caller-supplied tshark/capinfos. Without --pcap, only client readiness is verified.",
);
process.exit(0);
}
if (!(args.length === 0 || (args.length === 2 && args[0] === "--pcap")))
throw new Error("Expected --pcap PATH");
const path = args[1] === undefined ? null : resolve(args[1]);
// This parent is shared by independent CLI/MCP owners. A leased Windows
// runtime root would prevent their native admission of its directory handle.
const root = await mkdtemp(join(tmpdir(), "rea-pcap-verifier-"));
const environment = Object.fromEntries(
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
);
Object.assign(environment, {
TMPDIR: root,
TMP: root,
TEMP: root,
});
const entrypoint = resolve("scripts/rea.mjs");
const execute = promisify(execFile);
const client = new Client({ name: "pcap-real-verifier", version: "1" });
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env: environment,
stderr: "pipe",
});
transport.stderr?.resume();
try {
await client.connect(transport);
const readiness = await run("inspect_pcap_client", {});
assert(readiness.client.tshark_version && readiness.client.capinfos_version);
if (path === null)
console.log("[unverified] Capture inspection: no --pcap was supplied");
else {
const identity = await hashStableFile(path);
const result = await run("inspect_pcap_capture", { path });
assert.deepEqual(result.target, { path, ...identity });
assert(result.packet_count >= 0);
assert(
result.endpoints.every(
(entry) => entry.packet_count <= result.packet_count,
),
);
assert(
result.protocol_distribution.every(
(entry) => entry.packet_count <= result.packet_count,
),
);
const partial =
result.endpoint_count > result.endpoints.length ||
result.dns_query_count > result.dns_queries.length ||
result.tls_sni_count > result.tls_sni_values.length ||
result.protocol_count > result.protocol_distribution.length ||
result.http_request_uri_count > result.http_request_uris.length ||
result.unattributed_dns_packet_count > 0 ||
result.dns_queries.some(
(entry) =>
entry.response_address_count > entry.response_addresses.length,
);
assert.equal(result.coverage, partial ? "partial" : "complete");
assert.deepEqual(
await hashStableFile(path),
identity,
"Input capture must remain unchanged",
);
console.log(
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
);
}
} finally {
try {
await client.close();
} finally {
await transport.close();
}
assert.deepEqual(
await readdir(root),
[],
`Owned capture resources remain under ${root}`,
);
await rmdir(root);
}
console.log("All asserted pcap CLI/MCP checks passed");
async function run(name, input) {
const contract = toolContract(name);
const { stdout } = await execute(
process.execPath,
[
entrypoint,
name.replaceAll("_", "-"),
...(input.path === undefined ? [] : [input.path]),
"--json",
],
{ env: environment, timeout: 360_000, maxBuffer: 8 * 1024 * 1024 },
);
const cli = contract.outputSchema.parse(JSON.parse(stdout));
const response = await client.callTool(
{ name, arguments: input },
{ timeout: 360_000 },
);
assert.notEqual(response.isError, true, mcpTextValue(response));
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
assert.deepEqual(mcp.normalized_result, cli.normalized_result);
assert.deepEqual(mcp.subject, cli.subject);
if (input.path !== undefined) {
assert.equal(cli.subject.local_path, input.path);
assert.equal(
cli.subject.digest.sha256,
cli.normalized_result.target.sha256,
);
}
console.log(`[ok] ${name}: named output schema and CLI/MCP Evidence parity`);
return cli.normalized_result;
}
+215
View File
@@ -0,0 +1,215 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { isAbsolute } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { Ajv2020 } from "ajv/dist/2020.js";
import { parseEvidence } from "../dist/domain/evidence.js";
// Caller-supplied producer evidence stays local; no fixture capture or secrets are bundled.
const [controlPath, lossPath, reportPath] = process.argv.slice(2);
if (
![controlPath, lossPath, reportPath].every(
(value) => typeof value === "string" && isAbsolute(value),
)
)
throw new Error(
"Usage: node scripts/verify-qlog-captures.mjs /absolute/control/origin.qlog /absolute/loss/origin.qlog /absolute/new-report.json",
);
const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url));
const client = new Client({ name: "qlog-producer-verifier", version: "1" });
const environment = {
PATH: process.env.PATH ?? "/usr/bin:/bin",
REA_LOG_LEVEL: "silent",
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env: environment,
stderr: "pipe",
});
let diagnostics = "";
transport.stderr?.on("data", (bytes) => {
if (diagnostics.length < 65536) diagnostics += bytes.toString("utf8");
});
const ajv = new Ajv2020({ strict: false, validateFormats: false });
const measurements = [];
try {
try {
await client.connect(transport);
} catch (cause) {
throw new Error(`MCP startup failed: ${diagnostics}`, { cause });
}
const tool = (await client.listTools()).tools.find(
(tool) => tool.name === "inspect_web_network_capture",
);
assert.ok(tool?.outputSchema);
for (const schema of [tool.inputSchema, tool.outputSchema])
assert.ok(ajv.validateSchema(schema));
const validateOutput = ajv.compile(tool.outputSchema);
for (const [lane, path] of [
["control", controlPath],
["loss", lossPath],
]) {
const source = await readFile(path);
const raw = JSON.parse(source.toString("utf8"));
const sha256 = createHash("sha256").update(source).digest("hex");
assert.equal(raw.qlog_format, "JSON");
assert.equal(raw.qlog_version, "0.3");
const input = { capture_path: path, format: "qlog" };
assert.ok(ajv.validate(tool.inputSchema, input));
const response = await client.callTool({
name: tool.name,
arguments: input,
});
assert.notEqual(response.isError, true);
assert.ok(
validateOutput(response.structuredContent),
JSON.stringify(validateOutput.errors),
);
const mcp = parseEvidence(response.structuredContent);
const completed = await promisify(execFile)(
process.execPath,
[entrypoint, "inspect-web-network-capture", path, "qlog", "--json"],
{ env: environment, timeout: 40_000, maxBuffer: 96 * 1024 * 1024 },
);
const cli = parseEvidence(JSON.parse(completed.stdout));
assert.equal(cli.evidence_id, mcp.evidence_id);
assert.deepEqual(cli.normalized_result, mcp.normalized_result);
const capture = mcp.normalized_result;
assert.equal(capture.artifact.sha256, sha256);
assert.equal(
capture.total_records,
raw.traces.reduce((sum, trace) => sum + trace.events.length, 0),
);
for (const [ordinal, original] of raw.traces
.flatMap((trace) => trace.events)
.entries()) {
const record = capture.records[ordinal];
assert.equal(record.ordinal, ordinal);
assert.equal(record.reported.name, original.name);
assert.equal(record.reported.time, original.time);
assert.deepEqual(record.binary_fields, []);
assert.ok(
record.numeric_literals.some(
(number) =>
number.pointer === "/time" &&
Number(number.literal) === original.time,
),
);
}
assert.equal(capture.qlog.body_bytes, "not-reported");
assert.equal(mcp.authority, "historical-reference");
assert.equal(
mcp.analysis_profile.parameters.qlog_profile,
"aioquic-1.5.0-json-0.3",
);
const trace = capture.qlog.traces[0];
assert.equal(trace.vantage_point, "server");
const stream = trace.streams.find(
(stream) => stream.stream_id === "0" && stream.direction === "sent",
);
assert.ok(stream);
assert.equal(stream.unique_bytes, "32803");
assert.equal(
stream.frame_length_sum,
lane === "control" ? "32803" : "33972",
);
assert.equal(stream.repeated_bytes, lane === "control" ? "0" : "1169");
assert.equal(trace.loss_declarations.length, lane === "control" ? 0 : 1);
if (lane === "loss")
assert.deepEqual(trace.loss_declarations[0].packet_number, {
state: "reported",
value: "5",
});
// Structurally-known secrets may not survive in Evidence, including sidecars.
const serialized = JSON.stringify(mcp);
for (const original of raw.traces.flatMap((trace) => trace.events)) {
if (
original.name === "transport:parameters_set" &&
typeof original.data.stateless_reset_token === "string"
)
assert.ok(!serialized.includes(original.data.stateless_reset_token));
for (const frame of original.data.frames ?? []) {
if (
frame.frame_type === "new_connection_id" &&
typeof frame.reset_token === "string"
)
assert.ok(!serialized.includes(frame.reset_token));
if (frame.frame_type === "new_token" && typeof frame.token === "string")
assert.ok(!serialized.includes(frame.token));
}
}
const retained = await client.callTool({
name: "inspect_analysis_view",
arguments: {
source: { kind: "retained-evidence", evidence_id: mcp.evidence_id },
view: { kind: "api-surface" },
},
});
assert.equal(retained.isError, true);
assert.equal(
JSON.parse(retained.content.find((item) => item.type === "text").text)
.error.code,
"unsupported_target",
);
const selected = await client.callTool({
name: tool.name,
arguments: { ...input, record_ordinals: [capture.total_records - 1, 0] },
});
const subset = parseEvidence(selected.structuredContent).normalized_result;
assert.deepEqual(
subset.records.map((record) => record.ordinal),
[capture.total_records - 1, 0],
);
assert.deepEqual(subset.qlog, capture.qlog);
measurements.push({
lane,
path,
sha256,
evidence_id: mcp.evidence_id,
profile_digest: mcp.analysis_profile.digest,
events: capture.total_records,
stream_id: stream.stream_id,
unique_bytes: stream.unique_bytes,
frame_length_sum: stream.frame_length_sum,
repeated_bytes: stream.repeated_bytes,
loss_declarations: trace.loss_declarations.length,
cli_mcp_parity: true,
retained_reference: true,
selected_scope_preserved: true,
auth_exclusions: true,
});
}
await client.ping();
const report = {
status: "passed",
profile: "aioquic-1.5.0-json-0.3",
measurements,
limitations: [
"Caller supplies provenance; format selection does not authenticate the producer.",
"These known loopback cases do not establish WAN behavior, other qlog producers, platforms or general completeness.",
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
],
};
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
flag: "wx",
mode: 0o600,
});
console.log(
JSON.stringify({
status: report.status,
cases: measurements.length,
report: reportPath,
}),
);
} finally {
await client.close();
await transport.close();
}
+30 -17
View File
@@ -8,7 +8,7 @@ import {
realpath,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { join, resolve, sep } from "node:path";
import { fileURLToPath } from "node:url";
import { createHash } from "node:crypto";
import { exec } from "./lib/verify-package-core.mjs";
@@ -18,10 +18,6 @@ import {
toolContract,
} from "../dist/contracts/toolContracts.js";
assert(
process.platform !== "win32",
"The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
);
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
@@ -32,6 +28,13 @@ assert(
"Usage: node scripts/verify-qwen-client.mjs [chat|call]",
);
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
// npm exposes only qwen.cmd on Windows. Resolve a selected JavaScript entry
// before the verifier switches to its disposable workspace.
const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined;
const launch = (args) =>
entryPoint === undefined
? [command, args]
: [process.execPath, [entryPoint, ...args]];
const verifier = createVerifierRun();
const profile = join(lab, "selected profile");
const account = join(lab, "account");
@@ -54,14 +57,18 @@ const baseEnvironment = {
let clientVersion;
try {
clientVersion = (
await exec(command, ["--version"], {
await exec(...launch(["--version"]), {
env: baseEnvironment,
timeout: 30_000,
})
).stdout.trim();
} catch (cause) {
const windowsHint =
process.platform === "win32"
? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js."
: "";
throw new Error(
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`,
);
}
const original = JSON.stringify(
@@ -93,8 +100,8 @@ for (const step of [
assert(
result.plannedActions.every(
(a) =>
a.target.startsWith(profile + "/") ||
a.target.startsWith(sharedSkills + "/"),
a.target.startsWith(profile + sep) ||
a.target.startsWith(sharedSkills + sep),
),
);
await writeFile(
@@ -163,7 +170,14 @@ const fixture = await createOpenAiModelFixture({
skillSeen ||= JSON.stringify(body.messages ?? []).includes(
"reverse-engineer-anything",
);
selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
const messageText = (body.messages ?? [])
.map((message) =>
typeof message.content === "string"
? message.content
: (message.content ?? []).map((part) => part.text ?? "").join("\n"),
)
.join("\n");
selectedSkillLoaded ||= messageText.includes(
join(sharedSkills, "reverse-engineer-anything"),
);
const toolText = JSON.stringify(
@@ -243,9 +257,7 @@ const fixture = await createOpenAiModelFixture({
const invoke = tools.find((t) => t.name === "tool_call");
const skill = tools.find((t) => t.name === "skill");
const shell = tools.find((t) => t.name === "run_shell_command");
const persisted = toolText.match(
/Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
);
const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u);
const call = (name, args) => ({
role: "assistant",
content: null,
@@ -268,13 +280,14 @@ const fixture = await createOpenAiModelFixture({
artifactPath = persisted[1];
assert(
(await realpath(artifactPath)).startsWith(
(await realpath(runtime)) + "/",
(await realpath(runtime)) + sep,
),
);
const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
// Keep paths and JSON quotes out of shell syntax on Windows and POSIX.
const encodedScript = Buffer.from(script).toString("base64");
message = call(shell.name, {
command: `node -e ${quote(script)}`,
command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`,
description:
"Read the complete saved REA Evidence and verify the fixture facts.",
});
@@ -329,7 +342,7 @@ const args = [
let response;
let failure;
try {
response = await exec(command, args, {
response = await exec(...launch(args), {
env,
cwd: workspace,
timeout: 120_000,
+479
View File
@@ -0,0 +1,479 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import {
mkdir,
mkdtemp,
readFile,
readdir,
rm,
symlink,
unlink,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../dist/domain/evidence.js";
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.argv.length !== 2)
throw new Error("Usage: node scripts/verify-real-ghidra-6502.mjs");
if (process.env.GHIDRA_INSTALL_DIR === undefined)
throw new Error(
"Set GHIDRA_INSTALL_DIR to a bring-your-own Ghidra installation.",
);
const exec = promisify(execFile);
const run = createVerifierRun();
const root = fileURLToPath(new URL("..", import.meta.url));
const ca65 = process.env.REA_CA65 ?? "ca65";
const ld65 = process.env.REA_LD65 ?? "ld65";
const toolchain = {};
for (const [name, command] of Object.entries({ ca65, ld65 })) {
const version = await exec(command, ["--version"], { timeout: 10000 });
toolchain[name] = (version.stdout + version.stderr).trim();
}
await mkdir(join(root, "build"), { recursive: true });
const workspace = await mkdtemp(join(root, "build", "rea-6502-proof-"));
const runtime = join(workspace, "runtime");
await mkdir(runtime);
const path = join(workspace, "image.bin");
const object = join(workspace, "image.o");
const entrypoint = join(root, "scripts", "rea.mjs");
const env = {
...process.env,
TMPDIR: runtime,
TMP: runtime,
TEMP: runtime,
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "ghidra",
REA_PROCESS_RUN_ID: run.run_id,
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env,
stderr: "pipe",
});
const client = new Client({ name: "rea-real-6502-proof", version: "1" });
let stderr = "";
transport.stderr?.on("data", (chunk) => {
stderr = (stderr + chunk.toString()).slice(-65536);
});
let report;
try {
await exec(
ca65,
[join(root, "tests/conformance/ghidra/6502.s"), "-o", object],
{ timeout: 10000 },
);
await exec(
ld65,
["-C", join(root, "tests/conformance/ghidra/6502.cfg"), "-o", path, object],
{ timeout: 10000 },
);
const bytes = await readFile(path);
// Independently known opcodes and link addresses from the source fixture.
assert.equal(bytes.toString("hex"), "a907201080c90ad003a92a60a9006018690360");
const sha256 = createHash("sha256").update(bytes).digest("hex");
await client.connect(transport);
const opened = await call("open_binary", {
path,
format: "raw-binary",
architecture: "6502",
entry_point: "0x008001",
provider_id: "ghidra",
});
assert.equal(opened.architecture, "6502");
assert.equal(opened.entry_point, "0x8001");
const status = await call("binary_session");
const image = evidence(await call("inspect_native_load_image"), sha256);
assert.equal(
image.normalized_result.status,
"verified",
JSON.stringify(image.normalized_result),
);
assert.equal(image.normalized_result.architecture, "6502");
assert.equal(image.normalized_result.base_address, "0x8001");
assert.equal(image.normalized_result.language_id, "6502:LE:16:default");
assert.equal(image.normalized_result.compiler_spec_id, "default");
assert.equal(image.normalized_result.file_bytes, bytes.length);
const memory = evidence(
await call("read_bytes", { address: "0x8001", length: bytes.length }),
sha256,
);
assert.equal(memory.normalized_result.bytes_hex, bytes.toString("hex"));
const procedures = evidence(await call("list_procedures"), sha256);
for (const address of ["0x8001", "0x8010"])
assert.ok(
procedures.normalized_result.some(
(procedure) => procedure.address === address,
),
);
const instruction = evidence(
await call("inspect_native_instruction", { address: "0x8003" }),
sha256,
);
assert.equal(instruction.normalized_result.mnemonic.toUpperCase(), "JSR");
assert.ok(
instruction.normalized_result.references.some(
(reference) => reference.call && reference.target_address === "0x8010",
),
);
const pseudo = evidence(
await call("procedure_pseudo_code", { procedure: "0x8010" }),
sha256,
);
assert.match(pseudo.normalized_result, /\+\s*(?:3|0x3|'\\x03')/u);
const filePseudo = evidence(
await call("procedure_pseudo_code", { procedure: "file-offset:0xf" }),
sha256,
);
assert.equal(filePseudo.normalized_result, pseudo.normalized_result);
assert.ok(
filePseudo.locations.some(
(location) => location.kind === "file-offset" && location.offset === 15,
),
);
const rejected = await client.callTool({
name: "open_binary",
arguments: {
path,
format: "raw-binary",
architecture: "6502",
entry_point: "0xffff",
},
});
assert.equal(rejected.isError, true);
assert.match(JSON.stringify(rejected.content), /6502 address width/u);
const preserved = await call("binary_session");
assert.equal(preserved.analysis_run.run_id, status.analysis_run.run_id);
assert.equal(preserved.entry_point, "0x8001");
await call("close_binary");
await client.ping();
const { stdout } = await exec(
process.execPath,
[
entrypoint,
"inspect-native-load-image",
path,
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
"0x8001",
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const cli = parseEvidence(JSON.parse(stdout));
assert.deepEqual(cli.normalized_result, image.normalized_result);
assert.equal(cli.analysis_profile.digest, image.analysis_profile.digest);
assert.equal(cli.subject.digest.sha256, sha256);
const fileCli = await exec(
process.execPath,
[
entrypoint,
"decompile",
path,
"--file-offset=0xf",
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
"0x8001",
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const fileCliEvidence = parseEvidence(JSON.parse(fileCli.stdout));
assert.equal(fileCliEvidence.normalized_result, filePseudo.normalized_result);
assert.deepEqual(fileCliEvidence.locations, filePseudo.locations);
assert.equal(fileCliEvidence.subject.digest.sha256, sha256);
assert.deepEqual(await readFile(path), bytes);
assert.deepEqual(await readdir(runtime), []);
const lowMemory = [];
// BinaryLoader maps file bytes before trying processor-defined blocks. Prove
// that overlapping zero-page/stack definitions never overlay the selected image.
for (const [entry, fullAddressSpace] of [
[0xff, false],
[0, true],
]) {
const lowPath = join(workspace, `low-${entry}.bin`);
await exec(
ld65,
[
"-C",
join(root, "tests/conformance/ghidra/6502.cfg"),
"--start-addr",
String(entry),
"-o",
lowPath,
object,
],
{ timeout: 10000 },
);
const linked = await readFile(lowPath);
const expectedCode = Buffer.from(
"a907200000c90ad003a92a60a9006018690360",
"hex",
);
expectedCode.writeUInt16LE(entry + 15, 3);
assert.deepEqual(linked, expectedCode);
const lowBytes = fullAddressSpace
? Buffer.concat([linked, Buffer.alloc(65536 - linked.length)])
: linked;
await writeFile(lowPath, lowBytes);
const lowSha256 = createHash("sha256").update(lowBytes).digest("hex");
const lowEntry = `0x${entry.toString(16)}`;
const lowOpened = await call("open_binary", {
path: lowPath,
format: "raw-binary",
architecture: "6502",
entry_point: lowEntry,
provider_id: "ghidra",
});
assert.equal(lowOpened.entry_point, lowEntry);
const lowImage = evidence(
await call("inspect_native_load_image"),
lowSha256,
);
assert.equal(
lowImage.normalized_result.status,
"verified",
JSON.stringify(lowImage.normalized_result),
);
assert.equal(lowImage.normalized_result.base_address, lowEntry);
assert.equal(lowImage.normalized_result.file_bytes, lowBytes.length);
const fileMappings =
lowImage.normalized_result.observations.mappings.filter(
(mapping) => mapping.source_file_index !== null,
);
assert.equal(fileMappings.length, 1);
assert.equal(
fileMappings[0].address_space,
lowImage.normalized_result.observations.default_address_space,
);
assert.equal(fileMappings[0].overlay, false);
assert.equal(fileMappings[0].start, lowEntry);
assert.equal(fileMappings[0].length, lowBytes.length);
const lowRead = evidence(
await call("read_bytes", { address: lowEntry, length: linked.length }),
lowSha256,
);
assert.equal(lowRead.normalized_result.bytes_hex, linked.toString("hex"));
const lowCall = evidence(
await call("inspect_native_instruction", {
address: `0x${(entry + 2).toString(16)}`,
}),
lowSha256,
);
assert.equal(lowCall.normalized_result.mnemonic.toUpperCase(), "JSR");
assert.ok(
lowCall.normalized_result.references.some(
(reference) =>
reference.call &&
reference.target_address === `0x${(entry + 15).toString(16)}`,
),
);
await call("close_binary");
const lowCli = await exec(
process.execPath,
[
entrypoint,
"inspect-native-load-image",
lowPath,
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
lowEntry,
"--provider",
"ghidra",
"--json",
],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const lowCliEvidence = evidence(JSON.parse(lowCli.stdout), lowSha256);
assert.deepEqual(
lowCliEvidence.normalized_result,
lowImage.normalized_result,
);
assert.equal(
lowCliEvidence.analysis_profile.digest,
lowImage.analysis_profile.digest,
);
assert.deepEqual(await readFile(lowPath), lowBytes);
assert.deepEqual(await readdir(runtime), []);
lowMemory.push({
entry: lowEntry,
bytes: lowBytes.length,
sha256: lowSha256,
});
}
let windowsReparseAdmission = "not-applicable";
if (process.platform === "win32") {
const junction = join(workspace, "linked");
await symlink(workspace, junction, "junction");
try {
const linkedPath = join(junction, "image.bin");
// Reopening the same bytes through a junction must replace the direct session.
await call("open_binary", {
path,
format: "raw-binary",
architecture: "6502",
entry_point: "0x8001",
provider_id: "ghidra",
});
const directImage = evidence(
await call("inspect_native_load_image"),
sha256,
);
assert.equal(directImage.normalized_result.status, "verified");
// Canonical file identity must not bypass admission of the new path.
await call("open_binary", {
path: linkedPath,
format: "raw-binary",
architecture: "6502",
entry_point: "0x8001",
provider_id: "ghidra",
});
// open_binary selects a lazy session; the first analysis admits the snapshot.
const linkedImage = await client.callTool(
{ name: "inspect_native_load_image", arguments: {} },
{ timeout: 240000 },
);
assert.equal(linkedImage.isError, true);
assert.match(
JSON.stringify(linkedImage),
/Reparse path component is unsupported/u,
);
await call("close_binary");
const snapshotPath = join(workspace, "direct-snapshot.json");
const cliArgs = (selectedPath) => [
entrypoint,
"analyze",
selectedPath,
"--target-format",
"raw-binary",
"--architecture",
"6502",
"--entry-point",
"0x8001",
"--provider",
"ghidra",
"--snapshot",
snapshotPath,
"--json",
];
const saved = await exec(process.execPath, cliArgs(path), {
env,
timeout: 240000,
maxBuffer: 16 * 1024 * 1024,
});
assert.equal(
evidence(JSON.parse(saved.stdout), sha256).operation,
"binary_overview",
);
const savedSnapshot = JSON.parse(await readFile(snapshotPath, "utf8"));
assert.ok(
savedSnapshot.workflow_entries.some(
({ operation }) => operation === "binary_overview",
),
);
assert.deepEqual(await readdir(runtime), []);
// Replaying a matching snapshot must still reach native path admission.
await assert.rejects(
exec(process.execPath, cliArgs(linkedPath), {
env,
timeout: 240000,
maxBuffer: 16 * 1024 * 1024,
}),
(cause) => {
assert.match(
cause.stdout + cause.stderr,
/Reparse path component is unsupported/u,
);
return true;
},
);
assert.deepEqual(await readFile(path), bytes);
assert.deepEqual(await readdir(runtime), []);
windowsReparseAdmission = "cli-snapshot-mcp-rejected";
} finally {
await unlink(junction);
}
}
report = {
ok: true,
mocked: false,
lane: "ghidra-6502",
host: process.platform,
toolchain,
provider: image.provider,
target: {
sha256,
bytes: bytes.length,
architecture: "6502",
entry: "0x8001",
},
cli_mcp_load_image_parity: true,
instructions_and_call_reference: true,
decompilation: true,
file_offset_decompilation_cli_mcp: true,
invalid_mapping_preserves_session: true,
low_memory_cli_mcp_parity: lowMemory,
windows_reparse_admission: windowsReparseAdmission,
source_unchanged: true,
runtime_cleanup: "complete",
};
} catch (cause) {
if (stderr) process.stderr.write(stderr);
throw cause;
} finally {
try {
await client.close();
} finally {
await transport.close();
}
}
const verifierRun = await completeVerifierRun(run);
if (process.platform !== "win32") {
assert.equal(verifierRun.process_lineage.status, "verified");
assert.deepEqual(verifierRun.process_lineage.descendants, []);
}
// Keep the workspace for diagnosis unless verification and cleanup both pass.
await rm(workspace, { recursive: true, force: true });
process.stdout.write(
JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n",
);
async function call(name, args = {}) {
const result = await client.callTool(
{ name, arguments: args },
{ timeout: 240000 },
);
assert.notEqual(result.isError, true, JSON.stringify(result));
if (["open_binary", "binary_session", "close_binary"].includes(name))
return requireMcpOperationResult(result, name);
return result.structuredContent;
}
function evidence(value, sha256) {
const result = parseEvidence(value);
assert.equal(result.subject.digest.sha256, sha256);
assert.equal(result.subject.architecture, "6502");
assert.equal(result.subject.format, "raw-binary");
return result;
}
@@ -0,0 +1,313 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { parseEvidence } from "../dist/domain/evidence.js";
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
if (process.argv.length !== 2 || process.platform !== "linux")
throw new Error("Run this ELF verifier on Linux without arguments.");
if (!process.env.GHIDRA_INSTALL_DIR)
throw new Error("Set GHIDRA_INSTALL_DIR to a caller-installed Ghidra.");
const exec = promisify(execFile);
const run = createVerifierRun();
const root = fileURLToPath(new URL("..", import.meta.url));
const workspace = await mkdtemp(join(tmpdir(), "rea-address-context-"));
const runtime = join(workspace, "runtime");
await mkdir(runtime);
const path = join(workspace, "pie.elf");
const entrypoint = join(root, "scripts/rea.mjs");
const env = {
...process.env,
TMPDIR: runtime,
REA_ANALYSIS_PROVIDER: "ghidra",
REA_LOG_LEVEL: "silent",
};
const transport = new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
env,
stderr: "pipe",
});
const client = new Client({ name: "rea-real-address-context", version: "1" });
let stderr = "";
transport.stderr?.on("data", (chunk) => {
stderr = (stderr + chunk.toString()).slice(-65536);
});
let report;
let connected = false;
let primaryFailure;
let verifierRun;
try {
const compiler = process.env.CC ?? "gcc";
const version = await exec(compiler, ["--version"], { timeout: 10000 });
await exec(
compiler,
[
"-O0",
"-g",
"-fPIE",
"-pie",
"-nostdlib",
"-Wl,-e,rea_pie_entry",
"-Wl,--build-id=none",
join(root, "tests/conformance/c/pie-address-context.c"),
"-o",
path,
],
{ timeout: 30000 },
);
const bytes = await readFile(path);
assert.equal(bytes.subarray(0, 6).toString("hex"), "7f454c460201");
assert.equal(bytes.readUInt16LE(16), 3); // ET_DYN: source-owned x86-64 PIE.
assert.equal(bytes.readUInt16LE(18), 62);
const declaredEntry = `0x${bytes.readBigUInt64LE(24).toString(16)}`;
const sha256 = createHash("sha256").update(bytes).digest("hex");
await client.connect(transport);
connected = true;
await call("open_binary", { path, provider_id: "ghidra" });
const overview = parseEvidence(await call("binary_overview"));
assert.equal(overview.subject.digest.sha256, sha256);
const base = overview.normalized_result.image_base;
assert.match(base, /^0x[0-9a-f]+$/u);
assert.notEqual(base, "0x0");
const active = await call("binary_session");
const image = parseEvidence(await call("inspect_native_load_image"));
assert.equal(base, image.normalized_result.observations.image_base);
assert.ok(overview.normalized_result.segments.length > 0);
assert.ok(
overview.normalized_result.segments.every(
(segment) =>
segment.image_base === base &&
typeof segment.address_space === "string",
),
);
const procedures = parseEvidence(await call("list_procedures"));
const entry = procedures.normalized_result.find(
(item) => item.value === "rea_pie_entry",
);
const addition = procedures.normalized_result.find(
(item) => item.value === "rea_pie_add",
);
assert.ok(entry && addition);
assert.notEqual(entry.address, declaredEntry);
assert.ok(
image.normalized_result.observations.entry_points.includes(entry.address),
);
const mapping = image.normalized_result.observations.mappings.find(
(item) =>
item.source_file_index === 0 &&
item.file_offset !== null &&
item.address_space === "ram" &&
BigInt(item.start) <= BigInt(entry.address) &&
BigInt(item.end) >= BigInt(entry.address),
);
assert.ok(mapping);
const offset =
mapping.file_offset + Number(BigInt(entry.address) - BigInt(mapping.start));
const reverseMapping = parseEvidence(
await call("address_to_file_offset", { address: entry.address }),
);
assert.equal(reverseMapping.normalized_result.file_offset, offset);
const memory = parseEvidence(
await call("read_bytes", { address: entry.address, length: 8 }),
);
assert.equal(
memory.normalized_result.bytes_hex,
bytes.subarray(offset, offset + 8).toString("hex"),
);
const fromOffset = parseEvidence(
await call("procedure_pseudo_code", {
procedure: `file-offset:0x${offset.toString(16)}`,
}),
);
assert.ok(
fromOffset.locations.some(
(location) =>
location.kind === "file-offset" && location.offset === offset,
),
);
assert.equal(fromOffset.raw_result.selection.mapped_address, entry.address);
const fromNumericOffset = parseEvidence(
await call("procedure_pseudo_code", {
procedure: `0x${offset.toString(16)}`,
}),
);
assert.equal(
fromNumericOffset.normalized_result,
fromOffset.normalized_result,
);
const missingAddress = "ram:0xffffffff";
const rejected = await client.callTool(
{ name: "procedure_pseudo_code", arguments: { procedure: missingAddress } },
{ timeout: 240000 },
);
assert.equal(rejected.isError, true);
assert.ok(
JSON.stringify(rejected.content).includes(`Ghidra image base: ${base}`),
);
assert.ok(JSON.stringify(rejected.content).includes("list_procedures"));
assert.ok(
JSON.stringify(rejected.content).includes("inspect_native_load_image"),
);
const pseudo = parseEvidence(
await call("procedure_pseudo_code", { procedure: addition.address }),
);
assert.match(pseudo.normalized_result, /\+\s*(?:7|0x7)\b/u);
const retained = await call("binary_session");
assert.equal(retained.analysis_run.run_id, active.analysis_run.run_id);
await call("close_binary");
const cli = async (args) =>
exec(
process.execPath,
[entrypoint, ...args, "--provider", "ghidra", "--json"],
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
);
const cliOverview = parseEvidence(
JSON.parse((await cli(["analyze", path])).stdout),
);
assert.deepEqual(cliOverview.normalized_result, overview.normalized_result);
let cliFailure;
try {
await cli(["decompile", path, missingAddress]);
} catch (cause) {
cliFailure = cause;
}
assert.ok(cliFailure && typeof cliFailure.stdout === "string");
const error = JSON.parse(cliFailure.stdout);
assert.equal(error.code, "invalid_request");
assert.ok(JSON.stringify(error).includes(`Ghidra image base: ${base}`));
for (const command of [
"rea search <path> . --kind procedures --mode regex",
"rea analyze <path>",
"rea inspect-native-load-image <path>",
])
assert.ok(JSON.stringify(error).includes(command));
const cliProcedures = parseEvidence(
JSON.parse(
(
await cli([
"search",
path,
".",
"--kind",
"procedures",
"--mode",
"regex",
])
).stdout,
),
);
const cliEntry = cliProcedures.normalized_result.find(
(item) => item.value === "rea_pie_entry",
);
assert.ok(cliEntry);
assert.equal(cliEntry.address, entry.address);
const cliImage = parseEvidence(
JSON.parse((await cli(["inspect-native-load-image", path])).stdout),
);
assert.equal(cliImage.normalized_result.observations.image_base, base);
const cliPseudo = parseEvidence(
JSON.parse((await cli(["decompile", path, cliEntry.address])).stdout),
);
const cliOffset = parseEvidence(
JSON.parse(
(await cli(["decompile", path, `--file-offset=0x${offset.toString(16)}`]))
.stdout,
),
);
assert.equal(cliOffset.normalized_result, fromOffset.normalized_result);
assert.deepEqual(cliOffset.locations, fromOffset.locations);
assert.match(cliPseudo.normalized_result, /rea_pie_add/u);
assert.deepEqual(await readFile(path), bytes);
assert.deepEqual(await readdir(runtime), []);
report = {
ok: true,
mocked: false,
provider: image.provider,
compiler: version.stdout.split("\n")[0],
sha256,
declared_elf_entry: declaredEntry,
loaded_entry: entry.address,
observed_image_base: base,
cli_mcp_overview_parity: true,
failed_lookup_guidance: true,
cli_recovery_commands: true,
loaded_decompilation: true,
file_offset_decompilation: true,
address_file_offset_round_trip: true,
source_unchanged: true,
runtime_cleanup: "complete",
};
} catch (cause) {
primaryFailure = { cause };
if (stderr) process.stderr.write(stderr);
}
const cleanupFailures = [];
// Await provider-owned cleanup before the SDK can terminate its MCP parent.
if (connected) {
try {
await call("close_binary");
} catch (cause) {
cleanupFailures.push(cause);
}
}
try {
await client.close();
} catch (cause) {
cleanupFailures.push(cause);
}
try {
await transport.close();
} catch (cause) {
cleanupFailures.push(cause);
}
try {
assert.deepEqual(await readdir(runtime), []);
verifierRun = await completeVerifierRun(run);
assert.equal(verifierRun.process_lineage.status, "verified");
assert.deepEqual(verifierRun.process_lineage.descendants, []);
} catch (cause) {
cleanupFailures.push(cause);
}
if (cleanupFailures.length === 0) {
try {
await rm(workspace, { recursive: true, force: true });
} catch (cause) {
cleanupFailures.push(cause);
}
}
if (cleanupFailures.length > 0) {
process.stderr.write(
`Retained verifier workspace after cleanup failure: ${workspace}\n`,
);
throw new AggregateError(
primaryFailure === undefined
? cleanupFailures
: [primaryFailure.cause, ...cleanupFailures],
"Ghidra address-context verifier cleanup failed",
);
}
if (primaryFailure !== undefined) throw primaryFailure.cause;
process.stdout.write(
JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n",
);
async function call(name, args = {}) {
const result = await client.callTool(
{ name, arguments: args },
{ timeout: 240000 },
);
assert.notEqual(result.isError, true, JSON.stringify(result));
return ["open_binary", "binary_session", "close_binary"].includes(name)
? requireMcpOperationResult(result, name)
: result.structuredContent;
}
+68
View File
@@ -45,8 +45,13 @@ const transport = new StdioClientTransport({
const source = fileURLToPath(
new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url),
);
const literalSource = fileURLToPath(
new URL("../tests/conformance/swift/string-literals.swift", import.meta.url),
);
const binary = join(root, "swift-inventory");
const literalBinary = join(root, "swift-literals");
const reports = [];
let literalReport;
try {
await execute("swiftc", ["-O", "-g", source, "-o", binary], {
timeout: 60_000,
@@ -54,6 +59,19 @@ try {
await mkdir(join(root, "copied"));
const copied = join(root, "copied", "swift-inventory");
await copyFile(binary, copied);
// Decoding covers AArch64 register pairs, so build that slice on any Mac host.
await execute(
"swiftc",
[
"-O",
"-target",
"arm64-apple-macosx13.0",
literalSource,
"-o",
literalBinary,
],
{ timeout: 60_000 },
);
await client.connect(transport);
for (const [variant, path] of [
["debug", binary],
@@ -132,6 +150,7 @@ try {
await call("close_binary", {});
}
}
literalReport = await verifyStringLiterals();
} finally {
await client.close();
await transport.close();
@@ -143,6 +162,7 @@ console.log(
ok: true,
provider: "ghidra",
fixtures: reports,
swift_string_literals: literalReport,
verifier_run: await completeVerifierRun(run),
},
null,
@@ -159,3 +179,51 @@ async function call(name, arguments_) {
toolContract(name).outputSchema.parse(response.structuredContent);
return requireMcpOperationResult(response, name);
}
// Literal text lives only in register constants, so it must come from the decoded pairs.
async function verifyStringLiterals() {
await call("open_binary", { path: literalBinary, provider_id: "ghidra" });
try {
const procedures = await call("list_procedures", {});
const expected = {
smallLiteral: [["SleepDisabled", "small"]],
largeLiteral: [["The power setting could not be changed.", "large"]],
unicodeLiterals: [
["café", "small"],
["Résumé of the naïve décor, written twice", "large"],
],
emptyAndControlLiterals: [
["", "small"],
["tab\there", "small"],
],
returnedLiteral: [["Holding awake", "small"]],
};
const decoded = {};
for (const [name, literals] of Object.entries(expected)) {
const procedure = procedures.find(({ value }) => value.includes(name));
assert(procedure, `${name} is missing from the procedure inventory`);
const dossier = await call("analyze_function", {
procedure: procedure.address,
});
const observed = dossier.swift_string_literals;
assert(Array.isArray(observed), `${name} recorded no Swift literals`);
assert.deepEqual(dossier.swift_string_literal_omissions, []);
assert.deepEqual(
observed.map(({ value, representation }) => [value, representation]),
literals,
);
for (const literal of observed) {
assert.deepEqual(literal.registers, ["x0", "x1"]);
assert.equal(
literal.address === null,
literal.representation === "small",
`${literal.value} storage does not match its representation`,
);
}
decoded[name] = observed.length;
}
return decoded;
} finally {
await call("close_binary", {});
}
}
+6
View File
@@ -28,6 +28,7 @@ import {
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs";
import { verifyGhidraFileOffsets } from "./lib/real-ghidra-file-offsets.mjs";
const verifierRun = createVerifierRun();
const SESSION_CAPABILITIES = GHIDRA_SESSION_CAPABILITIES.filter(
@@ -300,6 +301,11 @@ try {
}
if (report === undefined)
throw new Error("Windows Ghidra report was not produced");
report.file_offsets = await verifyGhidraFileOffsets(
targetPath,
resolve(root, "scripts", "rea.mjs"),
architecture === "x86" ? "0x401010" : "0x140001010",
);
process.stdout.write(
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
);
+4 -1
View File
@@ -11,7 +11,8 @@ createVerifierRun();
if (process.platform !== "linux")
throw new Error("Linux real-Hopper verification must run on Linux");
const launcher = process.env.HOPPER_LAUNCHER_PATH ?? "/opt/hopper/bin/Hopper";
const configuredLauncher = process.env.HOPPER_LAUNCHER_PATH;
const launcher = configuredLauncher ?? "/opt/hopper/bin/Hopper";
await access(launcher);
await access("/usr/bin/Xvfb");
await access("/usr/bin/xauth");
@@ -28,5 +29,7 @@ try {
process.env.HOPPER_LAUNCHER_PATH = alternateLauncher;
await import("./verify-real-hopper.mjs");
} finally {
if (configuredLauncher === undefined) delete process.env.HOPPER_LAUNCHER_PATH;
else process.env.HOPPER_LAUNCHER_PATH = configuredLauncher;
await rm(alternateDirectory, { recursive: true, force: true });
}
+9
View File
@@ -47,6 +47,7 @@ import {
} from "./lib/real-hopper-boundaries.mjs";
import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs";
import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs";
import { verifyLinuxHopperOwnerExit } from "./lib/real-hopper-owner-exit.mjs";
const execFileAsync = promisify(execFile);
const verifierRun = createVerifierRun();
const timeout = 180_000;
@@ -577,6 +578,14 @@ if (sessionsAfter.length > 0) {
}
if (summary === undefined)
throw new Error("Real-Hopper verification did not produce a summary");
if (process.platform === "linux")
summary.ownerExit = await verifyLinuxHopperOwnerExit({
primary: targetA,
secondary: targetB,
serverCommand,
serverArgs,
environment: serverEnvironment,
});
const completedVerifierRun = await completeVerifierRun(verifierRun);
if (
completedVerifierRun.process_lineage.status !== "verified" ||
+60 -1
View File
@@ -1,4 +1,8 @@
#!/usr/bin/env node
import {
assertFeatureTrace,
findMappedNonFunction,
} from "./lib/ida-feature-trace.mjs";
import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs";
import assert from "node:assert/strict";
@@ -6,7 +10,7 @@ import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { access, writeFile } from "node:fs/promises";
import { dirname, join, resolve } from "node:path";
import { basename, dirname, join, resolve } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { parseArgs, promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@@ -261,6 +265,61 @@ const mcp = async (cliDossier) => {
break;
}
}
if (config.mode === "headless") {
const documents = observe(await call("list_documents", {}));
assert.equal(documents.length, 1);
assert.equal(basename(documents[0]), basename(target));
assert.ok(
!workspaces.has(dirname(documents[0])),
"list_documents reported the private workspace copy.",
);
const resolveAt = async (address) =>
observe(await call("resolve_containing_procedure", { address }));
const instructionAddresses = dossier.assembly
.map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1])
.filter((address) => address !== undefined);
const queries = [
dossier.procedure.address,
...instructionAddresses.slice(1, 2),
];
for (const queried of queries) {
const containing = await resolveAt(queried);
assert.equal(containing.found, true);
assert.equal(containing.procedure.address, dossier.procedure.address);
assert.equal(containing.procedure.name, dossier.procedure.name);
}
const unmapped = await resolveAt("0xffffffffffffff00");
assert.equal(unmapped.found, false);
assert.equal(unmapped.reason, "outside_segments");
const strings = observe(await call("list_strings", {}));
const mappedString = await findMappedNonFunction(strings, resolveAt);
if (mappedString === undefined)
report.unverified.push(
"Mapped non-function containment: no string of the target is mapped outside every function.",
);
const context = await client.callTool({
name: "inspect_address_context",
arguments: { address: dossier.procedure.address },
});
assert.notEqual(context.isError, true);
const contextResult = context.structuredContent.result;
assert.equal(contextResult.document, documents[0]);
assert.equal(contextResult.procedure.state, "available");
assert.equal(
contextResult.procedure.value.procedure.address,
dossier.procedure.address,
);
const traced = await client.callTool({
name: "trace_feature",
arguments: { query: dossier.procedure.name },
});
assert.notEqual(traced.isError, true);
const trace = traced.structuredContent.normalized_result;
assertFeatureTrace(trace, dossier.procedure);
report.checks.push(
"headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing",
);
}
if (values["expected-string"] === undefined)
report.unverified.push(
"String inventory/search correctness: no independent expected literal supplied.",
+104 -2
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { estimateTokenCount } from "tokenx";
import { execFile } from "node:child_process";
import { createHash } from "node:crypto";
import { readFile, writeFile, readdir } from "node:fs/promises";
@@ -68,6 +69,20 @@ for (const byte of Buffer.from("REA")) {
lines.push(`#${tick + 1000}`, "1!");
const source = lines.join("\n") + "\n";
let cases = 0;
const discoveryMeasurements = [];
const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8"))
.split("\n")
.find((line) => line.startsWith("Max address space "))
?.trim()
.match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u);
assert.ok(inheritedAddressSpace);
const expectedAddressSpace = Math.min(
1073741824,
...inheritedAddressSpace
.slice(1)
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
);
try {
await writeFile(path, source);
await client.connect(transport);
@@ -104,14 +119,16 @@ try {
createHash("sha256").update(source).digest("hex"),
);
assert.equal(report.hardware_acquisition, "not-performed");
assert.equal(report.limits.address_space_bytes, 1073741824);
assert.equal(report.limits.address_space_bytes, expectedAddressSpace);
assert.equal(report.tool.configured_path, process.env.REA_SIGROK_COMMAND);
assert.equal(
report.resource_limiter.configured_path,
process.env.REA_SIGROK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit",
);
assert.ok(
report.resource_limiter.arguments.includes("--as=1073741824:1073741824"),
report.resource_limiter.arguments.includes(
`--as=${expectedAddressSpace}:${expectedAddressSpace}`,
),
);
assert.equal(report.diagnostics, "");
const flags = Object.entries(input)
@@ -245,6 +262,88 @@ try {
});
assert.equal(invalid.isError, true);
cases++;
const discoveryTool = (await client.listTools()).tools.find(
({ name }) => name === "inspect_signal_decoders",
);
assert.ok(discoveryTool?.outputSchema);
assert.ok(ajv.validateSchema(discoveryTool.inputSchema));
assert.ok(ajv.validateSchema(discoveryTool.outputSchema));
const nativeHelp = (
await execute(
process.env.REA_SIGROK_COMMAND,
["--dont-scan", "--protocol-decoders=uart", "--show"],
{ env: environment, timeout: 30000, maxBuffer: 4 * 1024 * 1024 },
)
).stdout.split(/\r?\n/);
if (nativeHelp.at(-1) === "") nativeHelp.pop();
for (const input of [
{ query: "uart" },
{ limit: 1 },
{ offset: 1, limit: 2 },
{ decoder: "uart", query: "BaUd" },
{ decoder: "uart", offset: 50, limit: null },
{ query: "rea_no_such_decoder" },
]) {
const started = performance.now();
const response = await client.callTool({
name: discoveryTool.name,
arguments: input,
});
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(
ajv.validate(discoveryTool.outputSchema, response.structuredContent),
JSON.stringify(ajv.errors),
);
const evidence = parseEvidence(response.structuredContent);
const report = evidence.normalized_result;
discoveryMeasurements.push({
input,
elapsed_ms: +(performance.now() - started).toFixed(2),
response_bytes: Buffer.byteLength(JSON.stringify(response)),
estimated_tokens_one_tool_request_response: estimateTokenCount(
JSON.stringify(discoveryTool) +
JSON.stringify(input) +
JSON.stringify(response),
),
});
if (input.decoder) {
const matched = nativeHelp
.map((text, i) => ({ line_number: i + 1, text }))
.filter(({ text }) =>
text.toLowerCase().includes((input.query ?? "").toLowerCase()),
);
assert.deepEqual(
report.help_lines,
matched.slice(
input.offset ?? 0,
input.limit === null
? undefined
: (input.offset ?? 0) + (input.limit ?? 50),
),
);
} else if (input.query === "uart")
assert.ok(report.decoders.some(({ id }) => id === "uart"));
const flags = Object.entries(input).flatMap(([key, value]) =>
value === null ? ["--all"] : [`--${key}`, String(value)],
);
const cli = JSON.parse(
(
await execute(
process.execPath,
[entrypoint, "inspect-signal-decoders", ...flags, "--format", "json"],
{ env: environment, timeout: 60000, maxBuffer: 8 * 1024 * 1024 },
)
).stdout,
);
assert.deepEqual(cli.normalized_result, report);
cases++;
}
const unavailable = await client.callTool({
name: discoveryTool.name,
arguments: { decoder: "rea_no_such_decoder" },
});
assert.equal(unavailable.isError, true);
cases++;
await client.close();
assert.deepEqual((await readdir(root.path)).sort(), [
"paged.vcd",
@@ -260,6 +359,9 @@ console.log(
{
status: "passed",
cases,
discovery_measurements: discoveryMeasurements,
measurement_scope:
"Warm MCP server, fresh sigrok processes; tokenx estimates include one advertised tool/request/response, not billed model usage. No model-driven selection benchmark, CPU or peak process-family memory measurement; no cache or speedup claim.",
host: `${process.platform}/${process.arch}`,
scope:
"real sigrok UART VCD through compiled CLI/MCP; no physical acquisition",
+2 -2
View File
@@ -27,9 +27,9 @@ if (
);
const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length;
if (
bytes.toString("ascii", 0, 2) !== "MZ" ||
bytes.toString("latin1", 0, 2) !== "MZ" ||
offset > bytes.length - 26 ||
bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" ||
bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" ||
bytes.readUInt16LE(offset + 4) !== 0x8664 ||
bytes.readUInt16LE(offset + 24) !== 0x20b
)
+69
View File
@@ -11,6 +11,7 @@ import {
mkdtemp,
readFile,
rename,
readdir,
rm,
symlink,
writeFile,
@@ -37,6 +38,14 @@ const { WindowsPrivateRuntime, windowsPrivateRuntime } = await import(
const { ghidraSessionRoot } = await import(
pathToFileURL(join(packageRoot, "dist/ghidra/GhidraSessionRoot.js"))
);
const { PrivateRuntimeRoot } = await import(
pathToFileURL(join(packageRoot, "dist/process/PrivateRuntimeRoot.js"))
);
const { PrivateRuntimeRootUnavailableError } = await import(
pathToFileURL(
join(packageRoot, "dist/process/PrivateRuntimeRootUnavailableError.js"),
)
);
const workspace = await mkdtemp(join(tmpdir(), "rea-native-conformance-"));
const environment = Object.entries(process.env)
.filter(([, value]) => value !== undefined)
@@ -140,6 +149,30 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
])
assert.throws(() => native.call("open", [rejected]));
report.controls.ordinaryDriveSeparatorsAndRequestedIdentity = true;
// An owned runtime's DELETE lease conflicts with a fresh parent admission.
// This produces a genuine sharing failure after successful authority caching,
// without another executable, and proves that no child runtime was created.
const sharingParent = WindowsPrivateRuntime.create(
workspace,
"rea-sharing-parent-",
);
try {
await assert.rejects(
PrivateRuntimeRoot.create({ parent: sharingParent.observation.path }),
(failure) => {
assert.ok(failure instanceof PrivateRuntimeRootUnavailableError);
assert.equal(failure.retryable, true);
assert.ok(failure.cause instanceof Error);
assert.equal(failure.cause.win32Code, 32);
assert.equal(failure.cause.runtimeAllocation, "not-started");
return true;
},
);
assert.deepEqual(await readdir(sharingParent.observation.path), []);
report.controls.transientRuntimeAdmissionBeforeAllocation = true;
} finally {
await sharingParent.close();
}
const driveRoot = parse(workspace).root;
const runtimeParent = ghidraSessionRoot({
base: join(driveRoot, ".Tmp", "scratch"),
@@ -629,6 +662,42 @@ setInterval(()=>{},1000);\n`,
unicodeNamesPreserved: true,
});
report.processes.windowsEnvironmentSemantics = true;
// Observed through this process's own id: a separate probe process would
// get its own processor group, so it cannot stand in for the process that
// spawns the child.
const callerAffinity = JSON.parse(
execFileSync(boundaryFixture, ["affinity", String(process.pid)], {
encoding: "utf8",
}),
);
const affinityChild = native.call("process_spawn", [
boundaryFixture,
`${quote(boundaryFixture)} affinity`,
workspace,
environment,
]);
const observedAffinity = await wait(affinityChild);
native.call("process_close", [affinityChild.handle]);
assert.equal(observedAffinity.exitCode, 0);
const childAffinity = JSON.parse(observedAffinity.output);
// Masks cross as hexadecimal strings because a mask can exceed the
// safe-integer range. Sharing one processor group makes the caller's mask
// applicable, so it must be inherited exactly. Windows placing the child in
// another group is only permitted when the caller asked for no bound of its
// own, which its group's whole active set establishes; a narrower mask has
// to fail the spawn rather than widen to that group's default.
const sharedGroup =
callerAffinity.singleProcessorGroup &&
childAffinity.singleProcessorGroup &&
callerAffinity.processorGroup === childAffinity.processorGroup;
if (sharedGroup)
assert.equal(childAffinity.affinityMask, callerAffinity.affinityMask);
else assert.equal(callerAffinity.affinityMask, callerAffinity.systemMask);
report.processes.callerAffinityBoundsOwnedProcessOrRefusesToWiden = true;
report.callerAffinity = {
...callerAffinity,
sharedGroupWithChild: sharedGroup,
};
}
const ownerFixture = join(workspace, "owner.mjs");
const ownerPidFile = join(workspace, "crash-child.pid");
+298 -204
View File
@@ -1,15 +1,8 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import {
access,
mkdir,
mkdtemp,
readFile,
rm,
symlink,
} from "node:fs/promises";
import * as fs from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import * as nodePath from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
@@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
import { parseEvidence } from "../../../dist/domain/evidence.js";
import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js";
const exec = promisify(execFile);
const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const entrypoint = resolve(
const execFileAsync = promisify(execFile);
const { join, resolve, dirname } = nodePath;
const CALL_TIMEOUT_MS = 150_000;
const CONNECT_TIMEOUT_MS = 30_000;
const SEVENZIP_TIMEOUT_MS = 10_000;
const MAX_STDOUT_BYTES = 16 * 1024 * 1024;
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
const ENTRYPOINT = resolve(
process.env.REA_FIRMWARE_TEST_ENTRYPOINT ??
join(repository, "scripts/rea.mjs"),
join(REPO_ROOT, "scripts/rea.mjs"),
);
if (process.platform !== "linux")
throw new Error(
"verify:firmware requires Linux; other hosts are not verified",
);
for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
const path = process.env[variable];
if (path === undefined)
throw new Error(
`verify:firmware requires ${variable}; see docs/firmware-analysis.md`,
);
await access(path);
}
try {
await exec("7z", ["i"], { timeout: 10_000 });
} catch (cause) {
throw new Error(
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
{ cause },
);
}
const fixtureRoot = resolve(
const FIXTURE_ROOT = resolve(
process.env.REA_FIRMWARE_FIXTURE_ROOT ??
join(repository, "_reference/firmware-integration/generated"),
join(REPO_ROOT, "_reference/firmware-integration/generated"),
);
await access(join(fixtureRoot, "oracle.json")).catch((cause) => {
throw new Error(
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
{ cause },
);
});
// ---------------------------------------------------------------------------
// Prerequisite checks
// ---------------------------------------------------------------------------
async function checkPrerequisites() {
if (process.platform !== "linux") {
throw new Error(
"verify:firmware requires Linux; other hosts are not verified",
);
}
for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
const value = process.env[name];
if (value === undefined) {
throw new Error(
`verify:firmware requires ${name}; see docs/firmware-analysis.md`,
);
}
await fs.access(value);
}
try {
await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS });
} catch (cause) {
throw new Error(
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
{ cause },
);
}
try {
await fs.access(join(FIXTURE_ROOT, "oracle.json"));
} catch (cause) {
throw new Error(
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
{ cause },
);
}
}
await checkPrerequisites();
// ---------------------------------------------------------------------------
// Shared state
// ---------------------------------------------------------------------------
const oracle = JSON.parse(
await readFile(join(fixtureRoot, "oracle.json"), "utf8"),
await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"),
);
const path = join(fixtureRoot, "firmware.bin");
assert.equal(await hashFirmwareFile(path), oracle.sha256);
const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-"));
const environment = Object.fromEntries(
Object.entries(process.env).filter(([, value]) => value !== undefined),
const firmwarePath = join(FIXTURE_ROOT, "firmware.bin");
assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256);
const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-"));
const baseEnv = Object.fromEntries(
Object.entries(process.env).filter(([, v]) => v !== undefined),
);
const client = new Client({ name: "rea-real-firmware-verifier", version: "1" });
const cli = async (args, env = environment) => {
const response = await exec(
const mcpClient = new Client({
name: "rea-real-firmware-verifier",
version: "1",
});
// ---------------------------------------------------------------------------
// Invocation helpers
// ---------------------------------------------------------------------------
async function runCli(args, env = baseEnv) {
const { stdout } = await execFileAsync(
process.execPath,
[entrypoint, ...args, "--format", "json"],
[ENTRYPOINT, ...args, "--format", "json"],
{
env,
cwd: repository,
timeout: 150_000,
maxBuffer: 16 * 1024 * 1024,
cwd: REPO_ROOT,
timeout: CALL_TIMEOUT_MS,
maxBuffer: MAX_STDOUT_BYTES,
},
);
return parseEvidence(JSON.parse(response.stdout));
};
const call = async (name, args) => {
const response = await client.callTool(
return parseEvidence(JSON.parse(stdout));
}
async function callTool(name, args) {
const result = await mcpClient.callTool(
{ name, arguments: args },
{ timeout: 150_000 },
{ timeout: CALL_TIMEOUT_MS },
);
assert.notEqual(response.isError, true, JSON.stringify(response));
assert.ok(response.structuredContent !== undefined);
return response.structuredContent;
};
const verifyRegions = (evidence) => {
assert.notEqual(result.isError, true, JSON.stringify(result));
assert.ok(result.structuredContent !== undefined);
return result.structuredContent;
}
// ---------------------------------------------------------------------------
// Result verifiers
// ---------------------------------------------------------------------------
function verifyRegions(evidence) {
assert.equal(evidence.subject.digest.sha256, oracle.sha256);
const result = firmwareResultSchemas.inspect_firmware_regions.parse(
const parsed = firmwareResultSchemas.inspect_firmware_regions.parse(
evidence.normalized_result,
);
const gzip = result.regions.find(
(region) => region.signature === "gzip" && region.offset === oracle.offset,
const gzipRegion = parsed.regions.find(
(r) => r.signature === "gzip" && r.offset === oracle.offset,
);
assert.ok(
gzip !== undefined,
gzipRegion !== undefined,
"Binwalk must find the independently generated gzip offset",
);
assert.equal(gzip.reported_size, oracle.length);
assert.equal(gzip.size_basis, "provider_reported_validation_unknown");
return result.regions.map(
({ provider_id: _provider_id, ...region }) => region,
);
};
const verifyFiles = async (evidence) => {
const result = firmwareResultSchemas.extract_firmware.parse(
assert.equal(gzipRegion.reported_size, oracle.length);
assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown");
// Strip provider-specific ids so CLI and MCP results compare equal.
return parsed.regions.map((region) => {
const { provider_id: _ignored, ...rest } = region;
return rest;
});
}
async function verifyFiles(evidence) {
const parsed = firmwareResultSchemas.extract_firmware.parse(
evidence.normalized_result,
);
assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip"));
assert.ok(parsed.chunks.some((c) => c.handler === "gzip"));
for (const [suffix, expected] of Object.entries(oracle.files)) {
const file = result.files.find((candidate) =>
candidate.relative_path.endsWith(`/${suffix}`),
const entry = parsed.files.find((f) =>
f.relative_path.endsWith(`/${suffix}`),
);
assert.ok(file !== undefined, `Missing extracted ${suffix}`);
assert.equal(file.sha256, expected.sha256);
assert.equal(file.size, expected.size);
assert.equal(await hashFirmwareFile(file.path), expected.sha256);
assert.equal(file.original_file_range, null);
assert.equal(file.runtime_address, null);
assert.ok(entry !== undefined, `Missing extracted ${suffix}`);
assert.equal(entry.sha256, expected.sha256);
assert.equal(entry.size, expected.size);
assert.equal(await hashFirmwareFile(entry.path), expected.sha256);
assert.equal(entry.original_file_range, null);
assert.equal(entry.runtime_address, null);
}
return result;
};
try {
const inspected = await cli(["inspect-firmware-regions", path]);
const cliRegions = verifyRegions(inspected);
const extracted = await cli([
"extract-firmware",
path,
join(root, "cli-output"),
return parsed;
}
const toContentIdentity = (parsed) =>
parsed.files.map(({ relative_path, sha256, size }) => ({
relative_path,
sha256,
size,
}));
// ---------------------------------------------------------------------------
// Verification stages
// ---------------------------------------------------------------------------
async function stageCli() {
const regionsEvidence = await runCli([
"inspect-firmware-regions",
firmwarePath,
]);
const cliFiles = await verifyFiles(extracted);
assert.equal(cliFiles.coverage, "partial");
const regions = verifyRegions(regionsEvidence);
const extractEvidence = await runCli([
"extract-firmware",
firmwarePath,
join(workDir, "cli-output"),
]);
const files = await verifyFiles(extractEvidence);
assert.equal(files.coverage, "partial");
assert.ok(
cliFiles.chunks.some(
(chunk) =>
chunk.handler === null &&
chunk.range.offset === 0 &&
chunk.range.length === 64,
files.chunks.some(
(c) =>
c.handler === null && c.range.offset === 0 && c.range.length === 64,
),
);
await client.connect(
new StdioClientTransport({
command: process.execPath,
args: [entrypoint, "mcp"],
cwd: repository,
env: environment,
stderr: "pipe",
}),
{ timeout: 30_000 },
);
const inspectedMcp = await call("inspect_firmware_regions", { path });
const mcpEvidence = parseEvidence(inspectedMcp);
assert.deepEqual(verifyRegions(mcpEvidence), cliRegions);
const extractedMcp = await call("extract_firmware", {
path,
output_directory: join(root, "mcp-output"),
return { regions, files };
}
async function connectMcp() {
const transport = new StdioClientTransport({
command: process.execPath,
args: [ENTRYPOINT, "mcp"],
cwd: REPO_ROOT,
env: baseEnv,
stderr: "pipe",
});
const mcpFiles = await verifyFiles(parseEvidence(extractedMcp));
const contentIdentity = (result) =>
result.files.map(({ relative_path, sha256, size }) => ({
relative_path,
sha256,
size,
}));
assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles));
const selected = await call("extract_firmware", {
path,
output_directory: join(root, "selected-output"),
await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS });
}
async function stageMcp(cli) {
const regionsRaw = await callTool("inspect_firmware_regions", {
path: firmwarePath,
});
assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions);
const extractRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "mcp-output"),
});
const mcpFiles = await verifyFiles(parseEvidence(extractRaw));
assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files));
const rangedRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "selected-output"),
range: { offset: oracle.offset, length: oracle.length },
});
const selectedFiles = await verifyFiles(parseEvidence(selected));
assert.equal(selectedFiles.selection.offset, oracle.offset);
assert.equal(selectedFiles.selection.length, oracle.length);
const depth = await call("extract_firmware", {
path,
output_directory: join(root, "depth-output"),
const ranged = await verifyFiles(parseEvidence(rangedRaw));
assert.equal(ranged.selection.offset, oracle.offset);
assert.equal(ranged.selection.length, oracle.length);
const depthRaw = await callTool("extract_firmware", {
path: firmwarePath,
output_directory: join(workDir, "depth-output"),
max_depth: 1,
});
const depthResult = firmwareResultSchemas.extract_firmware.parse(
depth.normalized_result,
const depthParsed = firmwareResultSchemas.extract_firmware.parse(
depthRaw.normalized_result,
);
assert.equal(depthResult.coverage, "partial");
assert.ok(depthResult.depth_limited_paths.length > 0);
assert.equal(depthParsed.coverage, "partial");
assert.ok(depthParsed.depth_limited_paths.length > 0);
console.log(
"PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits",
);
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") {
const ext4 = join(fixtureRoot, "ext4.bin");
await access(ext4);
const response = await call("extract_firmware", {
path: ext4,
output_directory: join(root, "ext4-output"),
});
const result = firmwareResultSchemas.extract_firmware.parse(
response.normalized_result,
return mcpFiles;
}
async function stageExt4() {
const ext4Path = join(FIXTURE_ROOT, "ext4.bin");
await fs.access(ext4Path);
const raw = await callTool("extract_firmware", {
path: ext4Path,
output_directory: join(workDir, "ext4-output"),
});
const parsed = firmwareResultSchemas.extract_firmware.parse(
raw.normalized_result,
);
assert.ok(parsed.chunks.some((c) => c.handler === "extfs"));
assert.equal(parsed.coverage, "complete");
for (const [suffix, expected] of Object.entries(oracle.files)) {
const match = parsed.files.find((f) =>
f.relative_path.endsWith(`/${suffix}`),
);
assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs"));
assert.equal(result.coverage, "complete");
for (const [suffix, expected] of Object.entries(oracle.files))
assert.equal(
result.files.find((file) => file.relative_path.endsWith(`/${suffix}`))
?.sha256,
expected.sha256,
);
console.log("PASS real ext4 extraction with debugfs; no mounts");
// Preserve the process-ownership utility while withholding all extractors.
const withoutExtractors = join(root, "without-extractors");
await access("/bin/ps");
await mkdir(withoutExtractors);
await symlink("/bin/ps", join(withoutExtractors, "ps"));
const missing = await cli(
["extract-firmware", ext4, join(root, "missing-extractor-output")],
{ ...environment, PATH: withoutExtractors },
);
const partial = firmwareResultSchemas.extract_firmware.parse(
missing.normalized_result,
);
assert.equal(partial.coverage, "partial");
assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs"));
console.log(
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
);
assert.equal(match?.sha256, expected.sha256);
}
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") {
if (process.env.GHIDRA_INSTALL_DIR === undefined)
throw new Error(
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
);
const child = mcpFiles.files.find((file) =>
file.relative_path.endsWith("/usr/bin/probe"),
);
assert.ok(child !== undefined);
await call("open_binary", { path: child.path, provider_id: "ghidra" });
const dossier = await call("analyze_function", {
procedure: "firmware_probe",
});
const evidence = parseEvidence(dossier);
assert.equal(
evidence.subject.digest.sha256,
oracle.files["usr/bin/probe"].sha256,
);
const pseudocode = await call("procedure_pseudo_code", {
procedure: "firmware_probe",
});
assert.equal(parseEvidence(pseudocode).provider.id, "ghidra");
assert.ok(
JSON.stringify(evidence.normalized_result).includes("firmware_probe"),
);
await call("close_binary", {});
console.log(
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
console.log("PASS real ext4 extraction with debugfs; no mounts");
// Keep the process-ownership utility available but hide every extractor.
const emptyBin = join(workDir, "without-extractors");
await fs.access("/bin/ps");
await fs.mkdir(emptyBin);
await fs.symlink("/bin/ps", join(emptyBin, "ps"));
const degraded = await runCli(
["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")],
{ ...baseEnv, PATH: emptyBin },
);
const degradedParsed = firmwareResultSchemas.extract_firmware.parse(
degraded.normalized_result,
);
assert.equal(degradedParsed.coverage, "partial");
assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs"));
console.log(
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
);
}
async function stageGhidra(mcpFiles) {
if (process.env.GHIDRA_INSTALL_DIR === undefined) {
throw new Error(
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
);
}
const probe = mcpFiles.files.find((f) =>
f.relative_path.endsWith("/usr/bin/probe"),
);
assert.ok(probe !== undefined);
await callTool("open_binary", { path: probe.path, provider_id: "ghidra" });
const dossierRaw = await callTool("analyze_function", {
procedure: "firmware_probe",
});
const dossier = parseEvidence(dossierRaw);
assert.equal(
await hashFirmwareFile(path),
dossier.subject.digest.sha256,
oracle.files["usr/bin/probe"].sha256,
);
const pseudoRaw = await callTool("procedure_pseudo_code", {
procedure: "firmware_probe",
});
assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra");
assert.ok(
JSON.stringify(dossier.normalized_result).includes("firmware_probe"),
);
await callTool("close_binary", {});
console.log(
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
);
}
// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
try {
const cli = await stageCli();
await connectMcp();
const mcpFiles = await stageMcp(cli);
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4();
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1")
await stageGhidra(mcpFiles);
assert.equal(
await hashFirmwareFile(firmwarePath),
oracle.sha256,
"Original firmware bytes must remain unchanged",
);
} finally {
await client.close();
await rm(root, { recursive: true, force: true });
await mcpClient.close();
await fs.rm(workDir, { recursive: true, force: true });
}
+44 -1
View File
@@ -153,10 +153,36 @@ try {
});
try {
await client.connect(transport, { timeout: 30_000 });
await client.listTools();
const opened = await client.callTool(
{ name: "open_binary", arguments: { path } },
{ timeout: 150_000 },
);
assert.notEqual(opened.isError, true, JSON.stringify(opened));
assert.equal(opened.structuredContent?.result?.format, "hap");
assert.equal(opened.structuredContent?.result?.sha256, fixture.sha256);
const status = await client.callTool(
{ name: "binary_session", arguments: {} },
{ timeout: 150_000 },
);
assert.notEqual(status.isError, true, JSON.stringify(status));
assert.equal(status.structuredContent?.result?.open, true);
assert.equal(status.structuredContent?.result?.format, "hap");
const inspected = await client.callTool(
{ name: "inspect_artifact", arguments: {} },
{ timeout: 150_000 },
);
assert.notEqual(inspected.isError, true, JSON.stringify(inspected));
const mcpInspection = parseEvidence(inspected.structuredContent);
assert.deepEqual(
mcpInspection.normalized_result,
inspection.normalized_result,
);
console.log("PASS MCP open/status/inspect_artifact parity");
const response = await client.callTool(
{
name: "project_harmony_application_graph",
arguments: { inventory_evidence: [inspection] },
arguments: { inventory_evidence: [mcpInspection] },
},
{ timeout: 150_000 },
);
@@ -168,6 +194,23 @@ try {
projectionEvidence.normalized_result,
);
console.log("PASS MCP/CLI parity project_harmony_application_graph");
const closed = await client.callTool(
{ name: "close_binary", arguments: {} },
{ timeout: 150_000 },
);
assert.notEqual(closed.isError, true, JSON.stringify(closed));
const empty = await client.callTool(
{ name: "binary_session", arguments: {} },
{ timeout: 150_000 },
);
assert.notEqual(empty.isError, true, JSON.stringify(empty));
assert.equal(empty.structuredContent?.result?.open, false);
assert.equal(
await hashFile(path),
fixture.sha256,
"MCP workflow must preserve source bytes",
);
console.log("PASS MCP close and unchanged source bytes");
} catch (cause) {
process.stderr.write(stderr);
throw cause;
+1 -1
View File
@@ -114,7 +114,7 @@ def shared_blocks(root, path, source):
}
from website_i18n import language_markup
selector, alternates = language_markup(root, path.relative_to(site).as_posix())
values["language_picker"] = selector if not metadata.noindex else ""
values["language_picker"] = indent(selector, " ") if not metadata.noindex else ""
values["alternates"] = alternates if not metadata.noindex else ""
templates = root / "website/templates"
return {