chore: reconcile captured main with PR #2801
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const branch = process.env.CI_DEFAULT_BRANCH;
|
||||
const head = process.env.HEAD_SHA;
|
||||
const shaPattern = /^[0-9a-f]{40}$/i;
|
||||
|
||||
function ancestor(base) {
|
||||
try {
|
||||
execFileSync("git", ["merge-base", "--is-ancestor", base, head], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function write(base, reason) {
|
||||
const output = { base, reason };
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`base=${base}\nreason=${reason.replace(/[\r\n]/g, " ")}\n`,
|
||||
);
|
||||
}
|
||||
console.log(JSON.stringify(output));
|
||||
}
|
||||
|
||||
if (!repository || !branch || !head || !shaPattern.test(head)) {
|
||||
write("", "Repository, default branch, or checked-out head is invalid");
|
||||
} else {
|
||||
try {
|
||||
const response = execFileSync(
|
||||
"gh",
|
||||
[
|
||||
"api",
|
||||
"--method",
|
||||
"GET",
|
||||
`repos/${repository}/actions/workflows/ci.yml/runs`,
|
||||
"-f",
|
||||
`branch=${branch}`,
|
||||
"-f",
|
||||
"event=push",
|
||||
"-f",
|
||||
"status=success",
|
||||
"-F",
|
||||
"per_page=100",
|
||||
],
|
||||
{ encoding: "utf8", timeout: 15_000, maxBuffer: 4 * 1024 * 1024 },
|
||||
);
|
||||
const runs = JSON.parse(response).workflow_runs;
|
||||
if (!Array.isArray(runs))
|
||||
throw new Error("GitHub returned no workflow run list");
|
||||
const base = runs.find(
|
||||
(run) =>
|
||||
run?.event === "push" &&
|
||||
run?.conclusion === "success" &&
|
||||
run?.head_branch === branch &&
|
||||
run?.head_repository?.full_name === repository &&
|
||||
typeof run?.head_sha === "string" &&
|
||||
shaPattern.test(run.head_sha) &&
|
||||
run.head_sha.toLowerCase() !== head.toLowerCase() &&
|
||||
ancestor(run.head_sha),
|
||||
)?.head_sha;
|
||||
if (base) write(base, "");
|
||||
else
|
||||
write(
|
||||
"",
|
||||
"No trusted successful CI push baseline is an ancestor of this head",
|
||||
);
|
||||
} catch {
|
||||
write("", "Could not read a successful CI push baseline from GitHub");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { appendFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { collectJobTimings } from "./job-timings.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings");
|
||||
const MAX_BUFFER = 16 * 1024 * 1024;
|
||||
const API_TIMEOUT_MS = 30_000;
|
||||
const WORKFLOWS = [
|
||||
{ id: "ci.yml", name: "CI" },
|
||||
{ id: "release.yml", name: "Release" },
|
||||
];
|
||||
const LOOKBACK_MS = 24 * 60 * 60 * 1_000;
|
||||
const MAX_RUNS_PER_WORKFLOW = 100;
|
||||
const TIME_BUDGET_MS = 240_000;
|
||||
|
||||
/** Collect a bounded recent sample of completed CI and Release attempts. */
|
||||
export async function collectJobTimingsBatch({
|
||||
repository,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
observedAt = new Date().toISOString(),
|
||||
env = process.env,
|
||||
now = Date.now,
|
||||
collect = collectJobTimings,
|
||||
} = {}) {
|
||||
const identity = validateRepository(repository);
|
||||
const started = now();
|
||||
const cutoff = new Date(started - LOOKBACK_MS).toISOString();
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
observedAt,
|
||||
repository: identity,
|
||||
selection: {
|
||||
workflows: WORKFLOWS,
|
||||
status: "completed",
|
||||
createdAfter: cutoff,
|
||||
lookbackHours: 24,
|
||||
overlap: "Each hourly run re-reads the preceding 24 hours.",
|
||||
pagination:
|
||||
"Run listings use the first 100; jobs are paginated within the remaining time and 16 MiB response limits.",
|
||||
maxRunsPerWorkflow: MAX_RUNS_PER_WORKFLOW,
|
||||
timeBudgetMs: TIME_BUDGET_MS,
|
||||
exhaustive: false,
|
||||
},
|
||||
coverage: {
|
||||
workflowsQueried: [],
|
||||
runsListed: 0,
|
||||
selectedRuns: 0,
|
||||
collectedRuns: 0,
|
||||
},
|
||||
truncation: [],
|
||||
errors: [],
|
||||
runs: [],
|
||||
elapsedMs: 0,
|
||||
};
|
||||
const budgetExpired = () => now() - started >= TIME_BUDGET_MS;
|
||||
|
||||
const selected = [];
|
||||
for (const workflow of WORKFLOWS) {
|
||||
if (budgetExpired()) {
|
||||
manifest.truncation.push("Time budget reached while listing workflows.");
|
||||
break;
|
||||
}
|
||||
try {
|
||||
const response = await apiJson(
|
||||
`repos/${identity}/actions/workflows/${workflow.id}/runs?status=completed&created=%3E%3D${encodeURIComponent(cutoff)}&per_page=${String(MAX_RUNS_PER_WORKFLOW)}&page=1`,
|
||||
env,
|
||||
started,
|
||||
now,
|
||||
);
|
||||
if (!Array.isArray(response.workflow_runs))
|
||||
throw new Error("run listing must contain a workflow_runs array");
|
||||
manifest.coverage.workflowsQueried.push(workflow.name);
|
||||
manifest.coverage.runsListed += response.workflow_runs.length;
|
||||
if (response.workflow_runs.length >= MAX_RUNS_PER_WORKFLOW)
|
||||
manifest.truncation.push(
|
||||
`${workflow.name} run listing reached the 100-run first-page limit.`,
|
||||
);
|
||||
for (const run of response.workflow_runs.slice(
|
||||
0,
|
||||
MAX_RUNS_PER_WORKFLOW,
|
||||
)) {
|
||||
if (
|
||||
run.status !== "completed" ||
|
||||
!Number.isSafeInteger(run.id) ||
|
||||
!Number.isSafeInteger(run.run_attempt)
|
||||
) {
|
||||
manifest.errors.push({
|
||||
scope: `workflow:${workflow.name}`,
|
||||
runId: run.id ?? null,
|
||||
message:
|
||||
"Run record missing completed status, numeric id, or numeric latest attempt.",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (run.created_at && Date.parse(run.created_at) < Date.parse(cutoff))
|
||||
continue;
|
||||
selected.push({
|
||||
id: run.id,
|
||||
runAttempt: run.run_attempt,
|
||||
workflow: workflow.name,
|
||||
conclusion: run.conclusion ?? null,
|
||||
createdAt: run.created_at ?? null,
|
||||
});
|
||||
}
|
||||
} catch (cause) {
|
||||
manifest.errors.push({
|
||||
scope: `workflow:${workflow.name}`,
|
||||
message: errorMessage(cause),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
selected.sort(
|
||||
(left, right) =>
|
||||
(Date.parse(right.createdAt ?? "") || 0) -
|
||||
(Date.parse(left.createdAt ?? "") || 0),
|
||||
);
|
||||
manifest.coverage.selectedRuns = selected.length;
|
||||
for (const run of selected) {
|
||||
if (budgetExpired()) {
|
||||
manifest.truncation.push(
|
||||
"Time budget reached before all selected runs were collected.",
|
||||
);
|
||||
break;
|
||||
}
|
||||
const runRecord = {
|
||||
...run,
|
||||
status: "pending",
|
||||
rawPath: null,
|
||||
reportPath: null,
|
||||
error: null,
|
||||
};
|
||||
manifest.runs.push(runRecord);
|
||||
const runDirectory = join(
|
||||
outputDirectory,
|
||||
"runs",
|
||||
`${String(run.id)}-${String(run.runAttempt)}`,
|
||||
);
|
||||
try {
|
||||
const result = await collect({
|
||||
repository: identity,
|
||||
runId: String(run.id),
|
||||
runAttempt: String(run.runAttempt),
|
||||
outputDirectory: runDirectory,
|
||||
env,
|
||||
timeoutMs: Math.max(
|
||||
1,
|
||||
Math.min(API_TIMEOUT_MS, TIME_BUDGET_MS - (now() - started)),
|
||||
),
|
||||
});
|
||||
runRecord.status = "collected";
|
||||
runRecord.rawPath = result.rawPath;
|
||||
runRecord.reportPath = result.reportPath;
|
||||
manifest.coverage.collectedRuns += 1;
|
||||
} catch (cause) {
|
||||
runRecord.status = "error";
|
||||
runRecord.error = errorMessage(cause);
|
||||
manifest.errors.push({
|
||||
scope: "run",
|
||||
runId: run.id,
|
||||
runAttempt: run.runAttempt,
|
||||
message: runRecord.error,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (budgetExpired() && manifest.runs.length < manifest.coverage.selectedRuns)
|
||||
manifest.truncation.push(
|
||||
"Time budget reached; selected runs remain unattempted.",
|
||||
);
|
||||
return await writeManifest(outputDirectory, manifest, started, now, env);
|
||||
}
|
||||
|
||||
function validateRepository(repository) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
return repository;
|
||||
}
|
||||
|
||||
async function apiJson(endpoint, env, started, now) {
|
||||
const timeout = Math.max(
|
||||
1,
|
||||
Math.min(API_TIMEOUT_MS, TIME_BUDGET_MS - (now() - started)),
|
||||
);
|
||||
const { stdout } = await execFileAsync("gh", ["api", endpoint], {
|
||||
env,
|
||||
encoding: "utf8",
|
||||
maxBuffer: MAX_BUFFER,
|
||||
timeout,
|
||||
});
|
||||
try {
|
||||
return JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("GitHub Actions API returned malformed JSON", { cause });
|
||||
}
|
||||
}
|
||||
|
||||
async function writeManifest(outputDirectory, manifest, started, now, env) {
|
||||
manifest.elapsedMs = Math.max(0, now() - started);
|
||||
if (
|
||||
manifest.elapsedMs >= TIME_BUDGET_MS &&
|
||||
!manifest.truncation.some((item) => item.includes("Time budget"))
|
||||
)
|
||||
manifest.truncation.push("Time budget reached.");
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const manifestPath = join(outputDirectory, "batch-manifest.json");
|
||||
await writeFile(
|
||||
manifestPath,
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
if (env.GITHUB_STEP_SUMMARY)
|
||||
await appendBatchSummary(env.GITHUB_STEP_SUMMARY, manifest, manifestPath);
|
||||
return { manifestPath, manifest };
|
||||
}
|
||||
|
||||
async function appendBatchSummary(summaryPath, manifest, manifestPath) {
|
||||
const collected = manifest.runs.filter(
|
||||
({ status }) => status === "collected",
|
||||
).length;
|
||||
const failed = manifest.runs.filter(
|
||||
({ status }) => status === "error",
|
||||
).length;
|
||||
const lines = [
|
||||
"### Batched CI timing snapshot",
|
||||
"",
|
||||
`Observed ${manifest.observedAt}; collected ${String(collected)} of ${String(manifest.coverage.selectedRuns)} selected recent completed runs (${String(failed)} run errors).`,
|
||||
`Coverage uses the first ${String(MAX_RUNS_PER_WORKFLOW)} runs per workflow created in the preceding 24 hours and a ${String(TIME_BUDGET_MS / 1_000)} second total budget. This sample is bounded and is not exhaustive.`,
|
||||
`Manifest: \`${manifestPath}\`.`,
|
||||
...manifest.truncation.map((message) => `- Truncated: ${message}`),
|
||||
...manifest.errors.map(
|
||||
({ scope, message }) => `- Error (${scope}): ${message}`,
|
||||
),
|
||||
"",
|
||||
];
|
||||
await appendFile(summaryPath, lines.join("\n"), "utf8");
|
||||
}
|
||||
|
||||
function errorMessage(cause) {
|
||||
return cause instanceof Error ? cause.message : String(cause);
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
const result = await collectJobTimingsBatch({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
});
|
||||
process.stdout.write(
|
||||
`Wrote batch manifest for ${String(result.manifest.coverage.collectedRuns)} collected runs to ${result.manifestPath}\n`,
|
||||
);
|
||||
} catch (cause) {
|
||||
process.stderr.write(`CI timing batch failed: ${errorMessage(cause)}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,354 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MAX_BUFFER = 16 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings");
|
||||
const STEP_COSTS = [
|
||||
{ key: "npmCi", pattern: /\bnpm ci\b/iu },
|
||||
{ key: "artifactUpload", pattern: /upload.*artifact/iu },
|
||||
{ key: "artifactDownload", pattern: /download.*artifact/iu },
|
||||
];
|
||||
|
||||
/** Fetch and summarize one workflow attempt without turning telemetry into a gate. */
|
||||
export async function collectJobTimings({
|
||||
repository,
|
||||
runId,
|
||||
runAttempt,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
observedAt = new Date().toISOString(),
|
||||
env = process.env,
|
||||
timeoutMs = TIMEOUT_MS,
|
||||
}) {
|
||||
const identity = validateIdentity({ repository, runId, runAttempt });
|
||||
const endpoint = `repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100`;
|
||||
const { stdout } = await execFileAsync(
|
||||
"gh",
|
||||
["api", "--paginate", "--slurp", endpoint],
|
||||
{ env, encoding: "utf8", maxBuffer: MAX_BUFFER, timeout: timeoutMs },
|
||||
);
|
||||
const rawPath = join(outputDirectory, "jobs.json");
|
||||
const reportPath = join(outputDirectory, "report.json");
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(rawPath, stdout, "utf8");
|
||||
const pages = parsePages(stdout);
|
||||
const jobs = pages.flatMap(({ jobs: pageJobs }, index) =>
|
||||
validateJobs(pageJobs, index),
|
||||
);
|
||||
const report = normalizeReport({ identity, jobs, observedAt });
|
||||
|
||||
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
if (summaryPath !== undefined)
|
||||
await appendSummary(summaryPath, report, rawPath, reportPath);
|
||||
return { rawPath, reportPath, report };
|
||||
}
|
||||
|
||||
function validateIdentity({ repository, runId, runAttempt }) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
return {
|
||||
repository,
|
||||
runId: parsePositiveInteger(runId, "CI_TIMING_RUN_ID"),
|
||||
runAttempt: parsePositiveInteger(runAttempt, "CI_TIMING_RUN_ATTEMPT"),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
if (typeof value === "string" && /^[1-9]\d*$/u.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
|
||||
function parsePages(stdout) {
|
||||
let pages;
|
||||
try {
|
||||
pages = JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("GitHub jobs API returned malformed JSON", { cause });
|
||||
}
|
||||
if (!Array.isArray(pages) || pages.length === 0)
|
||||
throw new Error("GitHub jobs API slurp must contain a nonempty page array");
|
||||
for (const page of pages)
|
||||
if (
|
||||
typeof page !== "object" ||
|
||||
page === null ||
|
||||
Array.isArray(page) ||
|
||||
!Array.isArray(page.jobs)
|
||||
)
|
||||
throw new Error("GitHub jobs API page must contain a jobs array");
|
||||
return pages;
|
||||
}
|
||||
|
||||
function validateJobs(jobs, pageIndex) {
|
||||
return jobs.map((job, jobIndex) => {
|
||||
if (
|
||||
typeof job !== "object" ||
|
||||
job === null ||
|
||||
Array.isArray(job) ||
|
||||
typeof job.name !== "string" ||
|
||||
job.name.length === 0 ||
|
||||
typeof job.status !== "string"
|
||||
)
|
||||
throw new Error(
|
||||
`GitHub jobs API page ${String(pageIndex + 1)} job ${String(jobIndex + 1)} is malformed`,
|
||||
);
|
||||
return job;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeReport({ identity, jobs, observedAt }) {
|
||||
const normalizedJobs = jobs.map(normalizeJob);
|
||||
const queues = normalizedJobs
|
||||
.map(({ queue }) => queue.seconds)
|
||||
.filter((value) => value !== null);
|
||||
const completedExecutions = normalizedJobs
|
||||
.filter(
|
||||
({ status, conclusion }) =>
|
||||
status === "completed" && conclusion !== "skipped",
|
||||
)
|
||||
.map(({ execution }) => execution.seconds)
|
||||
.filter((value) => value !== null);
|
||||
const completedDurationSumSeconds = completedExecutions.reduce(
|
||||
(sum, seconds) => sum + seconds,
|
||||
0,
|
||||
);
|
||||
const completedExecutionUnknown = normalizedJobs.filter(
|
||||
({ status, conclusion, execution }) =>
|
||||
status === "completed" &&
|
||||
conclusion !== "skipped" &&
|
||||
execution.seconds === null,
|
||||
).length;
|
||||
const skipped = normalizedJobs.filter(
|
||||
({ conclusion }) => conclusion === "skipped",
|
||||
).length;
|
||||
const costs = Object.fromEntries(STEP_COSTS.map(({ key }) => [key, []]));
|
||||
for (const job of normalizedJobs)
|
||||
for (const step of job.steps)
|
||||
if (step.costKey !== null)
|
||||
costs[step.costKey].push({ job: job.name, ...step });
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
run: {
|
||||
...identity,
|
||||
observedAt,
|
||||
snapshot:
|
||||
normalizedJobs.length > 0 &&
|
||||
normalizedJobs.every(({ status }) => status === "completed")
|
||||
? "completed job status snapshot; unknown timestamps remain unmeasured"
|
||||
: "partial; job statuses may still be queued or in progress",
|
||||
},
|
||||
counts: {
|
||||
jobs: normalizedJobs.length,
|
||||
completed: normalizedJobs.filter(({ status }) => status === "completed")
|
||||
.length,
|
||||
skipped,
|
||||
queueUnknown: normalizedJobs.filter(({ queue }) => queue.seconds === null)
|
||||
.length,
|
||||
executionUnknown: normalizedJobs.filter(
|
||||
({ execution }) => execution.seconds === null,
|
||||
).length,
|
||||
completedExecutionUnknown,
|
||||
},
|
||||
aggregates: {
|
||||
queueSeconds: percentileSummary(queues),
|
||||
completedExecutionSeconds: percentileSummary(completedExecutions),
|
||||
summedCompletedJobElapsedSeconds: {
|
||||
value:
|
||||
completedExecutions.length === 0
|
||||
? null
|
||||
: round(completedDurationSumSeconds),
|
||||
knownCount: completedExecutions.length,
|
||||
unknownCount: completedExecutionUnknown,
|
||||
label:
|
||||
"sum of known completed job elapsed durations; incomplete when timestamps are unknown; not CPU time, billing, or workflow wall time",
|
||||
},
|
||||
},
|
||||
jobs: normalizedJobs,
|
||||
stepCostInterpretation:
|
||||
"Matched install/artifact steps by step name; these are not total artifact transfer costs.",
|
||||
stepCosts: costs,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeJob(job) {
|
||||
const skipped = job.conclusion === "skipped";
|
||||
const hasStarted = ["in_progress", "completed"].includes(job.status);
|
||||
const queue = skipped
|
||||
? unknownDuration("job was skipped")
|
||||
: !hasStarted
|
||||
? unknownDuration("job start is not established by its status")
|
||||
: duration(job.created_at, job.started_at);
|
||||
const execution = skipped
|
||||
? unknownDuration("job was skipped")
|
||||
: job.status === "completed"
|
||||
? duration(job.started_at, job.completed_at)
|
||||
: unknownDuration("job is not completed");
|
||||
const steps = Array.isArray(job.steps) ? job.steps.map(normalizeStep) : [];
|
||||
return {
|
||||
id: Number.isSafeInteger(job.id) && job.id > 0 ? job.id : null,
|
||||
url: stringOrNull(job.html_url),
|
||||
name: job.name,
|
||||
workflowName: stringOrNull(job.workflow_name),
|
||||
status: job.status,
|
||||
conclusion: stringOrNull(job.conclusion),
|
||||
createdAt: stringOrNull(job.created_at),
|
||||
startedAt: stringOrNull(job.started_at),
|
||||
completedAt: stringOrNull(job.completed_at),
|
||||
queue,
|
||||
execution,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeStep(step) {
|
||||
if (
|
||||
typeof step !== "object" ||
|
||||
step === null ||
|
||||
Array.isArray(step) ||
|
||||
typeof step.name !== "string"
|
||||
)
|
||||
throw new Error("GitHub jobs API contains a malformed job step");
|
||||
const cost = STEP_COSTS.find(({ pattern }) => pattern.test(step.name));
|
||||
return {
|
||||
name: step.name,
|
||||
costKey: cost?.key ?? null,
|
||||
duration: duration(step.started_at, step.completed_at),
|
||||
};
|
||||
}
|
||||
|
||||
function duration(startValue, endValue) {
|
||||
if (typeof startValue !== "string" || typeof endValue !== "string")
|
||||
return unknownDuration("timestamp unavailable");
|
||||
const start = Date.parse(startValue);
|
||||
const end = Date.parse(endValue);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end))
|
||||
return unknownDuration("invalid timestamp");
|
||||
const milliseconds = end - start;
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0)
|
||||
return unknownDuration("timestamps out of order");
|
||||
return { seconds: round(milliseconds / 1_000), reason: null };
|
||||
}
|
||||
|
||||
function unknownDuration(reason) {
|
||||
return { seconds: null, reason };
|
||||
}
|
||||
|
||||
function percentileSummary(values) {
|
||||
return {
|
||||
count: values.length,
|
||||
p50: nearestRank(values, 0.5),
|
||||
p95: nearestRank(values, 0.95),
|
||||
unit: "seconds",
|
||||
};
|
||||
}
|
||||
|
||||
function nearestRank(values, percentile) {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
return round(sorted[Math.ceil(percentile * sorted.length) - 1]);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 1_000) / 1_000;
|
||||
}
|
||||
|
||||
function stringOrNull(value) {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
async function appendSummary(summaryPath, report, rawPath, reportPath) {
|
||||
await mkdir(dirname(summaryPath), { recursive: true });
|
||||
const rows = [
|
||||
["Queue", report.aggregates.queueSeconds],
|
||||
["Completed execution", report.aggregates.completedExecutionSeconds],
|
||||
]
|
||||
.map(
|
||||
([label, item]) =>
|
||||
`| ${label} | ${display(item.p50)} | ${display(item.p95)} | ${String(item.count)} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const jobRows = report.jobs
|
||||
.map(
|
||||
(job) =>
|
||||
`| ${escapeCell(job.name)} | ${job.status} | ${job.conclusion ?? "unknown"} | ${display(job.queue.seconds)} | ${job.queue.seconds === null ? job.queue.reason : ""} | ${display(job.execution.seconds)} | ${job.execution.seconds === null ? job.execution.reason : ""} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const costRows = STEP_COSTS.map(({ key }) => {
|
||||
const samples = report.stepCosts[key];
|
||||
const values = samples
|
||||
.map(({ duration: item }) => item.seconds)
|
||||
.filter((value) => value !== null);
|
||||
return `| ${key} | ${String(samples.length)} | ${display(nearestRank(values, 0.5))} | ${samples.filter(({ duration: item }) => item.seconds === null).length} |`;
|
||||
}).join("\n");
|
||||
const text = [
|
||||
"### CI job timing snapshot",
|
||||
"",
|
||||
`Observed at ${report.run.observedAt} for run ${String(report.run.runId)}, attempt ${String(report.run.runAttempt)}. This is a point-in-time snapshot of fetched jobs. Queue starts at job creation and excludes preceding dependency waiting. Missing, queued, in-progress, invalid, or out-of-order timestamps are reported as unknown. Elapsed durations are not CPU time or billing data.`,
|
||||
"",
|
||||
`Fetched ${String(report.counts.jobs)} jobs; ${String(report.counts.completed)} are completed (${String(report.counts.skipped)} skipped); ${String(report.counts.queueUnknown)} queue durations and ${String(report.counts.executionUnknown)} execution durations (${String(report.counts.completedExecutionUnknown)} for non-skipped completed jobs) are unknown.`,
|
||||
"",
|
||||
"| Metric | p50 (s) | p95 (s) | Samples |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
rows,
|
||||
"",
|
||||
"| Job | Status | Conclusion | Queue (s) | Queue detail | Execution (s) | Execution detail |",
|
||||
"| --- | --- | --- | ---: | --- | ---: | --- |",
|
||||
jobRows,
|
||||
"",
|
||||
`Sum of known completed job elapsed durations: **${display(report.aggregates.summedCompletedJobElapsedSeconds.value)} s** across ${String(report.aggregates.summedCompletedJobElapsedSeconds.knownCount)} known and ${String(report.aggregates.summedCompletedJobElapsedSeconds.unknownCount)} unknown jobs; this is not CPU time, billing, or workflow wall time.`,
|
||||
"",
|
||||
"Matched install/artifact steps by name; these are not total artifact transfer costs.",
|
||||
"",
|
||||
"| Matched step group | Samples | p50 (s) | Unknown |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
costRows,
|
||||
"",
|
||||
`Reports: \`${rawPath}\`, \`${reportPath}\`.`,
|
||||
"",
|
||||
].join("\n");
|
||||
await writeFile(summaryPath, text, { encoding: "utf8", flag: "a" });
|
||||
}
|
||||
|
||||
function display(value) {
|
||||
return value === null ? "unknown" : String(value);
|
||||
}
|
||||
|
||||
function escapeCell(value) {
|
||||
return value.replaceAll("|", "\\|").replaceAll("\n", " ");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
const result = await collectJobTimings({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
runId: process.env.CI_TIMING_RUN_ID,
|
||||
runAttempt: process.env.CI_TIMING_RUN_ATTEMPT,
|
||||
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
summaryPath: process.env.GITHUB_STEP_SUMMARY,
|
||||
});
|
||||
process.stdout.write(
|
||||
`Wrote ${result.report.jobs.length} job timing records to ${result.reportPath}\n`,
|
||||
);
|
||||
} catch (cause) {
|
||||
process.stderr.write(
|
||||
`CI timing telemetry failed: ${cause instanceof Error ? cause.message : String(cause)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
+151
-106
@@ -1,30 +1,20 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
import { planTestSelection } from "./test-selection.mjs";
|
||||
|
||||
const flags = [
|
||||
...new Set([
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"apple",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
]),
|
||||
];
|
||||
// Test files select the native-host lanes that run them explicitly. Provider
|
||||
// scopes remain implementation-owned, so test-only edits stay off those lanes.
|
||||
const NATIVE_TEST_OWNERS = ["apple", "package"];
|
||||
// Test files select the host/runtime lanes that execute their focused suites.
|
||||
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
|
||||
// Actionlint validates workflow syntax only. The semantic suite asserts these
|
||||
// workflows' release, publication and self-hosted runner safeguards.
|
||||
const GUARDED_WORKFLOWS = new Set([
|
||||
".github/workflows/pr-title.yml",
|
||||
".github/workflows/ci-full-pr.yml",
|
||||
".github/workflows/release.yml",
|
||||
".github/workflows/real-ghidra-windows.yml",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
".github/workflows/ci-timings.yml",
|
||||
]);
|
||||
// Sources compiled by build-conformance-fixtures.mjs. The readiness journey
|
||||
// analyzes the built C fixture as well as its own applications.
|
||||
@@ -36,6 +26,15 @@ const FIXTURE_SOURCES = [
|
||||
"tests/conformance/versions/**",
|
||||
];
|
||||
const READINESS_SOURCES = ["tests/conformance/readiness/**"];
|
||||
// These reporters affect CI output only; they do not select a host or provider
|
||||
// verification lane when changed on their own.
|
||||
const PORTABLE_CI_REPORTERS = new Set([
|
||||
"scripts/ci/job-timings.mjs",
|
||||
"scripts/ci/job-timings-batch.mjs",
|
||||
"scripts/ci/test-report.mjs",
|
||||
"scripts/ci/test-timing-reporter.mjs",
|
||||
"scripts/ci/test-history.mjs",
|
||||
]);
|
||||
const matchesAny = (file, patterns) =>
|
||||
patterns.some((pattern) => matchesGlob(file, pattern));
|
||||
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
|
||||
@@ -63,6 +62,10 @@ function classifyPackage(base, head) {
|
||||
]),
|
||||
].filter((key) => beforeScripts?.[key] !== afterScripts?.[key]);
|
||||
for (const script of changed) {
|
||||
if (script === "test:sqlite" || script === "test:sqlite:run") {
|
||||
selected.sqlite = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
/^(test(?::|$)|check(?::|$)|lint(?::|$)|format(?::|$)|knip$|jscpd$)/.test(
|
||||
script,
|
||||
@@ -108,101 +111,137 @@ function classifyPackage(base, head) {
|
||||
}
|
||||
|
||||
const event = process.env.GITHUB_EVENT_NAME;
|
||||
let files = [];
|
||||
if (
|
||||
event === "push" ||
|
||||
if (!["pull_request", "push", "schedule", "workflow_dispatch"].includes(event))
|
||||
throw new Error(`Unsupported CI event: ${event}`);
|
||||
const requestedFull =
|
||||
event === "schedule" ||
|
||||
event === "workflow_dispatch" ||
|
||||
process.env.CI_FULL === "true"
|
||||
) {
|
||||
process.env.CI_FULL === "true";
|
||||
const tier =
|
||||
event === "schedule" &&
|
||||
process.env.CI_SKIP_FULL === "true" &&
|
||||
process.env.CI_FULL !== "true"
|
||||
? "reused"
|
||||
: requestedFull
|
||||
? "full"
|
||||
: "fast";
|
||||
let files = [];
|
||||
if (tier === "reused") {
|
||||
reasons.push(
|
||||
"Recent exact-SHA full qualification reused; no fresh provider observation",
|
||||
);
|
||||
} else if (tier === "full") {
|
||||
full("Full baseline requested by event or ci:full label");
|
||||
} else if (event === "pull_request") {
|
||||
const base = git("merge-base", process.env.BASE_SHA, process.env.HEAD_SHA);
|
||||
} else if (event === "pull_request" || event === "push") {
|
||||
const head = process.env.HEAD_SHA;
|
||||
// Disable rename detection so both the removed and added paths participate.
|
||||
files = git("diff", "--name-only", "--no-renames", "-z", base, head)
|
||||
.split("\0")
|
||||
.filter(Boolean);
|
||||
for (const file of files) {
|
||||
if (
|
||||
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
|
||||
file,
|
||||
) ||
|
||||
file.startsWith("docs/") ||
|
||||
file.startsWith(".github/ISSUE_TEMPLATE/")
|
||||
) {
|
||||
selected.docs = true;
|
||||
selected.website ||= matchesAny(file, SCOPES.website);
|
||||
continue;
|
||||
}
|
||||
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
|
||||
selected.code = true;
|
||||
for (const owner of NATIVE_TEST_OWNERS)
|
||||
selected[owner] ||= matchesAny(file, SCOPES[owner]);
|
||||
continue;
|
||||
}
|
||||
if (matchesAny(file, MANUAL_VERIFIERS)) {
|
||||
selected.code = true;
|
||||
continue;
|
||||
}
|
||||
const owners = Object.entries(SCOPES)
|
||||
.filter(([, patterns]) =>
|
||||
patterns.some((pattern) => matchesGlob(file, pattern)),
|
||||
)
|
||||
.map(([owner]) => owner);
|
||||
for (const owner of owners) selected[owner] = true;
|
||||
if (
|
||||
file.startsWith(".github/workflows/") ||
|
||||
file === ".github/actionlint.yaml"
|
||||
) {
|
||||
selected.workflows = true;
|
||||
if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
|
||||
if (file !== ".github/workflows/ci.yml") continue;
|
||||
}
|
||||
// The Notes example is also the source-owned agent evaluation fixture.
|
||||
if (file.startsWith("website/public/examples/notes-electron/"))
|
||||
selected.code = true;
|
||||
if (owners.length === 1 && owners[0] === "website") continue;
|
||||
if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
|
||||
if (file === "package.json") {
|
||||
classifyPackage(base, head);
|
||||
continue;
|
||||
}
|
||||
selected.code = true;
|
||||
if (matchesAny(file, FIXTURE_SOURCES)) {
|
||||
selected.fixtures = true;
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
|
||||
if (file.startsWith("tests/")) continue;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
)
|
||||
) {
|
||||
full(`Shared boundary: ${file}`);
|
||||
} else if (owners.length) {
|
||||
if (file.startsWith("src/") || file.startsWith("bridge/"))
|
||||
selected.runtime = true;
|
||||
} else if (/^(src|bridge)\//.test(file)) {
|
||||
// Unowned runtime sources keep the Linux x64 installed-package baseline;
|
||||
// ci:full, main pushes and manual dispatch retain every native host.
|
||||
selected.runtime = true;
|
||||
reasons.push(`Unowned runtime source: ${file}`);
|
||||
} else if (file.startsWith(".agents/skills/")) {
|
||||
selected.runtime = true;
|
||||
selected.docs = true;
|
||||
} else {
|
||||
full(`Unclassified input: ${file}`);
|
||||
let base;
|
||||
if (event === "pull_request") {
|
||||
base = git("merge-base", process.env.BASE_SHA, head);
|
||||
} else {
|
||||
const candidate = process.env.BASE_SHA;
|
||||
const baselineReason = process.env.BASELINE_REASON;
|
||||
try {
|
||||
if (!candidate || !/^[0-9a-f]{40}$/i.test(candidate))
|
||||
throw new Error(
|
||||
baselineReason || "No valid successful push baseline was found",
|
||||
);
|
||||
execFileSync("git", ["merge-base", "--is-ancestor", candidate, head], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
base = candidate;
|
||||
} catch {
|
||||
full(
|
||||
baselineReason ||
|
||||
"No valid successful push baseline ancestor; selecting all lanes",
|
||||
);
|
||||
}
|
||||
}
|
||||
} else throw new Error(`Unsupported CI event: ${event}`);
|
||||
if (base) {
|
||||
// Disable rename detection so both the removed and added paths participate.
|
||||
files = git("diff", "--name-only", "--no-renames", "-z", base, head)
|
||||
.split("\0")
|
||||
.filter(Boolean);
|
||||
for (const file of files) {
|
||||
if (
|
||||
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
|
||||
file,
|
||||
) ||
|
||||
file.startsWith("docs/") ||
|
||||
file.startsWith(".github/ISSUE_TEMPLATE/")
|
||||
) {
|
||||
selected.docs = true;
|
||||
selected.website ||= matchesAny(file, SCOPES.website);
|
||||
continue;
|
||||
}
|
||||
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
|
||||
selected.code = true;
|
||||
for (const owner of HOST_TEST_OWNERS)
|
||||
selected[owner] ||= matchesAny(file, SCOPES[owner]);
|
||||
continue;
|
||||
}
|
||||
if (matchesAny(file, MANUAL_VERIFIERS)) {
|
||||
selected.code = true;
|
||||
continue;
|
||||
}
|
||||
const owners = Object.entries(SCOPES)
|
||||
.filter(([, patterns]) =>
|
||||
patterns.some((pattern) => matchesGlob(file, pattern)),
|
||||
)
|
||||
.map(([owner]) => owner);
|
||||
for (const owner of owners) selected[owner] = true;
|
||||
if (
|
||||
file.startsWith(".github/workflows/") ||
|
||||
file === ".github/actionlint.yaml"
|
||||
) {
|
||||
selected.workflows = true;
|
||||
if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
|
||||
if (file !== ".github/workflows/ci.yml") continue;
|
||||
}
|
||||
// The Notes example is also the source-owned agent evaluation fixture.
|
||||
if (file.startsWith("website/public/examples/notes-electron/"))
|
||||
selected.code = true;
|
||||
if (owners.length === 1 && owners[0] === "website") continue;
|
||||
if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
|
||||
if (file === "package.json") {
|
||||
classifyPackage(base, head);
|
||||
continue;
|
||||
}
|
||||
selected.code = true;
|
||||
if (matchesAny(file, FIXTURE_SOURCES)) {
|
||||
selected.fixtures = true;
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
|
||||
if (file.startsWith("tests/")) continue;
|
||||
if (PORTABLE_CI_REPORTERS.has(file)) continue;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
)
|
||||
) {
|
||||
full(`Shared boundary: ${file}`);
|
||||
} else if (owners.length) {
|
||||
if (file.startsWith("src/") || file.startsWith("bridge/"))
|
||||
selected.runtime = true;
|
||||
} else if (/^(src|bridge)\//.test(file)) {
|
||||
// Unowned runtime sources keep the Linux x64 installed-package baseline;
|
||||
// full qualification retains every native host.
|
||||
selected.runtime = true;
|
||||
reasons.push(`Unowned runtime source: ${file}`);
|
||||
} else if (file.startsWith(".agents/skills/")) {
|
||||
selected.runtime = true;
|
||||
selected.docs = true;
|
||||
} else {
|
||||
full(`Unclassified input: ${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
selected.runtime ||= selected.package;
|
||||
const linux = { os: "ubuntu-latest", platform: "linux", architecture: "x64" };
|
||||
// macOS x64 is the slowest package host. Pull requests run it only with ci:full;
|
||||
// main pushes, schedules and manual dispatch always include it.
|
||||
// macOS x64 is the slowest package host. Fast qualification omits it.
|
||||
const intel =
|
||||
event !== "pull_request" || process.env.CI_FULL === "true"
|
||||
tier === "full"
|
||||
? [{ os: "macos-15-intel", platform: "darwin", architecture: "x64" }]
|
||||
: [];
|
||||
const matrix = {
|
||||
@@ -215,17 +254,23 @@ const matrix = {
|
||||
]
|
||||
: [linux],
|
||||
};
|
||||
const plan = { ...selected, matrix, files, reasons };
|
||||
const testSelection = planTestSelection(files, event, tier);
|
||||
const plan = { ...selected, tier, matrix, files, reasons, testSelection };
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries({ ...selected, matrix: JSON.stringify(matrix) })
|
||||
Object.entries({
|
||||
...selected,
|
||||
tier,
|
||||
matrix: JSON.stringify(matrix),
|
||||
test_selection: JSON.stringify(testSelection),
|
||||
})
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Selected CI lanes\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
|
||||
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
|
||||
);
|
||||
console.log(JSON.stringify(plan));
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ARTIFACT_NAME = "ci-full-qualification-v1";
|
||||
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
const positiveId = (value) =>
|
||||
(typeof value === "string" || typeof value === "number") &&
|
||||
/^[1-9]\d*$/.test(String(value)) &&
|
||||
Number.isSafeInteger(Number(value));
|
||||
const commitSha = (value) =>
|
||||
typeof value === "string" && /^[0-9a-f]{40}$/.test(value);
|
||||
|
||||
/** Record only after the required gate has verified every full-tier obligation. */
|
||||
export function recordQualification(directory) {
|
||||
const sourceSha = process.env.GITHUB_SHA;
|
||||
const runId = process.env.GITHUB_RUN_ID;
|
||||
const runAttempt = process.env.GITHUB_RUN_ATTEMPT;
|
||||
const repository = process.env.GITHUB_REPOSITORY;
|
||||
if (
|
||||
!commitSha(sourceSha) ||
|
||||
!positiveId(runId) ||
|
||||
!positiveId(runAttempt) ||
|
||||
!repository
|
||||
)
|
||||
throw new Error(
|
||||
"Full qualification receipt requires source and workflow identity",
|
||||
);
|
||||
const checkoutSha = execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
if (checkoutSha !== sourceSha)
|
||||
throw new Error("Qualification checkout differs from workflow source");
|
||||
mkdirSync(directory, { recursive: true });
|
||||
const path = resolve(directory, "receipt.json");
|
||||
writeFileSync(
|
||||
path,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
tier: "full",
|
||||
sourceSha,
|
||||
repository,
|
||||
runId: Number(runId),
|
||||
runAttempt: Number(runAttempt),
|
||||
qualifiedAt: new Date().toISOString(),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`receipt_name=${ARTIFACT_NAME}\nreceipt_path=${path}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
function admit() {
|
||||
if (process.env.CI_FORCE_FULL === "true")
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Forced daily runner and dependency refresh",
|
||||
};
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const repositoryId = Number(process.env.CI_REPOSITORY_ID);
|
||||
const branch = process.env.CI_DEFAULT_BRANCH;
|
||||
const head = process.env.HEAD_SHA;
|
||||
if (!repository || !positiveId(repositoryId) || !branch || !commitSha(head))
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Qualification lookup identity is incomplete",
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const deadline = now + 20_000;
|
||||
const api = (...arguments_) => {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0)
|
||||
throw new Error("Qualification lookup budget exhausted");
|
||||
return JSON.parse(
|
||||
execFileSync("gh", ["api", "--method", "GET", ...arguments_], {
|
||||
encoding: "utf8",
|
||||
timeout: Math.min(15_000, remaining),
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}),
|
||||
);
|
||||
};
|
||||
try {
|
||||
const { artifacts } = api(
|
||||
`repos/${repository}/actions/artifacts`,
|
||||
"-f",
|
||||
`name=${ARTIFACT_NAME}`,
|
||||
"-F",
|
||||
"per_page=100",
|
||||
);
|
||||
if (!Array.isArray(artifacts))
|
||||
throw new Error("Missing qualification artifact inventory");
|
||||
const candidates = artifacts.filter((artifact) => {
|
||||
const lineage = artifact?.workflow_run;
|
||||
const age = now - Date.parse(artifact?.created_at);
|
||||
return (
|
||||
artifact?.name === ARTIFACT_NAME &&
|
||||
artifact?.expired === false &&
|
||||
typeof artifact?.created_at === "string" &&
|
||||
Number.isFinite(age) &&
|
||||
age >= 0 &&
|
||||
age < MAX_AGE_MS &&
|
||||
positiveId(lineage?.id) &&
|
||||
String(lineage.id) !== process.env.GITHUB_RUN_ID &&
|
||||
lineage?.repository_id === repositoryId &&
|
||||
lineage?.head_repository_id === repositoryId &&
|
||||
lineage?.head_branch === branch &&
|
||||
lineage?.head_sha === head
|
||||
);
|
||||
});
|
||||
// Artifact metadata narrows the search before run lookups. Never scan an
|
||||
// unbounded history or treat a missing receipt as a successful qualification.
|
||||
for (const artifact of candidates.slice(0, 3)) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
);
|
||||
// An old run can upload a new artifact when only its gate is rerun.
|
||||
// Bound the original run age as well so retries cannot renew stale proof.
|
||||
const runAge = Date.now() - Date.parse(run.created_at);
|
||||
if (
|
||||
typeof run.created_at === "string" &&
|
||||
Number.isFinite(runAge) &&
|
||||
runAge >= 0 &&
|
||||
runAge < MAX_AGE_MS &&
|
||||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_sha === head &&
|
||||
run.head_branch === branch &&
|
||||
run.head_repository?.full_name === repository
|
||||
) {
|
||||
return {
|
||||
run_full: false,
|
||||
run_id: run.id,
|
||||
reason: `Reusing full qualification from run ${run.id}; no fresh provider observation`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "No trusted full qualification for this SHA within 24 hours",
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Qualification lookup unavailable; running full validation",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) {
|
||||
const decision = admit();
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries(decision)
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Scheduled qualification\n\n${decision.reason}.\n`,
|
||||
);
|
||||
console.log(JSON.stringify(decision));
|
||||
}
|
||||
+40
-6
@@ -1,18 +1,46 @@
|
||||
import { CI_FLAGS } from "./scopes.mjs";
|
||||
import { recordQualification } from "./qualification.mjs";
|
||||
|
||||
const jobs = JSON.parse(process.env.CI_JOBS);
|
||||
const outputs = jobs.changes.outputs;
|
||||
const required = ["changes", "static", "test"];
|
||||
const outputs = jobs?.changes?.outputs;
|
||||
if (!["fast", "full", "reused"].includes(outputs?.tier))
|
||||
throw new Error(
|
||||
"CI classification requires a valid tier: fast, full, or reused",
|
||||
);
|
||||
const invalidFlags = CI_FLAGS.filter(
|
||||
(flag) => outputs[flag] !== "true" && outputs[flag] !== "false",
|
||||
);
|
||||
if (invalidFlags.length)
|
||||
throw new Error(
|
||||
`CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`,
|
||||
);
|
||||
const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true");
|
||||
const reused = outputs.tier === "reused";
|
||||
if (
|
||||
reused &&
|
||||
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
|
||||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
|
||||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
|
||||
selectedScopes.length !== 0)
|
||||
)
|
||||
throw new Error(
|
||||
"Reused qualification requires a scheduled admission receipt and no selected lanes",
|
||||
);
|
||||
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
const lanes = {
|
||||
code: ["build", "test-shard", "sqlite-runtime"],
|
||||
code: ["build", "test-shard"],
|
||||
runtime: ["windows-native-build", "package-e2e"],
|
||||
package: ["windows-curated"],
|
||||
sqlite: ["sqlite-runtime"],
|
||||
apple: ["apple-artifacts"],
|
||||
fixtures: ["conformance-fixtures"],
|
||||
readiness: ["reconstruction-readiness"],
|
||||
docs: ["docs"],
|
||||
workflows: ["static"],
|
||||
};
|
||||
for (const [scope, value] of Object.entries(outputs)) {
|
||||
if (value === "true") required.push(...(lanes[scope] ?? [scope]));
|
||||
}
|
||||
for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope]));
|
||||
const failed = [...new Set(required)].filter(
|
||||
(name) => jobs[name]?.result !== "success",
|
||||
);
|
||||
@@ -23,3 +51,9 @@ for (const [name, job] of Object.entries(jobs)) {
|
||||
if (failed.length)
|
||||
throw new Error(`CI lanes did not pass: ${failed.join(", ")}`);
|
||||
console.log("Every selected CI lane passed");
|
||||
if (reused)
|
||||
console.log(
|
||||
`Qualification reused from run ${outputs.qualification_run_id}; no fresh provider observation`,
|
||||
);
|
||||
if (outputs.tier === "full" && process.env.CI_QUALIFICATION_DIRECTORY)
|
||||
recordQualification(process.env.CI_QUALIFICATION_DIRECTORY);
|
||||
|
||||
@@ -55,6 +55,9 @@ export const SCOPES = {
|
||||
"tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts",
|
||||
"tests/boundary/filesystem/setupUnavailableProvider.test.ts",
|
||||
"tests/boundary/filesystem/windowsNpxRegistration.test.ts",
|
||||
"tests/acceptance/setup/homeDirectoryFallback.test.ts",
|
||||
"tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts",
|
||||
"src/domain/javascript/javascriptRuntimeStaticCandidates.ts",
|
||||
"tests/boundary/filesystem/webScriptManifestPaths.test.ts",
|
||||
"tests/boundary/process/ProcessEvidence.host.test.ts",
|
||||
"tests/acceptance/webModuleCli.test.ts",
|
||||
@@ -84,6 +87,55 @@ export const SCOPES = {
|
||||
"tests/boundary/filesystem/javascriptArtifactContentIdentity.test.ts",
|
||||
"tests/boundary/process/sourceMapUncHosts.test.ts",
|
||||
],
|
||||
// Native SQLite runtime compatibility for the producer, public adapters,
|
||||
// fixtures, shared lifecycle, and the package script that defines its suite.
|
||||
sqlite: [
|
||||
"src/sqlite/**",
|
||||
"src/domain/sqlite/**",
|
||||
"src/contracts/sqlite/**",
|
||||
"src/application/sqlite/**",
|
||||
"src/composition/sqlite.ts",
|
||||
"src/cli/sqliteCommands.ts",
|
||||
"src/server/registerSqliteTools.ts",
|
||||
"src/application/DirectAnalysisStatus*.ts",
|
||||
"src/application/CapabilityInventory*.ts",
|
||||
"src/application/AnalysisProvider*.ts",
|
||||
"src/contracts/toolContracts.ts",
|
||||
"src/contracts/toolContractTypes.ts",
|
||||
"src/contracts/toolEffects*.ts",
|
||||
"src/contracts/toolOutputSchemaPrimitives.ts",
|
||||
"src/domain/analysisError*.ts",
|
||||
"src/domain/evidence*.ts",
|
||||
"src/domain/inputIssueProjection*.ts",
|
||||
"src/domain/jsonSerialization*.ts",
|
||||
"src/domain/jsonValue*.ts",
|
||||
"src/domain/localPath*.ts",
|
||||
"src/domain/providerAdapterError*.ts",
|
||||
"src/domain/providerCleanupError*.ts",
|
||||
"src/domain/result*.ts",
|
||||
"src/process/OwnedCommand*.ts",
|
||||
"src/process/ProviderProcess*.ts",
|
||||
"src/process/PrivateRuntimeRoot*.ts",
|
||||
"src/artifacts/ArtifactReader*.ts",
|
||||
"src/artifacts/ArtifactResourceScope*.ts",
|
||||
"src/artifacts/ArtifactPaths*.ts",
|
||||
"src/artifacts/readStableArtifact*.ts",
|
||||
"src/filesystem/OwnedFileHandle*.ts",
|
||||
"src/server/EvidenceMcpServer*.ts",
|
||||
"src/server/analysisAdmission*.ts",
|
||||
"src/server/contractToolHandler*.ts",
|
||||
"src/server/toolRegistrationOptions*.ts",
|
||||
"src/server/createServer*.ts",
|
||||
"src/cli.ts",
|
||||
"src/cliCommandNames.ts",
|
||||
"src/cliLogging.ts",
|
||||
"src/cli/commandCancellation.ts",
|
||||
"tests/boundary/sqlite/**",
|
||||
"tests/fixtures/sqlite/**",
|
||||
"tests/acceptance/sqliteDatabaseCli.test.ts",
|
||||
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
],
|
||||
browser: [
|
||||
".github/workflows/real-browser.yml",
|
||||
"scripts/verify-real-browser.mjs",
|
||||
@@ -299,6 +351,7 @@ export const SCOPES = {
|
||||
nativeaot: [
|
||||
"bridge/nativeaot/**",
|
||||
"scripts/generate-nativeaot-fixtures.mjs",
|
||||
"scripts/lib/nativeaot-build.*",
|
||||
"tests/fixtures/nativeaot/**",
|
||||
".github/workflows/nativeaot-fixtures.yml",
|
||||
],
|
||||
@@ -311,6 +364,20 @@ export const SCOPES = {
|
||||
],
|
||||
};
|
||||
|
||||
export const CI_FLAGS = [
|
||||
...new Set([
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"apple",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
]),
|
||||
];
|
||||
|
||||
// Verifier entrypoints that only manual, licensed, or self-hosted workflows
|
||||
// run. No selected CI lane executes them, so their edits need only the static
|
||||
// checks and the source-test suite. Shared helpers under scripts/lib remain
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ARTIFACT_NAME = "test-timings";
|
||||
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
const MAX_CANDIDATES = 5;
|
||||
const positiveId = (value) =>
|
||||
(typeof value === "string" || typeof value === "number") &&
|
||||
/^[1-9]\d*$/.test(String(value)) &&
|
||||
Number.isSafeInteger(Number(value));
|
||||
|
||||
function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
|
||||
if (
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository ?? "") ||
|
||||
!branch ||
|
||||
!positiveId(repositoryId)
|
||||
)
|
||||
return "";
|
||||
const deadline = Date.now() + 20_000;
|
||||
const api = (...arguments_) => {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0)
|
||||
throw new Error("Timing history lookup budget exhausted");
|
||||
return JSON.parse(
|
||||
execFileSync("gh", ["api", "--method", "GET", ...arguments_], {
|
||||
encoding: "utf8",
|
||||
timeout: Math.min(10_000, remaining),
|
||||
maxBuffer: 4 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const { artifacts } = api(
|
||||
`repos/${repository}/actions/artifacts`,
|
||||
"-f",
|
||||
`name=${ARTIFACT_NAME}`,
|
||||
"-F",
|
||||
"per_page=100",
|
||||
);
|
||||
if (!Array.isArray(artifacts))
|
||||
throw new Error("Missing timing artifact inventory");
|
||||
const candidates = artifacts
|
||||
.filter((artifact) => {
|
||||
const lineage = artifact?.workflow_run;
|
||||
const age = now - Date.parse(artifact?.created_at);
|
||||
return (
|
||||
artifact?.name === ARTIFACT_NAME &&
|
||||
artifact?.expired === false &&
|
||||
typeof artifact?.created_at === "string" &&
|
||||
Number.isFinite(age) &&
|
||||
age >= 0 &&
|
||||
age < MAX_AGE_MS &&
|
||||
positiveId(lineage?.id) &&
|
||||
positiveId(lineage?.repository_id) &&
|
||||
positiveId(lineage?.head_repository_id) &&
|
||||
lineage.repository_id === lineage.head_repository_id &&
|
||||
lineage.repository_id === Number(repositoryId) &&
|
||||
lineage.head_branch === branch
|
||||
);
|
||||
})
|
||||
.sort(
|
||||
(left, right) =>
|
||||
Date.parse(right.created_at) - Date.parse(left.created_at),
|
||||
)
|
||||
.slice(0, MAX_CANDIDATES);
|
||||
|
||||
for (const artifact of candidates) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
);
|
||||
const runAge = now - Date.parse(run?.created_at);
|
||||
if (
|
||||
typeof run?.created_at === "string" &&
|
||||
Number.isFinite(runAge) &&
|
||||
runAge >= 0 &&
|
||||
runAge < MAX_AGE_MS &&
|
||||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_branch === branch &&
|
||||
run.head_repository?.full_name === repository
|
||||
)
|
||||
return String(run.id);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function write(runId) {
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(process.env.GITHUB_OUTPUT, `run-id=${runId}\n`);
|
||||
console.log(JSON.stringify({ run_id: runId }));
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
fileURLToPath(import.meta.url) === resolve(process.argv[1])
|
||||
) {
|
||||
let runId = "";
|
||||
try {
|
||||
runId = findTimingRun({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
branch: process.env.CI_DEFAULT_BRANCH,
|
||||
repositoryId: process.env.CI_REPOSITORY_ID,
|
||||
});
|
||||
} catch {
|
||||
// History improves shard ordering but must never block CI.
|
||||
}
|
||||
write(runId);
|
||||
}
|
||||
+82
-17
@@ -1,28 +1,36 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { compareTestSelection } from "./test-selection.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const historyDirectory = join(root, ".cache/ci-test-history");
|
||||
const historyPath = join(historyDirectory, "test-results.json");
|
||||
const selectionDirectory = join(root, ".cache/ci-test-selection");
|
||||
const selectionPath = join(selectionDirectory, "report.json");
|
||||
// Never let a failed or partial rerun leave a previous successful history file.
|
||||
await rm(historyPath, { force: true });
|
||||
await rm(selectionPath, { force: true });
|
||||
const report = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
success: z.boolean(),
|
||||
numFailedTests: z.number().int().nonnegative(),
|
||||
testResults: z.array(
|
||||
reason: z.string().optional(),
|
||||
unhandledErrors: z.number().int().nonnegative().optional(),
|
||||
files: z.array(
|
||||
z.object({
|
||||
name: z.string().min(1),
|
||||
startTime: z.number().finite(),
|
||||
endTime: z.number().finite(),
|
||||
durationMs: z.number().finite().nonnegative(),
|
||||
state: z.string().optional(),
|
||||
}),
|
||||
),
|
||||
})
|
||||
.parse(JSON.parse(await readFile("test-results.json", "utf8")));
|
||||
.parse(JSON.parse(await readFile("test-timings.json", "utf8")));
|
||||
|
||||
const files = report.testResults.map(({ name, startTime, endTime }) => {
|
||||
const files = report.files.map(({ name, durationMs, state }) => {
|
||||
const path = relative(root, resolve(root, name));
|
||||
if (
|
||||
path === "" ||
|
||||
@@ -31,32 +39,85 @@ const files = report.testResults.map(({ name, startTime, endTime }) => {
|
||||
isAbsolute(path)
|
||||
)
|
||||
throw new Error(`Test report path is outside the checkout: ${name}`);
|
||||
const duration = endTime - startTime;
|
||||
if (!Number.isFinite(duration) || duration < 0)
|
||||
throw new Error(`Test report has an invalid duration: ${name}`);
|
||||
return { name: path.split(sep).join("/"), startTime: 0, endTime: duration };
|
||||
return { name: path.split(sep).join("/"), durationMs, state };
|
||||
});
|
||||
const complete = process.env.SHARD_RESULT === "success";
|
||||
const passed =
|
||||
complete &&
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success &&
|
||||
report.numFailedTests === 0;
|
||||
complete && process.env.MERGE_RESULT === "success" && report.success;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
|
||||
const expected =
|
||||
verifyInventory || process.env.CI_TEST_SELECTION
|
||||
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
})
|
||||
.split("\0")
|
||||
.filter((path) => path.endsWith(".test.ts"))
|
||||
: [];
|
||||
let selection;
|
||||
if (process.env.CI_TEST_SELECTION) {
|
||||
const comparison = compareTestSelection(
|
||||
JSON.parse(process.env.CI_TEST_SELECTION),
|
||||
expected,
|
||||
files,
|
||||
);
|
||||
selection = {
|
||||
...comparison,
|
||||
complete:
|
||||
comparison.complete &&
|
||||
report.unhandledErrors === 0 &&
|
||||
["passed", "failed"].includes(report.reason) &&
|
||||
["success", "failure"].includes(process.env.SHARD_RESULT) &&
|
||||
["success", "failure"].includes(process.env.MERGE_RESULT),
|
||||
executionSha: process.env.GITHUB_SHA ?? null,
|
||||
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
|
||||
runId: process.env.GITHUB_RUN_ID ?? null,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
|
||||
};
|
||||
await mkdir(selectionDirectory, { recursive: true });
|
||||
await writeFile(selectionPath, JSON.stringify(selection));
|
||||
}
|
||||
if (passed && verifyInventory) {
|
||||
const actual = files.map(({ name }) => name);
|
||||
const expectedSet = new Set(expected);
|
||||
const actualSet = new Set(actual);
|
||||
const missing = expected.filter((path) => !actualSet.has(path));
|
||||
const unexpected = actual.filter((path) => !expectedSet.has(path));
|
||||
if (missing.length || unexpected.length || actualSet.size !== actual.length)
|
||||
throw new Error(
|
||||
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
|
||||
);
|
||||
}
|
||||
const summary = [
|
||||
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
"### Observational PR test selection",
|
||||
"",
|
||||
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
|
||||
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
|
||||
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
|
||||
passed
|
||||
? "Successful timing history is available."
|
||||
: "This report is diagnostic only and is not a successful timing baseline.",
|
||||
"",
|
||||
"Durations include tests and hooks, worker preparation, environment, imports, and setup. They are scheduling weights, not summed workflow wall time.",
|
||||
"",
|
||||
"| Slowest file | Duration (ms) |",
|
||||
"| --- | ---: |",
|
||||
...files
|
||||
.toSorted((left, right) => right.endTime - left.endTime)
|
||||
.toSorted((left, right) => right.durationMs - left.durationMs)
|
||||
.slice(0, 20)
|
||||
.map(
|
||||
({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`,
|
||||
({ name, durationMs }) =>
|
||||
`| ${name.replaceAll("|", "\\|")} | ${durationMs} |`,
|
||||
),
|
||||
"",
|
||||
].join("\n");
|
||||
@@ -68,6 +129,10 @@ if (passed) {
|
||||
await mkdir(historyDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
historyPath,
|
||||
JSON.stringify({ success: true, numFailedTests: 0, testResults: files }),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
success: true,
|
||||
files: files.map(({ name, durationMs }) => ({ name, durationMs })),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
export interface TestSelectionPlan {
|
||||
readonly version: 1;
|
||||
readonly mode: "full" | "candidate";
|
||||
readonly owners: readonly string[];
|
||||
readonly changedTests: readonly string[];
|
||||
readonly changedPaths: readonly string[];
|
||||
readonly reason: string;
|
||||
}
|
||||
export function planTestSelection(
|
||||
files: readonly string[],
|
||||
event: string,
|
||||
tier: string,
|
||||
): TestSelectionPlan;
|
||||
export function compareTestSelection(
|
||||
plan: unknown,
|
||||
inventory: readonly string[],
|
||||
results: readonly {
|
||||
readonly name: string;
|
||||
readonly durationMs: number;
|
||||
readonly state?: string;
|
||||
}[],
|
||||
): {
|
||||
readonly version: 1;
|
||||
readonly observational: true;
|
||||
readonly plan: TestSelectionPlan;
|
||||
readonly selected: readonly string[];
|
||||
readonly omitted: readonly string[];
|
||||
readonly missing: readonly string[];
|
||||
readonly unexpected: readonly string[];
|
||||
readonly duplicateFiles: number;
|
||||
readonly unobserved: readonly string[];
|
||||
readonly missedFailures: readonly string[];
|
||||
readonly complete: boolean;
|
||||
readonly omittedDurationMs: number;
|
||||
};
|
||||
@@ -0,0 +1,151 @@
|
||||
import { matchesGlob } from "node:path";
|
||||
|
||||
// Candidate ownership only: these leaf adapters and their producer tests can be
|
||||
// compared with full-suite outcomes. This table never changes test execution.
|
||||
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
|
||||
const OWNERS = {
|
||||
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
|
||||
apktool: ["src/apktool/**"],
|
||||
dotnet: ["src/dotnet/**"],
|
||||
evm: ["src/evm/**", "tests/boundary/evm/**"],
|
||||
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
|
||||
flutter: ["src/flutter/**"],
|
||||
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
|
||||
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
|
||||
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
|
||||
jeb: ["src/jeb/**"],
|
||||
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
|
||||
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
|
||||
wasm: ["src/wasm/**"],
|
||||
};
|
||||
const ownersOf = (path) =>
|
||||
Object.entries(OWNERS)
|
||||
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
|
||||
.map(([owner]) => owner);
|
||||
const validPath = (path) =>
|
||||
typeof path === "string" &&
|
||||
/^(src|tests)\//.test(path) &&
|
||||
!/[\\\0\r\n]/.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part));
|
||||
const full = (reason, changedPaths = []) => ({
|
||||
version: 1,
|
||||
mode: "full",
|
||||
owners: [],
|
||||
changedTests: [],
|
||||
changedPaths,
|
||||
reason,
|
||||
});
|
||||
|
||||
/** Plan an observational PR subset from the classifier's existing Git diff. */
|
||||
export function planTestSelection(files, event, tier) {
|
||||
if (event !== "pull_request" || tier !== "fast")
|
||||
return full("Only ordinary PRs are candidates", files);
|
||||
if (!files.length) return full("No changed paths available");
|
||||
const owners = new Set();
|
||||
const changedTests = [];
|
||||
for (const path of files) {
|
||||
if (!validPath(path))
|
||||
return full(`Shared or unknown input: ${path}`, files);
|
||||
const matches = ownersOf(path);
|
||||
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
|
||||
for (const owner of matches) owners.add(owner);
|
||||
if (path.endsWith(".test.ts")) changedTests.push(path);
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
mode: "candidate",
|
||||
owners: [...owners].sort(),
|
||||
changedTests: [...new Set(changedTests)].sort(),
|
||||
changedPaths: files,
|
||||
reason:
|
||||
"Observe affected leaf owners; retain integration and unowned tests",
|
||||
};
|
||||
}
|
||||
|
||||
/** Compare a proposal with the full run; missing observations never prove parity. */
|
||||
export function compareTestSelection(plan, inventory, results) {
|
||||
if (
|
||||
plan?.version !== 1 ||
|
||||
!["full", "candidate"].includes(plan.mode) ||
|
||||
!Array.isArray(plan.owners) ||
|
||||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
|
||||
!Array.isArray(plan.changedTests) ||
|
||||
!Array.isArray(plan.changedPaths) ||
|
||||
plan.changedPaths.some((path) => typeof path !== "string") ||
|
||||
plan.changedTests.some(
|
||||
(path) => !validPath(path) || !path.endsWith(".test.ts"),
|
||||
) ||
|
||||
typeof plan.reason !== "string" ||
|
||||
(plan.mode === "candidate" && !plan.owners.length) ||
|
||||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
|
||||
)
|
||||
throw new Error("Invalid observational test selection plan");
|
||||
const expected = new Set(inventory);
|
||||
if (!expected.size || expected.size !== inventory.length)
|
||||
throw new Error("Test selection requires a nonempty unique inventory");
|
||||
const ownersByFile = new Map(
|
||||
plan.mode === "candidate"
|
||||
? inventory.map((path) => [path, ownersOf(path)])
|
||||
: [],
|
||||
);
|
||||
// Removed/renamed tests have no current executable identity. Keep all tests.
|
||||
const removed = plan.changedTests.filter((path) => !expected.has(path));
|
||||
const missingOwners = plan.owners.filter(
|
||||
(owner) =>
|
||||
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
|
||||
);
|
||||
const effective = removed.length
|
||||
? full(
|
||||
`Changed tests absent from current inventory: ${removed.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: missingOwners.length
|
||||
? full(
|
||||
`Owners without executable tests: ${missingOwners.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: plan;
|
||||
const selected = [];
|
||||
const omitted = [];
|
||||
for (const path of inventory.toSorted()) {
|
||||
const owners = ownersByFile.get(path) ?? [];
|
||||
const keep =
|
||||
effective.mode === "full" ||
|
||||
!owners.length ||
|
||||
effective.changedTests.includes(path) ||
|
||||
owners.some((owner) => effective.owners.includes(owner));
|
||||
(keep ? selected : omitted).push(path);
|
||||
}
|
||||
const actual = new Set(results.map((file) => file.name));
|
||||
const missing = inventory.filter((path) => !actual.has(path));
|
||||
const unexpected = [...actual].filter((path) => !expected.has(path));
|
||||
const duplicateFiles = results.length - actual.size;
|
||||
const unobserved = results
|
||||
.filter((file) => !["passed", "failed"].includes(file.state))
|
||||
.map((file) => file.name);
|
||||
const omittedSet = new Set(omitted);
|
||||
const missedFailures = results
|
||||
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
|
||||
.map((file) => file.name);
|
||||
return {
|
||||
version: 1,
|
||||
observational: true,
|
||||
plan: effective,
|
||||
selected,
|
||||
omitted,
|
||||
missing,
|
||||
unexpected,
|
||||
duplicateFiles,
|
||||
unobserved,
|
||||
missedFailures,
|
||||
complete:
|
||||
!missing.length &&
|
||||
!unexpected.length &&
|
||||
!duplicateFiles &&
|
||||
!unobserved.length,
|
||||
// Phase totals are scheduling weights, not an estimate of wall-time savings.
|
||||
omittedDurationMs: results
|
||||
.filter((file) => omittedSet.has(file.name))
|
||||
.reduce((sum, file) => sum + file.durationMs, 0),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
|
||||
/** @implements {import("vitest/node").Reporter} */
|
||||
export default class TestTimingReporter {
|
||||
/**
|
||||
* @param {ReadonlyArray<import("vitest/node").TestModule>} modules
|
||||
* @param {ReadonlyArray<import("vitest/node").SerializedError>} errors
|
||||
* @param {import("vitest/node").TestRunEndReason} reason
|
||||
*/
|
||||
async onTestRunEnd(modules, errors, reason) {
|
||||
const files = modules.map((module) => {
|
||||
const diagnostic = module.diagnostic();
|
||||
// These disjoint phases match Vitest's per-file wall-time accounting.
|
||||
// Import timings already include transform waits; do not add them again.
|
||||
const phases = {
|
||||
execution: diagnostic.duration,
|
||||
preparation: diagnostic.prepareDuration,
|
||||
environment: diagnostic.environmentSetupDuration,
|
||||
imports: diagnostic.collectDuration,
|
||||
setup: diagnostic.setupDuration,
|
||||
};
|
||||
return {
|
||||
name: module.moduleId,
|
||||
state: module.state(),
|
||||
durationMs: Object.values(phases).reduce(
|
||||
(total, value) => total + value,
|
||||
0,
|
||||
),
|
||||
phases,
|
||||
};
|
||||
});
|
||||
await writeFile(
|
||||
"test-timings.json",
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
success: reason === "passed" && errors.length === 0,
|
||||
reason,
|
||||
unhandledErrors: errors.length,
|
||||
files,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { copyFile, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const MANIFEST_NAME = "manifest.json";
|
||||
const TARBALL_NAME = "package.tgz";
|
||||
const MANIFEST_KEYS = [
|
||||
"schemaVersion",
|
||||
"sourceSha",
|
||||
"runId",
|
||||
"runAttempt",
|
||||
"platform",
|
||||
"architecture",
|
||||
"packageVersion",
|
||||
"nodeVersion",
|
||||
"sha256",
|
||||
].sort();
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Normalize workflow identity and host facts into the manifest-owned fields. */
|
||||
export function createVerifiedPackageMetadata({
|
||||
sourceSha,
|
||||
runId,
|
||||
runAttempt,
|
||||
platform,
|
||||
architecture,
|
||||
packageVersion,
|
||||
nodeVersion,
|
||||
}) {
|
||||
if (!/^[a-f0-9]{40}$/u.test(sourceSha))
|
||||
throw new Error("Source SHA must be a full commit SHA");
|
||||
return {
|
||||
sourceSha,
|
||||
runId: parsePositiveInteger(runId, "GITHUB_RUN_ID"),
|
||||
runAttempt: parsePositiveInteger(runAttempt, "GITHUB_RUN_ATTEMPT"),
|
||||
platform,
|
||||
architecture,
|
||||
packageVersion,
|
||||
nodeVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/** Build the single supported manifest representation for a verified tarball. */
|
||||
function createVerifiedPackageManifest(metadata) {
|
||||
const manifest = { schemaVersion: 1, ...metadata };
|
||||
validateManifestShape(manifest);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/** Write a manifest after the caller has completed qualification and cleanup. */
|
||||
export async function writeVerifiedPackageManifest(directory, metadata) {
|
||||
const manifest = createVerifiedPackageManifest(metadata);
|
||||
await writeFile(
|
||||
join(directory, MANIFEST_NAME),
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/** Copy the qualified archive under its fixed name and prepare its manifest. */
|
||||
export async function retainVerifiedPackage({
|
||||
sourceTarball,
|
||||
directory,
|
||||
metadata,
|
||||
}) {
|
||||
const tarball = join(directory, TARBALL_NAME);
|
||||
await copyFile(sourceTarball, tarball);
|
||||
const sha256 = await sha256File(tarball);
|
||||
return {
|
||||
directory,
|
||||
metadata: createVerifiedPackageManifest({ ...metadata, sha256 }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Validate a retained package against the current checkout and workflow attempt. */
|
||||
async function validateVerifiedPackage({
|
||||
directory,
|
||||
sourceSha,
|
||||
runId,
|
||||
runAttempt,
|
||||
packageVersion,
|
||||
nodeVersion,
|
||||
}) {
|
||||
if (!/^[a-f0-9]{40}$/u.test(sourceSha))
|
||||
throw new Error("Expected source SHA must be a full commit SHA");
|
||||
if (!Number.isSafeInteger(runId) || runId <= 0)
|
||||
throw new Error("Expected workflow run ID must be a positive integer");
|
||||
if (!Number.isSafeInteger(runAttempt) || runAttempt <= 0)
|
||||
throw new Error("Expected workflow run attempt must be a positive integer");
|
||||
const manifestPath = join(directory, MANIFEST_NAME);
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
||||
} catch (cause) {
|
||||
throw new Error(`Could not read verified package manifest: ${cause}`, {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
validateManifestShape(manifest);
|
||||
assertEqual(manifest.sourceSha, sourceSha, "source SHA");
|
||||
assertEqual(manifest.runId, runId, "workflow run ID");
|
||||
// A successful producer may belong to an earlier attempt when only failed
|
||||
// publisher jobs are rerun. The workflow still requires same-run qualification.
|
||||
if (manifest.runAttempt > runAttempt)
|
||||
throw new Error("Verified package workflow run attempt is from the future");
|
||||
assertEqual(manifest.platform, "linux", "verifier platform");
|
||||
assertEqual(manifest.architecture, "x64", "verifier architecture");
|
||||
assertEqual(manifest.packageVersion, packageVersion, "package version");
|
||||
assertEqual(manifest.nodeVersion, nodeVersion, "Node.js version");
|
||||
|
||||
// The artifact format deliberately fixes the name; no manifest value can
|
||||
// redirect validation to another path.
|
||||
const tarball = resolve(directory, TARBALL_NAME);
|
||||
const digest = await sha256File(tarball);
|
||||
assertEqual(manifest.sha256, digest, "tarball SHA-256");
|
||||
return tarball;
|
||||
}
|
||||
|
||||
function validateManifestShape(manifest) {
|
||||
if (
|
||||
typeof manifest !== "object" ||
|
||||
manifest === null ||
|
||||
Array.isArray(manifest)
|
||||
)
|
||||
throw new Error("Verified package manifest must be a JSON object");
|
||||
const keys = Object.keys(manifest).sort();
|
||||
if (
|
||||
keys.length !== MANIFEST_KEYS.length ||
|
||||
keys.some((key, index) => key !== MANIFEST_KEYS[index])
|
||||
)
|
||||
throw new Error("Verified package manifest has missing or unknown fields");
|
||||
if (manifest.schemaVersion !== 1)
|
||||
throw new Error("Unsupported verified package manifest schema version");
|
||||
if (
|
||||
typeof manifest.sourceSha !== "string" ||
|
||||
!/^[a-f0-9]{40}$/u.test(manifest.sourceSha)
|
||||
)
|
||||
throw new Error("Verified package manifest has an invalid source SHA");
|
||||
if (!Number.isSafeInteger(manifest.runId) || manifest.runId <= 0)
|
||||
throw new Error("Verified package manifest has an invalid workflow run ID");
|
||||
if (!Number.isSafeInteger(manifest.runAttempt) || manifest.runAttempt <= 0)
|
||||
throw new Error(
|
||||
"Verified package manifest has an invalid workflow run attempt",
|
||||
);
|
||||
for (const key of [
|
||||
"platform",
|
||||
"architecture",
|
||||
"packageVersion",
|
||||
"nodeVersion",
|
||||
])
|
||||
if (typeof manifest[key] !== "string" || manifest[key].length === 0)
|
||||
throw new Error(`Verified package manifest has an invalid ${key}`);
|
||||
if (
|
||||
typeof manifest.sha256 !== "string" ||
|
||||
!/^[a-f0-9]{64}$/u.test(manifest.sha256)
|
||||
)
|
||||
throw new Error("Verified package manifest has an invalid tarball SHA-256");
|
||||
}
|
||||
|
||||
function assertEqual(actual, expected, label) {
|
||||
if (actual !== expected)
|
||||
throw new Error(`Verified package ${label} mismatch`);
|
||||
}
|
||||
|
||||
async function sha256File(path) {
|
||||
const hash = createHash("sha256");
|
||||
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function validateFromCli(directory) {
|
||||
const sourceSha = process.env.GITHUB_SHA;
|
||||
const runId = parsePositiveInteger(
|
||||
process.env.GITHUB_RUN_ID,
|
||||
"GITHUB_RUN_ID",
|
||||
);
|
||||
const runAttempt = parsePositiveInteger(
|
||||
process.env.GITHUB_RUN_ATTEMPT,
|
||||
"GITHUB_RUN_ATTEMPT",
|
||||
);
|
||||
if (sourceSha === undefined || !/^[a-f0-9]{40}$/u.test(sourceSha))
|
||||
throw new Error("GITHUB_SHA must be a full commit SHA");
|
||||
const checkoutSha = (
|
||||
await execFileAsync("git", ["rev-parse", "HEAD"])
|
||||
).stdout.trim();
|
||||
assertEqual(checkoutSha, sourceSha, "checked out source SHA");
|
||||
const packageJson = JSON.parse(await readFile("package.json", "utf8"));
|
||||
const tarball = await validateVerifiedPackage({
|
||||
directory,
|
||||
sourceSha,
|
||||
runId,
|
||||
runAttempt,
|
||||
packageVersion: packageJson.version,
|
||||
nodeVersion: process.version,
|
||||
});
|
||||
if (process.env.GITHUB_OUTPUT === undefined)
|
||||
throw new Error(
|
||||
"GITHUB_OUTPUT is required to publish the validated tarball path",
|
||||
);
|
||||
await writeFile(process.env.GITHUB_OUTPUT, `tarball=${tarball}\n`, {
|
||||
encoding: "utf8",
|
||||
flag: "a",
|
||||
});
|
||||
process.stdout.write(`${tarball}\n`);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
if (value === undefined || !/^[1-9]\d*$/u.test(value))
|
||||
throw new Error(`${name} must be a positive integer`);
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed))
|
||||
throw new Error(`${name} must be a safe positive integer`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const invokedPath = process.argv[1];
|
||||
if (
|
||||
invokedPath !== undefined &&
|
||||
resolve(invokedPath) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
const [command, directory, ...extra] = process.argv.slice(2);
|
||||
if (command !== "validate" || directory === undefined || extra.length > 0) {
|
||||
process.stderr.write(
|
||||
"Usage: node scripts/ci/verified-package.mjs validate <artifact-directory>\n",
|
||||
);
|
||||
process.exitCode = 2;
|
||||
} else {
|
||||
await validateFromCli(directory);
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,20 @@ class Generator:
|
||||
assert "ows-secret" not in json.dumps(projected), projected
|
||||
if isinstance(value, bytes):
|
||||
assert projected["binary_fields"][-1]["state"] == "redacted", projected
|
||||
body = b"A" * 12345
|
||||
state = {"response": {"content": body}}
|
||||
control = adapter.project_record(state, [])
|
||||
field = control["binary_fields"][0]
|
||||
assert field["state"] == "retained" and field["bytes"] == len(body), control
|
||||
for literal in ["12345", "2345"]:
|
||||
# Only the generated byte count matches this exclusion.
|
||||
assert literal not in control["reported"]["response"]["content"]
|
||||
assert literal not in field["content_base64"] and literal not in field["sha256"]
|
||||
projected = adapter.project_record(state, [literal])
|
||||
assert projected["reported"]["response"]["content"] is None, projected
|
||||
assert projected["binary_fields"] == [{"pointer": "/response/content", "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}], projected
|
||||
assert {"pointer": "/response/content", "reason": "explicit-sensitive-value"} in projected["redactions"], projected
|
||||
assert literal not in json.dumps(projected), projected
|
||||
(root / "native-error-classifications.json").write_text(json.dumps({"passed": len(classifications) + 2}))
|
||||
first = http.HTTPFlow(connection.Client(peername=("127.0.0.1", 1), sockname=("127.0.0.1", 2)), connection.Server(address=("example.test", 80)))
|
||||
first.id = "original-producer-id"
|
||||
|
||||
@@ -4,9 +4,14 @@ import { createHash } from "node:crypto";
|
||||
import { copyFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
buildNativeAotFixture,
|
||||
nativeAotCommandError,
|
||||
supportsNativeAotFixtureHost,
|
||||
} from "./lib/nativeaot-build.mjs";
|
||||
|
||||
const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
if (process.arch !== "x64" || !["linux", "win32"].includes(process.platform))
|
||||
if (!supportsNativeAotFixtureHost(process.platform, process.arch))
|
||||
throw new Error(
|
||||
"NativeAOT fixture lane requires Linux or Windows x64; it does not cross-compile NativeAOT.",
|
||||
);
|
||||
@@ -40,9 +45,7 @@ const run = (command, args) => {
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
if (result.error || result.status !== 0)
|
||||
throw new Error(
|
||||
`NativeAOT fixture lane requires/action failed ${command}: ${result.error?.message ?? ""}\n${result.stdout ?? ""}${result.stderr ?? ""}`,
|
||||
);
|
||||
throw nativeAotCommandError(command, args[0], result);
|
||||
return result.stdout;
|
||||
};
|
||||
const dotnet = process.env.REA_NATIVEAOT_DOTNET ?? "dotnet";
|
||||
@@ -57,20 +60,12 @@ if (process.platform === "linux") {
|
||||
run("clang", ["--version"]);
|
||||
}
|
||||
const map = join(output, "NativeAotFixture.map");
|
||||
run(dotnet, [
|
||||
"publish",
|
||||
join(root, "tests/fixtures/nativeaot/NativeAotFixture.csproj"),
|
||||
"-r",
|
||||
await buildNativeAotFixture(run, {
|
||||
dotnet,
|
||||
project: join(root, "tests/fixtures/nativeaot/NativeAotFixture.csproj"),
|
||||
runtime,
|
||||
"-c",
|
||||
"Release",
|
||||
"-o",
|
||||
join(output, "symbols"),
|
||||
"--maxcpucount:1",
|
||||
`-p:BaseIntermediateOutputPath=${join(output, "obj")}/`,
|
||||
`-p:BaseOutputPath=${join(output, "bin")}/`,
|
||||
`-p:NativeAotMapPath=${map}`,
|
||||
]);
|
||||
output,
|
||||
});
|
||||
const name =
|
||||
process.platform === "win32" ? "NativeAotFixture.exe" : "NativeAotFixture";
|
||||
const binary = join(output, "symbols", name);
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { ensureGeneratedFile } from "./lib/generated-file.mjs";
|
||||
|
||||
const arguments_ = new Set(process.argv.slice(2));
|
||||
for (const argument of arguments_)
|
||||
if (argument !== "--check")
|
||||
throw new Error(`Unknown inventory option: ${argument}`);
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const lock = JSON.parse(
|
||||
await readFile(join(root, "package-lock.json"), "utf8"),
|
||||
);
|
||||
const entries = Object.entries(lock.packages)
|
||||
.filter(([path]) => path !== "")
|
||||
.map(([path, record]) => ({
|
||||
path,
|
||||
name: record.name ?? path.slice(path.lastIndexOf("node_modules/") + 13),
|
||||
version: record.version,
|
||||
license: record.license ?? "UNDECLARED",
|
||||
resolved: record.resolved,
|
||||
integrity: record.integrity,
|
||||
dev: record.dev === true,
|
||||
optional: record.optional === true,
|
||||
}))
|
||||
.sort((left, right) =>
|
||||
left.path < right.path ? -1 : left.path > right.path ? 1 : 0,
|
||||
);
|
||||
for (const entry of entries)
|
||||
for (const field of ["name", "version", "resolved", "integrity"])
|
||||
if (typeof entry[field] !== "string" || entry[field].length === 0)
|
||||
throw new Error(`Missing locked ${field}: ${entry.path}`);
|
||||
const runtime = entries.filter(({ dev }) => !dev);
|
||||
const runtimeFingerprint = createHash("sha256")
|
||||
.update(JSON.stringify(runtime))
|
||||
.digest("hex");
|
||||
const source = [
|
||||
"REA npm dependency inventory",
|
||||
"Prepared from the root package-lock.json; refresh this inventory when locked dependencies change.",
|
||||
"License identifiers are upstream lockfile declarations, not a legal assessment.",
|
||||
"Runtime includes transitive and optional platform packages; development includes build, test and website tooling.",
|
||||
`Locked package entries: ${entries.length}; runtime: ${runtime.length}.`,
|
||||
"Full runtime copyright/license/NOTICE texts: npm-runtime-notices.txt.",
|
||||
"",
|
||||
...entries.flatMap((entry) => [
|
||||
`${entry.name}@${entry.version} | ${entry.dev ? "development" : "runtime"}${entry.optional ? ", optional" : ""} | ${entry.license}`,
|
||||
` Lockfile path: ${entry.path}`,
|
||||
` Package: https://www.npmjs.com/package/${entry.name}/v/${entry.version}`,
|
||||
` Archive: ${entry.resolved}`,
|
||||
` Integrity: ${entry.integrity}`,
|
||||
]),
|
||||
"",
|
||||
].join("\n");
|
||||
await ensureGeneratedFile({
|
||||
path: join(root, "licenses/npm-dependencies.txt"),
|
||||
source,
|
||||
check: arguments_.has("--check"),
|
||||
generateCommand: "npm run licenses:generate",
|
||||
});
|
||||
|
||||
if (arguments_.has("--check")) {
|
||||
const notices = await readFile(
|
||||
join(root, "licenses/npm-runtime-notices.txt"),
|
||||
"utf8",
|
||||
);
|
||||
const headings = [...notices.matchAll(/^===== (.+) =====$/gmu)];
|
||||
const sections = new Map();
|
||||
for (const [index, heading] of headings.entries()) {
|
||||
if (sections.has(heading[1]))
|
||||
throw new Error(`Duplicate runtime notice: ${heading[1]}`);
|
||||
sections.set(
|
||||
heading[1],
|
||||
notices.slice(
|
||||
heading.index + heading[0].length + 1,
|
||||
headings[index + 1]?.index,
|
||||
),
|
||||
);
|
||||
}
|
||||
for (const entry of runtime) {
|
||||
const identity = `${entry.name}@${entry.version}`;
|
||||
const section = sections.get(identity);
|
||||
if (section === undefined)
|
||||
throw new Error(`Missing runtime notice: ${identity}`);
|
||||
const metadata = [
|
||||
`Declared license: ${entry.license}`,
|
||||
`Archive: ${entry.resolved}`,
|
||||
`Integrity: ${entry.integrity}`,
|
||||
"",
|
||||
].join("\n");
|
||||
if (!section.startsWith(metadata))
|
||||
throw new Error(`Runtime notice metadata differs for ${identity}`);
|
||||
}
|
||||
if (
|
||||
!notices
|
||||
.slice(0, headings[0]?.index)
|
||||
.includes(
|
||||
`Runtime dependency fingerprint (SHA-256): ${runtimeFingerprint}\n`,
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
`Runtime notice fingerprint differs; refresh verified notices for ${runtimeFingerprint}`,
|
||||
);
|
||||
const references = JSON.parse(
|
||||
await readFile(join(root, "licenses/upstream-sources.json"), "utf8"),
|
||||
);
|
||||
for (const reference of references) {
|
||||
const bytes = await readFile(join(root, reference.file));
|
||||
if (createHash("sha256").update(bytes).digest("hex") !== reference.sha256)
|
||||
throw new Error(`Preserved notice digest differs: ${reference.file}`);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
locked_packages: entries.length,
|
||||
runtime_packages: runtime.length,
|
||||
runtime_dependency_fingerprint: runtimeFingerprint,
|
||||
}),
|
||||
);
|
||||
@@ -10,6 +10,7 @@ import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import select
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -118,6 +119,7 @@ class XWindowAttributes(ctypes.Structure):
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--probe", action="store_true")
|
||||
parser.add_argument("--owner-stdin", action="store_true")
|
||||
parser.add_argument("--hopper")
|
||||
parser.add_argument("--socket")
|
||||
parser.add_argument(
|
||||
@@ -133,6 +135,43 @@ def parse_args() -> argparse.Namespace:
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
class OwnerLifetime:
|
||||
"""The launcher's private pipe is a lifetime capability, independent of PIDs."""
|
||||
|
||||
def __init__(self, enabled: bool) -> None:
|
||||
self._poll = select.poll() if enabled else None
|
||||
if self._poll is not None:
|
||||
# Node/libuv can implement piped stdio with a Unix socket pair.
|
||||
mode = os.fstat(0).st_mode
|
||||
if not (stat.S_ISFIFO(mode) or stat.S_ISSOCK(mode)):
|
||||
raise AdapterFailure(
|
||||
INVALID_LAUNCH_COMMAND,
|
||||
"invalid_launch_command",
|
||||
"owner lifetime requires a private stdin pipe",
|
||||
)
|
||||
self._poll.register(0, select.POLLIN | select.POLLHUP | select.POLLERR)
|
||||
|
||||
def alive(self) -> bool:
|
||||
if self._poll is None:
|
||||
return True
|
||||
for _, events in self._poll.poll(0):
|
||||
if events & (select.POLLHUP | select.POLLERR | select.POLLNVAL):
|
||||
return False
|
||||
if events & select.POLLIN and not os.read(0, 4096):
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def wait_for_hopper(child: subprocess.Popen[bytes], owner: OwnerLifetime) -> int:
|
||||
while owner.alive():
|
||||
try:
|
||||
return child.wait(timeout=0.1)
|
||||
except subprocess.TimeoutExpired:
|
||||
continue
|
||||
# run_launch's finally owns cleanup of both direct, unreaped children.
|
||||
return 0
|
||||
|
||||
|
||||
def digest(path: Path) -> str:
|
||||
value = hashlib.sha256()
|
||||
with path.open("rb") as file:
|
||||
@@ -572,6 +611,7 @@ def start_xvfb(
|
||||
"-auth",
|
||||
str(authority),
|
||||
],
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
@@ -627,6 +667,9 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
"invalid_launch_command",
|
||||
"missing Hopper launch coordinates",
|
||||
)
|
||||
owner = OwnerLifetime(args.owner_stdin)
|
||||
if not owner.alive():
|
||||
return 0
|
||||
hopper = Path(args.hopper)
|
||||
socket = Path(args.socket)
|
||||
try:
|
||||
@@ -654,6 +697,8 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
child: subprocess.Popen[bytes] | None = None
|
||||
display: int | None = None
|
||||
try:
|
||||
if not owner.alive():
|
||||
return 0
|
||||
hopper_environment = dict(os.environ)
|
||||
for name in ("WAYLAND_DISPLAY", "WAYLAND_SOCKET"):
|
||||
hopper_environment.pop(name, None)
|
||||
@@ -666,7 +711,10 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
"XDG_SESSION_TYPE": "x11",
|
||||
}
|
||||
)
|
||||
child = subprocess.Popen(command, env=hopper_environment)
|
||||
# Children must not consume or retain REA's lifetime pipe.
|
||||
child = subprocess.Popen(
|
||||
command, env=hopper_environment, stdin=subprocess.DEVNULL
|
||||
)
|
||||
leader = os.getpid()
|
||||
os.environ["XAUTHORITY"] = str(authority)
|
||||
|
||||
@@ -702,10 +750,12 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
observed: list[tuple[int, int, int, int, int]] = []
|
||||
dialog: tuple[int, int, int, int, int] | None = None
|
||||
while time.monotonic() < deadline:
|
||||
if not owner.alive():
|
||||
return 0
|
||||
if socket.exists():
|
||||
x11.XCloseDisplay(display)
|
||||
display = None
|
||||
return child.wait()
|
||||
return wait_for_hopper(child, owner)
|
||||
if not hopper_is_owned(leader, hopper):
|
||||
if child.poll() is None:
|
||||
time.sleep(0.1)
|
||||
@@ -749,7 +799,7 @@ def run_launch(args: argparse.Namespace) -> int:
|
||||
x11.XFlush(display)
|
||||
x11.XCloseDisplay(display)
|
||||
display = None
|
||||
return child.wait()
|
||||
return wait_for_hopper(child, owner)
|
||||
except AdapterFailure as error:
|
||||
raise error.with_stderr(capture)
|
||||
finally:
|
||||
|
||||
@@ -52,6 +52,11 @@ export const toolFamilyCatalog = (sources) => {
|
||||
surface: "android-provider",
|
||||
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
id: "pcap",
|
||||
surface: "pcap-provider",
|
||||
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
id: "flutter",
|
||||
surface: "flutter-provider",
|
||||
@@ -251,6 +256,10 @@ export const providerCatalog = (sources) => {
|
||||
identity: sources.androidProvider.JADX_PROVIDER_IDENTITY,
|
||||
contracts: sources.androidContracts.ANDROID_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.pcapProvider.PCAP_PROVIDER_IDENTITY,
|
||||
contracts: sources.pcapContracts.PCAP_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.flutterProvider.FLUTTER_PROVIDER_IDENTITY,
|
||||
contracts: sources.flutterContracts.FLUTTER_TOOL_CONTRACTS,
|
||||
@@ -272,6 +281,11 @@ export const providerCatalog = (sources) => {
|
||||
contracts:
|
||||
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity: sources.webNetworkCaptureProvider.SAZ_CAPTURE_PROVIDER_IDENTITY,
|
||||
contracts:
|
||||
sources.webNetworkCaptureContracts.WEB_NETWORK_CAPTURE_TOOL_CONTRACTS,
|
||||
},
|
||||
{
|
||||
identity:
|
||||
sources.webNetworkCaptureProvider.MITMPROXY_CAPTURE_PROVIDER_IDENTITY,
|
||||
|
||||
@@ -68,8 +68,8 @@ const SOURCE_PATHS = {
|
||||
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
|
||||
androidProvider: "dist/android/JadxRelease.js",
|
||||
apktoolContracts: "dist/contracts/apktool/apktoolToolContracts.js",
|
||||
flutterContracts: "dist/contracts/flutter/flutterToolContracts.js",
|
||||
flutterProvider: "dist/flutter/FlutterBuildProvider.js",
|
||||
pcapContracts: "dist/contracts/pcap/pcapToolContracts.js",
|
||||
pcapProvider: "dist/pcap/PcapProvider.js",
|
||||
apktoolProvider: "dist/apktool/ApktoolProvider.js",
|
||||
jebContracts: "dist/contracts/jeb/jebToolContracts.js",
|
||||
jebProvider: "dist/jeb/JebProvider.js",
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
export function assertCleanLinuxHopperRuntime(
|
||||
launcherPaths?: readonly string[],
|
||||
): Promise<void>;
|
||||
@@ -0,0 +1,64 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { readdir, stat } from "node:fs/promises";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* Observe leftover demo processes on a dedicated clean Linux runner, including
|
||||
* groups reparented after their launcher exits. This is not ownership proof and
|
||||
* never authorizes signaling; unrelated runtime processes also block this lane.
|
||||
*/
|
||||
export const assertCleanLinuxHopperRuntime = async (
|
||||
launcherPaths = [
|
||||
"/opt/hopper/bin/Hopper",
|
||||
...(process.env.HOPPER_LAUNCHER_PATH === undefined
|
||||
? []
|
||||
: [process.env.HOPPER_LAUNCHER_PATH]),
|
||||
],
|
||||
) => {
|
||||
const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]);
|
||||
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout))
|
||||
throw new Error("Clean Linux runner retained a Hopper demo process");
|
||||
const launchers = [];
|
||||
for (const path of launcherPaths) {
|
||||
try {
|
||||
launchers.push(await stat(path, { bigint: true }));
|
||||
} catch (error) {
|
||||
if (error?.code !== "ENOENT") throw error;
|
||||
}
|
||||
}
|
||||
if (launchers.length === 0) return;
|
||||
// /proc exposes the executed file even when argv[0] names a removed symlink.
|
||||
// Compare file identity, not a display path or a process name.
|
||||
let inspectionError;
|
||||
for (const entry of await readdir("/proc", { withFileTypes: true })) {
|
||||
if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue;
|
||||
let executable;
|
||||
try {
|
||||
executable = await stat(`/proc/${entry.name}/exe`, { bigint: true });
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT" || error?.code === "ESRCH") continue;
|
||||
// Other users' executables may be unreadable. The setup lane launches
|
||||
// providers as the caller; unreadable caller-owned processes fail closed.
|
||||
if (error?.code === "EACCES") {
|
||||
const owner = await stat(`/proc/${entry.name}`).catch((cause) => {
|
||||
if (cause?.code === "ENOENT") return undefined;
|
||||
throw cause;
|
||||
});
|
||||
if (owner === undefined || owner.uid !== process.getuid()) continue;
|
||||
}
|
||||
// Finish the inventory so an unrelated restricted process cannot mask a
|
||||
// positively identified leak. Unknown caller processes still fail closed.
|
||||
inspectionError ??= error;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
launchers.some(
|
||||
({ dev, ino }) => dev === executable.dev && ino === executable.ino,
|
||||
)
|
||||
)
|
||||
throw new Error("Clean Linux runner retained a Hopper demo process");
|
||||
}
|
||||
if (inspectionError !== undefined) throw inspectionError;
|
||||
};
|
||||
@@ -0,0 +1,9 @@
|
||||
export function assertFeatureTrace(
|
||||
trace: unknown,
|
||||
procedure: { readonly address: string },
|
||||
): void;
|
||||
|
||||
export function findMappedNonFunction(
|
||||
strings: readonly { readonly address: string }[],
|
||||
resolveAt: (address: string) => Promise<unknown>,
|
||||
): Promise<string | undefined>;
|
||||
@@ -0,0 +1,52 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]);
|
||||
|
||||
/**
|
||||
* Assert the `trace_feature` result of the real IDA lane for one selected
|
||||
* procedure. The query is a substring search, so the trace may also match other
|
||||
* procedures and strings; each reference is checked against its own matched
|
||||
* target, and a source outside every function (a data reference) is valid.
|
||||
*/
|
||||
export function assertFeatureTrace(trace, procedure) {
|
||||
assert.ok(
|
||||
trace.matches.some(
|
||||
({ type, address }) =>
|
||||
type === "procedure" && address === procedure.address,
|
||||
),
|
||||
"The trace omitted the selected procedure's match.",
|
||||
);
|
||||
assert.equal(typeof trace.truncated, "boolean");
|
||||
const matched = new Set(trace.matches.map(({ address }) => address));
|
||||
for (const reference of trace.references) {
|
||||
assert.ok(
|
||||
matched.has(reference.target_address),
|
||||
`A reference targets ${reference.target_address}, which is not a match.`,
|
||||
);
|
||||
const { containing_procedure: containing } = reference;
|
||||
if (containing.found)
|
||||
assert.equal(typeof containing.procedure.address, "string");
|
||||
else assert.ok(NEGATIVE_REASONS.has(containing.reason));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Find a string that IDA maps outside every function. A string inside a
|
||||
* function chunk (inline data in code) is valid and skipped, so the strings are
|
||||
* tried in order. Any other answer must be "not in a procedure": IDA lists only
|
||||
* mapped strings, so an unmapped answer is a containment defect and fails the
|
||||
* lane. Returns the address, or undefined when every string is inside a function.
|
||||
*/
|
||||
export async function findMappedNonFunction(strings, resolveAt) {
|
||||
for (const { address } of strings) {
|
||||
const resolved = await resolveAt(address);
|
||||
if (resolved.found) continue;
|
||||
assert.equal(
|
||||
resolved.reason,
|
||||
"not_in_procedure",
|
||||
`The listed string at ${address} resolved as ${resolved.reason}.`,
|
||||
);
|
||||
return address;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
export function supportsNativeAotFixtureHost(
|
||||
platform: string,
|
||||
architecture: string,
|
||||
): boolean;
|
||||
|
||||
export function buildNativeAotFixture(
|
||||
run: (command: string, args: string[]) => unknown,
|
||||
options: {
|
||||
dotnet: string;
|
||||
project: string;
|
||||
runtime: string;
|
||||
output: string;
|
||||
},
|
||||
): Promise<void>;
|
||||
|
||||
export function nativeAotCommandError(
|
||||
command: string,
|
||||
operation: string,
|
||||
result: {
|
||||
error?: { message: string };
|
||||
status: number | null;
|
||||
stdout?: string | null;
|
||||
stderr?: string | null;
|
||||
},
|
||||
): Error;
|
||||
@@ -0,0 +1,60 @@
|
||||
import { join } from "node:path";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
|
||||
/** NativeAOT fixtures use native toolchains, not cross-compilation. */
|
||||
export function supportsNativeAotFixtureHost(platform, architecture) {
|
||||
return architecture === "x64" && ["linux", "win32"].includes(platform);
|
||||
}
|
||||
|
||||
/** Keep network recovery separate from compilation and preserve restore failures. */
|
||||
export async function buildNativeAotFixture(run, options) {
|
||||
const { dotnet, project, runtime, output } = options;
|
||||
const common = [
|
||||
project,
|
||||
"-r",
|
||||
runtime,
|
||||
"--maxcpucount:1",
|
||||
"-p:Configuration=Release",
|
||||
`-p:BaseIntermediateOutputPath=${join(output, "obj")}/`,
|
||||
`-p:BaseOutputPath=${join(output, "bin")}/`,
|
||||
`-p:NativeAotMapPath=${join(output, "NativeAotFixture.map")}`,
|
||||
];
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
console.error(`NativeAOT dependency restore: attempt ${attempt}/2`);
|
||||
try {
|
||||
run(dotnet, ["restore", ...common, "--verbosity", "normal"]);
|
||||
break;
|
||||
} catch (cause) {
|
||||
if (attempt === 2)
|
||||
throw new Error(
|
||||
"NativeAOT dependency restore failed after two attempts",
|
||||
{
|
||||
cause,
|
||||
},
|
||||
);
|
||||
// Child diagnostics can contain authenticated feed URLs; keep retry logs static.
|
||||
console.error(
|
||||
"NativeAOT dependency restore failed; retrying in 1 second",
|
||||
);
|
||||
await delay(1_000);
|
||||
}
|
||||
}
|
||||
console.error("NativeAOT compilation (dependencies already restored)");
|
||||
run(dotnet, [
|
||||
"publish",
|
||||
...common,
|
||||
"--no-restore",
|
||||
"-o",
|
||||
join(output, "symbols"),
|
||||
]);
|
||||
}
|
||||
|
||||
/** Sanitize child diagnostics before any retry, cause chain, or uncaught output. */
|
||||
export function nativeAotCommandError(command, operation, result) {
|
||||
const diagnostic = `NativeAOT fixture lane requires/action failed ${command} ${operation}: ${result.error?.message ?? `exit ${result.status}`}\n${result.stdout ?? ""}${result.stderr ?? ""}`;
|
||||
// Only URL authority userinfo is authentication; preserve host, path, query,
|
||||
// and unrelated diagnostic values. Never retain the raw result as a cause.
|
||||
return new Error(
|
||||
diagnostic.replace(/\b(https?:\/\/)[^\\\s/?#]*@/giu, "$1[redacted]@"),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../../dist/domain/evidence.js";
|
||||
import {
|
||||
requireMcpOperationResult,
|
||||
requireMcpToolError,
|
||||
mcpTextValue,
|
||||
} from "./mcp-verifier-results.mjs";
|
||||
|
||||
/** Exercise file coordinates through public CLI/MCP using the source-owned PE fixture. */
|
||||
export async function verifyGhidraFileOffsets(
|
||||
targetPath,
|
||||
entrypoint,
|
||||
procedure,
|
||||
) {
|
||||
const original = await readFile(targetPath);
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-file-offsets-"));
|
||||
const runtime = join(workspace, "runtime");
|
||||
await mkdir(runtime);
|
||||
const env = {
|
||||
...process.env,
|
||||
TEMP: runtime,
|
||||
TMP: runtime,
|
||||
TMPDIR: runtime,
|
||||
REA_ANALYSIS_PROVIDER: "ghidra",
|
||||
REA_LOG_LEVEL: "silent",
|
||||
};
|
||||
const client = new Client({ name: "rea-file-offsets", version: "1" });
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
let connected = false;
|
||||
let complete = false;
|
||||
let primaryFailure;
|
||||
const call = async (name, arguments_ = {}) => {
|
||||
const result = await client.callTool(
|
||||
{ name, arguments: arguments_ },
|
||||
{ timeout: 240_000 },
|
||||
);
|
||||
const projected = requireMcpOperationResult(result, name);
|
||||
return ["open_binary", "binary_session", "close_binary"].includes(name)
|
||||
? projected
|
||||
: (result.structuredContent ?? JSON.parse(mcpTextValue(result)));
|
||||
};
|
||||
try {
|
||||
await client.connect(transport);
|
||||
connected = true;
|
||||
await call("open_binary", { path: targetPath, provider_id: "ghidra" });
|
||||
const session = await call("binary_session");
|
||||
const mapping = parseEvidence(
|
||||
await call("address_to_file_offset", { address: procedure }),
|
||||
);
|
||||
const offset = mapping.normalized_result.file_offset;
|
||||
const coordinate = `0x${offset.toString(16)}`;
|
||||
const expected = parseEvidence(
|
||||
await call("procedure_pseudo_code", { procedure }),
|
||||
);
|
||||
const explicit = parseEvidence(
|
||||
await call("procedure_pseudo_code", {
|
||||
procedure: `file-offset:${coordinate}`,
|
||||
}),
|
||||
);
|
||||
const automatic = parseEvidence(
|
||||
await call("procedure_pseudo_code", { procedure: coordinate }),
|
||||
);
|
||||
for (const actual of [explicit, automatic]) {
|
||||
assert.equal(actual.normalized_result, expected.normalized_result);
|
||||
assert.equal(actual.raw_result.selection.file_offset, offset);
|
||||
assert.equal(actual.raw_result.selection.mapped_address, procedure);
|
||||
assert.ok(
|
||||
actual.locations.some(
|
||||
(location) =>
|
||||
location.kind === "file-offset" && location.offset === offset,
|
||||
),
|
||||
);
|
||||
}
|
||||
const invalid = await client.callTool({
|
||||
name: "procedure_pseudo_code",
|
||||
arguments: { procedure: "file-offset:0x20000000000000" },
|
||||
});
|
||||
assert.equal(requireMcpToolError(invalid).code, "invalid_request");
|
||||
const unmapped = await client.callTool({
|
||||
name: "procedure_pseudo_code",
|
||||
arguments: { procedure: "file-offset:0x100000" },
|
||||
});
|
||||
assert.ok(
|
||||
JSON.stringify(requireMcpToolError(unmapped)).includes(
|
||||
"no loaded source mapping",
|
||||
),
|
||||
);
|
||||
assert.equal(
|
||||
(await call("binary_session")).analysis_run.run_id,
|
||||
session.analysis_run.run_id,
|
||||
);
|
||||
const image = parseEvidence(await call("inspect_native_load_image"));
|
||||
const headerMapping = image.normalized_result.observations.mappings.find(
|
||||
(item) => item.initialized && item.file_offset === 0,
|
||||
);
|
||||
assert.ok(
|
||||
headerMapping,
|
||||
"Fixture header must have an observed source mapping",
|
||||
);
|
||||
const headerOffset = parseEvidence(
|
||||
await call("address_to_file_offset", { address: headerMapping.start }),
|
||||
);
|
||||
assert.equal(headerOffset.normalized_result.file_offset, 0);
|
||||
for (const selector of ["file-offset:0x0", "0x0"]) {
|
||||
const missing = requireMcpToolError(
|
||||
await client.callTool({
|
||||
name: "procedure_pseudo_code",
|
||||
arguments: { procedure: selector },
|
||||
}),
|
||||
);
|
||||
assertMissingMappedProcedure(missing, headerMapping.start);
|
||||
}
|
||||
assert.equal(
|
||||
(await call("binary_session")).analysis_run.run_id,
|
||||
session.analysis_run.run_id,
|
||||
);
|
||||
await call("close_binary");
|
||||
const cli = JSON.parse(
|
||||
(
|
||||
await promisify(execFile)(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"decompile",
|
||||
targetPath,
|
||||
`--file-offset=${coordinate}`,
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
)
|
||||
).stdout,
|
||||
);
|
||||
assert.equal(cli.normalized_result, explicit.normalized_result);
|
||||
assert.deepEqual(cli.locations, explicit.locations);
|
||||
const missingCli = await promisify(execFile)(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"decompile",
|
||||
targetPath,
|
||||
"--file-offset=0x0",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
).then(
|
||||
() => {
|
||||
throw new Error(
|
||||
"Mapped header bytes must not decompile as a procedure",
|
||||
);
|
||||
},
|
||||
(cause) => JSON.parse(cause.stdout),
|
||||
);
|
||||
assertMissingMappedProcedure(missingCli, headerMapping.start);
|
||||
const alias = Buffer.from(original);
|
||||
const pe = alias.readUInt32LE(0x3c);
|
||||
if (alias.readUInt16LE(pe + 6) >= 2) {
|
||||
const secondSection = pe + 24 + alias.readUInt16LE(pe + 20) + 40;
|
||||
// Both sections now carry the same original bytes at distinct virtual addresses.
|
||||
alias.writeUInt32LE(0x200, secondSection + 20);
|
||||
const ambiguousPath = join(workspace, "ambiguous.exe");
|
||||
await writeFile(ambiguousPath, alias);
|
||||
await call("open_binary", { path: ambiguousPath, provider_id: "ghidra" });
|
||||
const ambiguous = await client.callTool(
|
||||
{
|
||||
name: "procedure_pseudo_code",
|
||||
arguments: { procedure: `file-offset:${coordinate}` },
|
||||
},
|
||||
{ timeout: 240_000 },
|
||||
);
|
||||
assert.ok(
|
||||
JSON.stringify(requireMcpToolError(ambiguous)).includes(
|
||||
"multiple loaded mappings",
|
||||
),
|
||||
);
|
||||
await call("close_binary");
|
||||
}
|
||||
assert.deepEqual(await readFile(targetPath), original);
|
||||
await client.close();
|
||||
connected = false;
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
complete = true;
|
||||
return {
|
||||
cli_mcp_parity: true,
|
||||
explicit_and_numeric_offsets: true,
|
||||
invalid_and_unmapped_offsets: true,
|
||||
mapped_offset_without_procedure: true,
|
||||
ambiguous_mappings:
|
||||
alias.readUInt16LE(pe + 6) >= 2
|
||||
? "verified"
|
||||
: "unavailable-single-section",
|
||||
source_unchanged: true,
|
||||
cleanup: "complete",
|
||||
};
|
||||
} catch (cause) {
|
||||
primaryFailure = cause;
|
||||
throw cause;
|
||||
} finally {
|
||||
const cleanupFailures = [];
|
||||
if (connected) {
|
||||
try {
|
||||
await call("close_binary");
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await client.close();
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
if (cleanupFailures.length > 0) {
|
||||
process.stderr.write(
|
||||
`Retained file-offset verifier workspace: ${workspace}\n`,
|
||||
);
|
||||
throw new AggregateError(
|
||||
primaryFailure === undefined
|
||||
? cleanupFailures
|
||||
: [primaryFailure, ...cleanupFailures],
|
||||
"File-offset verifier cleanup failed",
|
||||
);
|
||||
}
|
||||
if (complete) await rm(workspace, { recursive: true, force: true });
|
||||
else
|
||||
process.stderr.write(
|
||||
`Retained file-offset verifier workspace: ${workspace}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function assertMissingMappedProcedure(error, mappedAddress) {
|
||||
assert.equal(error.code, "invalid_request");
|
||||
const message = error.details?.issues?.[0]?.message;
|
||||
assert.equal(typeof message, "string");
|
||||
assert.ok(
|
||||
message.includes("File offset 0x0 maps to analysis address"),
|
||||
message,
|
||||
);
|
||||
assert.ok(message.includes(mappedAddress), message);
|
||||
assert.ok(message.includes("no procedure starts at or contains"), message);
|
||||
assert.ok(!message.includes("--file-offset"), message);
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { verifyNoTokenOwnedProcesses } from "../../dist/process/ProcessOwnership.js";
|
||||
import {
|
||||
observeProcessStartIdentity,
|
||||
signalProcessWithStartIdentity,
|
||||
} from "../../dist/process/ProcessOwnershipObservation.js";
|
||||
import { startUnrelatedHopperSentinel } from "./unrelated-hopper-sentinel.mjs";
|
||||
import {
|
||||
requireMcpEvidenceResult,
|
||||
requireMcpLifecycleResult,
|
||||
} from "./mcp-verifier-results.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Prove actual MCP owner exit releases the demo singleton for subsequent CLI/MCP sessions. */
|
||||
export async function verifyLinuxHopperOwnerExit({
|
||||
primary,
|
||||
secondary,
|
||||
serverCommand,
|
||||
serverArgs,
|
||||
environment,
|
||||
}) {
|
||||
const sentinel = await startUnrelatedHopperSentinel();
|
||||
const outcomes = [];
|
||||
try {
|
||||
for (const mode of ["graceful", "SIGKILL"]) {
|
||||
const transport = new StdioClientTransport({
|
||||
command: serverCommand,
|
||||
args: serverArgs,
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
// Drain launcher diagnostics even on the deliberately interrupted owner.
|
||||
transport.stderr?.resume();
|
||||
const client = new Client({ name: "hopper-owner-exit", version: "1" });
|
||||
const options = { timeout: 180_000 };
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const ownerPid = transport.pid;
|
||||
assert.ok(ownerPid !== null);
|
||||
const owner = await observeProcessStartIdentity(ownerPid);
|
||||
assert.equal(owner?.state, "readable");
|
||||
requireMcpLifecycleResult(
|
||||
await client.callTool(
|
||||
{
|
||||
name: "open_binary",
|
||||
arguments: { path: primary },
|
||||
},
|
||||
options,
|
||||
),
|
||||
"open_binary",
|
||||
);
|
||||
const procedures = requireMcpEvidenceResult(
|
||||
await client.callTool(
|
||||
{ name: "list_procedures", arguments: {} },
|
||||
options,
|
||||
),
|
||||
"list_procedures",
|
||||
);
|
||||
assert.ok(procedures.length > 0);
|
||||
const status = requireMcpLifecycleResult(
|
||||
await client.callTool(
|
||||
{ name: "binary_session", arguments: {} },
|
||||
options,
|
||||
),
|
||||
"binary_session",
|
||||
);
|
||||
const runId = status.analysis_run?.run_id;
|
||||
assert.equal(typeof runId, "string");
|
||||
assert.equal(status.analysis_provider_binding?.provider.id, "hopper");
|
||||
assert.equal((await verifyNoTokenOwnedProcesses(runId)).cleaned, false);
|
||||
if (mode === "graceful") await client.close();
|
||||
else {
|
||||
assert.equal(
|
||||
await signalProcessWithStartIdentity(
|
||||
ownerPid,
|
||||
owner.identity,
|
||||
"SIGKILL",
|
||||
),
|
||||
"signaled",
|
||||
);
|
||||
}
|
||||
const deadline = Date.now() + 10_000;
|
||||
let cleanup;
|
||||
do {
|
||||
cleanup = await verifyNoTokenOwnedProcesses(runId);
|
||||
if (cleanup.cleaned) break;
|
||||
await delay(100);
|
||||
} while (Date.now() < deadline);
|
||||
assert.equal(cleanup.cleaned, true, cleanup.reason);
|
||||
// The SIGKILL case must recover the stale lease for both target identities.
|
||||
for (const target of [primary, secondary]) {
|
||||
const result = await execFileAsync(
|
||||
process.execPath,
|
||||
[
|
||||
"scripts/rea.mjs",
|
||||
"inspect",
|
||||
target,
|
||||
"--provider",
|
||||
"hopper",
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
{ env: environment, timeout: 180_000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
assert.ok(JSON.parse(result.stdout).normalized_result);
|
||||
}
|
||||
outcomes.push({
|
||||
mode,
|
||||
ownedProcessesStopped: true,
|
||||
sameTargetReopened: true,
|
||||
differentTargetReopened: true,
|
||||
});
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
assert.equal(await sentinel.verifyAndClose(), true);
|
||||
}
|
||||
return { outcomes, unrelatedHopperSurvived: true };
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
type VerifierCall = (
|
||||
name: string,
|
||||
arguments_?: Record<string, unknown>,
|
||||
) => Promise<unknown>;
|
||||
|
||||
interface StringExpectation {
|
||||
readonly value: string;
|
||||
readonly encoding: string;
|
||||
readonly symbol?: string;
|
||||
readonly recognitionOptional?: boolean;
|
||||
}
|
||||
|
||||
/** Verify source bytes, observed decoding, address filters and literal searches. */
|
||||
export function verifyHopperStringObjects(
|
||||
call: VerifierCall,
|
||||
expectations: readonly StringExpectation[],
|
||||
inventory?: readonly z.output<typeof analysisStringSchema>[] | null,
|
||||
): Promise<{
|
||||
readonly sourceBytesVerified: true;
|
||||
readonly recognition: readonly {
|
||||
readonly value: string;
|
||||
readonly address: string;
|
||||
readonly state: "observed" | "not_observed";
|
||||
readonly decodedObject: "verified" | "unverified";
|
||||
}[];
|
||||
readonly displayEscapesExcludedFromMatching: true;
|
||||
}>;
|
||||
import type { z } from "zod";
|
||||
|
||||
import type { analysisStringSchema } from "../../src/domain/hopperValues.js";
|
||||
@@ -1,13 +1,16 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mcpTextValue } from "./mcp-verifier-results.mjs";
|
||||
|
||||
const fixtureSymbolPrefix = process.platform === "darwin" ? "_" : "";
|
||||
|
||||
/** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */
|
||||
export async function verifyHopperSearch(call, invalid, procedure) {
|
||||
const address = procedure.address;
|
||||
const original = await call("address_name", { address });
|
||||
let stringCoverage;
|
||||
try {
|
||||
await verifySymbolNames(call, invalid, address);
|
||||
await verifySearchModes(call, address);
|
||||
stringCoverage = await verifySearchModes(call, address);
|
||||
} finally {
|
||||
await call("set_address_name", { address, name: original });
|
||||
}
|
||||
@@ -26,7 +29,7 @@ export async function verifyHopperSearch(call, invalid, procedure) {
|
||||
symbolUtf16LimitValidatedBeforeMutation: true,
|
||||
regexAndCaseModes: true,
|
||||
annotationCacheInvalidation: true,
|
||||
escapedUtf8AndByteStrings: true,
|
||||
stringCoverage,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -103,11 +106,26 @@ async function verifySearchModes(call, address) {
|
||||
[{ address, value: "REA_Search_Upper" }],
|
||||
);
|
||||
const strings = await call("list_strings");
|
||||
const long = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle";
|
||||
const longItem = strings.find((item) =>
|
||||
item.value.startsWith("REA_LONG_LITERAL_"),
|
||||
const linux = process.platform === "linux";
|
||||
const fullLiteral = "REA_LONG_LITERAL_" + "x".repeat(4096) + "needle";
|
||||
// Recognition is a provider/loader capability, not an adapter guarantee.
|
||||
// Always retain the original size in raw-byte coverage; Linux additionally
|
||||
// requires an extent observed to produce native fragmented string objects.
|
||||
const fullBytes = Buffer.from(fullLiteral + "\0");
|
||||
const raw = await call("read_bytes", {
|
||||
address: `${fixtureSymbolPrefix}rea_long_literal`,
|
||||
length: fullBytes.length,
|
||||
});
|
||||
assert.equal(raw.complete, true);
|
||||
assert.equal(raw.bytes_hex, fullBytes.toString("hex"));
|
||||
const long = linux
|
||||
? "REA_FRAGMENTED_LITERAL_" + "x".repeat(512) + "fragment_tail"
|
||||
: fullLiteral;
|
||||
const longItem = strings.find(
|
||||
(item) => long.startsWith(item.value) && item.value.startsWith("REA_"),
|
||||
);
|
||||
assert.ok(longItem, "fixture omitted its real long literal");
|
||||
if (!linux) assert.equal(longItem.address, raw.address);
|
||||
assert.ok(long.startsWith(longItem.value));
|
||||
assert.ok(longItem.provider_value.endsWith("…"));
|
||||
assert.equal(longItem.string.termination, "missing");
|
||||
@@ -158,7 +176,7 @@ async function verifySearchModes(call, address) {
|
||||
}
|
||||
assert.deepEqual(
|
||||
await call("search_strings", {
|
||||
pattern: "needle",
|
||||
pattern: linux ? "fragment_tail" : "needle",
|
||||
mode: "literal",
|
||||
case_sensitive: true,
|
||||
}),
|
||||
@@ -176,36 +194,122 @@ async function verifySearchModes(call, address) {
|
||||
}),
|
||||
expected,
|
||||
);
|
||||
await verifyHopperStringObjects(call, [
|
||||
{ value: "REA_UTF8_é_😀", encoding: "utf-8" },
|
||||
{ value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" },
|
||||
{ value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" },
|
||||
{ value: "REA_LITERAL_…", encoding: "utf-8" },
|
||||
{ value: "REA_LATIN1_ÿ", encoding: "latin-1" },
|
||||
]);
|
||||
const objects = await verifyHopperStringObjects(
|
||||
call,
|
||||
[
|
||||
{
|
||||
value: "REA_UTF8_é_😀",
|
||||
encoding: "utf-8",
|
||||
symbol: `${fixtureSymbolPrefix}rea_utf8_literal`,
|
||||
recognitionOptional: linux,
|
||||
},
|
||||
{ value: 'REA_ESCAPED_"\\line\nend\t\r', encoding: "utf-8" },
|
||||
{ value: "REA_LITERAL_BACKSLASH_\\n", encoding: "utf-8" },
|
||||
{
|
||||
value: "REA_LITERAL_…",
|
||||
encoding: "utf-8",
|
||||
symbol: `${fixtureSymbolPrefix}rea_ellipsis_literal`,
|
||||
recognitionOptional: linux,
|
||||
},
|
||||
{
|
||||
value: "REA_LATIN1_ÿ",
|
||||
encoding: "latin-1",
|
||||
symbol: `${fixtureSymbolPrefix}rea_latin1_literal`,
|
||||
recognitionOptional: linux,
|
||||
},
|
||||
],
|
||||
strings,
|
||||
);
|
||||
return {
|
||||
fullLiteralSourceBytes: fullBytes.length,
|
||||
fragmentedLiteralSourceBytes: Buffer.byteLength(long),
|
||||
fullLiteralRecognition: {
|
||||
address: raw.address,
|
||||
state: strings.some((item) => item.address === raw.address)
|
||||
? "observed"
|
||||
: "not_observed",
|
||||
// Linux validates the original through raw bytes; its required decoded
|
||||
// fragment/dossier checks exercise the separate supported-size literal.
|
||||
decodedObject: linux ? "unverified" : "verified",
|
||||
},
|
||||
...objects,
|
||||
};
|
||||
}
|
||||
|
||||
/** Compare native string identities and values with the source fixture's bytes. */
|
||||
export async function verifyHopperStringObjects(call, expectations) {
|
||||
const strings = await call("list_strings");
|
||||
for (const { value, encoding } of expectations) {
|
||||
const item = strings.find((candidate) => candidate.value === value);
|
||||
assert.ok(
|
||||
item,
|
||||
`Hopper omitted the decoded string ${JSON.stringify(value)}`,
|
||||
);
|
||||
assert.equal(item.string.encoding, encoding);
|
||||
assert.equal(item.string.encoding_status, "inferred");
|
||||
assert.equal(item.string.termination, "present_or_not_required");
|
||||
export async function verifyHopperStringObjects(
|
||||
call,
|
||||
expectations,
|
||||
inventory = null,
|
||||
) {
|
||||
// Reuse only the caller's inventory for this unchanged document, never live
|
||||
// state from an earlier session. Address-filter and search checks remain real.
|
||||
const strings = inventory ?? (await call("list_strings"));
|
||||
const recognition = [];
|
||||
for (const {
|
||||
value,
|
||||
encoding,
|
||||
symbol,
|
||||
recognitionOptional = false,
|
||||
} of expectations) {
|
||||
const expectedBytes = Buffer.from(
|
||||
value + "\0",
|
||||
encoding.replaceAll("-", ""),
|
||||
);
|
||||
assert.equal(item.string.byte_length, expectedBytes.length);
|
||||
const bytes = await call("read_bytes", {
|
||||
let sourceBytes;
|
||||
if (symbol !== undefined) {
|
||||
const raw = await call("read_bytes", {
|
||||
address: symbol,
|
||||
length: expectedBytes.length,
|
||||
});
|
||||
sourceBytes = raw;
|
||||
assert.equal(raw.complete, true);
|
||||
assert.equal(raw.bytes_hex, expectedBytes.toString("hex"));
|
||||
}
|
||||
const item = strings.find((candidate) =>
|
||||
sourceBytes === undefined
|
||||
? candidate.value === value
|
||||
: candidate.address === sourceBytes.address,
|
||||
);
|
||||
if (item === undefined && recognitionOptional) {
|
||||
assert.ok(
|
||||
symbol,
|
||||
"optional recognition requires independent source-byte coverage",
|
||||
);
|
||||
recognition.push({
|
||||
value,
|
||||
address: sourceBytes.address,
|
||||
state: "not_observed",
|
||||
decodedObject: "unverified",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
assert.ok(
|
||||
item,
|
||||
`Hopper omitted the decoded string ${JSON.stringify(value)}`,
|
||||
);
|
||||
assert.equal(
|
||||
item.value,
|
||||
value,
|
||||
"native string decoding changed the source value",
|
||||
);
|
||||
recognition.push({
|
||||
value,
|
||||
address: item.address,
|
||||
length: expectedBytes.length,
|
||||
state: "observed",
|
||||
decodedObject: "verified",
|
||||
});
|
||||
assert.equal(item.string.encoding, encoding);
|
||||
assert.equal(item.string.encoding_status, "inferred");
|
||||
assert.equal(item.string.termination, "present_or_not_required");
|
||||
assert.equal(item.string.byte_length, expectedBytes.length);
|
||||
const bytes =
|
||||
sourceBytes ??
|
||||
(await call("read_bytes", {
|
||||
address: item.address,
|
||||
length: expectedBytes.length,
|
||||
}));
|
||||
assert.equal(bytes.complete, true);
|
||||
assert.equal(bytes.bytes_hex, expectedBytes.toString("hex"));
|
||||
assert.deepEqual(await call("list_strings", { address: item.address }), [
|
||||
item,
|
||||
@@ -225,7 +329,7 @@ export async function verifyHopperStringObjects(call, expectations) {
|
||||
);
|
||||
return {
|
||||
sourceBytesVerified: true,
|
||||
inferredEncodingReported: true,
|
||||
recognition,
|
||||
displayEscapesExcludedFromMatching: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -16,6 +16,12 @@ const isMcpMode =
|
||||
const isMcpDoctorMode = args[0] === "mcp" && args[1] === "doctor";
|
||||
const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
// Windows home lookup throws for an empty USERPROFILE. Dependencies resolve
|
||||
// home during import, before REA's own resolver can apply its account fallback.
|
||||
if (process.platform === "win32" && process.env.USERPROFILE === "") {
|
||||
delete process.env.USERPROFILE;
|
||||
}
|
||||
|
||||
// Playwright's registry assumes one of its three desktop host platforms when
|
||||
// it chooses the browser cache directory. Termux reports `android`, although
|
||||
// its Node runtime and filesystem are POSIX-compatible. Keep the cache path
|
||||
|
||||
@@ -12,6 +12,7 @@ import { tmpdir } from "node:os";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { assertCleanLinuxHopperRuntime } from "./lib/clean-linux-hopper-runtime.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const verifierRun = createVerifierRun();
|
||||
@@ -19,12 +20,12 @@ const command = process.env.REA_VERIFY_SERVER_COMMAND;
|
||||
if (process.platform !== "linux" || process.getuid?.() !== 0)
|
||||
throw new Error("Linux setup verification requires a root Linux runner");
|
||||
if (!command) throw new Error("REA_VERIFY_SERVER_COMMAND is required");
|
||||
await assertCleanLinuxHopperRuntime();
|
||||
try {
|
||||
await access("/opt/hopper/bin/Hopper");
|
||||
throw new Error("Linux setup verification requires a clean runner");
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.message.includes("clean runner"))
|
||||
throw error;
|
||||
if (error?.code !== "ENOENT") throw error;
|
||||
}
|
||||
|
||||
const runJson = async (args) => {
|
||||
@@ -35,14 +36,25 @@ const runJson = async (args) => {
|
||||
return JSON.parse(stdout);
|
||||
};
|
||||
|
||||
const plan = await runJson(["setup", "--json"]);
|
||||
if (plan.status !== "needs_confirmation")
|
||||
throw new Error("setup did not produce a read-only confirmation plan");
|
||||
const plan = await runJson([
|
||||
"setup",
|
||||
"--install-hopper",
|
||||
"--dry-run",
|
||||
"--json",
|
||||
]);
|
||||
if (
|
||||
plan.status !== "planned" ||
|
||||
!plan.plannedActions.some(({ kind }) => kind === "install_hopper") ||
|
||||
plan.appliedActions.length !== 0
|
||||
)
|
||||
throw new Error("setup did not produce a read-only Hopper installation plan");
|
||||
await access("/opt/hopper/bin/Hopper").then(
|
||||
() => {
|
||||
throw new Error("setup plan mutated the clean runner");
|
||||
},
|
||||
() => undefined,
|
||||
(error) => {
|
||||
if (error?.code !== "ENOENT") throw error;
|
||||
},
|
||||
);
|
||||
|
||||
const installed = await runJson([
|
||||
@@ -58,7 +70,7 @@ if (
|
||||
throw new Error(
|
||||
`approved setup did not become ready: ${JSON.stringify(installed)}`,
|
||||
);
|
||||
const doctor = await runJson(["doctor", "--json"]);
|
||||
const doctor = await runJson(["doctor", "--provider", "hopper", "--json"]);
|
||||
if (doctor.healthy !== true)
|
||||
throw new Error(`doctor is unhealthy: ${JSON.stringify(doctor)}`);
|
||||
|
||||
@@ -67,8 +79,6 @@ await execFileAsync(command, ["analyze", "/usr/bin/true"], {
|
||||
timeout: 180_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
});
|
||||
process.env.HOPPER_TARGET_PATH = "/usr/bin/true";
|
||||
process.env.HOPPER_SECOND_TARGET_PATH = "/usr/bin/false";
|
||||
process.env.REA_VERIFY_SERVER_ARGS_JSON = '["mcp"]';
|
||||
await import("./verify-real-hopper-linux.mjs");
|
||||
|
||||
@@ -95,28 +105,24 @@ try {
|
||||
await rm(modifiedDirectory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const repeated = await runJson([
|
||||
"setup",
|
||||
"--yes",
|
||||
"--install-hopper",
|
||||
"--json",
|
||||
]);
|
||||
const repeated = await runJson(["setup", "--yes", "--json"]);
|
||||
if (
|
||||
repeated.status !== "ready" ||
|
||||
repeated.appliedActions.includes("installed_hopper")
|
||||
repeated.plannedActions.length !== 0 ||
|
||||
repeated.appliedActions.length !== 0
|
||||
)
|
||||
throw new Error("a second setup was not idempotent");
|
||||
const { stdout: processes } = await execFileAsync("ps", [
|
||||
"-ax",
|
||||
"-o",
|
||||
"command=",
|
||||
]);
|
||||
if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(processes))
|
||||
throw new Error("Linux setup verification leaked a Hopper demo session");
|
||||
await assertCleanLinuxHopperRuntime();
|
||||
const completedVerifierRun = await completeVerifierRun(verifierRun);
|
||||
if (
|
||||
completedVerifierRun.process_lineage.status !== "verified" ||
|
||||
completedVerifierRun.process_lineage.descendants.length !== 0
|
||||
)
|
||||
throw new Error("Linux setup verification did not prove owned child cleanup");
|
||||
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
verifier_run: await completeVerifierRun(verifierRun),
|
||||
verifier_run: completedVerifierRun,
|
||||
status: "verified",
|
||||
setup: true,
|
||||
cli: true,
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, rm, unlink } from "node:fs/promises";
|
||||
import { execFile } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { PRODUCT_IDENTITY } from "../dist/identity.js";
|
||||
@@ -21,12 +23,19 @@ import { verifyRepositorySkill } from "./verify-repository-skill.mjs";
|
||||
import { verifyManaged } from "./verify-package-managed.mjs";
|
||||
import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import {
|
||||
createVerifiedPackageMetadata,
|
||||
retainVerifiedPackage,
|
||||
writeVerifiedPackageManifest,
|
||||
} from "./ci/verified-package.mjs";
|
||||
import {
|
||||
measureMcpStartup,
|
||||
profileMcpModuleLoading,
|
||||
} from "./lib/mcp-startup-probe.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const execFileAsync = promisify(execFile);
|
||||
const retentionDirectory = process.env.REA_VERIFIED_PACKAGE_DIRECTORY;
|
||||
const verifierRun = createVerifierRun();
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-package-"));
|
||||
const evidenceRoot = join(workspace, "evidence");
|
||||
@@ -37,6 +46,8 @@ let mcpModuleLoading;
|
||||
let update;
|
||||
let repositorySkill;
|
||||
let report;
|
||||
let retainedPackage;
|
||||
let retentionMetadata;
|
||||
|
||||
const runStage = async (name, action) => {
|
||||
const started = performance.now();
|
||||
@@ -49,6 +60,27 @@ const runStage = async (name, action) => {
|
||||
};
|
||||
|
||||
try {
|
||||
if (retentionDirectory !== undefined && retentionDirectory !== "") {
|
||||
const sourceSha = (
|
||||
await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root })
|
||||
).stdout.trim();
|
||||
retentionMetadata = createVerifiedPackageMetadata({
|
||||
sourceSha,
|
||||
runId: process.env.GITHUB_RUN_ID,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
packageVersion: PRODUCT_IDENTITY.packageVersion,
|
||||
nodeVersion: process.version,
|
||||
});
|
||||
await mkdir(retentionDirectory, { recursive: true });
|
||||
await unlink(join(retentionDirectory, "manifest.json")).catch((cause) => {
|
||||
if (
|
||||
!(cause instanceof Error && "code" in cause && cause.code === "ENOENT")
|
||||
)
|
||||
throw cause;
|
||||
});
|
||||
}
|
||||
repositorySkill = await runStage("repository skill", () =>
|
||||
verifyRepositorySkill({ root, workspace }),
|
||||
);
|
||||
@@ -215,6 +247,13 @@ try {
|
||||
mcpStartup,
|
||||
mcpModuleLoading,
|
||||
};
|
||||
if (retentionMetadata !== undefined) {
|
||||
retainedPackage = await retainVerifiedPackage({
|
||||
sourceTarball: tarball,
|
||||
directory: retentionDirectory,
|
||||
metadata: retentionMetadata,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await runStage("workspace cleanup", () =>
|
||||
rm(workspace, { recursive: true, force: true }),
|
||||
@@ -223,6 +262,11 @@ try {
|
||||
const verifierLineage = await runStage("verifier lineage", () =>
|
||||
completeVerifierRun(verifierRun),
|
||||
);
|
||||
if (retainedPackage !== undefined)
|
||||
await writeVerifiedPackageManifest(
|
||||
retainedPackage.directory,
|
||||
retainedPackage.metadata,
|
||||
);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ verifier_run: verifierLineage, ...report })}\n`,
|
||||
);
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { estimateTokenCount } from "tokenx";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile, readdir } from "node:fs/promises";
|
||||
import { readFile, writeFile, readdir, mkdir, rm } from "node:fs/promises";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
@@ -91,6 +92,20 @@ const source = pcap(1, [
|
||||
// LINKTYPE_BLUETOOTH_HCI_H4 = 187; H4 command packet, HCI Reset opcode 0x0c03, no parameters.
|
||||
const bluetoothSource = pcap(187, [Buffer.from("01030c00", "hex")]);
|
||||
let cases = 0;
|
||||
const discoveryMeasurements = [];
|
||||
const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8"))
|
||||
.split("\n")
|
||||
.find((line) => line.startsWith("Max address space "))
|
||||
?.trim()
|
||||
.match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u);
|
||||
assert.ok(inheritedAddressSpace);
|
||||
const expectedAddressSpace = Math.min(
|
||||
1073741824,
|
||||
...inheritedAddressSpace
|
||||
.slice(1)
|
||||
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
|
||||
);
|
||||
|
||||
try {
|
||||
await writeFile(path, source);
|
||||
await writeFile(bluetooth, bluetoothSource);
|
||||
@@ -121,14 +136,16 @@ try {
|
||||
.digest("hex"),
|
||||
);
|
||||
assert.equal(report.hardware_acquisition, "not-performed");
|
||||
assert.equal(report.limits.address_space_bytes, 1073741824);
|
||||
assert.equal(report.limits.address_space_bytes, expectedAddressSpace);
|
||||
assert.equal(report.tool.configured_path, process.env.REA_TSHARK_COMMAND);
|
||||
assert.equal(
|
||||
report.resource_limiter.configured_path,
|
||||
process.env.REA_TSHARK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit",
|
||||
);
|
||||
assert.ok(
|
||||
report.resource_limiter.arguments.includes("--as=1073741824:1073741824"),
|
||||
report.resource_limiter.arguments.includes(
|
||||
`--as=${expectedAddressSpace}:${expectedAddressSpace}`,
|
||||
),
|
||||
);
|
||||
assert.equal(report.configuration, "private-empty-personal-profile");
|
||||
const flags = Object.entries(input)
|
||||
@@ -236,6 +253,52 @@ try {
|
||||
const hci = await inspect({ path: bluetooth, fields: ["bthci_cmd.opcode"] });
|
||||
assert.deepEqual(hci.packets[0]["bthci_cmd.opcode"], ["0x0c03"]);
|
||||
assert.deepEqual(await readFile(path), source);
|
||||
// Exercise the incremental summary workflow against the same owned bytes
|
||||
// using real installed Wireshark tools; these inputs are format fixtures.
|
||||
const pcapTool = (await client.listTools()).tools.find(
|
||||
(t) => t.name === "inspect_pcap_capture",
|
||||
);
|
||||
assert.ok(pcapTool?.outputSchema);
|
||||
const pcapAjv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
assert.ok(pcapAjv.validateSchema(pcapTool.inputSchema));
|
||||
assert.ok(pcapAjv.validateSchema(pcapTool.outputSchema));
|
||||
for (const capturePath of [path, bluetooth]) {
|
||||
const response = await client.callTool({
|
||||
name: pcapTool.name,
|
||||
arguments: { path: capturePath, detail: "summary" },
|
||||
});
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(
|
||||
pcapAjv.validate(pcapTool.outputSchema, response.structuredContent),
|
||||
pcapAjv.errorsText(),
|
||||
);
|
||||
const summary = parseEvidence(response.structuredContent).normalized_result;
|
||||
assert.equal(
|
||||
summary.summary.target.sha256,
|
||||
createHash("sha256")
|
||||
.update(await readFile(capturePath))
|
||||
.digest("hex"),
|
||||
);
|
||||
const source = {
|
||||
kind: "retained-evidence",
|
||||
evidence_id: summary.parent_evidence_id,
|
||||
};
|
||||
const page = await client.callTool({
|
||||
name: "inspect_analysis_view",
|
||||
arguments: {
|
||||
source,
|
||||
view: { kind: "pcap", facet: "protocols", limit: 10 },
|
||||
},
|
||||
});
|
||||
assert.notEqual(page.isError, true, JSON.stringify(page));
|
||||
assert.equal(
|
||||
parseEvidence(page.structuredContent).normalized_result
|
||||
.parent_evidence_id,
|
||||
source.evidence_id,
|
||||
);
|
||||
cases++;
|
||||
}
|
||||
|
||||
assert.deepEqual(await readFile(bluetooth), bluetoothSource);
|
||||
for (const input of [
|
||||
{ path, fields: ["frame.number"], display_filter: "this is not a filter" },
|
||||
@@ -267,6 +330,133 @@ try {
|
||||
true,
|
||||
);
|
||||
cases++;
|
||||
const discovery = (await client.listTools()).tools.find(
|
||||
({ name }) => name === "inspect_packet_fields",
|
||||
);
|
||||
assert.ok(discovery?.outputSchema);
|
||||
assert.ok(ajv.validateSchema(discovery.inputSchema));
|
||||
assert.ok(ajv.validateSchema(discovery.outputSchema));
|
||||
const nativeProfile = join(root.path, "native-profile");
|
||||
await mkdir(nativeProfile);
|
||||
const native = (
|
||||
await execute(process.env.REA_TSHARK_COMMAND, ["-G", "fields"], {
|
||||
env: { ...environment, WIRESHARK_CONFIG_DIR: nativeProfile },
|
||||
timeout: 30000,
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
})
|
||||
).stdout;
|
||||
await rm(nativeProfile, { recursive: true });
|
||||
const nativeLines = native.split(/\r?\n/);
|
||||
if (nativeLines.at(-1) === "") nativeLines.pop();
|
||||
for (const input of [
|
||||
{ query: "usb" },
|
||||
{ limit: 1 },
|
||||
{ offset: 1, limit: 2 },
|
||||
{ kind: "fields", protocol: "usb", query: "idVendor" },
|
||||
{ kind: "fields", protocol: "udp", limit: null },
|
||||
{ kind: "fields", query: "rea_no_such_field" },
|
||||
]) {
|
||||
const started = performance.now();
|
||||
const response = await client.callTool({
|
||||
name: discovery.name,
|
||||
arguments: input,
|
||||
});
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(
|
||||
ajv.validate(discovery.outputSchema, response.structuredContent),
|
||||
JSON.stringify(ajv.errors),
|
||||
);
|
||||
const evidence = parseEvidence(response.structuredContent);
|
||||
const report = evidence.normalized_result;
|
||||
const kind = input.kind ?? "protocols";
|
||||
const query = (input.query ?? "").toLowerCase();
|
||||
const expectedMatches = nativeLines.flatMap((line, index) => {
|
||||
const columns = line.split("\t");
|
||||
if ((kind === "protocols") !== (columns[0] === "P")) return [];
|
||||
if (
|
||||
input.protocol !== undefined &&
|
||||
(columns[0] === "P" ? columns[2] : columns[4]) !== input.protocol
|
||||
)
|
||||
return [];
|
||||
if (
|
||||
![
|
||||
columns[1],
|
||||
columns[2],
|
||||
...(columns[0] === "F" ? [columns[7]] : []),
|
||||
].some((value) => value.toLowerCase().includes(query))
|
||||
)
|
||||
return [];
|
||||
return [{ sourceLine: index + 1, columns }];
|
||||
});
|
||||
const offset = input.offset ?? 0;
|
||||
const limit = input.limit === null ? undefined : (input.limit ?? 50);
|
||||
const expectedPage = expectedMatches.slice(
|
||||
offset,
|
||||
limit === undefined ? undefined : offset + limit,
|
||||
);
|
||||
assert.deepEqual(
|
||||
report.records.map((record) => record.source_line),
|
||||
expectedPage.map((record) => record.sourceLine),
|
||||
);
|
||||
assert.deepEqual(report.output_window, {
|
||||
examined: nativeLines.filter((line) =>
|
||||
line.startsWith(kind === "protocols" ? "P\t" : "F\t"),
|
||||
).length,
|
||||
matching: expectedMatches.length,
|
||||
returned: expectedPage.length,
|
||||
has_more:
|
||||
Math.max(0, expectedMatches.length - offset) > expectedPage.length,
|
||||
});
|
||||
discoveryMeasurements.push({
|
||||
input,
|
||||
elapsed_ms: +(performance.now() - started).toFixed(2),
|
||||
response_bytes: Buffer.byteLength(JSON.stringify(response)),
|
||||
estimated_tokens_one_tool_request_response: estimateTokenCount(
|
||||
JSON.stringify(discovery) +
|
||||
JSON.stringify(input) +
|
||||
JSON.stringify(response),
|
||||
),
|
||||
});
|
||||
for (const record of report.records) {
|
||||
const columns = nativeLines[record.source_line - 1].split("\t");
|
||||
assert.equal(record.name, columns[1]);
|
||||
assert.equal(record.abbreviation, columns[2]);
|
||||
assert.equal(record.field_type, columns[0] === "F" ? columns[3] : null);
|
||||
assert.equal(
|
||||
record.parent_protocol,
|
||||
columns[0] === "F" ? columns[4] : null,
|
||||
);
|
||||
assert.equal(record.description, columns[0] === "F" ? columns[7] : null);
|
||||
assert.equal(record.display_base, columns[0] === "F" ? columns[5] : null);
|
||||
assert.equal(record.bitmask, columns[0] === "F" ? columns[6] : null);
|
||||
}
|
||||
if (input.query === "idVendor")
|
||||
assert.ok(
|
||||
report.records.some(
|
||||
(row) =>
|
||||
row.abbreviation === "usb.idVendor" &&
|
||||
row.field_type === "FT_UINT16",
|
||||
),
|
||||
);
|
||||
if (input.query === "usb")
|
||||
assert.ok(report.records.some((row) => row.abbreviation === "usb"));
|
||||
if (input.query === "rea_no_such_field")
|
||||
assert.equal(report.output_window.matching, 0);
|
||||
const flags = Object.entries(input).flatMap(([key, value]) =>
|
||||
value === null ? ["--all"] : [`--${key}`, String(value)],
|
||||
);
|
||||
const cli = JSON.parse(
|
||||
(
|
||||
await execute(
|
||||
process.execPath,
|
||||
[entrypoint, "inspect-packet-fields", ...flags, "--format", "json"],
|
||||
{ env: environment, timeout: 60000, maxBuffer: 16 * 1024 * 1024 },
|
||||
)
|
||||
).stdout,
|
||||
);
|
||||
assert.deepEqual(cli.normalized_result, report);
|
||||
cases++;
|
||||
}
|
||||
await client.close();
|
||||
assert.deepEqual((await readdir(root.path)).sort(), [
|
||||
"bluetooth.pcap",
|
||||
@@ -283,6 +473,9 @@ console.log(
|
||||
{
|
||||
status: "passed",
|
||||
cases,
|
||||
discovery_measurements: discoveryMeasurements,
|
||||
measurement_scope:
|
||||
"Warm MCP server, fresh TShark processes; tokenx estimates include one advertised tool/request/response, not billed usage. Model-driven selection, CPU and peak process-family memory are unmeasured; no cache/speedup claim.",
|
||||
host: `${process.platform}/${process.arch}`,
|
||||
scope:
|
||||
"real TShark CLI/MCP; synthetic Ethernet/IPv4/UDP/DNS and Bluetooth HCI PCAP; no hardware acquisition",
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, readdir, rmdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { toolContract } from "../dist/contracts/toolContracts.js";
|
||||
import { hashStableFile } from "../dist/filesystem/StableFileHash.js";
|
||||
import { mcpTextValue } from "./lib/mcp-verifier-results.mjs";
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
if (args.includes("--help") || args.includes("-h")) {
|
||||
console.log(
|
||||
"Usage: npm run verify:pcap -- [--pcap PATH]\nRequires caller-supplied tshark/capinfos. Without --pcap, only client readiness is verified.",
|
||||
);
|
||||
process.exit(0);
|
||||
}
|
||||
if (!(args.length === 0 || (args.length === 2 && args[0] === "--pcap")))
|
||||
throw new Error("Expected --pcap PATH");
|
||||
const path = args[1] === undefined ? null : resolve(args[1]);
|
||||
// This parent is shared by independent CLI/MCP owners. A leased Windows
|
||||
// runtime root would prevent their native admission of its directory handle.
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-pcap-verifier-"));
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, value]) => typeof value === "string"),
|
||||
);
|
||||
Object.assign(environment, {
|
||||
TMPDIR: root,
|
||||
TMP: root,
|
||||
TEMP: root,
|
||||
});
|
||||
const entrypoint = resolve("scripts/rea.mjs");
|
||||
const execute = promisify(execFile);
|
||||
const client = new Client({ name: "pcap-real-verifier", version: "1" });
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.resume();
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const readiness = await run("inspect_pcap_client", {});
|
||||
assert(readiness.client.tshark_version && readiness.client.capinfos_version);
|
||||
if (path === null)
|
||||
console.log("[unverified] Capture inspection: no --pcap was supplied");
|
||||
else {
|
||||
const identity = await hashStableFile(path);
|
||||
const result = await run("inspect_pcap_capture", { path });
|
||||
assert.deepEqual(result.target, { path, ...identity });
|
||||
assert(result.packet_count >= 0);
|
||||
assert(
|
||||
result.endpoints.every(
|
||||
(entry) => entry.packet_count <= result.packet_count,
|
||||
),
|
||||
);
|
||||
assert(
|
||||
result.protocol_distribution.every(
|
||||
(entry) => entry.packet_count <= result.packet_count,
|
||||
),
|
||||
);
|
||||
const partial =
|
||||
result.endpoint_count > result.endpoints.length ||
|
||||
result.dns_query_count > result.dns_queries.length ||
|
||||
result.tls_sni_count > result.tls_sni_values.length ||
|
||||
result.protocol_count > result.protocol_distribution.length ||
|
||||
result.http_request_uri_count > result.http_request_uris.length ||
|
||||
result.unattributed_dns_packet_count > 0 ||
|
||||
result.dns_queries.some(
|
||||
(entry) =>
|
||||
entry.response_address_count > entry.response_addresses.length,
|
||||
);
|
||||
assert.equal(result.coverage, partial ? "partial" : "complete");
|
||||
assert.deepEqual(
|
||||
await hashStableFile(path),
|
||||
identity,
|
||||
"Input capture must remain unchanged",
|
||||
);
|
||||
console.log(
|
||||
`[ok] ${result.packet_count} packets; ${result.endpoint_count} addresses; coverage=${result.coverage}`,
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
assert.deepEqual(
|
||||
await readdir(root),
|
||||
[],
|
||||
`Owned capture resources remain under ${root}`,
|
||||
);
|
||||
await rmdir(root);
|
||||
}
|
||||
console.log("All asserted pcap CLI/MCP checks passed");
|
||||
|
||||
async function run(name, input) {
|
||||
const contract = toolContract(name);
|
||||
const { stdout } = await execute(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
name.replaceAll("_", "-"),
|
||||
...(input.path === undefined ? [] : [input.path]),
|
||||
"--json",
|
||||
],
|
||||
{ env: environment, timeout: 360_000, maxBuffer: 8 * 1024 * 1024 },
|
||||
);
|
||||
const cli = contract.outputSchema.parse(JSON.parse(stdout));
|
||||
const response = await client.callTool(
|
||||
{ name, arguments: input },
|
||||
{ timeout: 360_000 },
|
||||
);
|
||||
assert.notEqual(response.isError, true, mcpTextValue(response));
|
||||
const mcp = contract.outputSchema.parse(JSON.parse(mcpTextValue(response)));
|
||||
assert.deepEqual(mcp.normalized_result, cli.normalized_result);
|
||||
assert.deepEqual(mcp.subject, cli.subject);
|
||||
if (input.path !== undefined) {
|
||||
assert.equal(cli.subject.local_path, input.path);
|
||||
assert.equal(
|
||||
cli.subject.digest.sha256,
|
||||
cli.normalized_result.target.sha256,
|
||||
);
|
||||
}
|
||||
console.log(`[ok] ${name}: named output schema and CLI/MCP Evidence parity`);
|
||||
return cli.normalized_result;
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { isAbsolute } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { Ajv2020 } from "ajv/dist/2020.js";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
|
||||
// Caller-supplied producer evidence stays local; no fixture capture or secrets are bundled.
|
||||
const [controlPath, lossPath, reportPath] = process.argv.slice(2);
|
||||
if (
|
||||
![controlPath, lossPath, reportPath].every(
|
||||
(value) => typeof value === "string" && isAbsolute(value),
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
"Usage: node scripts/verify-qlog-captures.mjs /absolute/control/origin.qlog /absolute/loss/origin.qlog /absolute/new-report.json",
|
||||
);
|
||||
const entrypoint = fileURLToPath(new URL("./rea.mjs", import.meta.url));
|
||||
const client = new Client({ name: "qlog-producer-verifier", version: "1" });
|
||||
const environment = {
|
||||
PATH: process.env.PATH ?? "/usr/bin:/bin",
|
||||
REA_LOG_LEVEL: "silent",
|
||||
HOPPER_LAUNCHER_PATH: "/rea-unconfigured-deep-provider/hopper",
|
||||
};
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
let diagnostics = "";
|
||||
transport.stderr?.on("data", (bytes) => {
|
||||
if (diagnostics.length < 65536) diagnostics += bytes.toString("utf8");
|
||||
});
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const measurements = [];
|
||||
try {
|
||||
try {
|
||||
await client.connect(transport);
|
||||
} catch (cause) {
|
||||
throw new Error(`MCP startup failed: ${diagnostics}`, { cause });
|
||||
}
|
||||
const tool = (await client.listTools()).tools.find(
|
||||
(tool) => tool.name === "inspect_web_network_capture",
|
||||
);
|
||||
assert.ok(tool?.outputSchema);
|
||||
for (const schema of [tool.inputSchema, tool.outputSchema])
|
||||
assert.ok(ajv.validateSchema(schema));
|
||||
const validateOutput = ajv.compile(tool.outputSchema);
|
||||
for (const [lane, path] of [
|
||||
["control", controlPath],
|
||||
["loss", lossPath],
|
||||
]) {
|
||||
const source = await readFile(path);
|
||||
const raw = JSON.parse(source.toString("utf8"));
|
||||
const sha256 = createHash("sha256").update(source).digest("hex");
|
||||
assert.equal(raw.qlog_format, "JSON");
|
||||
assert.equal(raw.qlog_version, "0.3");
|
||||
const input = { capture_path: path, format: "qlog" };
|
||||
assert.ok(ajv.validate(tool.inputSchema, input));
|
||||
const response = await client.callTool({
|
||||
name: tool.name,
|
||||
arguments: input,
|
||||
});
|
||||
assert.notEqual(response.isError, true);
|
||||
assert.ok(
|
||||
validateOutput(response.structuredContent),
|
||||
JSON.stringify(validateOutput.errors),
|
||||
);
|
||||
const mcp = parseEvidence(response.structuredContent);
|
||||
const completed = await promisify(execFile)(
|
||||
process.execPath,
|
||||
[entrypoint, "inspect-web-network-capture", path, "qlog", "--json"],
|
||||
{ env: environment, timeout: 40_000, maxBuffer: 96 * 1024 * 1024 },
|
||||
);
|
||||
const cli = parseEvidence(JSON.parse(completed.stdout));
|
||||
assert.equal(cli.evidence_id, mcp.evidence_id);
|
||||
assert.deepEqual(cli.normalized_result, mcp.normalized_result);
|
||||
const capture = mcp.normalized_result;
|
||||
assert.equal(capture.artifact.sha256, sha256);
|
||||
assert.equal(
|
||||
capture.total_records,
|
||||
raw.traces.reduce((sum, trace) => sum + trace.events.length, 0),
|
||||
);
|
||||
for (const [ordinal, original] of raw.traces
|
||||
.flatMap((trace) => trace.events)
|
||||
.entries()) {
|
||||
const record = capture.records[ordinal];
|
||||
assert.equal(record.ordinal, ordinal);
|
||||
assert.equal(record.reported.name, original.name);
|
||||
assert.equal(record.reported.time, original.time);
|
||||
assert.deepEqual(record.binary_fields, []);
|
||||
assert.ok(
|
||||
record.numeric_literals.some(
|
||||
(number) =>
|
||||
number.pointer === "/time" &&
|
||||
Number(number.literal) === original.time,
|
||||
),
|
||||
);
|
||||
}
|
||||
assert.equal(capture.qlog.body_bytes, "not-reported");
|
||||
assert.equal(mcp.authority, "historical-reference");
|
||||
assert.equal(
|
||||
mcp.analysis_profile.parameters.qlog_profile,
|
||||
"aioquic-1.5.0-json-0.3",
|
||||
);
|
||||
const trace = capture.qlog.traces[0];
|
||||
assert.equal(trace.vantage_point, "server");
|
||||
const stream = trace.streams.find(
|
||||
(stream) => stream.stream_id === "0" && stream.direction === "sent",
|
||||
);
|
||||
assert.ok(stream);
|
||||
assert.equal(stream.unique_bytes, "32803");
|
||||
assert.equal(
|
||||
stream.frame_length_sum,
|
||||
lane === "control" ? "32803" : "33972",
|
||||
);
|
||||
assert.equal(stream.repeated_bytes, lane === "control" ? "0" : "1169");
|
||||
assert.equal(trace.loss_declarations.length, lane === "control" ? 0 : 1);
|
||||
if (lane === "loss")
|
||||
assert.deepEqual(trace.loss_declarations[0].packet_number, {
|
||||
state: "reported",
|
||||
value: "5",
|
||||
});
|
||||
// Structurally-known secrets may not survive in Evidence, including sidecars.
|
||||
const serialized = JSON.stringify(mcp);
|
||||
for (const original of raw.traces.flatMap((trace) => trace.events)) {
|
||||
if (
|
||||
original.name === "transport:parameters_set" &&
|
||||
typeof original.data.stateless_reset_token === "string"
|
||||
)
|
||||
assert.ok(!serialized.includes(original.data.stateless_reset_token));
|
||||
for (const frame of original.data.frames ?? []) {
|
||||
if (
|
||||
frame.frame_type === "new_connection_id" &&
|
||||
typeof frame.reset_token === "string"
|
||||
)
|
||||
assert.ok(!serialized.includes(frame.reset_token));
|
||||
if (frame.frame_type === "new_token" && typeof frame.token === "string")
|
||||
assert.ok(!serialized.includes(frame.token));
|
||||
}
|
||||
}
|
||||
const retained = await client.callTool({
|
||||
name: "inspect_analysis_view",
|
||||
arguments: {
|
||||
source: { kind: "retained-evidence", evidence_id: mcp.evidence_id },
|
||||
view: { kind: "api-surface" },
|
||||
},
|
||||
});
|
||||
assert.equal(retained.isError, true);
|
||||
assert.equal(
|
||||
JSON.parse(retained.content.find((item) => item.type === "text").text)
|
||||
.error.code,
|
||||
"unsupported_target",
|
||||
);
|
||||
const selected = await client.callTool({
|
||||
name: tool.name,
|
||||
arguments: { ...input, record_ordinals: [capture.total_records - 1, 0] },
|
||||
});
|
||||
const subset = parseEvidence(selected.structuredContent).normalized_result;
|
||||
assert.deepEqual(
|
||||
subset.records.map((record) => record.ordinal),
|
||||
[capture.total_records - 1, 0],
|
||||
);
|
||||
assert.deepEqual(subset.qlog, capture.qlog);
|
||||
measurements.push({
|
||||
lane,
|
||||
path,
|
||||
sha256,
|
||||
evidence_id: mcp.evidence_id,
|
||||
profile_digest: mcp.analysis_profile.digest,
|
||||
events: capture.total_records,
|
||||
stream_id: stream.stream_id,
|
||||
unique_bytes: stream.unique_bytes,
|
||||
frame_length_sum: stream.frame_length_sum,
|
||||
repeated_bytes: stream.repeated_bytes,
|
||||
loss_declarations: trace.loss_declarations.length,
|
||||
cli_mcp_parity: true,
|
||||
retained_reference: true,
|
||||
selected_scope_preserved: true,
|
||||
auth_exclusions: true,
|
||||
});
|
||||
}
|
||||
await client.ping();
|
||||
const report = {
|
||||
status: "passed",
|
||||
profile: "aioquic-1.5.0-json-0.3",
|
||||
measurements,
|
||||
limitations: [
|
||||
"Caller supplies provenance; format selection does not authenticate the producer.",
|
||||
"These known loopback cases do not establish WAN behavior, other qlog producers, platforms or general completeness.",
|
||||
"No PCAP or keylog is read by this verifier; repeated STREAM coverage is not proof of retransmission or network loss.",
|
||||
],
|
||||
};
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2) + "\n", {
|
||||
flag: "wx",
|
||||
mode: 0o600,
|
||||
});
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
status: report.status,
|
||||
cases: measurements.length,
|
||||
report: reportPath,
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
realpath,
|
||||
} from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { join, resolve, sep } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createHash } from "node:crypto";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
@@ -18,10 +18,6 @@ import {
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-"));
|
||||
@@ -32,6 +28,13 @@ assert(
|
||||
"Usage: node scripts/verify-qwen-client.mjs [chat|call]",
|
||||
);
|
||||
const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen";
|
||||
// npm exposes only qwen.cmd on Windows. Resolve a selected JavaScript entry
|
||||
// before the verifier switches to its disposable workspace.
|
||||
const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined;
|
||||
const launch = (args) =>
|
||||
entryPoint === undefined
|
||||
? [command, args]
|
||||
: [process.execPath, [entryPoint, ...args]];
|
||||
const verifier = createVerifierRun();
|
||||
const profile = join(lab, "selected profile");
|
||||
const account = join(lab, "account");
|
||||
@@ -54,14 +57,18 @@ const baseEnvironment = {
|
||||
let clientVersion;
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], {
|
||||
await exec(...launch(["--version"]), {
|
||||
env: baseEnvironment,
|
||||
timeout: 30_000,
|
||||
})
|
||||
).stdout.trim();
|
||||
} catch (cause) {
|
||||
const windowsHint =
|
||||
process.platform === "win32"
|
||||
? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js."
|
||||
: "";
|
||||
throw new Error(
|
||||
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`,
|
||||
`Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`,
|
||||
);
|
||||
}
|
||||
const original = JSON.stringify(
|
||||
@@ -93,8 +100,8 @@ for (const step of [
|
||||
assert(
|
||||
result.plannedActions.every(
|
||||
(a) =>
|
||||
a.target.startsWith(profile + "/") ||
|
||||
a.target.startsWith(sharedSkills + "/"),
|
||||
a.target.startsWith(profile + sep) ||
|
||||
a.target.startsWith(sharedSkills + sep),
|
||||
),
|
||||
);
|
||||
await writeFile(
|
||||
@@ -163,7 +170,14 @@ const fixture = await createOpenAiModelFixture({
|
||||
skillSeen ||= JSON.stringify(body.messages ?? []).includes(
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes(
|
||||
const messageText = (body.messages ?? [])
|
||||
.map((message) =>
|
||||
typeof message.content === "string"
|
||||
? message.content
|
||||
: (message.content ?? []).map((part) => part.text ?? "").join("\n"),
|
||||
)
|
||||
.join("\n");
|
||||
selectedSkillLoaded ||= messageText.includes(
|
||||
join(sharedSkills, "reverse-engineer-anything"),
|
||||
);
|
||||
const toolText = JSON.stringify(
|
||||
@@ -243,9 +257,7 @@ const fixture = await createOpenAiModelFixture({
|
||||
const invoke = tools.find((t) => t.name === "tool_call");
|
||||
const skill = tools.find((t) => t.name === "skill");
|
||||
const shell = tools.find((t) => t.name === "run_shell_command");
|
||||
const persisted = toolText.match(
|
||||
/Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u,
|
||||
);
|
||||
const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u);
|
||||
const call = (name, args) => ({
|
||||
role: "assistant",
|
||||
content: null,
|
||||
@@ -268,13 +280,14 @@ const fixture = await createOpenAiModelFixture({
|
||||
artifactPath = persisted[1];
|
||||
assert(
|
||||
(await realpath(artifactPath)).startsWith(
|
||||
(await realpath(runtime)) + "/",
|
||||
(await realpath(runtime)) + sep,
|
||||
),
|
||||
);
|
||||
const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`;
|
||||
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
||||
// Keep paths and JSON quotes out of shell syntax on Windows and POSIX.
|
||||
const encodedScript = Buffer.from(script).toString("base64");
|
||||
message = call(shell.name, {
|
||||
command: `node -e ${quote(script)}`,
|
||||
command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`,
|
||||
description:
|
||||
"Read the complete saved REA Evidence and verify the fixture facts.",
|
||||
});
|
||||
@@ -329,7 +342,7 @@ const args = [
|
||||
let response;
|
||||
let failure;
|
||||
try {
|
||||
response = await exec(command, args, {
|
||||
response = await exec(...launch(args), {
|
||||
env,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
|
||||
@@ -0,0 +1,479 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import {
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
readdir,
|
||||
rm,
|
||||
symlink,
|
||||
unlink,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2)
|
||||
throw new Error("Usage: node scripts/verify-real-ghidra-6502.mjs");
|
||||
if (process.env.GHIDRA_INSTALL_DIR === undefined)
|
||||
throw new Error(
|
||||
"Set GHIDRA_INSTALL_DIR to a bring-your-own Ghidra installation.",
|
||||
);
|
||||
const exec = promisify(execFile);
|
||||
const run = createVerifierRun();
|
||||
const root = fileURLToPath(new URL("..", import.meta.url));
|
||||
const ca65 = process.env.REA_CA65 ?? "ca65";
|
||||
const ld65 = process.env.REA_LD65 ?? "ld65";
|
||||
const toolchain = {};
|
||||
for (const [name, command] of Object.entries({ ca65, ld65 })) {
|
||||
const version = await exec(command, ["--version"], { timeout: 10000 });
|
||||
toolchain[name] = (version.stdout + version.stderr).trim();
|
||||
}
|
||||
await mkdir(join(root, "build"), { recursive: true });
|
||||
const workspace = await mkdtemp(join(root, "build", "rea-6502-proof-"));
|
||||
const runtime = join(workspace, "runtime");
|
||||
await mkdir(runtime);
|
||||
const path = join(workspace, "image.bin");
|
||||
const object = join(workspace, "image.o");
|
||||
const entrypoint = join(root, "scripts", "rea.mjs");
|
||||
const env = {
|
||||
...process.env,
|
||||
TMPDIR: runtime,
|
||||
TMP: runtime,
|
||||
TEMP: runtime,
|
||||
REA_LOG_LEVEL: "silent",
|
||||
REA_ANALYSIS_PROVIDER: "ghidra",
|
||||
REA_PROCESS_RUN_ID: run.run_id,
|
||||
};
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({ name: "rea-real-6502-proof", version: "1" });
|
||||
let stderr = "";
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
stderr = (stderr + chunk.toString()).slice(-65536);
|
||||
});
|
||||
let report;
|
||||
try {
|
||||
await exec(
|
||||
ca65,
|
||||
[join(root, "tests/conformance/ghidra/6502.s"), "-o", object],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
await exec(
|
||||
ld65,
|
||||
["-C", join(root, "tests/conformance/ghidra/6502.cfg"), "-o", path, object],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
const bytes = await readFile(path);
|
||||
// Independently known opcodes and link addresses from the source fixture.
|
||||
assert.equal(bytes.toString("hex"), "a907201080c90ad003a92a60a9006018690360");
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
await client.connect(transport);
|
||||
const opened = await call("open_binary", {
|
||||
path,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0x008001",
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
assert.equal(opened.architecture, "6502");
|
||||
assert.equal(opened.entry_point, "0x8001");
|
||||
const status = await call("binary_session");
|
||||
const image = evidence(await call("inspect_native_load_image"), sha256);
|
||||
assert.equal(
|
||||
image.normalized_result.status,
|
||||
"verified",
|
||||
JSON.stringify(image.normalized_result),
|
||||
);
|
||||
assert.equal(image.normalized_result.architecture, "6502");
|
||||
assert.equal(image.normalized_result.base_address, "0x8001");
|
||||
assert.equal(image.normalized_result.language_id, "6502:LE:16:default");
|
||||
assert.equal(image.normalized_result.compiler_spec_id, "default");
|
||||
assert.equal(image.normalized_result.file_bytes, bytes.length);
|
||||
const memory = evidence(
|
||||
await call("read_bytes", { address: "0x8001", length: bytes.length }),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(memory.normalized_result.bytes_hex, bytes.toString("hex"));
|
||||
const procedures = evidence(await call("list_procedures"), sha256);
|
||||
for (const address of ["0x8001", "0x8010"])
|
||||
assert.ok(
|
||||
procedures.normalized_result.some(
|
||||
(procedure) => procedure.address === address,
|
||||
),
|
||||
);
|
||||
const instruction = evidence(
|
||||
await call("inspect_native_instruction", { address: "0x8003" }),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(instruction.normalized_result.mnemonic.toUpperCase(), "JSR");
|
||||
assert.ok(
|
||||
instruction.normalized_result.references.some(
|
||||
(reference) => reference.call && reference.target_address === "0x8010",
|
||||
),
|
||||
);
|
||||
const pseudo = evidence(
|
||||
await call("procedure_pseudo_code", { procedure: "0x8010" }),
|
||||
sha256,
|
||||
);
|
||||
assert.match(pseudo.normalized_result, /\+\s*(?:3|0x3|'\\x03')/u);
|
||||
const filePseudo = evidence(
|
||||
await call("procedure_pseudo_code", { procedure: "file-offset:0xf" }),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(filePseudo.normalized_result, pseudo.normalized_result);
|
||||
assert.ok(
|
||||
filePseudo.locations.some(
|
||||
(location) => location.kind === "file-offset" && location.offset === 15,
|
||||
),
|
||||
);
|
||||
const rejected = await client.callTool({
|
||||
name: "open_binary",
|
||||
arguments: {
|
||||
path,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0xffff",
|
||||
},
|
||||
});
|
||||
assert.equal(rejected.isError, true);
|
||||
assert.match(JSON.stringify(rejected.content), /6502 address width/u);
|
||||
const preserved = await call("binary_session");
|
||||
assert.equal(preserved.analysis_run.run_id, status.analysis_run.run_id);
|
||||
assert.equal(preserved.entry_point, "0x8001");
|
||||
await call("close_binary");
|
||||
await client.ping();
|
||||
const { stdout } = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"inspect-native-load-image",
|
||||
path,
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
"0x8001",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const cli = parseEvidence(JSON.parse(stdout));
|
||||
assert.deepEqual(cli.normalized_result, image.normalized_result);
|
||||
assert.equal(cli.analysis_profile.digest, image.analysis_profile.digest);
|
||||
assert.equal(cli.subject.digest.sha256, sha256);
|
||||
const fileCli = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"decompile",
|
||||
path,
|
||||
"--file-offset=0xf",
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
"0x8001",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const fileCliEvidence = parseEvidence(JSON.parse(fileCli.stdout));
|
||||
assert.equal(fileCliEvidence.normalized_result, filePseudo.normalized_result);
|
||||
assert.deepEqual(fileCliEvidence.locations, filePseudo.locations);
|
||||
assert.equal(fileCliEvidence.subject.digest.sha256, sha256);
|
||||
assert.deepEqual(await readFile(path), bytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
const lowMemory = [];
|
||||
// BinaryLoader maps file bytes before trying processor-defined blocks. Prove
|
||||
// that overlapping zero-page/stack definitions never overlay the selected image.
|
||||
for (const [entry, fullAddressSpace] of [
|
||||
[0xff, false],
|
||||
[0, true],
|
||||
]) {
|
||||
const lowPath = join(workspace, `low-${entry}.bin`);
|
||||
await exec(
|
||||
ld65,
|
||||
[
|
||||
"-C",
|
||||
join(root, "tests/conformance/ghidra/6502.cfg"),
|
||||
"--start-addr",
|
||||
String(entry),
|
||||
"-o",
|
||||
lowPath,
|
||||
object,
|
||||
],
|
||||
{ timeout: 10000 },
|
||||
);
|
||||
const linked = await readFile(lowPath);
|
||||
const expectedCode = Buffer.from(
|
||||
"a907200000c90ad003a92a60a9006018690360",
|
||||
"hex",
|
||||
);
|
||||
expectedCode.writeUInt16LE(entry + 15, 3);
|
||||
assert.deepEqual(linked, expectedCode);
|
||||
const lowBytes = fullAddressSpace
|
||||
? Buffer.concat([linked, Buffer.alloc(65536 - linked.length)])
|
||||
: linked;
|
||||
await writeFile(lowPath, lowBytes);
|
||||
const lowSha256 = createHash("sha256").update(lowBytes).digest("hex");
|
||||
const lowEntry = `0x${entry.toString(16)}`;
|
||||
const lowOpened = await call("open_binary", {
|
||||
path: lowPath,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: lowEntry,
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
assert.equal(lowOpened.entry_point, lowEntry);
|
||||
const lowImage = evidence(
|
||||
await call("inspect_native_load_image"),
|
||||
lowSha256,
|
||||
);
|
||||
assert.equal(
|
||||
lowImage.normalized_result.status,
|
||||
"verified",
|
||||
JSON.stringify(lowImage.normalized_result),
|
||||
);
|
||||
assert.equal(lowImage.normalized_result.base_address, lowEntry);
|
||||
assert.equal(lowImage.normalized_result.file_bytes, lowBytes.length);
|
||||
const fileMappings =
|
||||
lowImage.normalized_result.observations.mappings.filter(
|
||||
(mapping) => mapping.source_file_index !== null,
|
||||
);
|
||||
assert.equal(fileMappings.length, 1);
|
||||
assert.equal(
|
||||
fileMappings[0].address_space,
|
||||
lowImage.normalized_result.observations.default_address_space,
|
||||
);
|
||||
assert.equal(fileMappings[0].overlay, false);
|
||||
assert.equal(fileMappings[0].start, lowEntry);
|
||||
assert.equal(fileMappings[0].length, lowBytes.length);
|
||||
const lowRead = evidence(
|
||||
await call("read_bytes", { address: lowEntry, length: linked.length }),
|
||||
lowSha256,
|
||||
);
|
||||
assert.equal(lowRead.normalized_result.bytes_hex, linked.toString("hex"));
|
||||
const lowCall = evidence(
|
||||
await call("inspect_native_instruction", {
|
||||
address: `0x${(entry + 2).toString(16)}`,
|
||||
}),
|
||||
lowSha256,
|
||||
);
|
||||
assert.equal(lowCall.normalized_result.mnemonic.toUpperCase(), "JSR");
|
||||
assert.ok(
|
||||
lowCall.normalized_result.references.some(
|
||||
(reference) =>
|
||||
reference.call &&
|
||||
reference.target_address === `0x${(entry + 15).toString(16)}`,
|
||||
),
|
||||
);
|
||||
await call("close_binary");
|
||||
const lowCli = await exec(
|
||||
process.execPath,
|
||||
[
|
||||
entrypoint,
|
||||
"inspect-native-load-image",
|
||||
lowPath,
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
lowEntry,
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--json",
|
||||
],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const lowCliEvidence = evidence(JSON.parse(lowCli.stdout), lowSha256);
|
||||
assert.deepEqual(
|
||||
lowCliEvidence.normalized_result,
|
||||
lowImage.normalized_result,
|
||||
);
|
||||
assert.equal(
|
||||
lowCliEvidence.analysis_profile.digest,
|
||||
lowImage.analysis_profile.digest,
|
||||
);
|
||||
assert.deepEqual(await readFile(lowPath), lowBytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
lowMemory.push({
|
||||
entry: lowEntry,
|
||||
bytes: lowBytes.length,
|
||||
sha256: lowSha256,
|
||||
});
|
||||
}
|
||||
let windowsReparseAdmission = "not-applicable";
|
||||
if (process.platform === "win32") {
|
||||
const junction = join(workspace, "linked");
|
||||
await symlink(workspace, junction, "junction");
|
||||
try {
|
||||
const linkedPath = join(junction, "image.bin");
|
||||
// Reopening the same bytes through a junction must replace the direct session.
|
||||
await call("open_binary", {
|
||||
path,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0x8001",
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
const directImage = evidence(
|
||||
await call("inspect_native_load_image"),
|
||||
sha256,
|
||||
);
|
||||
assert.equal(directImage.normalized_result.status, "verified");
|
||||
// Canonical file identity must not bypass admission of the new path.
|
||||
await call("open_binary", {
|
||||
path: linkedPath,
|
||||
format: "raw-binary",
|
||||
architecture: "6502",
|
||||
entry_point: "0x8001",
|
||||
provider_id: "ghidra",
|
||||
});
|
||||
// open_binary selects a lazy session; the first analysis admits the snapshot.
|
||||
const linkedImage = await client.callTool(
|
||||
{ name: "inspect_native_load_image", arguments: {} },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.equal(linkedImage.isError, true);
|
||||
assert.match(
|
||||
JSON.stringify(linkedImage),
|
||||
/Reparse path component is unsupported/u,
|
||||
);
|
||||
await call("close_binary");
|
||||
const snapshotPath = join(workspace, "direct-snapshot.json");
|
||||
const cliArgs = (selectedPath) => [
|
||||
entrypoint,
|
||||
"analyze",
|
||||
selectedPath,
|
||||
"--target-format",
|
||||
"raw-binary",
|
||||
"--architecture",
|
||||
"6502",
|
||||
"--entry-point",
|
||||
"0x8001",
|
||||
"--provider",
|
||||
"ghidra",
|
||||
"--snapshot",
|
||||
snapshotPath,
|
||||
"--json",
|
||||
];
|
||||
const saved = await exec(process.execPath, cliArgs(path), {
|
||||
env,
|
||||
timeout: 240000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
});
|
||||
assert.equal(
|
||||
evidence(JSON.parse(saved.stdout), sha256).operation,
|
||||
"binary_overview",
|
||||
);
|
||||
const savedSnapshot = JSON.parse(await readFile(snapshotPath, "utf8"));
|
||||
assert.ok(
|
||||
savedSnapshot.workflow_entries.some(
|
||||
({ operation }) => operation === "binary_overview",
|
||||
),
|
||||
);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
// Replaying a matching snapshot must still reach native path admission.
|
||||
await assert.rejects(
|
||||
exec(process.execPath, cliArgs(linkedPath), {
|
||||
env,
|
||||
timeout: 240000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
}),
|
||||
(cause) => {
|
||||
assert.match(
|
||||
cause.stdout + cause.stderr,
|
||||
/Reparse path component is unsupported/u,
|
||||
);
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.deepEqual(await readFile(path), bytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
windowsReparseAdmission = "cli-snapshot-mcp-rejected";
|
||||
} finally {
|
||||
await unlink(junction);
|
||||
}
|
||||
}
|
||||
report = {
|
||||
ok: true,
|
||||
mocked: false,
|
||||
lane: "ghidra-6502",
|
||||
host: process.platform,
|
||||
toolchain,
|
||||
provider: image.provider,
|
||||
target: {
|
||||
sha256,
|
||||
bytes: bytes.length,
|
||||
architecture: "6502",
|
||||
entry: "0x8001",
|
||||
},
|
||||
cli_mcp_load_image_parity: true,
|
||||
instructions_and_call_reference: true,
|
||||
decompilation: true,
|
||||
file_offset_decompilation_cli_mcp: true,
|
||||
invalid_mapping_preserves_session: true,
|
||||
low_memory_cli_mcp_parity: lowMemory,
|
||||
windows_reparse_admission: windowsReparseAdmission,
|
||||
source_unchanged: true,
|
||||
runtime_cleanup: "complete",
|
||||
};
|
||||
} catch (cause) {
|
||||
if (stderr) process.stderr.write(stderr);
|
||||
throw cause;
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
const verifierRun = await completeVerifierRun(run);
|
||||
if (process.platform !== "win32") {
|
||||
assert.equal(verifierRun.process_lineage.status, "verified");
|
||||
assert.deepEqual(verifierRun.process_lineage.descendants, []);
|
||||
}
|
||||
// Keep the workspace for diagnosis unless verification and cleanup both pass.
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
process.stdout.write(
|
||||
JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n",
|
||||
);
|
||||
|
||||
async function call(name, args = {}) {
|
||||
const result = await client.callTool(
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
if (["open_binary", "binary_session", "close_binary"].includes(name))
|
||||
return requireMcpOperationResult(result, name);
|
||||
return result.structuredContent;
|
||||
}
|
||||
|
||||
function evidence(value, sha256) {
|
||||
const result = parseEvidence(value);
|
||||
assert.equal(result.subject.digest.sha256, sha256);
|
||||
assert.equal(result.subject.architecture, "6502");
|
||||
assert.equal(result.subject.format, "raw-binary");
|
||||
return result;
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { parseEvidence } from "../dist/domain/evidence.js";
|
||||
import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
if (process.argv.length !== 2 || process.platform !== "linux")
|
||||
throw new Error("Run this ELF verifier on Linux without arguments.");
|
||||
if (!process.env.GHIDRA_INSTALL_DIR)
|
||||
throw new Error("Set GHIDRA_INSTALL_DIR to a caller-installed Ghidra.");
|
||||
const exec = promisify(execFile);
|
||||
const run = createVerifierRun();
|
||||
const root = fileURLToPath(new URL("..", import.meta.url));
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-address-context-"));
|
||||
const runtime = join(workspace, "runtime");
|
||||
await mkdir(runtime);
|
||||
const path = join(workspace, "pie.elf");
|
||||
const entrypoint = join(root, "scripts/rea.mjs");
|
||||
const env = {
|
||||
...process.env,
|
||||
TMPDIR: runtime,
|
||||
REA_ANALYSIS_PROVIDER: "ghidra",
|
||||
REA_LOG_LEVEL: "silent",
|
||||
};
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({ name: "rea-real-address-context", version: "1" });
|
||||
let stderr = "";
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
stderr = (stderr + chunk.toString()).slice(-65536);
|
||||
});
|
||||
let report;
|
||||
let connected = false;
|
||||
let primaryFailure;
|
||||
let verifierRun;
|
||||
try {
|
||||
const compiler = process.env.CC ?? "gcc";
|
||||
const version = await exec(compiler, ["--version"], { timeout: 10000 });
|
||||
await exec(
|
||||
compiler,
|
||||
[
|
||||
"-O0",
|
||||
"-g",
|
||||
"-fPIE",
|
||||
"-pie",
|
||||
"-nostdlib",
|
||||
"-Wl,-e,rea_pie_entry",
|
||||
"-Wl,--build-id=none",
|
||||
join(root, "tests/conformance/c/pie-address-context.c"),
|
||||
"-o",
|
||||
path,
|
||||
],
|
||||
{ timeout: 30000 },
|
||||
);
|
||||
const bytes = await readFile(path);
|
||||
assert.equal(bytes.subarray(0, 6).toString("hex"), "7f454c460201");
|
||||
assert.equal(bytes.readUInt16LE(16), 3); // ET_DYN: source-owned x86-64 PIE.
|
||||
assert.equal(bytes.readUInt16LE(18), 62);
|
||||
const declaredEntry = `0x${bytes.readBigUInt64LE(24).toString(16)}`;
|
||||
const sha256 = createHash("sha256").update(bytes).digest("hex");
|
||||
await client.connect(transport);
|
||||
connected = true;
|
||||
await call("open_binary", { path, provider_id: "ghidra" });
|
||||
const overview = parseEvidence(await call("binary_overview"));
|
||||
assert.equal(overview.subject.digest.sha256, sha256);
|
||||
const base = overview.normalized_result.image_base;
|
||||
assert.match(base, /^0x[0-9a-f]+$/u);
|
||||
assert.notEqual(base, "0x0");
|
||||
const active = await call("binary_session");
|
||||
const image = parseEvidence(await call("inspect_native_load_image"));
|
||||
assert.equal(base, image.normalized_result.observations.image_base);
|
||||
assert.ok(overview.normalized_result.segments.length > 0);
|
||||
assert.ok(
|
||||
overview.normalized_result.segments.every(
|
||||
(segment) =>
|
||||
segment.image_base === base &&
|
||||
typeof segment.address_space === "string",
|
||||
),
|
||||
);
|
||||
const procedures = parseEvidence(await call("list_procedures"));
|
||||
const entry = procedures.normalized_result.find(
|
||||
(item) => item.value === "rea_pie_entry",
|
||||
);
|
||||
const addition = procedures.normalized_result.find(
|
||||
(item) => item.value === "rea_pie_add",
|
||||
);
|
||||
assert.ok(entry && addition);
|
||||
assert.notEqual(entry.address, declaredEntry);
|
||||
assert.ok(
|
||||
image.normalized_result.observations.entry_points.includes(entry.address),
|
||||
);
|
||||
const mapping = image.normalized_result.observations.mappings.find(
|
||||
(item) =>
|
||||
item.source_file_index === 0 &&
|
||||
item.file_offset !== null &&
|
||||
item.address_space === "ram" &&
|
||||
BigInt(item.start) <= BigInt(entry.address) &&
|
||||
BigInt(item.end) >= BigInt(entry.address),
|
||||
);
|
||||
assert.ok(mapping);
|
||||
const offset =
|
||||
mapping.file_offset + Number(BigInt(entry.address) - BigInt(mapping.start));
|
||||
const reverseMapping = parseEvidence(
|
||||
await call("address_to_file_offset", { address: entry.address }),
|
||||
);
|
||||
assert.equal(reverseMapping.normalized_result.file_offset, offset);
|
||||
const memory = parseEvidence(
|
||||
await call("read_bytes", { address: entry.address, length: 8 }),
|
||||
);
|
||||
assert.equal(
|
||||
memory.normalized_result.bytes_hex,
|
||||
bytes.subarray(offset, offset + 8).toString("hex"),
|
||||
);
|
||||
const fromOffset = parseEvidence(
|
||||
await call("procedure_pseudo_code", {
|
||||
procedure: `file-offset:0x${offset.toString(16)}`,
|
||||
}),
|
||||
);
|
||||
assert.ok(
|
||||
fromOffset.locations.some(
|
||||
(location) =>
|
||||
location.kind === "file-offset" && location.offset === offset,
|
||||
),
|
||||
);
|
||||
assert.equal(fromOffset.raw_result.selection.mapped_address, entry.address);
|
||||
const fromNumericOffset = parseEvidence(
|
||||
await call("procedure_pseudo_code", {
|
||||
procedure: `0x${offset.toString(16)}`,
|
||||
}),
|
||||
);
|
||||
assert.equal(
|
||||
fromNumericOffset.normalized_result,
|
||||
fromOffset.normalized_result,
|
||||
);
|
||||
const missingAddress = "ram:0xffffffff";
|
||||
const rejected = await client.callTool(
|
||||
{ name: "procedure_pseudo_code", arguments: { procedure: missingAddress } },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.equal(rejected.isError, true);
|
||||
assert.ok(
|
||||
JSON.stringify(rejected.content).includes(`Ghidra image base: ${base}`),
|
||||
);
|
||||
assert.ok(JSON.stringify(rejected.content).includes("list_procedures"));
|
||||
assert.ok(
|
||||
JSON.stringify(rejected.content).includes("inspect_native_load_image"),
|
||||
);
|
||||
const pseudo = parseEvidence(
|
||||
await call("procedure_pseudo_code", { procedure: addition.address }),
|
||||
);
|
||||
assert.match(pseudo.normalized_result, /\+\s*(?:7|0x7)\b/u);
|
||||
const retained = await call("binary_session");
|
||||
assert.equal(retained.analysis_run.run_id, active.analysis_run.run_id);
|
||||
await call("close_binary");
|
||||
const cli = async (args) =>
|
||||
exec(
|
||||
process.execPath,
|
||||
[entrypoint, ...args, "--provider", "ghidra", "--json"],
|
||||
{ env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 },
|
||||
);
|
||||
const cliOverview = parseEvidence(
|
||||
JSON.parse((await cli(["analyze", path])).stdout),
|
||||
);
|
||||
assert.deepEqual(cliOverview.normalized_result, overview.normalized_result);
|
||||
let cliFailure;
|
||||
try {
|
||||
await cli(["decompile", path, missingAddress]);
|
||||
} catch (cause) {
|
||||
cliFailure = cause;
|
||||
}
|
||||
assert.ok(cliFailure && typeof cliFailure.stdout === "string");
|
||||
const error = JSON.parse(cliFailure.stdout);
|
||||
assert.equal(error.code, "invalid_request");
|
||||
assert.ok(JSON.stringify(error).includes(`Ghidra image base: ${base}`));
|
||||
for (const command of [
|
||||
"rea search <path> . --kind procedures --mode regex",
|
||||
"rea analyze <path>",
|
||||
"rea inspect-native-load-image <path>",
|
||||
])
|
||||
assert.ok(JSON.stringify(error).includes(command));
|
||||
const cliProcedures = parseEvidence(
|
||||
JSON.parse(
|
||||
(
|
||||
await cli([
|
||||
"search",
|
||||
path,
|
||||
".",
|
||||
"--kind",
|
||||
"procedures",
|
||||
"--mode",
|
||||
"regex",
|
||||
])
|
||||
).stdout,
|
||||
),
|
||||
);
|
||||
const cliEntry = cliProcedures.normalized_result.find(
|
||||
(item) => item.value === "rea_pie_entry",
|
||||
);
|
||||
assert.ok(cliEntry);
|
||||
assert.equal(cliEntry.address, entry.address);
|
||||
const cliImage = parseEvidence(
|
||||
JSON.parse((await cli(["inspect-native-load-image", path])).stdout),
|
||||
);
|
||||
assert.equal(cliImage.normalized_result.observations.image_base, base);
|
||||
const cliPseudo = parseEvidence(
|
||||
JSON.parse((await cli(["decompile", path, cliEntry.address])).stdout),
|
||||
);
|
||||
const cliOffset = parseEvidence(
|
||||
JSON.parse(
|
||||
(await cli(["decompile", path, `--file-offset=0x${offset.toString(16)}`]))
|
||||
.stdout,
|
||||
),
|
||||
);
|
||||
assert.equal(cliOffset.normalized_result, fromOffset.normalized_result);
|
||||
assert.deepEqual(cliOffset.locations, fromOffset.locations);
|
||||
assert.match(cliPseudo.normalized_result, /rea_pie_add/u);
|
||||
assert.deepEqual(await readFile(path), bytes);
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
report = {
|
||||
ok: true,
|
||||
mocked: false,
|
||||
provider: image.provider,
|
||||
compiler: version.stdout.split("\n")[0],
|
||||
sha256,
|
||||
declared_elf_entry: declaredEntry,
|
||||
loaded_entry: entry.address,
|
||||
observed_image_base: base,
|
||||
cli_mcp_overview_parity: true,
|
||||
failed_lookup_guidance: true,
|
||||
cli_recovery_commands: true,
|
||||
loaded_decompilation: true,
|
||||
file_offset_decompilation: true,
|
||||
address_file_offset_round_trip: true,
|
||||
source_unchanged: true,
|
||||
runtime_cleanup: "complete",
|
||||
};
|
||||
} catch (cause) {
|
||||
primaryFailure = { cause };
|
||||
if (stderr) process.stderr.write(stderr);
|
||||
}
|
||||
|
||||
const cleanupFailures = [];
|
||||
// Await provider-owned cleanup before the SDK can terminate its MCP parent.
|
||||
if (connected) {
|
||||
try {
|
||||
await call("close_binary");
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
}
|
||||
try {
|
||||
await client.close();
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
try {
|
||||
await transport.close();
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
try {
|
||||
assert.deepEqual(await readdir(runtime), []);
|
||||
verifierRun = await completeVerifierRun(run);
|
||||
assert.equal(verifierRun.process_lineage.status, "verified");
|
||||
assert.deepEqual(verifierRun.process_lineage.descendants, []);
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
if (cleanupFailures.length === 0) {
|
||||
try {
|
||||
await rm(workspace, { recursive: true, force: true });
|
||||
} catch (cause) {
|
||||
cleanupFailures.push(cause);
|
||||
}
|
||||
}
|
||||
if (cleanupFailures.length > 0) {
|
||||
process.stderr.write(
|
||||
`Retained verifier workspace after cleanup failure: ${workspace}\n`,
|
||||
);
|
||||
throw new AggregateError(
|
||||
primaryFailure === undefined
|
||||
? cleanupFailures
|
||||
: [primaryFailure.cause, ...cleanupFailures],
|
||||
"Ghidra address-context verifier cleanup failed",
|
||||
);
|
||||
}
|
||||
if (primaryFailure !== undefined) throw primaryFailure.cause;
|
||||
process.stdout.write(
|
||||
JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n",
|
||||
);
|
||||
|
||||
async function call(name, args = {}) {
|
||||
const result = await client.callTool(
|
||||
{ name, arguments: args },
|
||||
{ timeout: 240000 },
|
||||
);
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
return ["open_binary", "binary_session", "close_binary"].includes(name)
|
||||
? requireMcpOperationResult(result, name)
|
||||
: result.structuredContent;
|
||||
}
|
||||
@@ -45,8 +45,13 @@ const transport = new StdioClientTransport({
|
||||
const source = fileURLToPath(
|
||||
new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url),
|
||||
);
|
||||
const literalSource = fileURLToPath(
|
||||
new URL("../tests/conformance/swift/string-literals.swift", import.meta.url),
|
||||
);
|
||||
const binary = join(root, "swift-inventory");
|
||||
const literalBinary = join(root, "swift-literals");
|
||||
const reports = [];
|
||||
let literalReport;
|
||||
try {
|
||||
await execute("swiftc", ["-O", "-g", source, "-o", binary], {
|
||||
timeout: 60_000,
|
||||
@@ -54,6 +59,19 @@ try {
|
||||
await mkdir(join(root, "copied"));
|
||||
const copied = join(root, "copied", "swift-inventory");
|
||||
await copyFile(binary, copied);
|
||||
// Decoding covers AArch64 register pairs, so build that slice on any Mac host.
|
||||
await execute(
|
||||
"swiftc",
|
||||
[
|
||||
"-O",
|
||||
"-target",
|
||||
"arm64-apple-macosx13.0",
|
||||
literalSource,
|
||||
"-o",
|
||||
literalBinary,
|
||||
],
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
await client.connect(transport);
|
||||
for (const [variant, path] of [
|
||||
["debug", binary],
|
||||
@@ -132,6 +150,7 @@ try {
|
||||
await call("close_binary", {});
|
||||
}
|
||||
}
|
||||
literalReport = await verifyStringLiterals();
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
@@ -143,6 +162,7 @@ console.log(
|
||||
ok: true,
|
||||
provider: "ghidra",
|
||||
fixtures: reports,
|
||||
swift_string_literals: literalReport,
|
||||
verifier_run: await completeVerifierRun(run),
|
||||
},
|
||||
null,
|
||||
@@ -159,3 +179,51 @@ async function call(name, arguments_) {
|
||||
toolContract(name).outputSchema.parse(response.structuredContent);
|
||||
return requireMcpOperationResult(response, name);
|
||||
}
|
||||
|
||||
// Literal text lives only in register constants, so it must come from the decoded pairs.
|
||||
async function verifyStringLiterals() {
|
||||
await call("open_binary", { path: literalBinary, provider_id: "ghidra" });
|
||||
try {
|
||||
const procedures = await call("list_procedures", {});
|
||||
const expected = {
|
||||
smallLiteral: [["SleepDisabled", "small"]],
|
||||
largeLiteral: [["The power setting could not be changed.", "large"]],
|
||||
unicodeLiterals: [
|
||||
["café", "small"],
|
||||
["Résumé of the naïve décor, written twice", "large"],
|
||||
],
|
||||
emptyAndControlLiterals: [
|
||||
["", "small"],
|
||||
["tab\there", "small"],
|
||||
],
|
||||
returnedLiteral: [["Holding awake", "small"]],
|
||||
};
|
||||
const decoded = {};
|
||||
for (const [name, literals] of Object.entries(expected)) {
|
||||
const procedure = procedures.find(({ value }) => value.includes(name));
|
||||
assert(procedure, `${name} is missing from the procedure inventory`);
|
||||
const dossier = await call("analyze_function", {
|
||||
procedure: procedure.address,
|
||||
});
|
||||
const observed = dossier.swift_string_literals;
|
||||
assert(Array.isArray(observed), `${name} recorded no Swift literals`);
|
||||
assert.deepEqual(dossier.swift_string_literal_omissions, []);
|
||||
assert.deepEqual(
|
||||
observed.map(({ value, representation }) => [value, representation]),
|
||||
literals,
|
||||
);
|
||||
for (const literal of observed) {
|
||||
assert.deepEqual(literal.registers, ["x0", "x1"]);
|
||||
assert.equal(
|
||||
literal.address === null,
|
||||
literal.representation === "small",
|
||||
`${literal.value} storage does not match its representation`,
|
||||
);
|
||||
}
|
||||
decoded[name] = observed.length;
|
||||
}
|
||||
return decoded;
|
||||
} finally {
|
||||
await call("close_binary", {});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs";
|
||||
import { verifyGhidraFileOffsets } from "./lib/real-ghidra-file-offsets.mjs";
|
||||
|
||||
const verifierRun = createVerifierRun();
|
||||
const SESSION_CAPABILITIES = GHIDRA_SESSION_CAPABILITIES.filter(
|
||||
@@ -300,6 +301,11 @@ try {
|
||||
}
|
||||
if (report === undefined)
|
||||
throw new Error("Windows Ghidra report was not produced");
|
||||
report.file_offsets = await verifyGhidraFileOffsets(
|
||||
targetPath,
|
||||
resolve(root, "scripts", "rea.mjs"),
|
||||
architecture === "x86" ? "0x401010" : "0x140001010",
|
||||
);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), ...report })}\n`,
|
||||
);
|
||||
|
||||
@@ -11,7 +11,8 @@ createVerifierRun();
|
||||
|
||||
if (process.platform !== "linux")
|
||||
throw new Error("Linux real-Hopper verification must run on Linux");
|
||||
const launcher = process.env.HOPPER_LAUNCHER_PATH ?? "/opt/hopper/bin/Hopper";
|
||||
const configuredLauncher = process.env.HOPPER_LAUNCHER_PATH;
|
||||
const launcher = configuredLauncher ?? "/opt/hopper/bin/Hopper";
|
||||
await access(launcher);
|
||||
await access("/usr/bin/Xvfb");
|
||||
await access("/usr/bin/xauth");
|
||||
@@ -28,5 +29,7 @@ try {
|
||||
process.env.HOPPER_LAUNCHER_PATH = alternateLauncher;
|
||||
await import("./verify-real-hopper.mjs");
|
||||
} finally {
|
||||
if (configuredLauncher === undefined) delete process.env.HOPPER_LAUNCHER_PATH;
|
||||
else process.env.HOPPER_LAUNCHER_PATH = configuredLauncher;
|
||||
await rm(alternateDirectory, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -47,6 +47,7 @@ import {
|
||||
} from "./lib/real-hopper-boundaries.mjs";
|
||||
import { startUnrelatedHopperSentinel } from "./lib/unrelated-hopper-sentinel.mjs";
|
||||
import { snapshotHopperRuntime } from "./lib/real-hopper-cleanup.mjs";
|
||||
import { verifyLinuxHopperOwnerExit } from "./lib/real-hopper-owner-exit.mjs";
|
||||
const execFileAsync = promisify(execFile);
|
||||
const verifierRun = createVerifierRun();
|
||||
const timeout = 180_000;
|
||||
@@ -577,6 +578,14 @@ if (sessionsAfter.length > 0) {
|
||||
}
|
||||
if (summary === undefined)
|
||||
throw new Error("Real-Hopper verification did not produce a summary");
|
||||
if (process.platform === "linux")
|
||||
summary.ownerExit = await verifyLinuxHopperOwnerExit({
|
||||
primary: targetA,
|
||||
secondary: targetB,
|
||||
serverCommand,
|
||||
serverArgs,
|
||||
environment: serverEnvironment,
|
||||
});
|
||||
const completedVerifierRun = await completeVerifierRun(verifierRun);
|
||||
if (
|
||||
completedVerifierRun.process_lineage.status !== "verified" ||
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
#!/usr/bin/env node
|
||||
import {
|
||||
assertFeatureTrace,
|
||||
findMappedNonFunction,
|
||||
} from "./lib/ida-feature-trace.mjs";
|
||||
import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs";
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
@@ -6,7 +10,7 @@ import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { access, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { basename, dirname, join, resolve } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { parseArgs, promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
@@ -261,6 +265,61 @@ const mcp = async (cliDossier) => {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (config.mode === "headless") {
|
||||
const documents = observe(await call("list_documents", {}));
|
||||
assert.equal(documents.length, 1);
|
||||
assert.equal(basename(documents[0]), basename(target));
|
||||
assert.ok(
|
||||
!workspaces.has(dirname(documents[0])),
|
||||
"list_documents reported the private workspace copy.",
|
||||
);
|
||||
const resolveAt = async (address) =>
|
||||
observe(await call("resolve_containing_procedure", { address }));
|
||||
const instructionAddresses = dossier.assembly
|
||||
.map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1])
|
||||
.filter((address) => address !== undefined);
|
||||
const queries = [
|
||||
dossier.procedure.address,
|
||||
...instructionAddresses.slice(1, 2),
|
||||
];
|
||||
for (const queried of queries) {
|
||||
const containing = await resolveAt(queried);
|
||||
assert.equal(containing.found, true);
|
||||
assert.equal(containing.procedure.address, dossier.procedure.address);
|
||||
assert.equal(containing.procedure.name, dossier.procedure.name);
|
||||
}
|
||||
const unmapped = await resolveAt("0xffffffffffffff00");
|
||||
assert.equal(unmapped.found, false);
|
||||
assert.equal(unmapped.reason, "outside_segments");
|
||||
const strings = observe(await call("list_strings", {}));
|
||||
const mappedString = await findMappedNonFunction(strings, resolveAt);
|
||||
if (mappedString === undefined)
|
||||
report.unverified.push(
|
||||
"Mapped non-function containment: no string of the target is mapped outside every function.",
|
||||
);
|
||||
const context = await client.callTool({
|
||||
name: "inspect_address_context",
|
||||
arguments: { address: dossier.procedure.address },
|
||||
});
|
||||
assert.notEqual(context.isError, true);
|
||||
const contextResult = context.structuredContent.result;
|
||||
assert.equal(contextResult.document, documents[0]);
|
||||
assert.equal(contextResult.procedure.state, "available");
|
||||
assert.equal(
|
||||
contextResult.procedure.value.procedure.address,
|
||||
dossier.procedure.address,
|
||||
);
|
||||
const traced = await client.callTool({
|
||||
name: "trace_feature",
|
||||
arguments: { query: dossier.procedure.name },
|
||||
});
|
||||
assert.notEqual(traced.isError, true);
|
||||
const trace = traced.structuredContent.normalized_result;
|
||||
assertFeatureTrace(trace, dossier.procedure);
|
||||
report.checks.push(
|
||||
"headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing",
|
||||
);
|
||||
}
|
||||
if (values["expected-string"] === undefined)
|
||||
report.unverified.push(
|
||||
"String inventory/search correctness: no independent expected literal supplied.",
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { estimateTokenCount } from "tokenx";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile, readdir } from "node:fs/promises";
|
||||
@@ -68,6 +69,20 @@ for (const byte of Buffer.from("REA")) {
|
||||
lines.push(`#${tick + 1000}`, "1!");
|
||||
const source = lines.join("\n") + "\n";
|
||||
let cases = 0;
|
||||
const discoveryMeasurements = [];
|
||||
const inheritedAddressSpace = (await readFile("/proc/self/limits", "utf8"))
|
||||
.split("\n")
|
||||
.find((line) => line.startsWith("Max address space "))
|
||||
?.trim()
|
||||
.match(/^Max address space\s+(\S+)\s+(\S+)\s+bytes$/u);
|
||||
assert.ok(inheritedAddressSpace);
|
||||
const expectedAddressSpace = Math.min(
|
||||
1073741824,
|
||||
...inheritedAddressSpace
|
||||
.slice(1)
|
||||
.map((value) => (value === "unlimited" ? Infinity : Number(value))),
|
||||
);
|
||||
|
||||
try {
|
||||
await writeFile(path, source);
|
||||
await client.connect(transport);
|
||||
@@ -104,14 +119,16 @@ try {
|
||||
createHash("sha256").update(source).digest("hex"),
|
||||
);
|
||||
assert.equal(report.hardware_acquisition, "not-performed");
|
||||
assert.equal(report.limits.address_space_bytes, 1073741824);
|
||||
assert.equal(report.limits.address_space_bytes, expectedAddressSpace);
|
||||
assert.equal(report.tool.configured_path, process.env.REA_SIGROK_COMMAND);
|
||||
assert.equal(
|
||||
report.resource_limiter.configured_path,
|
||||
process.env.REA_SIGROK_PRLIMIT_COMMAND ?? "/usr/bin/prlimit",
|
||||
);
|
||||
assert.ok(
|
||||
report.resource_limiter.arguments.includes("--as=1073741824:1073741824"),
|
||||
report.resource_limiter.arguments.includes(
|
||||
`--as=${expectedAddressSpace}:${expectedAddressSpace}`,
|
||||
),
|
||||
);
|
||||
assert.equal(report.diagnostics, "");
|
||||
const flags = Object.entries(input)
|
||||
@@ -245,6 +262,88 @@ try {
|
||||
});
|
||||
assert.equal(invalid.isError, true);
|
||||
cases++;
|
||||
const discoveryTool = (await client.listTools()).tools.find(
|
||||
({ name }) => name === "inspect_signal_decoders",
|
||||
);
|
||||
assert.ok(discoveryTool?.outputSchema);
|
||||
assert.ok(ajv.validateSchema(discoveryTool.inputSchema));
|
||||
assert.ok(ajv.validateSchema(discoveryTool.outputSchema));
|
||||
const nativeHelp = (
|
||||
await execute(
|
||||
process.env.REA_SIGROK_COMMAND,
|
||||
["--dont-scan", "--protocol-decoders=uart", "--show"],
|
||||
{ env: environment, timeout: 30000, maxBuffer: 4 * 1024 * 1024 },
|
||||
)
|
||||
).stdout.split(/\r?\n/);
|
||||
if (nativeHelp.at(-1) === "") nativeHelp.pop();
|
||||
for (const input of [
|
||||
{ query: "uart" },
|
||||
{ limit: 1 },
|
||||
{ offset: 1, limit: 2 },
|
||||
{ decoder: "uart", query: "BaUd" },
|
||||
{ decoder: "uart", offset: 50, limit: null },
|
||||
{ query: "rea_no_such_decoder" },
|
||||
]) {
|
||||
const started = performance.now();
|
||||
const response = await client.callTool({
|
||||
name: discoveryTool.name,
|
||||
arguments: input,
|
||||
});
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(
|
||||
ajv.validate(discoveryTool.outputSchema, response.structuredContent),
|
||||
JSON.stringify(ajv.errors),
|
||||
);
|
||||
const evidence = parseEvidence(response.structuredContent);
|
||||
const report = evidence.normalized_result;
|
||||
discoveryMeasurements.push({
|
||||
input,
|
||||
elapsed_ms: +(performance.now() - started).toFixed(2),
|
||||
response_bytes: Buffer.byteLength(JSON.stringify(response)),
|
||||
estimated_tokens_one_tool_request_response: estimateTokenCount(
|
||||
JSON.stringify(discoveryTool) +
|
||||
JSON.stringify(input) +
|
||||
JSON.stringify(response),
|
||||
),
|
||||
});
|
||||
if (input.decoder) {
|
||||
const matched = nativeHelp
|
||||
.map((text, i) => ({ line_number: i + 1, text }))
|
||||
.filter(({ text }) =>
|
||||
text.toLowerCase().includes((input.query ?? "").toLowerCase()),
|
||||
);
|
||||
assert.deepEqual(
|
||||
report.help_lines,
|
||||
matched.slice(
|
||||
input.offset ?? 0,
|
||||
input.limit === null
|
||||
? undefined
|
||||
: (input.offset ?? 0) + (input.limit ?? 50),
|
||||
),
|
||||
);
|
||||
} else if (input.query === "uart")
|
||||
assert.ok(report.decoders.some(({ id }) => id === "uart"));
|
||||
const flags = Object.entries(input).flatMap(([key, value]) =>
|
||||
value === null ? ["--all"] : [`--${key}`, String(value)],
|
||||
);
|
||||
const cli = JSON.parse(
|
||||
(
|
||||
await execute(
|
||||
process.execPath,
|
||||
[entrypoint, "inspect-signal-decoders", ...flags, "--format", "json"],
|
||||
{ env: environment, timeout: 60000, maxBuffer: 8 * 1024 * 1024 },
|
||||
)
|
||||
).stdout,
|
||||
);
|
||||
assert.deepEqual(cli.normalized_result, report);
|
||||
cases++;
|
||||
}
|
||||
const unavailable = await client.callTool({
|
||||
name: discoveryTool.name,
|
||||
arguments: { decoder: "rea_no_such_decoder" },
|
||||
});
|
||||
assert.equal(unavailable.isError, true);
|
||||
cases++;
|
||||
await client.close();
|
||||
assert.deepEqual((await readdir(root.path)).sort(), [
|
||||
"paged.vcd",
|
||||
@@ -260,6 +359,9 @@ console.log(
|
||||
{
|
||||
status: "passed",
|
||||
cases,
|
||||
discovery_measurements: discoveryMeasurements,
|
||||
measurement_scope:
|
||||
"Warm MCP server, fresh sigrok processes; tokenx estimates include one advertised tool/request/response, not billed model usage. No model-driven selection benchmark, CPU or peak process-family memory measurement; no cache or speedup claim.",
|
||||
host: `${process.platform}/${process.arch}`,
|
||||
scope:
|
||||
"real sigrok UART VCD through compiled CLI/MCP; no physical acquisition",
|
||||
|
||||
@@ -27,9 +27,9 @@ if (
|
||||
);
|
||||
const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length;
|
||||
if (
|
||||
bytes.toString("ascii", 0, 2) !== "MZ" ||
|
||||
bytes.toString("latin1", 0, 2) !== "MZ" ||
|
||||
offset > bytes.length - 26 ||
|
||||
bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" ||
|
||||
bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" ||
|
||||
bytes.readUInt16LE(offset + 4) !== 0x8664 ||
|
||||
bytes.readUInt16LE(offset + 24) !== 0x20b
|
||||
)
|
||||
|
||||
@@ -11,6 +11,7 @@ import {
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rename,
|
||||
readdir,
|
||||
rm,
|
||||
symlink,
|
||||
writeFile,
|
||||
@@ -37,6 +38,14 @@ const { WindowsPrivateRuntime, windowsPrivateRuntime } = await import(
|
||||
const { ghidraSessionRoot } = await import(
|
||||
pathToFileURL(join(packageRoot, "dist/ghidra/GhidraSessionRoot.js"))
|
||||
);
|
||||
const { PrivateRuntimeRoot } = await import(
|
||||
pathToFileURL(join(packageRoot, "dist/process/PrivateRuntimeRoot.js"))
|
||||
);
|
||||
const { PrivateRuntimeRootUnavailableError } = await import(
|
||||
pathToFileURL(
|
||||
join(packageRoot, "dist/process/PrivateRuntimeRootUnavailableError.js"),
|
||||
)
|
||||
);
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-native-conformance-"));
|
||||
const environment = Object.entries(process.env)
|
||||
.filter(([, value]) => value !== undefined)
|
||||
@@ -140,6 +149,30 @@ process.stdout.write(JSON.stringify({executable:basename(process.execPath),abiVe
|
||||
])
|
||||
assert.throws(() => native.call("open", [rejected]));
|
||||
report.controls.ordinaryDriveSeparatorsAndRequestedIdentity = true;
|
||||
// An owned runtime's DELETE lease conflicts with a fresh parent admission.
|
||||
// This produces a genuine sharing failure after successful authority caching,
|
||||
// without another executable, and proves that no child runtime was created.
|
||||
const sharingParent = WindowsPrivateRuntime.create(
|
||||
workspace,
|
||||
"rea-sharing-parent-",
|
||||
);
|
||||
try {
|
||||
await assert.rejects(
|
||||
PrivateRuntimeRoot.create({ parent: sharingParent.observation.path }),
|
||||
(failure) => {
|
||||
assert.ok(failure instanceof PrivateRuntimeRootUnavailableError);
|
||||
assert.equal(failure.retryable, true);
|
||||
assert.ok(failure.cause instanceof Error);
|
||||
assert.equal(failure.cause.win32Code, 32);
|
||||
assert.equal(failure.cause.runtimeAllocation, "not-started");
|
||||
return true;
|
||||
},
|
||||
);
|
||||
assert.deepEqual(await readdir(sharingParent.observation.path), []);
|
||||
report.controls.transientRuntimeAdmissionBeforeAllocation = true;
|
||||
} finally {
|
||||
await sharingParent.close();
|
||||
}
|
||||
const driveRoot = parse(workspace).root;
|
||||
const runtimeParent = ghidraSessionRoot({
|
||||
base: join(driveRoot, ".Tmp", "scratch"),
|
||||
@@ -629,6 +662,42 @@ setInterval(()=>{},1000);\n`,
|
||||
unicodeNamesPreserved: true,
|
||||
});
|
||||
report.processes.windowsEnvironmentSemantics = true;
|
||||
// Observed through this process's own id: a separate probe process would
|
||||
// get its own processor group, so it cannot stand in for the process that
|
||||
// spawns the child.
|
||||
const callerAffinity = JSON.parse(
|
||||
execFileSync(boundaryFixture, ["affinity", String(process.pid)], {
|
||||
encoding: "utf8",
|
||||
}),
|
||||
);
|
||||
const affinityChild = native.call("process_spawn", [
|
||||
boundaryFixture,
|
||||
`${quote(boundaryFixture)} affinity`,
|
||||
workspace,
|
||||
environment,
|
||||
]);
|
||||
const observedAffinity = await wait(affinityChild);
|
||||
native.call("process_close", [affinityChild.handle]);
|
||||
assert.equal(observedAffinity.exitCode, 0);
|
||||
const childAffinity = JSON.parse(observedAffinity.output);
|
||||
// Masks cross as hexadecimal strings because a mask can exceed the
|
||||
// safe-integer range. Sharing one processor group makes the caller's mask
|
||||
// applicable, so it must be inherited exactly. Windows placing the child in
|
||||
// another group is only permitted when the caller asked for no bound of its
|
||||
// own, which its group's whole active set establishes; a narrower mask has
|
||||
// to fail the spawn rather than widen to that group's default.
|
||||
const sharedGroup =
|
||||
callerAffinity.singleProcessorGroup &&
|
||||
childAffinity.singleProcessorGroup &&
|
||||
callerAffinity.processorGroup === childAffinity.processorGroup;
|
||||
if (sharedGroup)
|
||||
assert.equal(childAffinity.affinityMask, callerAffinity.affinityMask);
|
||||
else assert.equal(callerAffinity.affinityMask, callerAffinity.systemMask);
|
||||
report.processes.callerAffinityBoundsOwnedProcessOrRefusesToWiden = true;
|
||||
report.callerAffinity = {
|
||||
...callerAffinity,
|
||||
sharedGroupWithChild: sharedGroup,
|
||||
};
|
||||
}
|
||||
const ownerFixture = join(workspace, "owner.mjs");
|
||||
const ownerPidFile = join(workspace, "crash-child.pid");
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import {
|
||||
access,
|
||||
mkdir,
|
||||
mkdtemp,
|
||||
readFile,
|
||||
rm,
|
||||
symlink,
|
||||
} from "node:fs/promises";
|
||||
import * as fs from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import * as nodePath from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
@@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna
|
||||
import { parseEvidence } from "../../../dist/domain/evidence.js";
|
||||
import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const entrypoint = resolve(
|
||||
const execFileAsync = promisify(execFile);
|
||||
const { join, resolve, dirname } = nodePath;
|
||||
|
||||
const CALL_TIMEOUT_MS = 150_000;
|
||||
const CONNECT_TIMEOUT_MS = 30_000;
|
||||
const SEVENZIP_TIMEOUT_MS = 10_000;
|
||||
const MAX_STDOUT_BYTES = 16 * 1024 * 1024;
|
||||
|
||||
const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
|
||||
const ENTRYPOINT = resolve(
|
||||
process.env.REA_FIRMWARE_TEST_ENTRYPOINT ??
|
||||
join(repository, "scripts/rea.mjs"),
|
||||
join(REPO_ROOT, "scripts/rea.mjs"),
|
||||
);
|
||||
if (process.platform !== "linux")
|
||||
throw new Error(
|
||||
"verify:firmware requires Linux; other hosts are not verified",
|
||||
);
|
||||
for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
|
||||
const path = process.env[variable];
|
||||
if (path === undefined)
|
||||
throw new Error(
|
||||
`verify:firmware requires ${variable}; see docs/firmware-analysis.md`,
|
||||
);
|
||||
await access(path);
|
||||
}
|
||||
try {
|
||||
await exec("7z", ["i"], { timeout: 10_000 });
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
const fixtureRoot = resolve(
|
||||
const FIXTURE_ROOT = resolve(
|
||||
process.env.REA_FIRMWARE_FIXTURE_ROOT ??
|
||||
join(repository, "_reference/firmware-integration/generated"),
|
||||
join(REPO_ROOT, "_reference/firmware-integration/generated"),
|
||||
);
|
||||
await access(join(fixtureRoot, "oracle.json")).catch((cause) => {
|
||||
throw new Error(
|
||||
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
|
||||
{ cause },
|
||||
);
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Prerequisite checks
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function checkPrerequisites() {
|
||||
if (process.platform !== "linux") {
|
||||
throw new Error(
|
||||
"verify:firmware requires Linux; other hosts are not verified",
|
||||
);
|
||||
}
|
||||
|
||||
for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) {
|
||||
const value = process.env[name];
|
||||
if (value === undefined) {
|
||||
throw new Error(
|
||||
`verify:firmware requires ${name}; see docs/firmware-analysis.md`,
|
||||
);
|
||||
}
|
||||
await fs.access(value);
|
||||
}
|
||||
|
||||
try {
|
||||
await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS });
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
"verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
await fs.access(join(FIXTURE_ROOT, "oracle.json"));
|
||||
} catch (cause) {
|
||||
throw new Error(
|
||||
"verify:firmware requires generated fixtures; run npm run fixtures:firmware",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await checkPrerequisites();
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared state
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const oracle = JSON.parse(
|
||||
await readFile(join(fixtureRoot, "oracle.json"), "utf8"),
|
||||
await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"),
|
||||
);
|
||||
const path = join(fixtureRoot, "firmware.bin");
|
||||
assert.equal(await hashFirmwareFile(path), oracle.sha256);
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-"));
|
||||
const environment = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, value]) => value !== undefined),
|
||||
const firmwarePath = join(FIXTURE_ROOT, "firmware.bin");
|
||||
assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256);
|
||||
|
||||
const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-"));
|
||||
const baseEnv = Object.fromEntries(
|
||||
Object.entries(process.env).filter(([, v]) => v !== undefined),
|
||||
);
|
||||
const client = new Client({ name: "rea-real-firmware-verifier", version: "1" });
|
||||
const cli = async (args, env = environment) => {
|
||||
const response = await exec(
|
||||
const mcpClient = new Client({
|
||||
name: "rea-real-firmware-verifier",
|
||||
version: "1",
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Invocation helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function runCli(args, env = baseEnv) {
|
||||
const { stdout } = await execFileAsync(
|
||||
process.execPath,
|
||||
[entrypoint, ...args, "--format", "json"],
|
||||
[ENTRYPOINT, ...args, "--format", "json"],
|
||||
{
|
||||
env,
|
||||
cwd: repository,
|
||||
timeout: 150_000,
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
cwd: REPO_ROOT,
|
||||
timeout: CALL_TIMEOUT_MS,
|
||||
maxBuffer: MAX_STDOUT_BYTES,
|
||||
},
|
||||
);
|
||||
return parseEvidence(JSON.parse(response.stdout));
|
||||
};
|
||||
const call = async (name, args) => {
|
||||
const response = await client.callTool(
|
||||
return parseEvidence(JSON.parse(stdout));
|
||||
}
|
||||
|
||||
async function callTool(name, args) {
|
||||
const result = await mcpClient.callTool(
|
||||
{ name, arguments: args },
|
||||
{ timeout: 150_000 },
|
||||
{ timeout: CALL_TIMEOUT_MS },
|
||||
);
|
||||
assert.notEqual(response.isError, true, JSON.stringify(response));
|
||||
assert.ok(response.structuredContent !== undefined);
|
||||
return response.structuredContent;
|
||||
};
|
||||
const verifyRegions = (evidence) => {
|
||||
assert.notEqual(result.isError, true, JSON.stringify(result));
|
||||
assert.ok(result.structuredContent !== undefined);
|
||||
return result.structuredContent;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Result verifiers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function verifyRegions(evidence) {
|
||||
assert.equal(evidence.subject.digest.sha256, oracle.sha256);
|
||||
const result = firmwareResultSchemas.inspect_firmware_regions.parse(
|
||||
|
||||
const parsed = firmwareResultSchemas.inspect_firmware_regions.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
const gzip = result.regions.find(
|
||||
(region) => region.signature === "gzip" && region.offset === oracle.offset,
|
||||
|
||||
const gzipRegion = parsed.regions.find(
|
||||
(r) => r.signature === "gzip" && r.offset === oracle.offset,
|
||||
);
|
||||
assert.ok(
|
||||
gzip !== undefined,
|
||||
gzipRegion !== undefined,
|
||||
"Binwalk must find the independently generated gzip offset",
|
||||
);
|
||||
assert.equal(gzip.reported_size, oracle.length);
|
||||
assert.equal(gzip.size_basis, "provider_reported_validation_unknown");
|
||||
return result.regions.map(
|
||||
({ provider_id: _provider_id, ...region }) => region,
|
||||
);
|
||||
};
|
||||
const verifyFiles = async (evidence) => {
|
||||
const result = firmwareResultSchemas.extract_firmware.parse(
|
||||
assert.equal(gzipRegion.reported_size, oracle.length);
|
||||
assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown");
|
||||
|
||||
// Strip provider-specific ids so CLI and MCP results compare equal.
|
||||
return parsed.regions.map((region) => {
|
||||
const { provider_id: _ignored, ...rest } = region;
|
||||
return rest;
|
||||
});
|
||||
}
|
||||
|
||||
async function verifyFiles(evidence) {
|
||||
const parsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip"));
|
||||
|
||||
assert.ok(parsed.chunks.some((c) => c.handler === "gzip"));
|
||||
|
||||
for (const [suffix, expected] of Object.entries(oracle.files)) {
|
||||
const file = result.files.find((candidate) =>
|
||||
candidate.relative_path.endsWith(`/${suffix}`),
|
||||
const entry = parsed.files.find((f) =>
|
||||
f.relative_path.endsWith(`/${suffix}`),
|
||||
);
|
||||
assert.ok(file !== undefined, `Missing extracted ${suffix}`);
|
||||
assert.equal(file.sha256, expected.sha256);
|
||||
assert.equal(file.size, expected.size);
|
||||
assert.equal(await hashFirmwareFile(file.path), expected.sha256);
|
||||
assert.equal(file.original_file_range, null);
|
||||
assert.equal(file.runtime_address, null);
|
||||
assert.ok(entry !== undefined, `Missing extracted ${suffix}`);
|
||||
assert.equal(entry.sha256, expected.sha256);
|
||||
assert.equal(entry.size, expected.size);
|
||||
assert.equal(await hashFirmwareFile(entry.path), expected.sha256);
|
||||
assert.equal(entry.original_file_range, null);
|
||||
assert.equal(entry.runtime_address, null);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
try {
|
||||
const inspected = await cli(["inspect-firmware-regions", path]);
|
||||
const cliRegions = verifyRegions(inspected);
|
||||
const extracted = await cli([
|
||||
"extract-firmware",
|
||||
path,
|
||||
join(root, "cli-output"),
|
||||
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const toContentIdentity = (parsed) =>
|
||||
parsed.files.map(({ relative_path, sha256, size }) => ({
|
||||
relative_path,
|
||||
sha256,
|
||||
size,
|
||||
}));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Verification stages
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function stageCli() {
|
||||
const regionsEvidence = await runCli([
|
||||
"inspect-firmware-regions",
|
||||
firmwarePath,
|
||||
]);
|
||||
const cliFiles = await verifyFiles(extracted);
|
||||
assert.equal(cliFiles.coverage, "partial");
|
||||
const regions = verifyRegions(regionsEvidence);
|
||||
|
||||
const extractEvidence = await runCli([
|
||||
"extract-firmware",
|
||||
firmwarePath,
|
||||
join(workDir, "cli-output"),
|
||||
]);
|
||||
const files = await verifyFiles(extractEvidence);
|
||||
|
||||
assert.equal(files.coverage, "partial");
|
||||
assert.ok(
|
||||
cliFiles.chunks.some(
|
||||
(chunk) =>
|
||||
chunk.handler === null &&
|
||||
chunk.range.offset === 0 &&
|
||||
chunk.range.length === 64,
|
||||
files.chunks.some(
|
||||
(c) =>
|
||||
c.handler === null && c.range.offset === 0 && c.range.length === 64,
|
||||
),
|
||||
);
|
||||
await client.connect(
|
||||
new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [entrypoint, "mcp"],
|
||||
cwd: repository,
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
}),
|
||||
{ timeout: 30_000 },
|
||||
);
|
||||
const inspectedMcp = await call("inspect_firmware_regions", { path });
|
||||
const mcpEvidence = parseEvidence(inspectedMcp);
|
||||
assert.deepEqual(verifyRegions(mcpEvidence), cliRegions);
|
||||
const extractedMcp = await call("extract_firmware", {
|
||||
path,
|
||||
output_directory: join(root, "mcp-output"),
|
||||
|
||||
return { regions, files };
|
||||
}
|
||||
|
||||
async function connectMcp() {
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [ENTRYPOINT, "mcp"],
|
||||
cwd: REPO_ROOT,
|
||||
env: baseEnv,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const mcpFiles = await verifyFiles(parseEvidence(extractedMcp));
|
||||
const contentIdentity = (result) =>
|
||||
result.files.map(({ relative_path, sha256, size }) => ({
|
||||
relative_path,
|
||||
sha256,
|
||||
size,
|
||||
}));
|
||||
assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles));
|
||||
const selected = await call("extract_firmware", {
|
||||
path,
|
||||
output_directory: join(root, "selected-output"),
|
||||
await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS });
|
||||
}
|
||||
|
||||
async function stageMcp(cli) {
|
||||
const regionsRaw = await callTool("inspect_firmware_regions", {
|
||||
path: firmwarePath,
|
||||
});
|
||||
assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions);
|
||||
|
||||
const extractRaw = await callTool("extract_firmware", {
|
||||
path: firmwarePath,
|
||||
output_directory: join(workDir, "mcp-output"),
|
||||
});
|
||||
const mcpFiles = await verifyFiles(parseEvidence(extractRaw));
|
||||
assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files));
|
||||
|
||||
const rangedRaw = await callTool("extract_firmware", {
|
||||
path: firmwarePath,
|
||||
output_directory: join(workDir, "selected-output"),
|
||||
range: { offset: oracle.offset, length: oracle.length },
|
||||
});
|
||||
const selectedFiles = await verifyFiles(parseEvidence(selected));
|
||||
assert.equal(selectedFiles.selection.offset, oracle.offset);
|
||||
assert.equal(selectedFiles.selection.length, oracle.length);
|
||||
const depth = await call("extract_firmware", {
|
||||
path,
|
||||
output_directory: join(root, "depth-output"),
|
||||
const ranged = await verifyFiles(parseEvidence(rangedRaw));
|
||||
assert.equal(ranged.selection.offset, oracle.offset);
|
||||
assert.equal(ranged.selection.length, oracle.length);
|
||||
|
||||
const depthRaw = await callTool("extract_firmware", {
|
||||
path: firmwarePath,
|
||||
output_directory: join(workDir, "depth-output"),
|
||||
max_depth: 1,
|
||||
});
|
||||
const depthResult = firmwareResultSchemas.extract_firmware.parse(
|
||||
depth.normalized_result,
|
||||
const depthParsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
depthRaw.normalized_result,
|
||||
);
|
||||
assert.equal(depthResult.coverage, "partial");
|
||||
assert.ok(depthResult.depth_limited_paths.length > 0);
|
||||
assert.equal(depthParsed.coverage, "partial");
|
||||
assert.ok(depthParsed.depth_limited_paths.length > 0);
|
||||
|
||||
console.log(
|
||||
"PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits",
|
||||
);
|
||||
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") {
|
||||
const ext4 = join(fixtureRoot, "ext4.bin");
|
||||
await access(ext4);
|
||||
const response = await call("extract_firmware", {
|
||||
path: ext4,
|
||||
output_directory: join(root, "ext4-output"),
|
||||
});
|
||||
const result = firmwareResultSchemas.extract_firmware.parse(
|
||||
response.normalized_result,
|
||||
|
||||
return mcpFiles;
|
||||
}
|
||||
|
||||
async function stageExt4() {
|
||||
const ext4Path = join(FIXTURE_ROOT, "ext4.bin");
|
||||
await fs.access(ext4Path);
|
||||
|
||||
const raw = await callTool("extract_firmware", {
|
||||
path: ext4Path,
|
||||
output_directory: join(workDir, "ext4-output"),
|
||||
});
|
||||
const parsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
raw.normalized_result,
|
||||
);
|
||||
assert.ok(parsed.chunks.some((c) => c.handler === "extfs"));
|
||||
assert.equal(parsed.coverage, "complete");
|
||||
|
||||
for (const [suffix, expected] of Object.entries(oracle.files)) {
|
||||
const match = parsed.files.find((f) =>
|
||||
f.relative_path.endsWith(`/${suffix}`),
|
||||
);
|
||||
assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs"));
|
||||
assert.equal(result.coverage, "complete");
|
||||
for (const [suffix, expected] of Object.entries(oracle.files))
|
||||
assert.equal(
|
||||
result.files.find((file) => file.relative_path.endsWith(`/${suffix}`))
|
||||
?.sha256,
|
||||
expected.sha256,
|
||||
);
|
||||
console.log("PASS real ext4 extraction with debugfs; no mounts");
|
||||
// Preserve the process-ownership utility while withholding all extractors.
|
||||
const withoutExtractors = join(root, "without-extractors");
|
||||
await access("/bin/ps");
|
||||
await mkdir(withoutExtractors);
|
||||
await symlink("/bin/ps", join(withoutExtractors, "ps"));
|
||||
const missing = await cli(
|
||||
["extract-firmware", ext4, join(root, "missing-extractor-output")],
|
||||
{ ...environment, PATH: withoutExtractors },
|
||||
);
|
||||
const partial = firmwareResultSchemas.extract_firmware.parse(
|
||||
missing.normalized_result,
|
||||
);
|
||||
assert.equal(partial.coverage, "partial");
|
||||
assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs"));
|
||||
console.log(
|
||||
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
|
||||
);
|
||||
assert.equal(match?.sha256, expected.sha256);
|
||||
}
|
||||
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") {
|
||||
if (process.env.GHIDRA_INSTALL_DIR === undefined)
|
||||
throw new Error(
|
||||
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
|
||||
);
|
||||
const child = mcpFiles.files.find((file) =>
|
||||
file.relative_path.endsWith("/usr/bin/probe"),
|
||||
);
|
||||
assert.ok(child !== undefined);
|
||||
await call("open_binary", { path: child.path, provider_id: "ghidra" });
|
||||
const dossier = await call("analyze_function", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
const evidence = parseEvidence(dossier);
|
||||
assert.equal(
|
||||
evidence.subject.digest.sha256,
|
||||
oracle.files["usr/bin/probe"].sha256,
|
||||
);
|
||||
const pseudocode = await call("procedure_pseudo_code", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
assert.equal(parseEvidence(pseudocode).provider.id, "ghidra");
|
||||
assert.ok(
|
||||
JSON.stringify(evidence.normalized_result).includes("firmware_probe"),
|
||||
);
|
||||
await call("close_binary", {});
|
||||
console.log(
|
||||
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
|
||||
console.log("PASS real ext4 extraction with debugfs; no mounts");
|
||||
|
||||
// Keep the process-ownership utility available but hide every extractor.
|
||||
const emptyBin = join(workDir, "without-extractors");
|
||||
await fs.access("/bin/ps");
|
||||
await fs.mkdir(emptyBin);
|
||||
await fs.symlink("/bin/ps", join(emptyBin, "ps"));
|
||||
|
||||
const degraded = await runCli(
|
||||
["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")],
|
||||
{ ...baseEnv, PATH: emptyBin },
|
||||
);
|
||||
const degradedParsed = firmwareResultSchemas.extract_firmware.parse(
|
||||
degraded.normalized_result,
|
||||
);
|
||||
assert.equal(degradedParsed.coverage, "partial");
|
||||
assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs"));
|
||||
console.log(
|
||||
"PASS real missing-extractor diagnostics retained despite Unblob exit 1",
|
||||
);
|
||||
}
|
||||
|
||||
async function stageGhidra(mcpFiles) {
|
||||
if (process.env.GHIDRA_INSTALL_DIR === undefined) {
|
||||
throw new Error(
|
||||
"Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md",
|
||||
);
|
||||
}
|
||||
|
||||
const probe = mcpFiles.files.find((f) =>
|
||||
f.relative_path.endsWith("/usr/bin/probe"),
|
||||
);
|
||||
assert.ok(probe !== undefined);
|
||||
|
||||
await callTool("open_binary", { path: probe.path, provider_id: "ghidra" });
|
||||
|
||||
const dossierRaw = await callTool("analyze_function", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
const dossier = parseEvidence(dossierRaw);
|
||||
assert.equal(
|
||||
await hashFirmwareFile(path),
|
||||
dossier.subject.digest.sha256,
|
||||
oracle.files["usr/bin/probe"].sha256,
|
||||
);
|
||||
|
||||
const pseudoRaw = await callTool("procedure_pseudo_code", {
|
||||
procedure: "firmware_probe",
|
||||
});
|
||||
assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra");
|
||||
assert.ok(
|
||||
JSON.stringify(dossier.normalized_result).includes("firmware_probe"),
|
||||
);
|
||||
|
||||
await callTool("close_binary", {});
|
||||
console.log(
|
||||
"PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest",
|
||||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Entry point
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
try {
|
||||
const cli = await stageCli();
|
||||
await connectMcp();
|
||||
const mcpFiles = await stageMcp(cli);
|
||||
|
||||
if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4();
|
||||
if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1")
|
||||
await stageGhidra(mcpFiles);
|
||||
|
||||
assert.equal(
|
||||
await hashFirmwareFile(firmwarePath),
|
||||
oracle.sha256,
|
||||
"Original firmware bytes must remain unchanged",
|
||||
);
|
||||
} finally {
|
||||
await client.close();
|
||||
await rm(root, { recursive: true, force: true });
|
||||
await mcpClient.close();
|
||||
await fs.rm(workDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -153,10 +153,36 @@ try {
|
||||
});
|
||||
try {
|
||||
await client.connect(transport, { timeout: 30_000 });
|
||||
await client.listTools();
|
||||
const opened = await client.callTool(
|
||||
{ name: "open_binary", arguments: { path } },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(opened.isError, true, JSON.stringify(opened));
|
||||
assert.equal(opened.structuredContent?.result?.format, "hap");
|
||||
assert.equal(opened.structuredContent?.result?.sha256, fixture.sha256);
|
||||
const status = await client.callTool(
|
||||
{ name: "binary_session", arguments: {} },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(status.isError, true, JSON.stringify(status));
|
||||
assert.equal(status.structuredContent?.result?.open, true);
|
||||
assert.equal(status.structuredContent?.result?.format, "hap");
|
||||
const inspected = await client.callTool(
|
||||
{ name: "inspect_artifact", arguments: {} },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(inspected.isError, true, JSON.stringify(inspected));
|
||||
const mcpInspection = parseEvidence(inspected.structuredContent);
|
||||
assert.deepEqual(
|
||||
mcpInspection.normalized_result,
|
||||
inspection.normalized_result,
|
||||
);
|
||||
console.log("PASS MCP open/status/inspect_artifact parity");
|
||||
const response = await client.callTool(
|
||||
{
|
||||
name: "project_harmony_application_graph",
|
||||
arguments: { inventory_evidence: [inspection] },
|
||||
arguments: { inventory_evidence: [mcpInspection] },
|
||||
},
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
@@ -168,6 +194,23 @@ try {
|
||||
projectionEvidence.normalized_result,
|
||||
);
|
||||
console.log("PASS MCP/CLI parity project_harmony_application_graph");
|
||||
const closed = await client.callTool(
|
||||
{ name: "close_binary", arguments: {} },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(closed.isError, true, JSON.stringify(closed));
|
||||
const empty = await client.callTool(
|
||||
{ name: "binary_session", arguments: {} },
|
||||
{ timeout: 150_000 },
|
||||
);
|
||||
assert.notEqual(empty.isError, true, JSON.stringify(empty));
|
||||
assert.equal(empty.structuredContent?.result?.open, false);
|
||||
assert.equal(
|
||||
await hashFile(path),
|
||||
fixture.sha256,
|
||||
"MCP workflow must preserve source bytes",
|
||||
);
|
||||
console.log("PASS MCP close and unchanged source bytes");
|
||||
} catch (cause) {
|
||||
process.stderr.write(stderr);
|
||||
throw cause;
|
||||
|
||||
+1
-1
@@ -114,7 +114,7 @@ def shared_blocks(root, path, source):
|
||||
}
|
||||
from website_i18n import language_markup
|
||||
selector, alternates = language_markup(root, path.relative_to(site).as_posix())
|
||||
values["language_picker"] = selector if not metadata.noindex else ""
|
||||
values["language_picker"] = indent(selector, " ") if not metadata.noindex else ""
|
||||
values["alternates"] = alternates if not metadata.noindex else ""
|
||||
templates = root / "website/templates"
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user