chore: integrate repaired verification and CI boundaries

Integrate upstream browser endpoint, Rizin fixture and cache-input repairs. Resolve the Windows verifier conflict by preserving the tested foreign-replacement identity/content checks through retry and teardown. All11 reviewed PR files remain unchanged.

Three directly affected selected boundary tests pass; real-model and unrelated suites are not repeated. Native Windows sources and fixture/build inputs remain unchanged from actual hosted success.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-13 22:26:48 +00:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
63 changed files with 5074 additions and 326 deletions
+67
View File
@@ -0,0 +1,67 @@
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
import type { StaticTaskContext } from "./static-task-context.mjs";
export type { StaticTaskContext } from "./static-task-context.mjs";
export interface StaticDiagnostic {
readonly kind: "lint" | "jscpd";
readonly path?: string;
readonly paths?: readonly string[];
readonly code?: string;
readonly message?: string;
readonly severity?: string;
}
export function classifyFailure(input: {
repository: string;
defaultBranch: string;
run: Record<string, unknown> & NormalizedWorkflowRun;
job: Record<string, unknown>;
diagnostics: readonly StaticDiagnostic[];
context: StaticTaskContext;
mainRun: Record<string, unknown> & NormalizedWorkflowRun;
mainJob: Record<string, unknown>;
mainDiagnostics: readonly StaticDiagnostic[];
mainContext: StaticTaskContext;
executionSourcesVerified: boolean;
ancestry: { status: string; mergeBaseSha: string | null };
blobs: Readonly<Record<string, boolean>>;
}): {
classification: "matching-main-diagnostics" | "unknown";
reason: string;
};
export function diagnosticPaths(item: StaticDiagnostic): readonly string[];
export function isFailedConclusion(
value: unknown,
): value is "failure" | "timed_out";
export function isEligibleMainRun(
run: Record<string, unknown>,
defaultBranch: string,
repository: string,
): boolean;
export function pullRequestBaseSha(
run: Record<string, unknown>,
repository: string,
): string | null;
export function safeSha(value: unknown): string | null;
export function validRepoPath(value: unknown): value is string;
export function selectStaticCheckArtifact(
page: unknown,
run: { id?: string | number; head_sha?: string },
runAttempt: number,
): unknown | null;
export function readStaticCheckArtifact(options: {
repository: string;
artifactId: number | string;
token: string;
fetchImpl?: typeof fetch;
}): Promise<{
context: StaticTaskContext;
diagnostics: readonly StaticDiagnostic[];
}>;
export function parseStaticArchive(archivePath: string): Promise<{
context: StaticTaskContext;
diagnostics: readonly StaticDiagnostic[];
}>;
+436
View File
@@ -0,0 +1,436 @@
import { execFile } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { isAbsolute, join } from "node:path";
import { promisify } from "node:util";
import { parseStaticTaskContext } from "./static-task-context.mjs";
const execute = promisify(execFile);
const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024;
const MAX_MEMBER_BYTES = 4 * 1024 * 1024;
const MAX_DIAGNOSTICS = 10_000;
const API_ROOT = "https://api.github.com/repos";
const SHA = /^[0-9a-f]{40}$/iu;
const TASK_HASH = /^[a-f0-9]{16}$/u;
/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */
export function classifyFailure({
repository,
defaultBranch,
run,
job,
diagnostics,
context,
mainRun,
mainJob,
mainDiagnostics,
mainContext,
executionSourcesVerified,
ancestry,
blobs,
}) {
const failedGate = firstFailure([
() => [
!isCompletedFailure(run) ||
job?.status !== "completed" ||
!isFailedConclusion(job?.conclusion),
"run_or_job_not_a_completed_failure",
],
() => [
!isEligiblePullRun(run, repository) ||
!isEligibleMainRun(mainRun, defaultBranch, repository),
"run_event_or_repository_identity_unverified",
],
() => [
job?.name !== mainJob?.name || job?.name !== "Static checks",
"failed_job_not_the_same_static_gate",
],
() => [
!Array.isArray(diagnostics) ||
diagnostics.length === 0 ||
!Array.isArray(mainDiagnostics),
"structured_diagnostics_missing_or_invalid",
],
() => [
!validContext(context) || !validContext(mainContext),
"static_task_context_missing_or_invalid",
],
() => [
executionSourcesVerified !== true,
"checkout_sources_not_verified_against_api_commits",
],
() => [
!sameExecutionContext(context, mainContext),
"task_or_toolchain_identity_differs",
],
() => [
ancestry?.status !== "ahead" && ancestry?.status !== "identical",
"main_head_not_proven_ancestor_of_tested_base",
],
() => [
ancestry?.mergeBaseSha !== mainRun?.head_sha,
"main_head_is_not_the_verified_merge_base",
],
() => [
!sameTaskHashes(diagnostics, context, mainContext),
"owning_task_hash_differs",
],
]);
if (failedGate) return outcome("unknown", failedGate.reason);
const keys = diagnostics.map(diagnosticKey);
const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean));
if (keys.some((key) => key === null))
return outcome("unknown", "diagnostic_shape_not_supported");
if (!keys.every((key) => mainKeys.has(key)))
return outcome(
"unknown",
"main_run_does_not_contain_the_same_structured_diagnostic",
);
if (!sameBlobEvidence(diagnostics, blobs))
return outcome(
"unknown",
"diagnostic_source_content_not_identical_at_main_and_tested_base",
);
return outcome(
"matching-main-diagnostics",
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
);
}
export function diagnosticPaths(item) {
return item.kind === "jscpd" ? item.paths : [item.path];
}
export function isFailedConclusion(value) {
return value === "failure" || value === "timed_out";
}
export function isEligibleMainRun(run, defaultBranch, repository) {
return (
isEligibleStaticRun(run) &&
run.repository.full_name === repository &&
run.head_branch === defaultBranch &&
["push", "schedule", "workflow_dispatch"].includes(run.event)
);
}
export function pullRequestBaseSha(run, repository) {
if (
run?.event !== "pull_request" ||
run.repository?.full_name !== repository ||
!Array.isArray(run.pull_requests) ||
run.pull_requests.length !== 1
)
return null;
const base = run.pull_requests[0].base;
return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "")
? base.sha
: null;
}
export function safeSha(value) {
return typeof value === "string" && SHA.test(value)
? value.toLowerCase()
: null;
}
export function selectStaticCheckArtifact(page, run, runAttempt) {
if (runAttempt !== 1) return null;
if (
!Array.isArray(page?.artifacts) ||
!Number.isSafeInteger(page.total_count) ||
page.total_count !== page.artifacts.length ||
page.artifacts.length >= 100
)
return null;
const matches = page.artifacts.filter(
(artifact) => artifact.name === "static-check-reports",
);
if (matches.length !== 1) return null;
const [artifact] = matches;
if (
artifact.expired !== false ||
String(artifact.workflow_run?.id) !== String(run.id) ||
artifact.workflow_run?.head_sha !== run.head_sha
)
return null;
return artifact;
}
export function validRepoPath(value) {
return (
typeof value === "string" &&
value.length > 0 &&
!isAbsolute(value) &&
!value.includes("\\") &&
value
.split("/")
.every((part) => part !== "" && part !== "." && part !== "..") &&
!value.startsWith("/")
);
}
/** Download one named static report archive and parse only whitelisted JSON members. */
export async function readStaticCheckArtifact({
repository,
artifactId,
token,
fetchImpl = fetch,
}) {
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? ""))
throw new Error("artifact repository identity is invalid");
if (!token) throw new Error("artifact download requires a GitHub token");
const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`;
let response = await fetchImpl(url, {
headers: {
Accept: "application/vnd.github+json",
Authorization: `Bearer ${token}`,
"X-GitHub-Api-Version": "2022-11-28",
},
signal: AbortSignal.timeout(10_000),
redirect: "manual",
});
if ([301, 302, 303, 307, 308].includes(response.status)) {
const location = response.headers.get("location");
if (!location || new URL(location).protocol !== "https:")
throw new Error("static_artifact_redirect_invalid");
response = await fetchImpl(location, {
signal: AbortSignal.timeout(10_000),
redirect: "error",
});
}
if (!response.ok) throw new Error(`github_http_${response.status}`);
const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES);
const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-"));
const archive = join(directory, "artifact.zip");
try {
await writeFile(archive, bytes, { flag: "wx" });
return await parseStaticArchive(archive);
} finally {
await rm(directory, { recursive: true, force: true });
}
}
/** Accept only the three named producer outputs; no archive paths are written to disk. */
export async function parseStaticArchive(archivePath) {
const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], {
encoding: "utf8",
maxBuffer: 128 * 1024,
timeout: 5_000,
});
const allowed = new Set([
"oxlint.json",
"jscpd/jscpd-report.json",
"static-task-context.json",
]);
const members = new Map();
for (const name of listing.split(/\r?\n/u).filter(Boolean)) {
const prefix = ".cache/static-checks/";
const member = name.startsWith(prefix) ? name.slice(prefix.length) : name;
if (!allowed.has(member)) continue;
if (members.has(member))
throw new Error("static artifact contains duplicate report members");
const { stdout } = await execute("unzip", ["-p", archivePath, name], {
encoding: "buffer",
maxBuffer: MAX_MEMBER_BYTES,
timeout: 5_000,
});
members.set(member, JSON.parse(stdout.toString("utf8")));
}
const context = parseStaticTaskContext(
members.get("static-task-context.json"),
);
const diagnostics = normalizeDiagnostics(
members.get("oxlint.json"),
members.get("jscpd/jscpd-report.json"),
);
if (diagnostics.length > MAX_DIAGNOSTICS)
throw new Error("static diagnostic count exceeds the reporting limit");
return { context, diagnostics };
}
function normalizeDiagnostics(lint, duplicate) {
return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)];
}
function normalizeLint(report) {
if (report === undefined) return [];
if (!Array.isArray(report?.diagnostics))
throw new Error("oxlint diagnostics are malformed");
return report.diagnostics
.filter((item) => item.severity === "error")
.map((item) => {
if (
typeof item.filename !== "string" ||
!repoPath(item.filename) ||
typeof item.code !== "string" ||
typeof item.message !== "string"
)
throw new Error("oxlint error diagnostic is malformed");
return {
kind: "lint",
path: item.filename,
code: item.code,
message: item.message,
severity: "error",
};
});
}
function normalizeDuplicates(report) {
if (report === undefined) return [];
if (
!Array.isArray(report?.duplicates) ||
!Number.isFinite(report.statistics?.total?.percentage) ||
report.statistics.total.percentage < 0
)
throw new Error("jscpd report is malformed");
// The producer owns the checker outcome; do not copy or infer its threshold.
const exitCode = report.reaCheck?.exitCode;
if (exitCode === undefined || exitCode === 0) return [];
if (!Number.isSafeInteger(exitCode) || exitCode < 0)
throw new Error("jscpd checker outcome is malformed");
if (exitCode !== 1) return [];
return report.duplicates.map((item) => {
const paths = [item.firstFile?.name, item.secondFile?.name].sort();
if (paths.some((path) => !repoPath(path)))
throw new Error("jscpd duplicate paths are malformed");
return { kind: "jscpd", paths };
});
}
async function readLimitedBytes(response, limit) {
const reader = response.body?.getReader();
if (!reader) throw new Error("static artifact response has no body");
const chunks = [];
let size = 0;
try {
for (;;) {
const { done, value } = await reader.read();
if (done) break;
size += value.byteLength;
if (size > limit) {
await reader.cancel();
throw new Error("static artifact exceeds the download limit");
}
chunks.push(value);
}
} finally {
reader.releaseLock();
}
const result = new Uint8Array(size);
let offset = 0;
for (const chunk of chunks) {
result.set(chunk, offset);
offset += chunk.byteLength;
}
return result;
}
function repoPath(value) {
return (
typeof value === "string" &&
value.length > 0 &&
!value.startsWith("/") &&
!value.includes("\\") &&
value
.split("/")
.every((part) => part !== "" && part !== "." && part !== "..")
);
}
function positiveId(value) {
const id = Number(value);
if (!Number.isSafeInteger(id) || id < 1)
throw new Error("artifact ID must be a positive integer");
return String(id);
}
function diagnosticKey(item) {
if (
item?.kind === "lint" &&
validRepoPath(item.path) &&
typeof item.code === "string" &&
typeof item.message === "string" &&
item.severity === "error"
)
return JSON.stringify([
item.kind,
item.path,
item.code,
item.message,
item.severity,
]);
if (
item?.kind === "jscpd" &&
Array.isArray(item.paths) &&
item.paths.length === 2 &&
item.paths.every(validRepoPath)
)
return JSON.stringify([item.kind, ...[...item.paths].sort()]);
return null;
}
function sameBlobEvidence(diagnostics, blobs) {
const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
return paths.length > 0 && paths.every((path) => blobs?.[path] === true);
}
function sameTaskHashes(diagnostics, left, right) {
return diagnostics.every((diagnostic) => {
const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built";
return (
TASK_HASH.test(left.taskHashes[task] ?? "") &&
left.taskHashes[task] === right.taskHashes[task]
);
});
}
function validContext(value) {
return (
value?.version === 1 &&
SHA.test(value.sourceSha ?? "") &&
value.sourceClean === true &&
typeof value.nodeVersion === "string" &&
typeof value.npmVersion === "string" &&
typeof value.turboVersion === "string" &&
typeof value.platform === "string" &&
typeof value.architecture === "string" &&
value.taskHashes &&
typeof value.taskHashes === "object" &&
!Array.isArray(value.taskHashes)
);
}
function sameExecutionContext(left, right) {
return [
"nodeVersion",
"npmVersion",
"turboVersion",
"platform",
"architecture",
].every((key) => left[key] === right[key]);
}
function isEligiblePullRun(run, repository) {
return (
isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null
);
}
function isEligibleStaticRun(run) {
return (
isRecord(run) &&
run.status === "completed" &&
isFailedConclusion(run.conclusion) &&
SHA.test(run.head_sha ?? "") &&
typeof run.repository?.full_name === "string" &&
run.workflowPath?.file === ".github/workflows/ci.yml"
);
}
function isCompletedFailure(run) {
return run?.status === "completed" && isFailedConclusion(run.conclusion);
}
function isRecord(value) {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function firstFailure(checks) {
for (const check of checks) {
const [failed, reason] = check();
if (failed) return { reason };
}
return null;
}
function outcome(classification, reason) {
return { classification, reason };
}
+159
View File
@@ -0,0 +1,159 @@
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
import type { StaticDiagnostic } from "./artifacts.mjs";
export type { StaticDiagnostic } from "./artifacts.mjs";
import type { StaticTaskContext } from "./static-task-context.mjs";
export type { StaticTaskContext } from "./static-task-context.mjs";
export interface CiRun {
readonly id?: string | number;
readonly run_attempt?: number;
readonly status?: string;
readonly conclusion?: string | null;
readonly event?: string;
readonly path?: string;
readonly head_sha?: string;
readonly head_branch?: string;
readonly repository?: { readonly full_name?: string };
readonly pull_requests?: readonly {
readonly base?: {
readonly sha?: string;
readonly repo?: { readonly full_name?: string };
};
}[];
}
export interface CiJob {
readonly name?: string;
readonly status?: string;
readonly conclusion?: string | null;
readonly run_id?: number;
readonly run_attempt?: number;
}
export interface StaticEvidence {
readonly context: StaticTaskContext;
readonly diagnostics: readonly StaticDiagnostic[];
}
export interface CiFailureReport {
readonly version: 1;
readonly observedAt: string;
readonly repository: string;
readonly defaultBranch: string;
readonly informationalOnly: true;
readonly gateEffect: string;
readonly boundedInput: {
readonly candidateRuns: number;
readonly limit: number;
readonly mainCandidateLimit: number;
readonly mainCandidateRuns: number;
readonly mainRunLookupLimit: number;
readonly manifestTruncated: boolean;
};
readonly counts: Readonly<
Record<"matching-main-diagnostics" | "unknown", number>
>;
readonly failures: readonly {
readonly runId: number | string;
readonly runAttempt: number;
readonly classification: "matching-main-diagnostics" | "unknown";
readonly reason: string;
readonly candidateFailures?: readonly {
readonly runId: number | string;
readonly runAttempt: number;
readonly reason: string;
}[];
readonly runHead?: string | null;
readonly event?: string | null;
readonly job?: string;
readonly conclusion?: string | null;
}[];
}
export interface FailureReportApi {
getRun(repository: string, id: string | number): Promise<CiRun>;
getJobs(
repository: string,
id: string | number,
attempt: number,
): Promise<readonly CiJob[]>;
getStaticEvidence(
repository: string,
run: CiRun,
job: CiJob,
): Promise<StaticEvidence | null>;
getAncestry(
repository: string,
ancestor: string,
descendant: string,
): Promise<{ status: string; mergeBaseSha: string | null }>;
verifyExecutionSource(
repository: string,
run: CiRun,
context: StaticTaskContext,
): Promise<boolean>;
getBlobIdentity(
repository: string,
ancestor: string,
descendant: string,
paths: readonly string[],
): Promise<Readonly<Record<string, boolean>>>;
}
export function classifyFailure(input: {
repository: string;
defaultBranch: string;
run: CiRun & NormalizedWorkflowRun;
job: CiJob;
diagnostics: readonly StaticDiagnostic[];
context: StaticTaskContext;
mainRun: CiRun & NormalizedWorkflowRun;
mainJob: CiJob;
mainDiagnostics: readonly StaticDiagnostic[];
mainContext: StaticTaskContext;
executionSourcesVerified: boolean;
ancestry: { status: string; mergeBaseSha: string | null };
blobs: Readonly<Record<string, boolean>>;
}): {
classification: "matching-main-diagnostics" | "unknown";
reason: string;
};
export function reportCiFailures(options: {
repository: string;
defaultBranch: string;
batchManifest: {
readonly repository: string;
readonly runs: readonly {
readonly id: number | string;
readonly runAttempt: number;
readonly workflow: string;
readonly event?: string | null;
readonly headBranch?: string | null;
readonly conclusion: string | null;
readonly status: string;
}[];
readonly truncation?: readonly unknown[];
readonly coverage?: { readonly exhaustive?: boolean };
readonly selection?: { readonly exhaustive?: boolean };
};
api: FailureReportApi;
outputDirectory?: string;
summaryPath?: string;
observedAt?: string;
}): Promise<CiFailureReport>;
export function createGitHubApi(options?: {
token?: string;
fetchImpl?: typeof fetch;
apiRequest?: (
method: string,
endpoint: string,
budget: { requests: number; startedAt: number; limitMs: number },
) => Promise<unknown>;
}): FailureReportApi;
export function writeFailureReport(
report: CiFailureReport,
outputDirectory?: string,
summaryPath?: string,
): Promise<{ jsonPath: string; markdownPath: string }>;
+529
View File
@@ -0,0 +1,529 @@
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import {
classifyFailure,
diagnosticPaths,
isEligibleMainRun,
isFailedConclusion,
pullRequestBaseSha,
readStaticCheckArtifact,
safeSha,
selectStaticCheckArtifact,
validRepoPath,
} from "./artifacts.mjs";
import { createGhApiRequest } from "./github-api.mjs";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const OUTPUT_DIRECTORY = join(".cache", "ci-failures");
const BATCH_DIRECTORY = join(".cache", "ci-job-timings");
const MAX_JSON_BYTES = 8 * 1024 * 1024;
const MAX_FAILED_RUNS = 20;
const MAX_MAIN_CANDIDATES = 5;
const MAX_MAIN_RUN_LOOKUPS = 100;
const MAX_PATHS_PER_COMPARISON = 50;
const MAX_API_REQUESTS = 250;
export { classifyFailure } from "./artifacts.mjs";
/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
export async function reportCiFailures({
repository,
defaultBranch,
batchManifest,
api,
outputDirectory = OUTPUT_DIRECTORY,
summaryPath,
observedAt = new Date().toISOString(),
}) {
validateRepository(repository);
if (typeof defaultBranch !== "string" || defaultBranch.length === 0)
throw new Error("defaultBranch is required");
if (
!batchManifest ||
batchManifest.repository !== repository ||
!Array.isArray(batchManifest.runs)
)
throw new Error("timing manifest repository or runs are invalid");
const failedRuns = batchManifest.runs.filter(
(entry) =>
entry.workflow === "CI" &&
entry.status === "collected" &&
isFailedConclusion(entry.conclusion),
);
const prRuns = failedRuns.filter((entry) => entry.event === "pull_request");
const candidates = prRuns.slice(0, MAX_FAILED_RUNS);
const mainEntries = failedRuns.filter(
(entry) =>
entry.headBranch === defaultBranch &&
["push", "schedule", "workflow_dispatch"].includes(entry.event),
);
const lookupRun = verifiedRunLookup(repository, api);
const mainCandidates = [];
for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) {
try {
const run = await lookupRun(entry);
if (isEligibleMainRun(run, defaultBranch, repository))
mainCandidates.push({
run,
getEvidence: mainEvidenceLookup(repository, api, entry, run),
});
if (mainCandidates.length === MAX_MAIN_CANDIDATES) break;
} catch {
// Unverified metadata cannot consume the eligible-main candidate budget.
}
}
const entries = [];
for (const entry of candidates) {
try {
entries.push(
...(await analyzeFailedRun({
repository,
defaultBranch,
entry,
run: await lookupRun(entry),
mainCandidates,
api,
})),
);
} catch (error) {
entries.push({
runId: entry.id,
runAttempt: entry.runAttempt,
classification: "unknown",
reason: errorCode(error),
});
}
}
const report = {
version: 1,
observedAt,
repository,
defaultBranch,
informationalOnly: true,
gateEffect: "none; this report never changes CI conclusions",
boundedInput: {
candidateRuns: candidates.length,
limit: MAX_FAILED_RUNS,
mainCandidateLimit: MAX_MAIN_CANDIDATES,
mainCandidateRuns: mainCandidates.length,
mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS,
manifestTruncated:
prRuns.length > MAX_FAILED_RUNS ||
batchManifest.truncation?.length > 0 ||
batchManifest.selection?.exhaustive === false ||
batchManifest.coverage?.exhaustive === false,
},
counts: Object.fromEntries(
["matching-main-diagnostics", "unknown"].map((category) => [
category,
entries.filter((item) => item.classification === category).length,
]),
),
failures: entries,
};
await writeFailureReport(report, outputDirectory, summaryPath);
return report;
}
async function analyzeFailedRun({
repository,
defaultBranch,
entry,
run,
mainCandidates,
api,
}) {
const jobs = await api.getJobs(repository, entry.id, entry.runAttempt);
if (jobs.some((job) => !hasJobIdentity(job, entry)))
throw new Error("workflow_job_identity_mismatch");
const failedJobs = jobs.filter(
(job) => job.status === "completed" && isFailedConclusion(job.conclusion),
);
if (failedJobs.length === 0)
return [
{
runId: entry.id,
runAttempt: entry.runAttempt,
classification: "unknown",
reason: "failed_jobs_not_present_in_api_snapshot",
},
];
const failures = [];
for (const job of failedJobs) {
const evidence = await api.getStaticEvidence(repository, run, job);
const classification = await attributeToMain({
repository,
defaultBranch,
run,
job,
evidence,
mainCandidates,
api,
});
failures.push({
runId: entry.id,
runAttempt: entry.runAttempt,
runHead: safeSha(run.head_sha),
event: run.event ?? null,
job: job.name,
conclusion: job.conclusion,
...classification,
});
}
return failures;
}
async function attributeToMain({
repository,
defaultBranch,
run,
job,
evidence,
mainCandidates,
api,
}) {
const baseSha = pullRequestBaseSha(run, repository);
if (baseSha === null || evidence === null)
return outcome("unknown", "no_matching_verified_main_failure");
let result = outcome("unknown", "no_matching_verified_main_failure");
const candidateFailures = [];
for (const candidate of mainCandidates) {
try {
const main = await compareCandidate({
repository,
defaultBranch,
run,
job,
evidence,
baseSha,
candidate,
api,
});
if (main !== null) result = main;
if (main?.classification === "matching-main-diagnostics") break;
} catch (error) {
candidateFailures.push({
runId: candidate.run.id,
runAttempt: candidate.run.run_attempt,
reason: errorCode(error),
});
}
}
return {
...result,
...(candidateFailures.length > 0 ? { candidateFailures } : {}),
};
}
async function compareCandidate({
repository,
defaultBranch,
run,
job,
evidence,
baseSha,
candidate,
api,
}) {
const mainRun = candidate.run;
const main = await candidate.getEvidence(job.name);
if (!main?.evidence) return null;
const { job: mainJob, evidence: mainEvidence } = main;
const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha);
const paths = [
...new Set(evidence.diagnostics.flatMap(diagnosticPaths)),
].slice(0, MAX_PATHS_PER_COMPARISON);
const blobs = await api.getBlobIdentity(
repository,
mainRun.head_sha,
baseSha,
paths,
);
const executionSourcesVerified =
(await api.verifyExecutionSource(repository, run, evidence.context)) &&
(await api.verifyExecutionSource(
repository,
mainRun,
mainEvidence.context,
));
return classifyFailure({
repository,
defaultBranch,
run,
job,
diagnostics: evidence.diagnostics,
context: evidence.context,
mainRun,
mainJob,
mainDiagnostics: mainEvidence.diagnostics,
mainContext: mainEvidence.context,
executionSourcesVerified,
ancestry,
blobs,
});
}
// Each verified main run/attempt owns its lazy jobs and parsed evidence for this
// report only. Cache nulls and rejected reads too; retries cannot spend the budget
// repeatedly on an immutable observation that was unavailable or invalid.
function mainEvidenceLookup(repository, api, entry, run) {
let jobs;
const evidence = new Map();
return (name) => {
jobs ??= Promise.resolve().then(async () => {
const items = await api.getJobs(repository, entry.id, entry.runAttempt);
return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items;
});
if (!evidence.has(name))
evidence.set(
name,
jobs.then(async (items) => {
const job = items.find(
(item) =>
item.name === name &&
item.status === "completed" &&
isFailedConclusion(item.conclusion),
);
if (!job) return null;
return {
job,
evidence: await api.getStaticEvidence(repository, run, job),
};
}),
);
return evidence.get(name);
};
}
// Selection and attribution share one immutable API observation per run/attempt.
function verifiedRunLookup(repository, api) {
const runs = new Map();
return (entry) => {
const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`;
if (!runs.has(key))
runs.set(
key,
Promise.resolve().then(async () => {
const run = normalizeWorkflowRun(
await api.getRun(repository, entry.id),
);
if (!hasRunIdentity(run, entry))
throw new Error("workflow_run_identity_mismatch");
return run;
}),
);
return runs.get(key);
};
}
function hasRunIdentity(run, entry) {
return (
String(run?.id) === String(entry.id) &&
Number(run.run_attempt) === Number(entry.runAttempt) &&
run.workflowPath?.file === ".github/workflows/ci.yml"
);
}
function hasJobIdentity(job, entry) {
return (
String(job.run_id) === String(entry.id) &&
Number(job.run_attempt) === Number(entry.runAttempt)
);
}
export function createGitHubApi({
token = process.env.GH_TOKEN,
fetchImpl = fetch,
apiRequest = createGhApiRequest(),
} = {}) {
if (!token) throw new Error("GH_TOKEN is required");
const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 };
const request = async (endpoint) => {
if (budget.requests >= MAX_API_REQUESTS)
throw new Error("github_api_request_budget_exhausted");
budget.requests += 1;
return await apiRequest("GET", endpoint, budget);
};
return {
async getRun(repository, id) {
return await request(
`repos/${repository}/actions/runs/${positiveId(id)}`,
);
},
async getJobs(repository, id, attempt) {
const page = await request(
`repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`,
);
if (!Array.isArray(page.jobs) || page.jobs.length >= 100)
throw new Error("job_list_incomplete_or_malformed");
return page.jobs;
},
async getStaticEvidence(repository, run, job) {
if (job.name !== "Static checks") return null;
if (run.run_attempt !== 1) return null;
const page = await request(
`repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`,
);
const artifact = selectStaticCheckArtifact(page, run, run.run_attempt);
if (!artifact) return null;
return await readStaticCheckArtifact({
repository,
artifactId: artifact.id,
token,
fetchImpl,
});
},
async getAncestry(repository, ancestor, descendant) {
const comparison = await request(
`repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`,
);
return {
status: comparison.status,
mergeBaseSha: safeSha(comparison.merge_base_commit?.sha),
};
},
async verifyExecutionSource(repository, run, context) {
const sourceSha = safeSha(context.sourceSha);
if (!sourceSha) return false;
const pr =
run.event === "pull_request"
? pullRequestBaseSha(run, repository)
: null;
if (
run.event === "pull_request" &&
(pr === null || context.prHeadSha !== run.head_sha)
)
return false;
const commit = await request(`repos/${repository}/commits/${sourceSha}`);
if (safeSha(commit.sha) !== sourceSha) return false;
if (sourceSha === run.head_sha) return true;
if (
pr === null ||
!Array.isArray(commit.parents) ||
commit.parents.length !== 2
)
return false;
return (
safeSha(commit.parents[0]?.sha) === pr &&
safeSha(commit.parents[1]?.sha) === run.head_sha
);
},
async getBlobIdentity(repository, ancestor, descendant, paths) {
const result = {};
for (const path of paths) {
if (!validRepoPath(path)) continue;
const [left, right] = await Promise.all([
getContent(repository, ancestor, path, request),
getContent(repository, descendant, path, request),
]);
result[path] = left !== null && left === right;
}
return result;
},
};
}
async function getContent(repository, ref, path, request) {
try {
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
const value = await request(
`repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
);
return typeof value.sha === "string" ? value.sha : null;
} catch {
return null;
}
}
function outcome(classification, reason) {
return { classification, reason };
}
function validateRepository(value) {
if (
typeof value !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)
)
throw new Error("repository must be owner/name");
}
function positiveId(value) {
if (!Number.isSafeInteger(Number(value)) || Number(value) < 1)
throw new Error(
"GitHub run and attempt IDs must be positive safe integers",
);
return String(Number(value));
}
function errorCode(error) {
return error instanceof Error ? error.message.slice(0, 160) : "unknown_error";
}
export async function writeFailureReport(
report,
outputDirectory = OUTPUT_DIRECTORY,
summaryPath,
) {
await mkdir(outputDirectory, { recursive: true });
const jsonPath = join(outputDirectory, "report.json");
const markdownPath = join(outputDirectory, "report.md");
const matchedDiagnostics = report.failures.filter(
(item) => item.classification === "matching-main-diagnostics",
);
const lines = [
"# CI failure attribution",
"",
"Informational only. This report never changes a CI conclusion or required gate.",
"",
`Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`,
"",
];
for (const item of report.failures)
lines.push(
`- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`,
);
if (report.failures.length === 0)
lines.push(
"- No completed failed CI attempts were present in the supplied timing snapshot.",
);
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8");
if (summaryPath)
await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8");
return { jsonPath, markdownPath };
}
async function main() {
const repository = process.env.CI_REPOSITORY;
const defaultBranch = process.env.CI_DEFAULT_BRANCH;
const batchPath =
process.env.CI_TIMING_MANIFEST ??
join(BATCH_DIRECTORY, "batch-manifest.json");
const batchManifest = JSON.parse(await readBoundedFile(batchPath));
const api = createGitHubApi();
await reportCiFailures({
repository,
defaultBranch,
batchManifest,
api,
outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY,
summaryPath: process.env.GITHUB_STEP_SUMMARY,
});
}
async function readBoundedFile(path) {
const bytes = await readFile(path);
if (bytes.byteLength > MAX_JSON_BYTES)
throw new Error("timing manifest exceeds size limit");
return bytes.toString("utf8");
}
if (
process.argv[1] &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
try {
await main();
} catch (error) {
console.error(error instanceof Error ? error.message : String(error));
process.exitCode = 1;
}
}
+16
View File
@@ -0,0 +1,16 @@
export interface GitHubApiBudget {
requests: number;
startedAt: number;
limitMs: number;
}
export type GitHubApiRequest = (
method: string,
endpoint: string,
budget: GitHubApiBudget,
) => Promise<unknown>;
export function createGhApiRequest(options?: {
env?: NodeJS.ProcessEnv;
now?: () => number;
}): GitHubApiRequest;
+33
View File
@@ -0,0 +1,33 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
const execFileAsync = promisify(execFile);
const MAX_BUFFER = 8 * 1024 * 1024;
const REQUEST_TIMEOUT_MS = 10_000;
/** Create a bounded GitHub CLI JSON transport with GET-only authority. */
export function createGhApiRequest({ env = process.env, now = Date.now } = {}) {
return async (method, endpoint, budget) => {
if (method !== "GET")
throw new Error(
"GitHub API transport supports read-only GET requests only",
);
const remaining = budget.limitMs - (now() - budget.startedAt);
if (remaining <= 0) throw new Error("lookup_time_budget_exhausted");
const { stdout } = await execFileAsync(
"gh",
["api", "--method", "GET", endpoint],
{
env,
encoding: "utf8",
maxBuffer: MAX_BUFFER,
timeout: Math.min(REQUEST_TIMEOUT_MS, remaining),
},
);
try {
return JSON.parse(stdout);
} catch (cause) {
throw new Error("github_api_malformed_json", { cause });
}
};
}
+3
View File
@@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({
id: run.id,
runAttempt: run.run_attempt,
workflow: workflow.name,
event: typeof run.event === "string" ? run.event : null,
headBranch:
typeof run.head_branch === "string" ? run.head_branch : null,
conclusion: run.conclusion ?? null,
createdAt: run.created_at ?? null,
});
+95
View File
@@ -0,0 +1,95 @@
export interface PrCiHead {
repositoryId: number | null;
repository?: string | null;
ref: string | null;
sha: string | null;
}
export interface PrCiPullRequest {
number: number | null;
draft?: boolean;
identityValid: boolean;
mergeable?: boolean;
head: PrCiHead;
[key: string]: unknown;
}
export interface PrCiWorkflowRun {
id?: number;
run_attempt?: number;
event?: string;
head_branch?: string;
head_repository?: { id?: number | null } | null;
head_sha?: string;
status?: string;
conclusion?: string | null;
created_at?: string;
html_url?: string;
pull_requests?: Array<{
number?: number;
head?: { sha?: string; repo?: { id?: number | null } } | null;
}>;
[key: string]: unknown;
}
export interface PrCiClassification {
category: string;
reason: string;
runs: Array<Record<string, unknown>>;
}
export interface PrCiHealthReport {
schemaVersion: 1;
observedAt: string;
repository: string;
repositoryId: number | null;
defaultBranch: string;
lookbackDays: number;
scan: {
status: "complete" | "partial";
cutoff: string;
pullRequestPages: number;
workflowRunPages: number;
pullRequestInventoryComplete: boolean;
workflowRunInventoryComplete: boolean;
pullRequestDetailsFetched: number;
pullRequestDetailsLimit: number;
apiRequests: number;
apiRequestLimit: number;
errors: string[];
};
counts: Record<string, number>;
pullRequests: Array<PrCiPullRequest & { ci: PrCiClassification }>;
}
export function collectPrCiHealth(options: {
repository: string;
repositoryId?: number | string | null;
defaultBranch: string;
lookbackDays?: number;
now?: number;
apiRequest?: GitHubApiRequest;
}): Promise<PrCiHealthReport>;
export function classifyPrCi(input: {
pull: PrCiPullRequest;
runs: Array<{ run: PrCiWorkflowRun; basis: string }>;
runsComplete: boolean;
pullRequestInventoryComplete: boolean;
ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>;
}): PrCiClassification;
export function matchRunToPullRequest(input: {
run: PrCiWorkflowRun;
pull: PrCiPullRequest;
identityCounts: Map<string, number>;
inventoryComplete?: boolean;
}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string };
export function writePrCiHealthReport(
report: PrCiHealthReport,
outputDirectory?: string,
): Promise<{ jsonPath: string; markdownPath: string }>;
export function renderMarkdownSummary(report: PrCiHealthReport): string;
import type { GitHubApiRequest } from "./github-api.mjs";
+718
View File
@@ -0,0 +1,718 @@
import { appendFile, mkdir, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { createGhApiRequest } from "./github-api.mjs";
const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health");
const PAGE_SIZE = 100;
const MAX_PULL_REQUEST_PAGES = 3;
const MAX_WORKFLOW_RUN_PAGES = 5;
const MAX_PULL_REQUEST_DETAILS = 40;
const MAX_API_REQUESTS =
MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS;
const LOOKBACK_DAYS = 14;
const MAX_LOOKBACK_DAYS = 30;
const MAX_TOTAL_TIME_MS = 60_000;
const SHA_PATTERN = /^[0-9a-f]{40}$/iu;
const COMPLETED_CATEGORIES = new Map([
["failure", "completed_failure"],
["timed_out", "completed_failure"],
["cancelled", "completed_canceled"],
["success", "completed_success"],
]);
/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */
export async function collectPrCiHealth({
repository,
repositoryId,
defaultBranch,
lookbackDays = LOOKBACK_DAYS,
now = Date.now(),
apiRequest = createGhApiRequest(),
}) {
const identity = validateInputs({
repository,
repositoryId,
defaultBranch,
lookbackDays,
now,
});
const observedAt = new Date(now).toISOString();
const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000;
const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff });
const classified = classifyPullRequests(snapshot);
const { pulls, runs, budget } = snapshot;
const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter(
Boolean,
);
return {
schemaVersion: 1,
observedAt,
repository: identity.repository,
repositoryId: identity.repositoryId,
defaultBranch: identity.defaultBranch,
lookbackDays: identity.lookbackDays,
scan: {
status:
errors.length === 0 && pulls.complete && runs.complete
? "complete"
: "partial",
cutoff: new Date(cutoff).toISOString(),
pullRequestPages: pulls.pages,
workflowRunPages: runs.pages,
pullRequestInventoryComplete: pulls.complete,
workflowRunInventoryComplete: runs.complete,
pullRequestDetailsFetched: snapshot.detailsFetched,
pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS,
apiRequests: budget.requests,
apiRequestLimit: MAX_API_REQUESTS,
errors,
},
counts: countCategories(classified),
pullRequests: classified,
};
}
async function collectApiSnapshot({ apiRequest, identity, cutoff }) {
const budget = {
requests: 0,
startedAt: Date.now(),
limitMs: MAX_TOTAL_TIME_MS,
};
const requestGet = async (endpoint) => {
if (budget.requests >= MAX_API_REQUESTS)
throw new Error("lookup_request_budget_exhausted");
if (Date.now() - budget.startedAt >= budget.limitMs)
throw new Error("lookup_time_budget_exhausted");
budget.requests += 1;
return await apiRequest("GET", endpoint, budget);
};
const pulls = await listOpenPullRequests({
requestGet,
identity,
});
const candidates = pulls.items.filter(
(pull) =>
!pull.draft &&
Number.isFinite(Date.parse(pull.createdAt ?? "")) &&
Date.parse(pull.createdAt ?? "") >= cutoff &&
pull.base.ref === identity.defaultBranch &&
(identity.repositoryId === null ||
pull.base.repositoryId === identity.repositoryId),
);
const runs = await listRecentWorkflowRuns({
requestGet,
identity,
cutoff,
});
const detailResult = await resolveMergeability({
candidates,
requestGet,
identity,
});
return {
budget,
pulls,
runs,
candidates,
detailsFetched: detailResult.fetched,
detailsErrors: detailResult.errors,
};
}
function classifyPullRequests(snapshot) {
const { candidates, pulls, runs } = snapshot;
const identityCounts = countHeadIdentities(pulls.items);
return candidates.map((pull) => {
const relevant = [];
const ambiguousRuns = [];
for (const run of runs.items) {
const match = matchRunToPullRequest({
run,
pull,
identityCounts,
inventoryComplete: pulls.complete,
});
if (match.kind === "match") relevant.push({ run, basis: match.basis });
if (match.kind === "ambiguous")
ambiguousRuns.push({ run, basis: "ambiguous_head_identity" });
}
return {
...pull,
ci: classifyPrCi({
pull,
runs: relevant,
runsComplete: runs.complete,
pullRequestInventoryComplete: pulls.complete,
ambiguousRuns,
}),
};
});
}
async function resolveMergeability({ candidates, requestGet, identity }) {
let detailsFetched = 0;
const errors = new Set();
for (const pull of candidates) {
if (pull.mergeable !== undefined) continue;
if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) {
errors.add("pull_request_mergeability_lookup_budget_exhausted");
continue;
}
const result = await getPullRequestDetails({ requestGet, identity, pull });
detailsFetched += 1;
if (result.error !== null) {
errors.add(result.error);
continue;
}
const { detail } = result;
const identityChanged =
detail.headSha !== pull.head.sha ||
detail.headRef !== pull.head.ref ||
detail.headRepositoryId !== pull.head.repositoryId ||
detail.baseRef !== pull.base.ref ||
detail.baseRepositoryId !== pull.base.repositoryId;
if (identityChanged) {
pull.identityValid = false;
pull.identityChangedDuringScan = true;
errors.add("pull_request_identity_changed_during_scan");
} else {
pull.mergeable = detail.mergeable;
pull.detailFetched = true;
if (detail.mergeable === undefined)
errors.add("pull_request_mergeability_unknown");
}
}
return { fetched: detailsFetched, errors: [...errors] };
}
function countCategories(pulls) {
const categories = [
"missing",
"startup_failure",
"queued_pending_executing",
"approval_required",
"completed_success",
"completed_failure",
"completed_canceled",
"completed_other",
"merge_conflict",
"unknown",
"deferred",
];
return Object.fromEntries(
categories.map((category) => [
category,
pulls.filter(({ ci }) => ci.category === category).length,
]),
);
}
/** Pure status classifier; missing requires complete, identity-safe inventories. */
export function classifyPrCi({
pull,
runs,
runsComplete,
pullRequestInventoryComplete,
ambiguousRuns,
}) {
if (!pull.identityValid)
return result("unknown", "pull_request_head_identity_invalid", []);
const evidence = [...runs, ...ambiguousRuns];
if (pull.mergeable === false)
return result("merge_conflict", "github_reports_merge_conflict", evidence);
// The newest collected run is not proven current until pagination completes:
// an older-created run outside this snapshot may have been retried recently.
if (!runsComplete)
return result(
evidence.length > 0 ? "unknown" : "deferred",
evidence.length > 0
? "workflow_run_inventory_incomplete"
: "lookup_inventory_incomplete",
evidence,
);
const selection = newestRun(runs);
if (!selection.known)
return result("unknown", "workflow_run_order_unknown", evidence);
const latest = selection.latest;
if (
ambiguousRuns.some((candidate) => {
const order = latest ? compareRunEvidence(candidate, latest) : null;
return order === null || order >= 0;
})
)
return result("unknown", "workflow_run_identity_ambiguous", evidence);
if (latest !== undefined) {
const run = latest.run;
if (run.conclusion === "startup_failure")
return result("startup_failure", "workflow_startup_failure", evidence);
if (run.status === "waiting" || run.conclusion === "action_required")
return result(
"approval_required",
"workflow_waiting_for_approval_or_gate",
evidence,
);
if (["queued", "requested", "pending", "in_progress"].includes(run.status))
return result(
"queued_pending_executing",
`workflow_${run.status}`,
evidence,
);
if (run.status === "completed")
return result(
COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other",
run.conclusion === "success"
? "workflow_succeeded"
: `workflow_${run.conclusion ?? "unknown_conclusion"}`,
evidence,
);
return result("unknown", "workflow_status_unrecognized", evidence);
}
if (!pullRequestInventoryComplete)
return result("deferred", "lookup_inventory_incomplete", []);
if (pull.mergeable !== true)
return result("unknown", "pull_request_mergeability_unknown", []);
return result("missing", "no_pull_request_event_run_for_current_head", []);
}
/** Match only a pull_request event proven to belong to this immutable PR head. */
export function matchRunToPullRequest({
run,
pull,
identityCounts,
inventoryComplete = true,
}) {
if (run?.event !== "pull_request") return { kind: "none" };
if (!pull.identityValid || !isRecord(run)) return { kind: "none" };
const association = matchExplicitPullAssociation(run, pull);
if (association === "match")
return { kind: "match", basis: "event_pr_number_and_head_sha" };
if (association === "ambiguous") return { kind: "ambiguous" };
if (association === "other" || !sameHeadRepositoryAndRef(run, pull))
return { kind: "none" };
if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete))
return { kind: "ambiguous" };
if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha)
return { kind: "match", basis: "event_unique_head_ref_and_repository_id" };
return { kind: "none" };
}
function matchExplicitPullAssociation(run, pull) {
const associations = Array.isArray(run.pull_requests)
? run.pull_requests
: [];
if (associations.length === 0) return "absent";
const matchingNumber = associations.filter(
(item) => item?.number === pull.number,
);
if (matchingNumber.length === 0) return "other";
const exactHead = matchingNumber.find(
(item) => item?.head?.sha === pull.head.sha,
);
if (exactHead === undefined) {
return matchingNumber.some(
(item) =>
typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha,
)
? "other"
: "absent";
}
const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id);
const runHeadRepositoryId = positiveInteger(run.head_repository?.id);
if (associatedHeadRepositoryId === null || runHeadRepositoryId === null)
return "ambiguous";
if (
associatedHeadRepositoryId !== pull.head.repositoryId ||
runHeadRepositoryId !== pull.head.repositoryId
)
return "other";
const baseRefMatches =
typeof exactHead.base?.ref !== "string" ||
exactHead.base.ref === pull.base.ref;
const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id);
const baseRepositoryMatches =
baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId;
return baseRefMatches && baseRepositoryMatches ? "match" : "other";
}
function sameHeadRepositoryAndRef(run, pull) {
return (
run.head_branch === pull.head.ref &&
run.head_repository?.id === pull.head.repositoryId
);
}
function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) {
if (!inventoryComplete) return false;
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
const unknownRepositoryKey = headIdentityKey(null, pull.head.ref);
return (
identityCounts.get(key) === 1 &&
(identityCounts.get(unknownRepositoryKey) ?? 0) === 0
);
}
function normalizePullRequest(raw, identity) {
const number = positiveInteger(raw?.number);
const base = normalizeBase(raw?.base);
const head = normalizeHead(raw?.head);
const draft = raw?.draft === true;
return {
number,
draft,
createdAt: typeof raw?.created_at === "string" ? raw.created_at : null,
url: typeof raw?.html_url === "string" ? raw.html_url : null,
base,
head,
mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined,
identityValid: hasValidPullIdentity({ raw, identity, number, base, head }),
};
}
function normalizeBase(raw) {
return {
ref: typeof raw?.ref === "string" ? raw.ref : null,
repositoryId: positiveInteger(raw?.repo?.id),
};
}
function normalizeHead(raw) {
return {
repositoryId: positiveInteger(raw?.repo?.id),
repository:
typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null,
ref: typeof raw?.ref === "string" ? raw.ref : null,
sha:
typeof raw?.sha === "string" && isSha(raw.sha)
? raw.sha.toLowerCase()
: null,
};
}
function hasValidPullIdentity({ raw, identity, number, base, head }) {
return (
number !== null &&
typeof raw?.draft === "boolean" &&
base.ref === identity.defaultBranch &&
(identity.repositoryId === null ||
base.repositoryId === identity.repositoryId) &&
head.repositoryId !== null &&
head.sha !== null &&
head.ref !== null
);
}
function countHeadIdentities(pulls) {
const counts = new Map();
for (const pull of pulls) {
if (typeof pull.head.ref !== "string") continue;
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
return counts;
}
function headIdentityKey(repositoryId, ref) {
return `${String(repositoryId)}\u0000${ref}`;
}
function newestRun(runs) {
let latest = runs[0];
for (const candidate of runs) {
if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0)
latest = candidate;
}
const known = runs.every((candidate) => {
if (candidate === latest) return true;
const order = compareRunEvidence(candidate, latest);
return order !== null && order <= 0;
});
return { known, latest };
}
// Attempt ordering is meaningful within one run; different run IDs are not clocks.
function compareRunEvidence(left, right) {
const leftRun = left.run;
const rightRun = right.run;
const id = positiveInteger(leftRun.id);
if (id !== null && id === positiveInteger(rightRun.id)) {
const leftAttempt = positiveInteger(leftRun.run_attempt);
const rightAttempt = positiveInteger(rightRun.run_attempt);
if (leftAttempt === null || rightAttempt === null) return null;
if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt;
return leftRun.status === rightRun.status &&
leftRun.conclusion === rightRun.conclusion
? 0
: null;
}
// A retry may start long after its original creation; without an attempt
// clock, creation dates only order first attempts from different runs.
if (
positiveInteger(leftRun.run_attempt) !== 1 ||
positiveInteger(rightRun.run_attempt) !== 1
)
return null;
const time =
Date.parse(leftRun.created_at ?? "") -
Date.parse(rightRun.created_at ?? "");
return Number.isFinite(time) && time !== 0 ? time : null;
}
function result(category, reason, runs) {
return {
category,
reason,
runs: runs.map(({ run, basis }) => ({
id: positiveInteger(run.id),
attempt: positiveInteger(run.run_attempt),
status: typeof run.status === "string" ? run.status : null,
conclusion: typeof run.conclusion === "string" ? run.conclusion : null,
headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null,
createdAt: typeof run.created_at === "string" ? run.created_at : null,
url: typeof run.html_url === "string" ? run.html_url : null,
identityBasis: basis,
})),
};
}
async function listOpenPullRequests({ requestGet, identity }) {
const items = [];
let complete = false;
let pages = 0;
let error = null;
for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) {
try {
const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`;
const response = await requestGet(endpoint);
if (!Array.isArray(response))
throw new Error("pull_request_page_malformed");
pages += 1;
const normalized = response.map((raw) =>
normalizePullRequest(raw, identity),
);
if (
normalized.some(
(pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")),
)
)
throw new Error("pull_request_page_malformed");
items.push(...normalized);
if (response.length < PAGE_SIZE) {
complete = true;
break;
}
} catch (cause) {
error = safeErrorCode(cause);
break;
}
}
return { items, pages, complete, error };
}
async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) {
const items = [];
let complete = false;
let pages = 0;
let error = null;
for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) {
try {
const cutoffDate = new Date(cutoff).toISOString().slice(0, 10);
const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`;
const response = await requestGet(endpoint);
if (!isRecord(response) || !Array.isArray(response.workflow_runs))
throw new Error("workflow_run_page_malformed");
pages += 1;
if (!response.workflow_runs.every(isRecord))
throw new Error("workflow_run_page_malformed");
items.push(...response.workflow_runs);
if (response.workflow_runs.length < PAGE_SIZE) {
complete = true;
break;
}
} catch (cause) {
error = safeErrorCode(cause);
break;
}
}
return { items, pages, complete, error };
}
async function getPullRequestDetails({ requestGet, identity, pull }) {
if (pull.number === null)
return { detail: null, error: "pull_request_detail_identity_missing" };
try {
const response = await requestGet(
`repos/${identity.repository}/pulls/${pull.number}`,
);
if (!isRecord(response))
return { detail: null, error: "pull_request_detail_malformed" };
return {
detail: {
mergeable:
typeof response.mergeable === "boolean"
? response.mergeable
: undefined,
headSha: isSha(response.head?.sha)
? response.head.sha.toLowerCase()
: null,
headRef:
typeof response.head?.ref === "string" ? response.head.ref : null,
headRepositoryId: positiveInteger(response.head?.repo?.id),
baseRef:
typeof response.base?.ref === "string" ? response.base.ref : null,
baseRepositoryId: positiveInteger(response.base?.repo?.id),
},
error: null,
};
} catch (cause) {
return { detail: null, error: safeErrorCode(cause) };
}
}
function validateInputs({
repository,
repositoryId,
defaultBranch,
lookbackDays,
now,
}) {
if (
typeof repository !== "string" ||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
)
throw new Error(
"CI_REPOSITORY must be an owner/name repository identifier",
);
if (
typeof defaultBranch !== "string" ||
defaultBranch.length === 0 ||
/[\r\n]/u.test(defaultBranch)
)
throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name");
const parsedRepositoryId =
repositoryId === undefined || repositoryId === null || repositoryId === ""
? null
: positiveInteger(repositoryId);
if (
repositoryId !== undefined &&
repositoryId !== null &&
repositoryId !== "" &&
parsedRepositoryId === null
)
throw new Error("CI_REPOSITORY_ID must be a positive safe integer");
if (
!Number.isInteger(lookbackDays) ||
lookbackDays < 1 ||
lookbackDays > MAX_LOOKBACK_DAYS
)
throw new Error(
`lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`,
);
if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp");
return {
repository,
repositoryId: parsedRepositoryId,
defaultBranch,
lookbackDays,
};
}
function positiveInteger(value) {
const number = typeof value === "number" ? value : Number(value);
return Number.isSafeInteger(number) && number > 0 ? number : null;
}
function isSha(value) {
return typeof value === "string" && SHA_PATTERN.test(value);
}
function isRecord(value) {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function safeErrorCode(cause) {
if (cause?.message === "lookup_time_budget_exhausted")
return "lookup_time_budget_exhausted";
if (cause?.message === "lookup_request_budget_exhausted")
return "lookup_request_budget_exhausted";
if (cause?.message === "github_api_malformed_json")
return "github_api_malformed_json";
if (cause?.message?.endsWith("_malformed")) return cause.message;
if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout";
return "github_api_request_failed";
}
export async function writePrCiHealthReport(
report,
outputDirectory = OUTPUT_DIRECTORY,
) {
const jsonPath = join(outputDirectory, "report.json");
const markdownPath = join(outputDirectory, "summary.md");
await mkdir(outputDirectory, { recursive: true });
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
await writeFile(markdownPath, renderMarkdownSummary(report), "utf8");
return { jsonPath, markdownPath };
}
export function renderMarkdownSummary(report) {
const rows = report.pullRequests.map(
(pull) =>
`| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`,
);
return [
"# Pull request CI health snapshot",
"",
`Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`,
"",
"This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.",
"",
`API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`,
"",
"| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |",
"| ---: | --- | --- | --- | --- | --- |",
...rows,
"",
"Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.",
"",
].join("\n");
}
function escapeCell(value) {
return String(value)
.replaceAll("|", "\\|")
.replaceAll("\n", " ")
.replaceAll("\r", " ");
}
if (
process.argv[1] !== undefined &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
) {
try {
const report = await collectPrCiHealth({
repository: process.env.CI_REPOSITORY,
repositoryId: process.env.CI_REPOSITORY_ID,
defaultBranch: process.env.CI_DEFAULT_BRANCH,
});
const paths = await writePrCiHealthReport(
report,
process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY,
);
if (process.env.GITHUB_STEP_SUMMARY)
await appendFile(
process.env.GITHUB_STEP_SUMMARY,
renderMarkdownSummary(report),
"utf8",
);
process.stdout.write(
`Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`,
);
} catch (cause) {
process.stderr.write(
`PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`,
);
process.exitCode = 1;
}
}
+4 -3
View File
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "ci-full-qualification-v1";
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
@@ -119,8 +120,8 @@ function admit() {
// Artifact metadata narrows the search before run lookups. Never scan an
// unbounded history or treat a missing receipt as a successful qualification.
for (const artifact of candidates.slice(0, 3)) {
const run = api(
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
const run = normalizeWorkflowRun(
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
// An old run can upload a new artifact when only its gate is rerun.
// Bound the original run age as well so retries cannot renew stale proof.
@@ -133,7 +134,7 @@ function admit() {
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.path === ".github/workflows/ci.yml" &&
run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_sha === head &&
run.head_branch === branch &&
+5 -1
View File
@@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
const required = reused ? ["changes"] : ["changes", "static", "test"];
const lanes = {
code: ["build", "test-shard"],
code: [
"build",
"test-shard",
...(outputs.tier === "fast" ? ["test-source"] : []),
],
reverse_tools: ["reverse-tools-quality"],
runtime: ["windows-native-build", "package-e2e"],
package: ["windows-curated"],
+2
View File
@@ -133,6 +133,8 @@ try {
for (const clone of report.duplicates)
for (const file of [clone.firstFile, clone.secondFile])
file.name = relative(process.cwd(), file.name).split(sep).join("/");
// Record the actual checker result, including caller-selected thresholds.
report.reaCheck = { exitCode: result.status };
await writeFile(reportPath, JSON.stringify(report, null, 2));
const { sources, clones, percentage } = report.statistics.total;
console.log(
+18
View File
@@ -0,0 +1,18 @@
export interface StaticTaskContext {
readonly version: 1;
readonly sourceSha: string;
readonly sourceClean: boolean;
readonly prHeadSha?: string;
readonly nodeVersion: string;
readonly npmVersion: string;
readonly turboVersion: string;
readonly platform: string;
readonly architecture: string;
readonly taskHashes: Readonly<Record<string, string>>;
}
export function parseStaticTaskContext(value: unknown): StaticTaskContext;
export function recordStaticTaskContext(options?: {
root?: string;
npmCli?: string;
outputDirectory?: string;
}): Promise<StaticTaskContext>;
+104
View File
@@ -0,0 +1,104 @@
import { execFile } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { z } from "zod";
const execute = promisify(execFile);
const contextSchema = z.object({
version: z.literal(1),
sourceSha: z.string().regex(/^[a-f0-9]{40}$/u),
sourceClean: z.boolean(),
prHeadSha: z
.string()
.regex(/^[a-f0-9]{40}$/u)
.optional(),
nodeVersion: z.string().min(1),
npmVersion: z.string().min(1),
turboVersion: z.string().min(1),
platform: z.string().min(1),
architecture: z.string().min(1),
taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)),
});
/** Parse the producing check graph's receipt; incomplete provenance is unknown. */
export const parseStaticTaskContext = (value) => contextSchema.parse(value);
/** Retain only named task hashes and tool/source identity, never the environment. */
export async function recordStaticTaskContext({
root = process.cwd(),
npmCli = process.env.npm_execpath,
outputDirectory = join(root, ".cache/static-checks"),
} = {}) {
if (typeof npmCli !== "string" || !npmCli)
throw new Error(
"Run static context recording through npm run ci:static-context",
);
const options = {
cwd: root,
encoding: "utf8",
timeout: 30_000,
maxBuffer: 32 * 1024 * 1024,
};
const [graph, source, status, npmVersion, turbo] = await Promise.all([
execute(
process.execPath,
[npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"],
options,
),
execute("git", ["rev-parse", "HEAD"], options),
execute(
"git",
["status", "--porcelain", "--untracked-files=normal"],
options,
),
execute(process.execPath, [npmCli, "--version"], options),
readFile(join(root, "node_modules/turbo/package.json"), "utf8"),
]);
const tasks = z
.object({
tasks: z
.array(
z.object({
task: z.string().min(1),
hash: z.string().regex(/^[a-f0-9]{16}$/u),
}),
)
.min(1),
})
.parse(JSON.parse(graph.stdout)).tasks;
if (new Set(tasks.map(({ task }) => task)).size !== tasks.length)
throw new Error("Static graph contains duplicate task identities");
const context = parseStaticTaskContext({
version: 1,
sourceSha: source.stdout.trim(),
sourceClean: status.stdout.length === 0,
...(process.env.CI_PR_HEAD_SHA
? { prHeadSha: process.env.CI_PR_HEAD_SHA }
: {}),
nodeVersion: process.version,
npmVersion: npmVersion.stdout.trim(),
turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo))
.version,
platform: process.platform,
architecture: process.arch,
taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])),
});
if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha)
throw new Error(
"Static context source does not match the workflow revision",
);
await mkdir(outputDirectory, { recursive: true });
await writeFile(
join(outputDirectory, "static-task-context.json"),
JSON.stringify(context, null, 2) + "\n",
);
return context;
}
if (
process.argv[1] &&
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
)
await recordStaticTaskContext();
+46
View File
@@ -0,0 +1,46 @@
import { spawn } from "node:child_process";
import { resolve } from "node:path";
import {
BUILT_TEST_PROJECTS,
SOURCE_TEST_PROJECTS,
} from "../lib/test-projects.mjs";
const [group, ...arguments_] = process.argv.slice(2);
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
group
];
if (!projects) throw new Error("Expected source or built test group");
// The group owns project selection; callers can request reporters or shards.
if (
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
)
throw new Error("Test groups cannot override their project inventory");
const child = spawn(
process.execPath,
[
resolve("node_modules/vitest/vitest.mjs"),
"run",
...projects.flatMap(({ name }) => ["--project", name]),
...arguments_,
],
{ stdio: "inherit" },
);
let shutdownSignal;
for (const signal of ["SIGINT", "SIGTERM"])
process.on(signal, () => {
shutdownSignal ??= signal;
child.kill(signal);
});
child.once("error", (error) => {
console.error(error.message);
process.exitCode = 1;
});
child.once("close", (code) => {
process.exitCode =
shutdownSignal === "SIGINT"
? 130
: shutdownSignal === "SIGTERM"
? 143
: (code ?? 1);
});
+4 -3
View File
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { normalizeWorkflowRun } from "./workflow-run.mjs";
const ARTIFACT_NAME = "test-timings";
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
@@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
.slice(0, MAX_CANDIDATES);
for (const artifact of candidates) {
const run = api(
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
const run = normalizeWorkflowRun(
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
);
const runAge = now - Date.parse(run?.created_at);
if (
@@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
run.id === artifact.workflow_run.id &&
run.status === "completed" &&
run.conclusion === "success" &&
run.path === ".github/workflows/ci.yml" &&
run.workflowPath?.file === ".github/workflows/ci.yml" &&
["schedule", "workflow_dispatch"].includes(run.event) &&
run.head_branch === branch &&
run.head_repository?.full_name === repository
+21 -5
View File
@@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => {
throw new Error(`Test report path is outside the checkout: ${name}`);
return { name: path.split(sep).join("/"), durationMs, state };
});
const complete = process.env.SHARD_RESULT === "success";
// Keep group outcomes intact: an unavailable group is not a completed failure.
const groupResults = [
process.env.SHARD_RESULT,
...(process.env.SOURCE_REQUIRED === "true"
? [process.env.SOURCE_RESULT]
: []),
];
const groupsCompleted = groupResults.every((state) =>
["success", "failure"].includes(state),
);
const passed =
complete && process.env.MERGE_RESULT === "success" && report.success;
groupResults.every((state) => state === "success") &&
process.env.MERGE_RESULT === "success" &&
report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
@@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) {
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
["success", "failure"].includes(process.env.SHARD_RESULT) &&
groupsCompleted &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
@@ -91,7 +102,9 @@ if (passed && verifyInventory) {
);
}
const summary = [
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
groupsCompleted
? "## Merged test diagnostics"
: "## Partial test diagnostics",
"",
...(selection
? [
@@ -99,11 +112,14 @@ const summary = [
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"",
]
: []),
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
...(process.env.SOURCE_REQUIRED === "true"
? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`]
: []),
passed
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
+10
View File
@@ -0,0 +1,10 @@
export interface WorkflowPath {
readonly file: string;
readonly ref: string | null;
}
export interface NormalizedWorkflowRun {
readonly workflowPath: WorkflowPath | null;
}
export function normalizeWorkflowRun<T extends Record<string, unknown>>(
value: T,
): Omit<T, "workflowPath"> & NormalizedWorkflowRun;
+15
View File
@@ -0,0 +1,15 @@
/** Keep the API path as evidence and parse its optional ref at the boundary. */
export function normalizeWorkflowRun(value) {
if (typeof value !== "object" || value === null || Array.isArray(value))
throw new Error("workflow_run_malformed");
const match =
typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path)
? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec(
value.path,
)
: null;
return {
...value,
workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null,
};
}
@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { TraceMap, eachMapping } from "@jridgewell/trace-mapping";
import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";
import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js";
@@ -89,10 +90,12 @@ async function verifyCase(executable, entrypoint, scenario) {
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const provider = new CdpBrowserProvider();
const targets = await provider.listTargets({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
});
const targets = await provider.listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "Owned fixture target missing");
+7 -4
View File
@@ -8,6 +8,7 @@ import { promisify } from "node:util";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";
import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js";
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
import { parseEvidence } from "../../dist/domain/evidence.js";
import { webExecutionSchema } from "../../dist/domain/webExecution.js";
@@ -45,10 +46,12 @@ export async function verifyBrowserRuntime(
try {
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
const endpoint = `http://127.0.0.1:${fixture.port}`;
const targets = await new CdpBrowserProvider().listTargets({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
});
const targets = await new CdpBrowserProvider().listTargets(
listBrowserTargetsInputSchema.parse({
cdp_endpoint: endpoint,
allowed_origins: [site.origin],
}),
);
if (!targets.ok) throw targets.error;
const target = targets.value.targets[0];
assert.ok(target, "owned fixture page target missing");
+11 -20
View File
@@ -1,17 +1,4 @@
const sourceProjects = [
"domain",
"services",
"adapters",
"composition",
"conformance",
"evaluation",
];
const sourceTest = (path) =>
path.startsWith("src/") ||
["composition", "conformance", "evaluation"].some((project) =>
path.startsWith(`tests/${project}/`),
);
import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs";
/** Parse explicit test paths and an optional branch-comparison base. */
export const parseDevelopmentTestRequest = (mode, arguments_) => {
@@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => {
path.split("/").some((part) => part === ".." || part === ".")
)
throw new Error(`Expected an exact repository test path: ${argument}`);
if (mode !== "focused" && !sourceTest(path))
throw new Error(
`Use test:focused for compiled boundary tests: ${path}`,
);
const project = projectForTest(path);
if (mode !== "focused" && project.group !== "source")
throw new Error(`Use test:focused for built test projects: ${path}`);
paths.push(path);
}
}
@@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => {
!/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "")
)
throw new Error("Changed tests require a resolved Git merge base");
const projects = request.mode === "focused" ? [] : sourceProjects;
const projects =
request.mode === "focused"
? []
: SOURCE_TEST_PROJECTS.map(({ name }) => name);
return {
needsBuild: request.paths.some((path) => !sourceTest(path)),
needsBuild: request.paths.some(
(path) => projectForTest(path).group === "built",
),
artifactTasks: Object.entries(artifactConsumers).flatMap(
([task, consumers]) =>
request.paths.some((path) => consumers.includes(path)) ? [task] : [],
+10
View File
@@ -0,0 +1,10 @@
export interface TestProjectDefinition {
readonly name: string;
readonly group: "source" | "built";
readonly include: readonly string[];
readonly exclude?: readonly string[];
}
export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[];
export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[];
export function projectForTest(testPath: string): TestProjectDefinition;
+107
View File
@@ -0,0 +1,107 @@
import { matchesGlob } from "node:path";
/** Canonical file ownership shared by Vitest and development-test planning. */
export const SOURCE_TEST_PROJECTS = [
{
name: "domain",
group: "source",
include: ["src/{contracts,domain}/**/*.test.ts"],
},
{
name: "services",
group: "source",
include: ["src/application/**/*.test.ts"],
},
{
name: "adapters",
group: "source",
include: [
"src/*.test.ts",
"src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
],
},
{
name: "composition",
group: "source",
include: ["tests/composition/**/*.test.ts"],
},
{
name: "conformance",
group: "source",
include: ["tests/conformance/**/*.test.ts"],
exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"],
},
{
name: "evaluation",
group: "source",
include: ["tests/evaluation/**/*.test.ts"],
},
];
export const BUILT_TEST_PROJECTS = [
{
name: "boundary",
group: "built",
include: [
"tests/boundary/**/*.test.ts",
"tests/conformance/providers/hopper/realHopperSearch.test.ts",
],
exclude: [
"tests/boundary/cli/**/*.test.ts",
"tests/boundary/mcp/**/*.test.ts",
"tests/boundary/process/**/*.test.ts",
],
},
{
name: "cli-boundary",
group: "built",
include: ["tests/boundary/cli/**/*.test.ts"],
},
{
name: "process-boundary",
group: "built",
include: ["tests/boundary/process/**/*.test.ts"],
},
{
name: "mcp-boundary",
group: "built",
include: ["tests/boundary/mcp/**/*.test.ts"],
},
{
name: "acceptance",
group: "built",
include: ["tests/acceptance/**/*.test.ts"],
},
{
name: "process-global",
group: "built",
include: ["tests/process-global/**/*.test.ts"],
},
];
const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
/** Return the unique Vitest owner for an exact repository-relative test path. */
export const projectForTest = (testPath) => {
if (
typeof testPath !== "string" ||
testPath.startsWith("/") ||
testPath.includes("\\") ||
testPath.split("/").some((part) => part === ".." || part === ".") ||
!testPath.endsWith(".test.ts")
)
throw new Error(`Expected a normalized repository test path: ${testPath}`);
const owners = ALL_TEST_PROJECTS.filter(
({ include, exclude = [] }) =>
include.some((pattern) => matchesGlob(testPath, pattern)) &&
!exclude.some((pattern) => matchesGlob(testPath, pattern)),
);
if (owners.length !== 1)
throw new Error(
owners.length === 0
? `No test project owns ${testPath}`
: `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`,
);
return owners[0];
};