chore: integrate repaired verification and CI boundaries
Integrate upstream browser endpoint, Rizin fixture and cache-input repairs. Resolve the Windows verifier conflict by preserving the tested foreign-replacement identity/content checks through retry and teardown. All11 reviewed PR files remain unchanged. Three directly affected selected boundary tests pass; real-model and unrelated suites are not repeated. Native Windows sources and fixture/build inputs remain unchanged from actual hosted success. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
commit
d6271f0020
@@ -0,0 +1,67 @@
|
||||
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
|
||||
import type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
export type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
|
||||
export interface StaticDiagnostic {
|
||||
readonly kind: "lint" | "jscpd";
|
||||
readonly path?: string;
|
||||
readonly paths?: readonly string[];
|
||||
readonly code?: string;
|
||||
readonly message?: string;
|
||||
readonly severity?: string;
|
||||
}
|
||||
|
||||
export function classifyFailure(input: {
|
||||
repository: string;
|
||||
defaultBranch: string;
|
||||
run: Record<string, unknown> & NormalizedWorkflowRun;
|
||||
job: Record<string, unknown>;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
context: StaticTaskContext;
|
||||
mainRun: Record<string, unknown> & NormalizedWorkflowRun;
|
||||
mainJob: Record<string, unknown>;
|
||||
mainDiagnostics: readonly StaticDiagnostic[];
|
||||
mainContext: StaticTaskContext;
|
||||
executionSourcesVerified: boolean;
|
||||
ancestry: { status: string; mergeBaseSha: string | null };
|
||||
blobs: Readonly<Record<string, boolean>>;
|
||||
}): {
|
||||
classification: "matching-main-diagnostics" | "unknown";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function diagnosticPaths(item: StaticDiagnostic): readonly string[];
|
||||
export function isFailedConclusion(
|
||||
value: unknown,
|
||||
): value is "failure" | "timed_out";
|
||||
export function isEligibleMainRun(
|
||||
run: Record<string, unknown>,
|
||||
defaultBranch: string,
|
||||
repository: string,
|
||||
): boolean;
|
||||
export function pullRequestBaseSha(
|
||||
run: Record<string, unknown>,
|
||||
repository: string,
|
||||
): string | null;
|
||||
export function safeSha(value: unknown): string | null;
|
||||
export function validRepoPath(value: unknown): value is string;
|
||||
export function selectStaticCheckArtifact(
|
||||
page: unknown,
|
||||
run: { id?: string | number; head_sha?: string },
|
||||
runAttempt: number,
|
||||
): unknown | null;
|
||||
|
||||
export function readStaticCheckArtifact(options: {
|
||||
repository: string;
|
||||
artifactId: number | string;
|
||||
token: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<{
|
||||
context: StaticTaskContext;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
}>;
|
||||
|
||||
export function parseStaticArchive(archivePath: string): Promise<{
|
||||
context: StaticTaskContext;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
}>;
|
||||
@@ -0,0 +1,436 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { parseStaticTaskContext } from "./static-task-context.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_MEMBER_BYTES = 4 * 1024 * 1024;
|
||||
const MAX_DIAGNOSTICS = 10_000;
|
||||
const API_ROOT = "https://api.github.com/repos";
|
||||
const SHA = /^[0-9a-f]{40}$/iu;
|
||||
const TASK_HASH = /^[a-f0-9]{16}$/u;
|
||||
|
||||
/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */
|
||||
export function classifyFailure({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
diagnostics,
|
||||
context,
|
||||
mainRun,
|
||||
mainJob,
|
||||
mainDiagnostics,
|
||||
mainContext,
|
||||
executionSourcesVerified,
|
||||
ancestry,
|
||||
blobs,
|
||||
}) {
|
||||
const failedGate = firstFailure([
|
||||
() => [
|
||||
!isCompletedFailure(run) ||
|
||||
job?.status !== "completed" ||
|
||||
!isFailedConclusion(job?.conclusion),
|
||||
"run_or_job_not_a_completed_failure",
|
||||
],
|
||||
() => [
|
||||
!isEligiblePullRun(run, repository) ||
|
||||
!isEligibleMainRun(mainRun, defaultBranch, repository),
|
||||
"run_event_or_repository_identity_unverified",
|
||||
],
|
||||
() => [
|
||||
job?.name !== mainJob?.name || job?.name !== "Static checks",
|
||||
"failed_job_not_the_same_static_gate",
|
||||
],
|
||||
() => [
|
||||
!Array.isArray(diagnostics) ||
|
||||
diagnostics.length === 0 ||
|
||||
!Array.isArray(mainDiagnostics),
|
||||
"structured_diagnostics_missing_or_invalid",
|
||||
],
|
||||
() => [
|
||||
!validContext(context) || !validContext(mainContext),
|
||||
"static_task_context_missing_or_invalid",
|
||||
],
|
||||
() => [
|
||||
executionSourcesVerified !== true,
|
||||
"checkout_sources_not_verified_against_api_commits",
|
||||
],
|
||||
() => [
|
||||
!sameExecutionContext(context, mainContext),
|
||||
"task_or_toolchain_identity_differs",
|
||||
],
|
||||
() => [
|
||||
ancestry?.status !== "ahead" && ancestry?.status !== "identical",
|
||||
"main_head_not_proven_ancestor_of_tested_base",
|
||||
],
|
||||
() => [
|
||||
ancestry?.mergeBaseSha !== mainRun?.head_sha,
|
||||
"main_head_is_not_the_verified_merge_base",
|
||||
],
|
||||
() => [
|
||||
!sameTaskHashes(diagnostics, context, mainContext),
|
||||
"owning_task_hash_differs",
|
||||
],
|
||||
]);
|
||||
if (failedGate) return outcome("unknown", failedGate.reason);
|
||||
const keys = diagnostics.map(diagnosticKey);
|
||||
const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean));
|
||||
if (keys.some((key) => key === null))
|
||||
return outcome("unknown", "diagnostic_shape_not_supported");
|
||||
if (!keys.every((key) => mainKeys.has(key)))
|
||||
return outcome(
|
||||
"unknown",
|
||||
"main_run_does_not_contain_the_same_structured_diagnostic",
|
||||
);
|
||||
if (!sameBlobEvidence(diagnostics, blobs))
|
||||
return outcome(
|
||||
"unknown",
|
||||
"diagnostic_source_content_not_identical_at_main_and_tested_base",
|
||||
);
|
||||
return outcome(
|
||||
"matching-main-diagnostics",
|
||||
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
|
||||
);
|
||||
}
|
||||
|
||||
export function diagnosticPaths(item) {
|
||||
return item.kind === "jscpd" ? item.paths : [item.path];
|
||||
}
|
||||
export function isFailedConclusion(value) {
|
||||
return value === "failure" || value === "timed_out";
|
||||
}
|
||||
export function isEligibleMainRun(run, defaultBranch, repository) {
|
||||
return (
|
||||
isEligibleStaticRun(run) &&
|
||||
run.repository.full_name === repository &&
|
||||
run.head_branch === defaultBranch &&
|
||||
["push", "schedule", "workflow_dispatch"].includes(run.event)
|
||||
);
|
||||
}
|
||||
export function pullRequestBaseSha(run, repository) {
|
||||
if (
|
||||
run?.event !== "pull_request" ||
|
||||
run.repository?.full_name !== repository ||
|
||||
!Array.isArray(run.pull_requests) ||
|
||||
run.pull_requests.length !== 1
|
||||
)
|
||||
return null;
|
||||
const base = run.pull_requests[0].base;
|
||||
return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "")
|
||||
? base.sha
|
||||
: null;
|
||||
}
|
||||
|
||||
export function safeSha(value) {
|
||||
return typeof value === "string" && SHA.test(value)
|
||||
? value.toLowerCase()
|
||||
: null;
|
||||
}
|
||||
export function selectStaticCheckArtifact(page, run, runAttempt) {
|
||||
if (runAttempt !== 1) return null;
|
||||
if (
|
||||
!Array.isArray(page?.artifacts) ||
|
||||
!Number.isSafeInteger(page.total_count) ||
|
||||
page.total_count !== page.artifacts.length ||
|
||||
page.artifacts.length >= 100
|
||||
)
|
||||
return null;
|
||||
const matches = page.artifacts.filter(
|
||||
(artifact) => artifact.name === "static-check-reports",
|
||||
);
|
||||
if (matches.length !== 1) return null;
|
||||
const [artifact] = matches;
|
||||
if (
|
||||
artifact.expired !== false ||
|
||||
String(artifact.workflow_run?.id) !== String(run.id) ||
|
||||
artifact.workflow_run?.head_sha !== run.head_sha
|
||||
)
|
||||
return null;
|
||||
return artifact;
|
||||
}
|
||||
export function validRepoPath(value) {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
!isAbsolute(value) &&
|
||||
!value.includes("\\") &&
|
||||
value
|
||||
.split("/")
|
||||
.every((part) => part !== "" && part !== "." && part !== "..") &&
|
||||
!value.startsWith("/")
|
||||
);
|
||||
}
|
||||
|
||||
/** Download one named static report archive and parse only whitelisted JSON members. */
|
||||
export async function readStaticCheckArtifact({
|
||||
repository,
|
||||
artifactId,
|
||||
token,
|
||||
fetchImpl = fetch,
|
||||
}) {
|
||||
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? ""))
|
||||
throw new Error("artifact repository identity is invalid");
|
||||
if (!token) throw new Error("artifact download requires a GitHub token");
|
||||
const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`;
|
||||
let response = await fetchImpl(url, {
|
||||
headers: {
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: `Bearer ${token}`,
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
redirect: "manual",
|
||||
});
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
const location = response.headers.get("location");
|
||||
if (!location || new URL(location).protocol !== "https:")
|
||||
throw new Error("static_artifact_redirect_invalid");
|
||||
response = await fetchImpl(location, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
redirect: "error",
|
||||
});
|
||||
}
|
||||
if (!response.ok) throw new Error(`github_http_${response.status}`);
|
||||
const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES);
|
||||
const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-"));
|
||||
const archive = join(directory, "artifact.zip");
|
||||
try {
|
||||
await writeFile(archive, bytes, { flag: "wx" });
|
||||
return await parseStaticArchive(archive);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Accept only the three named producer outputs; no archive paths are written to disk. */
|
||||
export async function parseStaticArchive(archivePath) {
|
||||
const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 128 * 1024,
|
||||
timeout: 5_000,
|
||||
});
|
||||
const allowed = new Set([
|
||||
"oxlint.json",
|
||||
"jscpd/jscpd-report.json",
|
||||
"static-task-context.json",
|
||||
]);
|
||||
const members = new Map();
|
||||
for (const name of listing.split(/\r?\n/u).filter(Boolean)) {
|
||||
const prefix = ".cache/static-checks/";
|
||||
const member = name.startsWith(prefix) ? name.slice(prefix.length) : name;
|
||||
if (!allowed.has(member)) continue;
|
||||
if (members.has(member))
|
||||
throw new Error("static artifact contains duplicate report members");
|
||||
const { stdout } = await execute("unzip", ["-p", archivePath, name], {
|
||||
encoding: "buffer",
|
||||
maxBuffer: MAX_MEMBER_BYTES,
|
||||
timeout: 5_000,
|
||||
});
|
||||
members.set(member, JSON.parse(stdout.toString("utf8")));
|
||||
}
|
||||
const context = parseStaticTaskContext(
|
||||
members.get("static-task-context.json"),
|
||||
);
|
||||
const diagnostics = normalizeDiagnostics(
|
||||
members.get("oxlint.json"),
|
||||
members.get("jscpd/jscpd-report.json"),
|
||||
);
|
||||
if (diagnostics.length > MAX_DIAGNOSTICS)
|
||||
throw new Error("static diagnostic count exceeds the reporting limit");
|
||||
return { context, diagnostics };
|
||||
}
|
||||
|
||||
function normalizeDiagnostics(lint, duplicate) {
|
||||
return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)];
|
||||
}
|
||||
|
||||
function normalizeLint(report) {
|
||||
if (report === undefined) return [];
|
||||
if (!Array.isArray(report?.diagnostics))
|
||||
throw new Error("oxlint diagnostics are malformed");
|
||||
return report.diagnostics
|
||||
.filter((item) => item.severity === "error")
|
||||
.map((item) => {
|
||||
if (
|
||||
typeof item.filename !== "string" ||
|
||||
!repoPath(item.filename) ||
|
||||
typeof item.code !== "string" ||
|
||||
typeof item.message !== "string"
|
||||
)
|
||||
throw new Error("oxlint error diagnostic is malformed");
|
||||
return {
|
||||
kind: "lint",
|
||||
path: item.filename,
|
||||
code: item.code,
|
||||
message: item.message,
|
||||
severity: "error",
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeDuplicates(report) {
|
||||
if (report === undefined) return [];
|
||||
if (
|
||||
!Array.isArray(report?.duplicates) ||
|
||||
!Number.isFinite(report.statistics?.total?.percentage) ||
|
||||
report.statistics.total.percentage < 0
|
||||
)
|
||||
throw new Error("jscpd report is malformed");
|
||||
// The producer owns the checker outcome; do not copy or infer its threshold.
|
||||
const exitCode = report.reaCheck?.exitCode;
|
||||
if (exitCode === undefined || exitCode === 0) return [];
|
||||
if (!Number.isSafeInteger(exitCode) || exitCode < 0)
|
||||
throw new Error("jscpd checker outcome is malformed");
|
||||
if (exitCode !== 1) return [];
|
||||
return report.duplicates.map((item) => {
|
||||
const paths = [item.firstFile?.name, item.secondFile?.name].sort();
|
||||
if (paths.some((path) => !repoPath(path)))
|
||||
throw new Error("jscpd duplicate paths are malformed");
|
||||
return { kind: "jscpd", paths };
|
||||
});
|
||||
}
|
||||
|
||||
async function readLimitedBytes(response, limit) {
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) throw new Error("static artifact response has no body");
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > limit) {
|
||||
await reader.cancel();
|
||||
throw new Error("static artifact exceeds the download limit");
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
const result = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function repoPath(value) {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
!value.startsWith("/") &&
|
||||
!value.includes("\\") &&
|
||||
value
|
||||
.split("/")
|
||||
.every((part) => part !== "" && part !== "." && part !== "..")
|
||||
);
|
||||
}
|
||||
|
||||
function positiveId(value) {
|
||||
const id = Number(value);
|
||||
if (!Number.isSafeInteger(id) || id < 1)
|
||||
throw new Error("artifact ID must be a positive integer");
|
||||
return String(id);
|
||||
}
|
||||
|
||||
function diagnosticKey(item) {
|
||||
if (
|
||||
item?.kind === "lint" &&
|
||||
validRepoPath(item.path) &&
|
||||
typeof item.code === "string" &&
|
||||
typeof item.message === "string" &&
|
||||
item.severity === "error"
|
||||
)
|
||||
return JSON.stringify([
|
||||
item.kind,
|
||||
item.path,
|
||||
item.code,
|
||||
item.message,
|
||||
item.severity,
|
||||
]);
|
||||
if (
|
||||
item?.kind === "jscpd" &&
|
||||
Array.isArray(item.paths) &&
|
||||
item.paths.length === 2 &&
|
||||
item.paths.every(validRepoPath)
|
||||
)
|
||||
return JSON.stringify([item.kind, ...[...item.paths].sort()]);
|
||||
return null;
|
||||
}
|
||||
function sameBlobEvidence(diagnostics, blobs) {
|
||||
const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
|
||||
return paths.length > 0 && paths.every((path) => blobs?.[path] === true);
|
||||
}
|
||||
function sameTaskHashes(diagnostics, left, right) {
|
||||
return diagnostics.every((diagnostic) => {
|
||||
const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built";
|
||||
return (
|
||||
TASK_HASH.test(left.taskHashes[task] ?? "") &&
|
||||
left.taskHashes[task] === right.taskHashes[task]
|
||||
);
|
||||
});
|
||||
}
|
||||
function validContext(value) {
|
||||
return (
|
||||
value?.version === 1 &&
|
||||
SHA.test(value.sourceSha ?? "") &&
|
||||
value.sourceClean === true &&
|
||||
typeof value.nodeVersion === "string" &&
|
||||
typeof value.npmVersion === "string" &&
|
||||
typeof value.turboVersion === "string" &&
|
||||
typeof value.platform === "string" &&
|
||||
typeof value.architecture === "string" &&
|
||||
value.taskHashes &&
|
||||
typeof value.taskHashes === "object" &&
|
||||
!Array.isArray(value.taskHashes)
|
||||
);
|
||||
}
|
||||
function sameExecutionContext(left, right) {
|
||||
return [
|
||||
"nodeVersion",
|
||||
"npmVersion",
|
||||
"turboVersion",
|
||||
"platform",
|
||||
"architecture",
|
||||
].every((key) => left[key] === right[key]);
|
||||
}
|
||||
function isEligiblePullRun(run, repository) {
|
||||
return (
|
||||
isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null
|
||||
);
|
||||
}
|
||||
function isEligibleStaticRun(run) {
|
||||
return (
|
||||
isRecord(run) &&
|
||||
run.status === "completed" &&
|
||||
isFailedConclusion(run.conclusion) &&
|
||||
SHA.test(run.head_sha ?? "") &&
|
||||
typeof run.repository?.full_name === "string" &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml"
|
||||
);
|
||||
}
|
||||
function isCompletedFailure(run) {
|
||||
return run?.status === "completed" && isFailedConclusion(run.conclusion);
|
||||
}
|
||||
function isRecord(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
function firstFailure(checks) {
|
||||
for (const check of checks) {
|
||||
const [failed, reason] = check();
|
||||
if (failed) return { reason };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function outcome(classification, reason) {
|
||||
return { classification, reason };
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
|
||||
import type { StaticDiagnostic } from "./artifacts.mjs";
|
||||
export type { StaticDiagnostic } from "./artifacts.mjs";
|
||||
import type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
export type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
export interface CiRun {
|
||||
readonly id?: string | number;
|
||||
readonly run_attempt?: number;
|
||||
readonly status?: string;
|
||||
readonly conclusion?: string | null;
|
||||
readonly event?: string;
|
||||
readonly path?: string;
|
||||
readonly head_sha?: string;
|
||||
readonly head_branch?: string;
|
||||
readonly repository?: { readonly full_name?: string };
|
||||
readonly pull_requests?: readonly {
|
||||
readonly base?: {
|
||||
readonly sha?: string;
|
||||
readonly repo?: { readonly full_name?: string };
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface CiJob {
|
||||
readonly name?: string;
|
||||
readonly status?: string;
|
||||
readonly conclusion?: string | null;
|
||||
readonly run_id?: number;
|
||||
readonly run_attempt?: number;
|
||||
}
|
||||
|
||||
export interface StaticEvidence {
|
||||
readonly context: StaticTaskContext;
|
||||
readonly diagnostics: readonly StaticDiagnostic[];
|
||||
}
|
||||
|
||||
export interface CiFailureReport {
|
||||
readonly version: 1;
|
||||
readonly observedAt: string;
|
||||
readonly repository: string;
|
||||
readonly defaultBranch: string;
|
||||
readonly informationalOnly: true;
|
||||
readonly gateEffect: string;
|
||||
readonly boundedInput: {
|
||||
readonly candidateRuns: number;
|
||||
readonly limit: number;
|
||||
readonly mainCandidateLimit: number;
|
||||
readonly mainCandidateRuns: number;
|
||||
readonly mainRunLookupLimit: number;
|
||||
readonly manifestTruncated: boolean;
|
||||
};
|
||||
readonly counts: Readonly<
|
||||
Record<"matching-main-diagnostics" | "unknown", number>
|
||||
>;
|
||||
readonly failures: readonly {
|
||||
readonly runId: number | string;
|
||||
readonly runAttempt: number;
|
||||
readonly classification: "matching-main-diagnostics" | "unknown";
|
||||
readonly reason: string;
|
||||
readonly candidateFailures?: readonly {
|
||||
readonly runId: number | string;
|
||||
readonly runAttempt: number;
|
||||
readonly reason: string;
|
||||
}[];
|
||||
readonly runHead?: string | null;
|
||||
readonly event?: string | null;
|
||||
readonly job?: string;
|
||||
readonly conclusion?: string | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface FailureReportApi {
|
||||
getRun(repository: string, id: string | number): Promise<CiRun>;
|
||||
getJobs(
|
||||
repository: string,
|
||||
id: string | number,
|
||||
attempt: number,
|
||||
): Promise<readonly CiJob[]>;
|
||||
getStaticEvidence(
|
||||
repository: string,
|
||||
run: CiRun,
|
||||
job: CiJob,
|
||||
): Promise<StaticEvidence | null>;
|
||||
getAncestry(
|
||||
repository: string,
|
||||
ancestor: string,
|
||||
descendant: string,
|
||||
): Promise<{ status: string; mergeBaseSha: string | null }>;
|
||||
verifyExecutionSource(
|
||||
repository: string,
|
||||
run: CiRun,
|
||||
context: StaticTaskContext,
|
||||
): Promise<boolean>;
|
||||
getBlobIdentity(
|
||||
repository: string,
|
||||
ancestor: string,
|
||||
descendant: string,
|
||||
paths: readonly string[],
|
||||
): Promise<Readonly<Record<string, boolean>>>;
|
||||
}
|
||||
|
||||
export function classifyFailure(input: {
|
||||
repository: string;
|
||||
defaultBranch: string;
|
||||
run: CiRun & NormalizedWorkflowRun;
|
||||
job: CiJob;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
context: StaticTaskContext;
|
||||
mainRun: CiRun & NormalizedWorkflowRun;
|
||||
mainJob: CiJob;
|
||||
mainDiagnostics: readonly StaticDiagnostic[];
|
||||
mainContext: StaticTaskContext;
|
||||
executionSourcesVerified: boolean;
|
||||
ancestry: { status: string; mergeBaseSha: string | null };
|
||||
blobs: Readonly<Record<string, boolean>>;
|
||||
}): {
|
||||
classification: "matching-main-diagnostics" | "unknown";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function reportCiFailures(options: {
|
||||
repository: string;
|
||||
defaultBranch: string;
|
||||
batchManifest: {
|
||||
readonly repository: string;
|
||||
readonly runs: readonly {
|
||||
readonly id: number | string;
|
||||
readonly runAttempt: number;
|
||||
readonly workflow: string;
|
||||
readonly event?: string | null;
|
||||
readonly headBranch?: string | null;
|
||||
readonly conclusion: string | null;
|
||||
readonly status: string;
|
||||
}[];
|
||||
readonly truncation?: readonly unknown[];
|
||||
readonly coverage?: { readonly exhaustive?: boolean };
|
||||
readonly selection?: { readonly exhaustive?: boolean };
|
||||
};
|
||||
api: FailureReportApi;
|
||||
outputDirectory?: string;
|
||||
summaryPath?: string;
|
||||
observedAt?: string;
|
||||
}): Promise<CiFailureReport>;
|
||||
|
||||
export function createGitHubApi(options?: {
|
||||
token?: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
apiRequest?: (
|
||||
method: string,
|
||||
endpoint: string,
|
||||
budget: { requests: number; startedAt: number; limitMs: number },
|
||||
) => Promise<unknown>;
|
||||
}): FailureReportApi;
|
||||
|
||||
export function writeFailureReport(
|
||||
report: CiFailureReport,
|
||||
outputDirectory?: string,
|
||||
summaryPath?: string,
|
||||
): Promise<{ jsonPath: string; markdownPath: string }>;
|
||||
@@ -0,0 +1,529 @@
|
||||
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
classifyFailure,
|
||||
diagnosticPaths,
|
||||
isEligibleMainRun,
|
||||
isFailedConclusion,
|
||||
pullRequestBaseSha,
|
||||
readStaticCheckArtifact,
|
||||
safeSha,
|
||||
selectStaticCheckArtifact,
|
||||
validRepoPath,
|
||||
} from "./artifacts.mjs";
|
||||
import { createGhApiRequest } from "./github-api.mjs";
|
||||
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
||||
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-failures");
|
||||
const BATCH_DIRECTORY = join(".cache", "ci-job-timings");
|
||||
const MAX_JSON_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_FAILED_RUNS = 20;
|
||||
const MAX_MAIN_CANDIDATES = 5;
|
||||
const MAX_MAIN_RUN_LOOKUPS = 100;
|
||||
const MAX_PATHS_PER_COMPARISON = 50;
|
||||
const MAX_API_REQUESTS = 250;
|
||||
|
||||
export { classifyFailure } from "./artifacts.mjs";
|
||||
|
||||
/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
|
||||
export async function reportCiFailures({
|
||||
repository,
|
||||
defaultBranch,
|
||||
batchManifest,
|
||||
api,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
observedAt = new Date().toISOString(),
|
||||
}) {
|
||||
validateRepository(repository);
|
||||
if (typeof defaultBranch !== "string" || defaultBranch.length === 0)
|
||||
throw new Error("defaultBranch is required");
|
||||
if (
|
||||
!batchManifest ||
|
||||
batchManifest.repository !== repository ||
|
||||
!Array.isArray(batchManifest.runs)
|
||||
)
|
||||
throw new Error("timing manifest repository or runs are invalid");
|
||||
|
||||
const failedRuns = batchManifest.runs.filter(
|
||||
(entry) =>
|
||||
entry.workflow === "CI" &&
|
||||
entry.status === "collected" &&
|
||||
isFailedConclusion(entry.conclusion),
|
||||
);
|
||||
const prRuns = failedRuns.filter((entry) => entry.event === "pull_request");
|
||||
const candidates = prRuns.slice(0, MAX_FAILED_RUNS);
|
||||
const mainEntries = failedRuns.filter(
|
||||
(entry) =>
|
||||
entry.headBranch === defaultBranch &&
|
||||
["push", "schedule", "workflow_dispatch"].includes(entry.event),
|
||||
);
|
||||
const lookupRun = verifiedRunLookup(repository, api);
|
||||
const mainCandidates = [];
|
||||
for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) {
|
||||
try {
|
||||
const run = await lookupRun(entry);
|
||||
if (isEligibleMainRun(run, defaultBranch, repository))
|
||||
mainCandidates.push({
|
||||
run,
|
||||
getEvidence: mainEvidenceLookup(repository, api, entry, run),
|
||||
});
|
||||
if (mainCandidates.length === MAX_MAIN_CANDIDATES) break;
|
||||
} catch {
|
||||
// Unverified metadata cannot consume the eligible-main candidate budget.
|
||||
}
|
||||
}
|
||||
const entries = [];
|
||||
for (const entry of candidates) {
|
||||
try {
|
||||
entries.push(
|
||||
...(await analyzeFailedRun({
|
||||
repository,
|
||||
defaultBranch,
|
||||
entry,
|
||||
run: await lookupRun(entry),
|
||||
mainCandidates,
|
||||
api,
|
||||
})),
|
||||
);
|
||||
} catch (error) {
|
||||
entries.push({
|
||||
runId: entry.id,
|
||||
runAttempt: entry.runAttempt,
|
||||
classification: "unknown",
|
||||
reason: errorCode(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
const report = {
|
||||
version: 1,
|
||||
observedAt,
|
||||
repository,
|
||||
defaultBranch,
|
||||
informationalOnly: true,
|
||||
gateEffect: "none; this report never changes CI conclusions",
|
||||
boundedInput: {
|
||||
candidateRuns: candidates.length,
|
||||
limit: MAX_FAILED_RUNS,
|
||||
mainCandidateLimit: MAX_MAIN_CANDIDATES,
|
||||
mainCandidateRuns: mainCandidates.length,
|
||||
mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS,
|
||||
manifestTruncated:
|
||||
prRuns.length > MAX_FAILED_RUNS ||
|
||||
batchManifest.truncation?.length > 0 ||
|
||||
batchManifest.selection?.exhaustive === false ||
|
||||
batchManifest.coverage?.exhaustive === false,
|
||||
},
|
||||
counts: Object.fromEntries(
|
||||
["matching-main-diagnostics", "unknown"].map((category) => [
|
||||
category,
|
||||
entries.filter((item) => item.classification === category).length,
|
||||
]),
|
||||
),
|
||||
failures: entries,
|
||||
};
|
||||
await writeFailureReport(report, outputDirectory, summaryPath);
|
||||
return report;
|
||||
}
|
||||
|
||||
async function analyzeFailedRun({
|
||||
repository,
|
||||
defaultBranch,
|
||||
entry,
|
||||
run,
|
||||
mainCandidates,
|
||||
api,
|
||||
}) {
|
||||
const jobs = await api.getJobs(repository, entry.id, entry.runAttempt);
|
||||
if (jobs.some((job) => !hasJobIdentity(job, entry)))
|
||||
throw new Error("workflow_job_identity_mismatch");
|
||||
const failedJobs = jobs.filter(
|
||||
(job) => job.status === "completed" && isFailedConclusion(job.conclusion),
|
||||
);
|
||||
if (failedJobs.length === 0)
|
||||
return [
|
||||
{
|
||||
runId: entry.id,
|
||||
runAttempt: entry.runAttempt,
|
||||
classification: "unknown",
|
||||
reason: "failed_jobs_not_present_in_api_snapshot",
|
||||
},
|
||||
];
|
||||
const failures = [];
|
||||
for (const job of failedJobs) {
|
||||
const evidence = await api.getStaticEvidence(repository, run, job);
|
||||
const classification = await attributeToMain({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
mainCandidates,
|
||||
api,
|
||||
});
|
||||
failures.push({
|
||||
runId: entry.id,
|
||||
runAttempt: entry.runAttempt,
|
||||
runHead: safeSha(run.head_sha),
|
||||
event: run.event ?? null,
|
||||
job: job.name,
|
||||
conclusion: job.conclusion,
|
||||
...classification,
|
||||
});
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function attributeToMain({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
mainCandidates,
|
||||
api,
|
||||
}) {
|
||||
const baseSha = pullRequestBaseSha(run, repository);
|
||||
if (baseSha === null || evidence === null)
|
||||
return outcome("unknown", "no_matching_verified_main_failure");
|
||||
let result = outcome("unknown", "no_matching_verified_main_failure");
|
||||
const candidateFailures = [];
|
||||
for (const candidate of mainCandidates) {
|
||||
try {
|
||||
const main = await compareCandidate({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
baseSha,
|
||||
candidate,
|
||||
api,
|
||||
});
|
||||
if (main !== null) result = main;
|
||||
if (main?.classification === "matching-main-diagnostics") break;
|
||||
} catch (error) {
|
||||
candidateFailures.push({
|
||||
runId: candidate.run.id,
|
||||
runAttempt: candidate.run.run_attempt,
|
||||
reason: errorCode(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
...(candidateFailures.length > 0 ? { candidateFailures } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function compareCandidate({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
baseSha,
|
||||
candidate,
|
||||
api,
|
||||
}) {
|
||||
const mainRun = candidate.run;
|
||||
const main = await candidate.getEvidence(job.name);
|
||||
if (!main?.evidence) return null;
|
||||
const { job: mainJob, evidence: mainEvidence } = main;
|
||||
const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha);
|
||||
const paths = [
|
||||
...new Set(evidence.diagnostics.flatMap(diagnosticPaths)),
|
||||
].slice(0, MAX_PATHS_PER_COMPARISON);
|
||||
const blobs = await api.getBlobIdentity(
|
||||
repository,
|
||||
mainRun.head_sha,
|
||||
baseSha,
|
||||
paths,
|
||||
);
|
||||
const executionSourcesVerified =
|
||||
(await api.verifyExecutionSource(repository, run, evidence.context)) &&
|
||||
(await api.verifyExecutionSource(
|
||||
repository,
|
||||
mainRun,
|
||||
mainEvidence.context,
|
||||
));
|
||||
return classifyFailure({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
diagnostics: evidence.diagnostics,
|
||||
context: evidence.context,
|
||||
mainRun,
|
||||
mainJob,
|
||||
mainDiagnostics: mainEvidence.diagnostics,
|
||||
mainContext: mainEvidence.context,
|
||||
executionSourcesVerified,
|
||||
ancestry,
|
||||
blobs,
|
||||
});
|
||||
}
|
||||
|
||||
// Each verified main run/attempt owns its lazy jobs and parsed evidence for this
|
||||
// report only. Cache nulls and rejected reads too; retries cannot spend the budget
|
||||
// repeatedly on an immutable observation that was unavailable or invalid.
|
||||
function mainEvidenceLookup(repository, api, entry, run) {
|
||||
let jobs;
|
||||
const evidence = new Map();
|
||||
return (name) => {
|
||||
jobs ??= Promise.resolve().then(async () => {
|
||||
const items = await api.getJobs(repository, entry.id, entry.runAttempt);
|
||||
return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items;
|
||||
});
|
||||
if (!evidence.has(name))
|
||||
evidence.set(
|
||||
name,
|
||||
jobs.then(async (items) => {
|
||||
const job = items.find(
|
||||
(item) =>
|
||||
item.name === name &&
|
||||
item.status === "completed" &&
|
||||
isFailedConclusion(item.conclusion),
|
||||
);
|
||||
if (!job) return null;
|
||||
return {
|
||||
job,
|
||||
evidence: await api.getStaticEvidence(repository, run, job),
|
||||
};
|
||||
}),
|
||||
);
|
||||
return evidence.get(name);
|
||||
};
|
||||
}
|
||||
|
||||
// Selection and attribution share one immutable API observation per run/attempt.
|
||||
function verifiedRunLookup(repository, api) {
|
||||
const runs = new Map();
|
||||
return (entry) => {
|
||||
const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`;
|
||||
if (!runs.has(key))
|
||||
runs.set(
|
||||
key,
|
||||
Promise.resolve().then(async () => {
|
||||
const run = normalizeWorkflowRun(
|
||||
await api.getRun(repository, entry.id),
|
||||
);
|
||||
if (!hasRunIdentity(run, entry))
|
||||
throw new Error("workflow_run_identity_mismatch");
|
||||
return run;
|
||||
}),
|
||||
);
|
||||
return runs.get(key);
|
||||
};
|
||||
}
|
||||
|
||||
function hasRunIdentity(run, entry) {
|
||||
return (
|
||||
String(run?.id) === String(entry.id) &&
|
||||
Number(run.run_attempt) === Number(entry.runAttempt) &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml"
|
||||
);
|
||||
}
|
||||
function hasJobIdentity(job, entry) {
|
||||
return (
|
||||
String(job.run_id) === String(entry.id) &&
|
||||
Number(job.run_attempt) === Number(entry.runAttempt)
|
||||
);
|
||||
}
|
||||
|
||||
export function createGitHubApi({
|
||||
token = process.env.GH_TOKEN,
|
||||
fetchImpl = fetch,
|
||||
apiRequest = createGhApiRequest(),
|
||||
} = {}) {
|
||||
if (!token) throw new Error("GH_TOKEN is required");
|
||||
const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 };
|
||||
const request = async (endpoint) => {
|
||||
if (budget.requests >= MAX_API_REQUESTS)
|
||||
throw new Error("github_api_request_budget_exhausted");
|
||||
budget.requests += 1;
|
||||
return await apiRequest("GET", endpoint, budget);
|
||||
};
|
||||
return {
|
||||
async getRun(repository, id) {
|
||||
return await request(
|
||||
`repos/${repository}/actions/runs/${positiveId(id)}`,
|
||||
);
|
||||
},
|
||||
async getJobs(repository, id, attempt) {
|
||||
const page = await request(
|
||||
`repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`,
|
||||
);
|
||||
if (!Array.isArray(page.jobs) || page.jobs.length >= 100)
|
||||
throw new Error("job_list_incomplete_or_malformed");
|
||||
return page.jobs;
|
||||
},
|
||||
async getStaticEvidence(repository, run, job) {
|
||||
if (job.name !== "Static checks") return null;
|
||||
if (run.run_attempt !== 1) return null;
|
||||
const page = await request(
|
||||
`repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`,
|
||||
);
|
||||
const artifact = selectStaticCheckArtifact(page, run, run.run_attempt);
|
||||
if (!artifact) return null;
|
||||
return await readStaticCheckArtifact({
|
||||
repository,
|
||||
artifactId: artifact.id,
|
||||
token,
|
||||
fetchImpl,
|
||||
});
|
||||
},
|
||||
async getAncestry(repository, ancestor, descendant) {
|
||||
const comparison = await request(
|
||||
`repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`,
|
||||
);
|
||||
return {
|
||||
status: comparison.status,
|
||||
mergeBaseSha: safeSha(comparison.merge_base_commit?.sha),
|
||||
};
|
||||
},
|
||||
async verifyExecutionSource(repository, run, context) {
|
||||
const sourceSha = safeSha(context.sourceSha);
|
||||
if (!sourceSha) return false;
|
||||
const pr =
|
||||
run.event === "pull_request"
|
||||
? pullRequestBaseSha(run, repository)
|
||||
: null;
|
||||
if (
|
||||
run.event === "pull_request" &&
|
||||
(pr === null || context.prHeadSha !== run.head_sha)
|
||||
)
|
||||
return false;
|
||||
const commit = await request(`repos/${repository}/commits/${sourceSha}`);
|
||||
if (safeSha(commit.sha) !== sourceSha) return false;
|
||||
if (sourceSha === run.head_sha) return true;
|
||||
if (
|
||||
pr === null ||
|
||||
!Array.isArray(commit.parents) ||
|
||||
commit.parents.length !== 2
|
||||
)
|
||||
return false;
|
||||
return (
|
||||
safeSha(commit.parents[0]?.sha) === pr &&
|
||||
safeSha(commit.parents[1]?.sha) === run.head_sha
|
||||
);
|
||||
},
|
||||
async getBlobIdentity(repository, ancestor, descendant, paths) {
|
||||
const result = {};
|
||||
for (const path of paths) {
|
||||
if (!validRepoPath(path)) continue;
|
||||
const [left, right] = await Promise.all([
|
||||
getContent(repository, ancestor, path, request),
|
||||
getContent(repository, descendant, path, request),
|
||||
]);
|
||||
result[path] = left !== null && left === right;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getContent(repository, ref, path, request) {
|
||||
try {
|
||||
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
|
||||
const value = await request(
|
||||
`repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
|
||||
);
|
||||
return typeof value.sha === "string" ? value.sha : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function outcome(classification, reason) {
|
||||
return { classification, reason };
|
||||
}
|
||||
function validateRepository(value) {
|
||||
if (
|
||||
typeof value !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)
|
||||
)
|
||||
throw new Error("repository must be owner/name");
|
||||
}
|
||||
function positiveId(value) {
|
||||
if (!Number.isSafeInteger(Number(value)) || Number(value) < 1)
|
||||
throw new Error(
|
||||
"GitHub run and attempt IDs must be positive safe integers",
|
||||
);
|
||||
return String(Number(value));
|
||||
}
|
||||
function errorCode(error) {
|
||||
return error instanceof Error ? error.message.slice(0, 160) : "unknown_error";
|
||||
}
|
||||
|
||||
export async function writeFailureReport(
|
||||
report,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
) {
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const jsonPath = join(outputDirectory, "report.json");
|
||||
const markdownPath = join(outputDirectory, "report.md");
|
||||
const matchedDiagnostics = report.failures.filter(
|
||||
(item) => item.classification === "matching-main-diagnostics",
|
||||
);
|
||||
const lines = [
|
||||
"# CI failure attribution",
|
||||
"",
|
||||
"Informational only. This report never changes a CI conclusion or required gate.",
|
||||
"",
|
||||
`Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`,
|
||||
"",
|
||||
];
|
||||
for (const item of report.failures)
|
||||
lines.push(
|
||||
`- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`,
|
||||
);
|
||||
if (report.failures.length === 0)
|
||||
lines.push(
|
||||
"- No completed failed CI attempts were present in the supplied timing snapshot.",
|
||||
);
|
||||
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8");
|
||||
if (summaryPath)
|
||||
await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8");
|
||||
return { jsonPath, markdownPath };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const defaultBranch = process.env.CI_DEFAULT_BRANCH;
|
||||
const batchPath =
|
||||
process.env.CI_TIMING_MANIFEST ??
|
||||
join(BATCH_DIRECTORY, "batch-manifest.json");
|
||||
const batchManifest = JSON.parse(await readBoundedFile(batchPath));
|
||||
const api = createGitHubApi();
|
||||
await reportCiFailures({
|
||||
repository,
|
||||
defaultBranch,
|
||||
batchManifest,
|
||||
api,
|
||||
outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
summaryPath: process.env.GITHUB_STEP_SUMMARY,
|
||||
});
|
||||
}
|
||||
async function readBoundedFile(path) {
|
||||
const bytes = await readFile(path);
|
||||
if (bytes.byteLength > MAX_JSON_BYTES)
|
||||
throw new Error("timing manifest exceeds size limit");
|
||||
return bytes.toString("utf8");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
await main();
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
export interface GitHubApiBudget {
|
||||
requests: number;
|
||||
startedAt: number;
|
||||
limitMs: number;
|
||||
}
|
||||
|
||||
export type GitHubApiRequest = (
|
||||
method: string,
|
||||
endpoint: string,
|
||||
budget: GitHubApiBudget,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export function createGhApiRequest(options?: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: () => number;
|
||||
}): GitHubApiRequest;
|
||||
@@ -0,0 +1,33 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MAX_BUFFER = 8 * 1024 * 1024;
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Create a bounded GitHub CLI JSON transport with GET-only authority. */
|
||||
export function createGhApiRequest({ env = process.env, now = Date.now } = {}) {
|
||||
return async (method, endpoint, budget) => {
|
||||
if (method !== "GET")
|
||||
throw new Error(
|
||||
"GitHub API transport supports read-only GET requests only",
|
||||
);
|
||||
const remaining = budget.limitMs - (now() - budget.startedAt);
|
||||
if (remaining <= 0) throw new Error("lookup_time_budget_exhausted");
|
||||
const { stdout } = await execFileAsync(
|
||||
"gh",
|
||||
["api", "--method", "GET", endpoint],
|
||||
{
|
||||
env,
|
||||
encoding: "utf8",
|
||||
maxBuffer: MAX_BUFFER,
|
||||
timeout: Math.min(REQUEST_TIMEOUT_MS, remaining),
|
||||
},
|
||||
);
|
||||
try {
|
||||
return JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("github_api_malformed_json", { cause });
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({
|
||||
id: run.id,
|
||||
runAttempt: run.run_attempt,
|
||||
workflow: workflow.name,
|
||||
event: typeof run.event === "string" ? run.event : null,
|
||||
headBranch:
|
||||
typeof run.head_branch === "string" ? run.head_branch : null,
|
||||
conclusion: run.conclusion ?? null,
|
||||
createdAt: run.created_at ?? null,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
export interface PrCiHead {
|
||||
repositoryId: number | null;
|
||||
repository?: string | null;
|
||||
ref: string | null;
|
||||
sha: string | null;
|
||||
}
|
||||
|
||||
export interface PrCiPullRequest {
|
||||
number: number | null;
|
||||
draft?: boolean;
|
||||
identityValid: boolean;
|
||||
mergeable?: boolean;
|
||||
head: PrCiHead;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface PrCiWorkflowRun {
|
||||
id?: number;
|
||||
run_attempt?: number;
|
||||
event?: string;
|
||||
head_branch?: string;
|
||||
head_repository?: { id?: number | null } | null;
|
||||
head_sha?: string;
|
||||
status?: string;
|
||||
conclusion?: string | null;
|
||||
created_at?: string;
|
||||
html_url?: string;
|
||||
pull_requests?: Array<{
|
||||
number?: number;
|
||||
head?: { sha?: string; repo?: { id?: number | null } } | null;
|
||||
}>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface PrCiClassification {
|
||||
category: string;
|
||||
reason: string;
|
||||
runs: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface PrCiHealthReport {
|
||||
schemaVersion: 1;
|
||||
observedAt: string;
|
||||
repository: string;
|
||||
repositoryId: number | null;
|
||||
defaultBranch: string;
|
||||
lookbackDays: number;
|
||||
scan: {
|
||||
status: "complete" | "partial";
|
||||
cutoff: string;
|
||||
pullRequestPages: number;
|
||||
workflowRunPages: number;
|
||||
pullRequestInventoryComplete: boolean;
|
||||
workflowRunInventoryComplete: boolean;
|
||||
pullRequestDetailsFetched: number;
|
||||
pullRequestDetailsLimit: number;
|
||||
apiRequests: number;
|
||||
apiRequestLimit: number;
|
||||
errors: string[];
|
||||
};
|
||||
counts: Record<string, number>;
|
||||
pullRequests: Array<PrCiPullRequest & { ci: PrCiClassification }>;
|
||||
}
|
||||
|
||||
export function collectPrCiHealth(options: {
|
||||
repository: string;
|
||||
repositoryId?: number | string | null;
|
||||
defaultBranch: string;
|
||||
lookbackDays?: number;
|
||||
now?: number;
|
||||
apiRequest?: GitHubApiRequest;
|
||||
}): Promise<PrCiHealthReport>;
|
||||
|
||||
export function classifyPrCi(input: {
|
||||
pull: PrCiPullRequest;
|
||||
runs: Array<{ run: PrCiWorkflowRun; basis: string }>;
|
||||
runsComplete: boolean;
|
||||
pullRequestInventoryComplete: boolean;
|
||||
ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>;
|
||||
}): PrCiClassification;
|
||||
|
||||
export function matchRunToPullRequest(input: {
|
||||
run: PrCiWorkflowRun;
|
||||
pull: PrCiPullRequest;
|
||||
identityCounts: Map<string, number>;
|
||||
inventoryComplete?: boolean;
|
||||
}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string };
|
||||
|
||||
export function writePrCiHealthReport(
|
||||
report: PrCiHealthReport,
|
||||
outputDirectory?: string,
|
||||
): Promise<{ jsonPath: string; markdownPath: string }>;
|
||||
|
||||
export function renderMarkdownSummary(report: PrCiHealthReport): string;
|
||||
import type { GitHubApiRequest } from "./github-api.mjs";
|
||||
@@ -0,0 +1,718 @@
|
||||
import { appendFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { createGhApiRequest } from "./github-api.mjs";
|
||||
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health");
|
||||
const PAGE_SIZE = 100;
|
||||
const MAX_PULL_REQUEST_PAGES = 3;
|
||||
const MAX_WORKFLOW_RUN_PAGES = 5;
|
||||
const MAX_PULL_REQUEST_DETAILS = 40;
|
||||
const MAX_API_REQUESTS =
|
||||
MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS;
|
||||
const LOOKBACK_DAYS = 14;
|
||||
const MAX_LOOKBACK_DAYS = 30;
|
||||
const MAX_TOTAL_TIME_MS = 60_000;
|
||||
const SHA_PATTERN = /^[0-9a-f]{40}$/iu;
|
||||
const COMPLETED_CATEGORIES = new Map([
|
||||
["failure", "completed_failure"],
|
||||
["timed_out", "completed_failure"],
|
||||
["cancelled", "completed_canceled"],
|
||||
["success", "completed_success"],
|
||||
]);
|
||||
|
||||
/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */
|
||||
export async function collectPrCiHealth({
|
||||
repository,
|
||||
repositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays = LOOKBACK_DAYS,
|
||||
now = Date.now(),
|
||||
apiRequest = createGhApiRequest(),
|
||||
}) {
|
||||
const identity = validateInputs({
|
||||
repository,
|
||||
repositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays,
|
||||
now,
|
||||
});
|
||||
const observedAt = new Date(now).toISOString();
|
||||
const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000;
|
||||
const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff });
|
||||
const classified = classifyPullRequests(snapshot);
|
||||
const { pulls, runs, budget } = snapshot;
|
||||
const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter(
|
||||
Boolean,
|
||||
);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
observedAt,
|
||||
repository: identity.repository,
|
||||
repositoryId: identity.repositoryId,
|
||||
defaultBranch: identity.defaultBranch,
|
||||
lookbackDays: identity.lookbackDays,
|
||||
scan: {
|
||||
status:
|
||||
errors.length === 0 && pulls.complete && runs.complete
|
||||
? "complete"
|
||||
: "partial",
|
||||
cutoff: new Date(cutoff).toISOString(),
|
||||
pullRequestPages: pulls.pages,
|
||||
workflowRunPages: runs.pages,
|
||||
pullRequestInventoryComplete: pulls.complete,
|
||||
workflowRunInventoryComplete: runs.complete,
|
||||
pullRequestDetailsFetched: snapshot.detailsFetched,
|
||||
pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS,
|
||||
apiRequests: budget.requests,
|
||||
apiRequestLimit: MAX_API_REQUESTS,
|
||||
errors,
|
||||
},
|
||||
counts: countCategories(classified),
|
||||
pullRequests: classified,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectApiSnapshot({ apiRequest, identity, cutoff }) {
|
||||
const budget = {
|
||||
requests: 0,
|
||||
startedAt: Date.now(),
|
||||
limitMs: MAX_TOTAL_TIME_MS,
|
||||
};
|
||||
const requestGet = async (endpoint) => {
|
||||
if (budget.requests >= MAX_API_REQUESTS)
|
||||
throw new Error("lookup_request_budget_exhausted");
|
||||
if (Date.now() - budget.startedAt >= budget.limitMs)
|
||||
throw new Error("lookup_time_budget_exhausted");
|
||||
budget.requests += 1;
|
||||
return await apiRequest("GET", endpoint, budget);
|
||||
};
|
||||
const pulls = await listOpenPullRequests({
|
||||
requestGet,
|
||||
identity,
|
||||
});
|
||||
const candidates = pulls.items.filter(
|
||||
(pull) =>
|
||||
!pull.draft &&
|
||||
Number.isFinite(Date.parse(pull.createdAt ?? "")) &&
|
||||
Date.parse(pull.createdAt ?? "") >= cutoff &&
|
||||
pull.base.ref === identity.defaultBranch &&
|
||||
(identity.repositoryId === null ||
|
||||
pull.base.repositoryId === identity.repositoryId),
|
||||
);
|
||||
const runs = await listRecentWorkflowRuns({
|
||||
requestGet,
|
||||
identity,
|
||||
cutoff,
|
||||
});
|
||||
|
||||
const detailResult = await resolveMergeability({
|
||||
candidates,
|
||||
requestGet,
|
||||
identity,
|
||||
});
|
||||
return {
|
||||
budget,
|
||||
pulls,
|
||||
runs,
|
||||
candidates,
|
||||
detailsFetched: detailResult.fetched,
|
||||
detailsErrors: detailResult.errors,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyPullRequests(snapshot) {
|
||||
const { candidates, pulls, runs } = snapshot;
|
||||
const identityCounts = countHeadIdentities(pulls.items);
|
||||
return candidates.map((pull) => {
|
||||
const relevant = [];
|
||||
const ambiguousRuns = [];
|
||||
for (const run of runs.items) {
|
||||
const match = matchRunToPullRequest({
|
||||
run,
|
||||
pull,
|
||||
identityCounts,
|
||||
inventoryComplete: pulls.complete,
|
||||
});
|
||||
if (match.kind === "match") relevant.push({ run, basis: match.basis });
|
||||
if (match.kind === "ambiguous")
|
||||
ambiguousRuns.push({ run, basis: "ambiguous_head_identity" });
|
||||
}
|
||||
return {
|
||||
...pull,
|
||||
ci: classifyPrCi({
|
||||
pull,
|
||||
runs: relevant,
|
||||
runsComplete: runs.complete,
|
||||
pullRequestInventoryComplete: pulls.complete,
|
||||
ambiguousRuns,
|
||||
}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveMergeability({ candidates, requestGet, identity }) {
|
||||
let detailsFetched = 0;
|
||||
const errors = new Set();
|
||||
for (const pull of candidates) {
|
||||
if (pull.mergeable !== undefined) continue;
|
||||
if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) {
|
||||
errors.add("pull_request_mergeability_lookup_budget_exhausted");
|
||||
continue;
|
||||
}
|
||||
const result = await getPullRequestDetails({ requestGet, identity, pull });
|
||||
detailsFetched += 1;
|
||||
if (result.error !== null) {
|
||||
errors.add(result.error);
|
||||
continue;
|
||||
}
|
||||
const { detail } = result;
|
||||
const identityChanged =
|
||||
detail.headSha !== pull.head.sha ||
|
||||
detail.headRef !== pull.head.ref ||
|
||||
detail.headRepositoryId !== pull.head.repositoryId ||
|
||||
detail.baseRef !== pull.base.ref ||
|
||||
detail.baseRepositoryId !== pull.base.repositoryId;
|
||||
if (identityChanged) {
|
||||
pull.identityValid = false;
|
||||
pull.identityChangedDuringScan = true;
|
||||
errors.add("pull_request_identity_changed_during_scan");
|
||||
} else {
|
||||
pull.mergeable = detail.mergeable;
|
||||
pull.detailFetched = true;
|
||||
if (detail.mergeable === undefined)
|
||||
errors.add("pull_request_mergeability_unknown");
|
||||
}
|
||||
}
|
||||
return { fetched: detailsFetched, errors: [...errors] };
|
||||
}
|
||||
|
||||
function countCategories(pulls) {
|
||||
const categories = [
|
||||
"missing",
|
||||
"startup_failure",
|
||||
"queued_pending_executing",
|
||||
"approval_required",
|
||||
"completed_success",
|
||||
"completed_failure",
|
||||
"completed_canceled",
|
||||
"completed_other",
|
||||
"merge_conflict",
|
||||
"unknown",
|
||||
"deferred",
|
||||
];
|
||||
return Object.fromEntries(
|
||||
categories.map((category) => [
|
||||
category,
|
||||
pulls.filter(({ ci }) => ci.category === category).length,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
/** Pure status classifier; missing requires complete, identity-safe inventories. */
|
||||
export function classifyPrCi({
|
||||
pull,
|
||||
runs,
|
||||
runsComplete,
|
||||
pullRequestInventoryComplete,
|
||||
ambiguousRuns,
|
||||
}) {
|
||||
if (!pull.identityValid)
|
||||
return result("unknown", "pull_request_head_identity_invalid", []);
|
||||
const evidence = [...runs, ...ambiguousRuns];
|
||||
if (pull.mergeable === false)
|
||||
return result("merge_conflict", "github_reports_merge_conflict", evidence);
|
||||
// The newest collected run is not proven current until pagination completes:
|
||||
// an older-created run outside this snapshot may have been retried recently.
|
||||
if (!runsComplete)
|
||||
return result(
|
||||
evidence.length > 0 ? "unknown" : "deferred",
|
||||
evidence.length > 0
|
||||
? "workflow_run_inventory_incomplete"
|
||||
: "lookup_inventory_incomplete",
|
||||
evidence,
|
||||
);
|
||||
const selection = newestRun(runs);
|
||||
if (!selection.known)
|
||||
return result("unknown", "workflow_run_order_unknown", evidence);
|
||||
const latest = selection.latest;
|
||||
if (
|
||||
ambiguousRuns.some((candidate) => {
|
||||
const order = latest ? compareRunEvidence(candidate, latest) : null;
|
||||
return order === null || order >= 0;
|
||||
})
|
||||
)
|
||||
return result("unknown", "workflow_run_identity_ambiguous", evidence);
|
||||
if (latest !== undefined) {
|
||||
const run = latest.run;
|
||||
if (run.conclusion === "startup_failure")
|
||||
return result("startup_failure", "workflow_startup_failure", evidence);
|
||||
if (run.status === "waiting" || run.conclusion === "action_required")
|
||||
return result(
|
||||
"approval_required",
|
||||
"workflow_waiting_for_approval_or_gate",
|
||||
evidence,
|
||||
);
|
||||
if (["queued", "requested", "pending", "in_progress"].includes(run.status))
|
||||
return result(
|
||||
"queued_pending_executing",
|
||||
`workflow_${run.status}`,
|
||||
evidence,
|
||||
);
|
||||
if (run.status === "completed")
|
||||
return result(
|
||||
COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other",
|
||||
run.conclusion === "success"
|
||||
? "workflow_succeeded"
|
||||
: `workflow_${run.conclusion ?? "unknown_conclusion"}`,
|
||||
evidence,
|
||||
);
|
||||
return result("unknown", "workflow_status_unrecognized", evidence);
|
||||
}
|
||||
if (!pullRequestInventoryComplete)
|
||||
return result("deferred", "lookup_inventory_incomplete", []);
|
||||
if (pull.mergeable !== true)
|
||||
return result("unknown", "pull_request_mergeability_unknown", []);
|
||||
return result("missing", "no_pull_request_event_run_for_current_head", []);
|
||||
}
|
||||
|
||||
/** Match only a pull_request event proven to belong to this immutable PR head. */
|
||||
export function matchRunToPullRequest({
|
||||
run,
|
||||
pull,
|
||||
identityCounts,
|
||||
inventoryComplete = true,
|
||||
}) {
|
||||
if (run?.event !== "pull_request") return { kind: "none" };
|
||||
if (!pull.identityValid || !isRecord(run)) return { kind: "none" };
|
||||
const association = matchExplicitPullAssociation(run, pull);
|
||||
if (association === "match")
|
||||
return { kind: "match", basis: "event_pr_number_and_head_sha" };
|
||||
if (association === "ambiguous") return { kind: "ambiguous" };
|
||||
if (association === "other" || !sameHeadRepositoryAndRef(run, pull))
|
||||
return { kind: "none" };
|
||||
if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete))
|
||||
return { kind: "ambiguous" };
|
||||
if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha)
|
||||
return { kind: "match", basis: "event_unique_head_ref_and_repository_id" };
|
||||
return { kind: "none" };
|
||||
}
|
||||
|
||||
function matchExplicitPullAssociation(run, pull) {
|
||||
const associations = Array.isArray(run.pull_requests)
|
||||
? run.pull_requests
|
||||
: [];
|
||||
if (associations.length === 0) return "absent";
|
||||
const matchingNumber = associations.filter(
|
||||
(item) => item?.number === pull.number,
|
||||
);
|
||||
if (matchingNumber.length === 0) return "other";
|
||||
const exactHead = matchingNumber.find(
|
||||
(item) => item?.head?.sha === pull.head.sha,
|
||||
);
|
||||
if (exactHead === undefined) {
|
||||
return matchingNumber.some(
|
||||
(item) =>
|
||||
typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha,
|
||||
)
|
||||
? "other"
|
||||
: "absent";
|
||||
}
|
||||
const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id);
|
||||
const runHeadRepositoryId = positiveInteger(run.head_repository?.id);
|
||||
if (associatedHeadRepositoryId === null || runHeadRepositoryId === null)
|
||||
return "ambiguous";
|
||||
if (
|
||||
associatedHeadRepositoryId !== pull.head.repositoryId ||
|
||||
runHeadRepositoryId !== pull.head.repositoryId
|
||||
)
|
||||
return "other";
|
||||
const baseRefMatches =
|
||||
typeof exactHead.base?.ref !== "string" ||
|
||||
exactHead.base.ref === pull.base.ref;
|
||||
const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id);
|
||||
const baseRepositoryMatches =
|
||||
baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId;
|
||||
return baseRefMatches && baseRepositoryMatches ? "match" : "other";
|
||||
}
|
||||
|
||||
function sameHeadRepositoryAndRef(run, pull) {
|
||||
return (
|
||||
run.head_branch === pull.head.ref &&
|
||||
run.head_repository?.id === pull.head.repositoryId
|
||||
);
|
||||
}
|
||||
|
||||
function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) {
|
||||
if (!inventoryComplete) return false;
|
||||
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
|
||||
const unknownRepositoryKey = headIdentityKey(null, pull.head.ref);
|
||||
return (
|
||||
identityCounts.get(key) === 1 &&
|
||||
(identityCounts.get(unknownRepositoryKey) ?? 0) === 0
|
||||
);
|
||||
}
|
||||
|
||||
function normalizePullRequest(raw, identity) {
|
||||
const number = positiveInteger(raw?.number);
|
||||
const base = normalizeBase(raw?.base);
|
||||
const head = normalizeHead(raw?.head);
|
||||
const draft = raw?.draft === true;
|
||||
return {
|
||||
number,
|
||||
draft,
|
||||
createdAt: typeof raw?.created_at === "string" ? raw.created_at : null,
|
||||
url: typeof raw?.html_url === "string" ? raw.html_url : null,
|
||||
base,
|
||||
head,
|
||||
mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined,
|
||||
identityValid: hasValidPullIdentity({ raw, identity, number, base, head }),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBase(raw) {
|
||||
return {
|
||||
ref: typeof raw?.ref === "string" ? raw.ref : null,
|
||||
repositoryId: positiveInteger(raw?.repo?.id),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeHead(raw) {
|
||||
return {
|
||||
repositoryId: positiveInteger(raw?.repo?.id),
|
||||
repository:
|
||||
typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null,
|
||||
ref: typeof raw?.ref === "string" ? raw.ref : null,
|
||||
sha:
|
||||
typeof raw?.sha === "string" && isSha(raw.sha)
|
||||
? raw.sha.toLowerCase()
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function hasValidPullIdentity({ raw, identity, number, base, head }) {
|
||||
return (
|
||||
number !== null &&
|
||||
typeof raw?.draft === "boolean" &&
|
||||
base.ref === identity.defaultBranch &&
|
||||
(identity.repositoryId === null ||
|
||||
base.repositoryId === identity.repositoryId) &&
|
||||
head.repositoryId !== null &&
|
||||
head.sha !== null &&
|
||||
head.ref !== null
|
||||
);
|
||||
}
|
||||
|
||||
function countHeadIdentities(pulls) {
|
||||
const counts = new Map();
|
||||
for (const pull of pulls) {
|
||||
if (typeof pull.head.ref !== "string") continue;
|
||||
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
function headIdentityKey(repositoryId, ref) {
|
||||
return `${String(repositoryId)}\u0000${ref}`;
|
||||
}
|
||||
|
||||
function newestRun(runs) {
|
||||
let latest = runs[0];
|
||||
for (const candidate of runs) {
|
||||
if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0)
|
||||
latest = candidate;
|
||||
}
|
||||
const known = runs.every((candidate) => {
|
||||
if (candidate === latest) return true;
|
||||
const order = compareRunEvidence(candidate, latest);
|
||||
return order !== null && order <= 0;
|
||||
});
|
||||
return { known, latest };
|
||||
}
|
||||
|
||||
// Attempt ordering is meaningful within one run; different run IDs are not clocks.
|
||||
function compareRunEvidence(left, right) {
|
||||
const leftRun = left.run;
|
||||
const rightRun = right.run;
|
||||
const id = positiveInteger(leftRun.id);
|
||||
if (id !== null && id === positiveInteger(rightRun.id)) {
|
||||
const leftAttempt = positiveInteger(leftRun.run_attempt);
|
||||
const rightAttempt = positiveInteger(rightRun.run_attempt);
|
||||
if (leftAttempt === null || rightAttempt === null) return null;
|
||||
if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt;
|
||||
return leftRun.status === rightRun.status &&
|
||||
leftRun.conclusion === rightRun.conclusion
|
||||
? 0
|
||||
: null;
|
||||
}
|
||||
// A retry may start long after its original creation; without an attempt
|
||||
// clock, creation dates only order first attempts from different runs.
|
||||
if (
|
||||
positiveInteger(leftRun.run_attempt) !== 1 ||
|
||||
positiveInteger(rightRun.run_attempt) !== 1
|
||||
)
|
||||
return null;
|
||||
const time =
|
||||
Date.parse(leftRun.created_at ?? "") -
|
||||
Date.parse(rightRun.created_at ?? "");
|
||||
return Number.isFinite(time) && time !== 0 ? time : null;
|
||||
}
|
||||
|
||||
function result(category, reason, runs) {
|
||||
return {
|
||||
category,
|
||||
reason,
|
||||
runs: runs.map(({ run, basis }) => ({
|
||||
id: positiveInteger(run.id),
|
||||
attempt: positiveInteger(run.run_attempt),
|
||||
status: typeof run.status === "string" ? run.status : null,
|
||||
conclusion: typeof run.conclusion === "string" ? run.conclusion : null,
|
||||
headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null,
|
||||
createdAt: typeof run.created_at === "string" ? run.created_at : null,
|
||||
url: typeof run.html_url === "string" ? run.html_url : null,
|
||||
identityBasis: basis,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async function listOpenPullRequests({ requestGet, identity }) {
|
||||
const items = [];
|
||||
let complete = false;
|
||||
let pages = 0;
|
||||
let error = null;
|
||||
for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) {
|
||||
try {
|
||||
const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`;
|
||||
const response = await requestGet(endpoint);
|
||||
if (!Array.isArray(response))
|
||||
throw new Error("pull_request_page_malformed");
|
||||
pages += 1;
|
||||
const normalized = response.map((raw) =>
|
||||
normalizePullRequest(raw, identity),
|
||||
);
|
||||
if (
|
||||
normalized.some(
|
||||
(pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")),
|
||||
)
|
||||
)
|
||||
throw new Error("pull_request_page_malformed");
|
||||
items.push(...normalized);
|
||||
if (response.length < PAGE_SIZE) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
} catch (cause) {
|
||||
error = safeErrorCode(cause);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { items, pages, complete, error };
|
||||
}
|
||||
|
||||
async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) {
|
||||
const items = [];
|
||||
let complete = false;
|
||||
let pages = 0;
|
||||
let error = null;
|
||||
for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) {
|
||||
try {
|
||||
const cutoffDate = new Date(cutoff).toISOString().slice(0, 10);
|
||||
const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`;
|
||||
const response = await requestGet(endpoint);
|
||||
if (!isRecord(response) || !Array.isArray(response.workflow_runs))
|
||||
throw new Error("workflow_run_page_malformed");
|
||||
pages += 1;
|
||||
if (!response.workflow_runs.every(isRecord))
|
||||
throw new Error("workflow_run_page_malformed");
|
||||
items.push(...response.workflow_runs);
|
||||
if (response.workflow_runs.length < PAGE_SIZE) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
} catch (cause) {
|
||||
error = safeErrorCode(cause);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { items, pages, complete, error };
|
||||
}
|
||||
|
||||
async function getPullRequestDetails({ requestGet, identity, pull }) {
|
||||
if (pull.number === null)
|
||||
return { detail: null, error: "pull_request_detail_identity_missing" };
|
||||
try {
|
||||
const response = await requestGet(
|
||||
`repos/${identity.repository}/pulls/${pull.number}`,
|
||||
);
|
||||
if (!isRecord(response))
|
||||
return { detail: null, error: "pull_request_detail_malformed" };
|
||||
return {
|
||||
detail: {
|
||||
mergeable:
|
||||
typeof response.mergeable === "boolean"
|
||||
? response.mergeable
|
||||
: undefined,
|
||||
headSha: isSha(response.head?.sha)
|
||||
? response.head.sha.toLowerCase()
|
||||
: null,
|
||||
headRef:
|
||||
typeof response.head?.ref === "string" ? response.head.ref : null,
|
||||
headRepositoryId: positiveInteger(response.head?.repo?.id),
|
||||
baseRef:
|
||||
typeof response.base?.ref === "string" ? response.base.ref : null,
|
||||
baseRepositoryId: positiveInteger(response.base?.repo?.id),
|
||||
},
|
||||
error: null,
|
||||
};
|
||||
} catch (cause) {
|
||||
return { detail: null, error: safeErrorCode(cause) };
|
||||
}
|
||||
}
|
||||
|
||||
function validateInputs({
|
||||
repository,
|
||||
repositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays,
|
||||
now,
|
||||
}) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
if (
|
||||
typeof defaultBranch !== "string" ||
|
||||
defaultBranch.length === 0 ||
|
||||
/[\r\n]/u.test(defaultBranch)
|
||||
)
|
||||
throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name");
|
||||
const parsedRepositoryId =
|
||||
repositoryId === undefined || repositoryId === null || repositoryId === ""
|
||||
? null
|
||||
: positiveInteger(repositoryId);
|
||||
if (
|
||||
repositoryId !== undefined &&
|
||||
repositoryId !== null &&
|
||||
repositoryId !== "" &&
|
||||
parsedRepositoryId === null
|
||||
)
|
||||
throw new Error("CI_REPOSITORY_ID must be a positive safe integer");
|
||||
if (
|
||||
!Number.isInteger(lookbackDays) ||
|
||||
lookbackDays < 1 ||
|
||||
lookbackDays > MAX_LOOKBACK_DAYS
|
||||
)
|
||||
throw new Error(
|
||||
`lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`,
|
||||
);
|
||||
if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp");
|
||||
return {
|
||||
repository,
|
||||
repositoryId: parsedRepositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays,
|
||||
};
|
||||
}
|
||||
|
||||
function positiveInteger(value) {
|
||||
const number = typeof value === "number" ? value : Number(value);
|
||||
return Number.isSafeInteger(number) && number > 0 ? number : null;
|
||||
}
|
||||
|
||||
function isSha(value) {
|
||||
return typeof value === "string" && SHA_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function isRecord(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function safeErrorCode(cause) {
|
||||
if (cause?.message === "lookup_time_budget_exhausted")
|
||||
return "lookup_time_budget_exhausted";
|
||||
if (cause?.message === "lookup_request_budget_exhausted")
|
||||
return "lookup_request_budget_exhausted";
|
||||
if (cause?.message === "github_api_malformed_json")
|
||||
return "github_api_malformed_json";
|
||||
if (cause?.message?.endsWith("_malformed")) return cause.message;
|
||||
if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout";
|
||||
return "github_api_request_failed";
|
||||
}
|
||||
|
||||
export async function writePrCiHealthReport(
|
||||
report,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
) {
|
||||
const jsonPath = join(outputDirectory, "report.json");
|
||||
const markdownPath = join(outputDirectory, "summary.md");
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
await writeFile(markdownPath, renderMarkdownSummary(report), "utf8");
|
||||
return { jsonPath, markdownPath };
|
||||
}
|
||||
|
||||
export function renderMarkdownSummary(report) {
|
||||
const rows = report.pullRequests.map(
|
||||
(pull) =>
|
||||
`| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`,
|
||||
);
|
||||
return [
|
||||
"# Pull request CI health snapshot",
|
||||
"",
|
||||
`Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`,
|
||||
"",
|
||||
"This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.",
|
||||
"",
|
||||
`API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`,
|
||||
"",
|
||||
"| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |",
|
||||
"| ---: | --- | --- | --- | --- | --- |",
|
||||
...rows,
|
||||
"",
|
||||
"Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function escapeCell(value) {
|
||||
return String(value)
|
||||
.replaceAll("|", "\\|")
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll("\r", " ");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
const report = await collectPrCiHealth({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
repositoryId: process.env.CI_REPOSITORY_ID,
|
||||
defaultBranch: process.env.CI_DEFAULT_BRANCH,
|
||||
});
|
||||
const paths = await writePrCiHealthReport(
|
||||
report,
|
||||
process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
await appendFile(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
renderMarkdownSummary(report),
|
||||
"utf8",
|
||||
);
|
||||
process.stdout.write(
|
||||
`Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`,
|
||||
);
|
||||
} catch (cause) {
|
||||
process.stderr.write(
|
||||
`PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
||||
|
||||
const ARTIFACT_NAME = "ci-full-qualification-v1";
|
||||
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
@@ -119,8 +120,8 @@ function admit() {
|
||||
// Artifact metadata narrows the search before run lookups. Never scan an
|
||||
// unbounded history or treat a missing receipt as a successful qualification.
|
||||
for (const artifact of candidates.slice(0, 3)) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
const run = normalizeWorkflowRun(
|
||||
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
|
||||
);
|
||||
// An old run can upload a new artifact when only its gate is rerun.
|
||||
// Bound the original run age as well so retries cannot renew stale proof.
|
||||
@@ -133,7 +134,7 @@ function admit() {
|
||||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_sha === head &&
|
||||
run.head_branch === branch &&
|
||||
|
||||
@@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
const lanes = {
|
||||
code: ["build", "test-shard"],
|
||||
code: [
|
||||
"build",
|
||||
"test-shard",
|
||||
...(outputs.tier === "fast" ? ["test-source"] : []),
|
||||
],
|
||||
reverse_tools: ["reverse-tools-quality"],
|
||||
runtime: ["windows-native-build", "package-e2e"],
|
||||
package: ["windows-curated"],
|
||||
|
||||
@@ -133,6 +133,8 @@ try {
|
||||
for (const clone of report.duplicates)
|
||||
for (const file of [clone.firstFile, clone.secondFile])
|
||||
file.name = relative(process.cwd(), file.name).split(sep).join("/");
|
||||
// Record the actual checker result, including caller-selected thresholds.
|
||||
report.reaCheck = { exitCode: result.status };
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2));
|
||||
const { sources, clones, percentage } = report.statistics.total;
|
||||
console.log(
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
export interface StaticTaskContext {
|
||||
readonly version: 1;
|
||||
readonly sourceSha: string;
|
||||
readonly sourceClean: boolean;
|
||||
readonly prHeadSha?: string;
|
||||
readonly nodeVersion: string;
|
||||
readonly npmVersion: string;
|
||||
readonly turboVersion: string;
|
||||
readonly platform: string;
|
||||
readonly architecture: string;
|
||||
readonly taskHashes: Readonly<Record<string, string>>;
|
||||
}
|
||||
export function parseStaticTaskContext(value: unknown): StaticTaskContext;
|
||||
export function recordStaticTaskContext(options?: {
|
||||
root?: string;
|
||||
npmCli?: string;
|
||||
outputDirectory?: string;
|
||||
}): Promise<StaticTaskContext>;
|
||||
@@ -0,0 +1,104 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { z } from "zod";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const contextSchema = z.object({
|
||||
version: z.literal(1),
|
||||
sourceSha: z.string().regex(/^[a-f0-9]{40}$/u),
|
||||
sourceClean: z.boolean(),
|
||||
prHeadSha: z
|
||||
.string()
|
||||
.regex(/^[a-f0-9]{40}$/u)
|
||||
.optional(),
|
||||
nodeVersion: z.string().min(1),
|
||||
npmVersion: z.string().min(1),
|
||||
turboVersion: z.string().min(1),
|
||||
platform: z.string().min(1),
|
||||
architecture: z.string().min(1),
|
||||
taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)),
|
||||
});
|
||||
|
||||
/** Parse the producing check graph's receipt; incomplete provenance is unknown. */
|
||||
export const parseStaticTaskContext = (value) => contextSchema.parse(value);
|
||||
|
||||
/** Retain only named task hashes and tool/source identity, never the environment. */
|
||||
export async function recordStaticTaskContext({
|
||||
root = process.cwd(),
|
||||
npmCli = process.env.npm_execpath,
|
||||
outputDirectory = join(root, ".cache/static-checks"),
|
||||
} = {}) {
|
||||
if (typeof npmCli !== "string" || !npmCli)
|
||||
throw new Error(
|
||||
"Run static context recording through npm run ci:static-context",
|
||||
);
|
||||
const options = {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
};
|
||||
const [graph, source, status, npmVersion, turbo] = await Promise.all([
|
||||
execute(
|
||||
process.execPath,
|
||||
[npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"],
|
||||
options,
|
||||
),
|
||||
execute("git", ["rev-parse", "HEAD"], options),
|
||||
execute(
|
||||
"git",
|
||||
["status", "--porcelain", "--untracked-files=normal"],
|
||||
options,
|
||||
),
|
||||
execute(process.execPath, [npmCli, "--version"], options),
|
||||
readFile(join(root, "node_modules/turbo/package.json"), "utf8"),
|
||||
]);
|
||||
const tasks = z
|
||||
.object({
|
||||
tasks: z
|
||||
.array(
|
||||
z.object({
|
||||
task: z.string().min(1),
|
||||
hash: z.string().regex(/^[a-f0-9]{16}$/u),
|
||||
}),
|
||||
)
|
||||
.min(1),
|
||||
})
|
||||
.parse(JSON.parse(graph.stdout)).tasks;
|
||||
if (new Set(tasks.map(({ task }) => task)).size !== tasks.length)
|
||||
throw new Error("Static graph contains duplicate task identities");
|
||||
const context = parseStaticTaskContext({
|
||||
version: 1,
|
||||
sourceSha: source.stdout.trim(),
|
||||
sourceClean: status.stdout.length === 0,
|
||||
...(process.env.CI_PR_HEAD_SHA
|
||||
? { prHeadSha: process.env.CI_PR_HEAD_SHA }
|
||||
: {}),
|
||||
nodeVersion: process.version,
|
||||
npmVersion: npmVersion.stdout.trim(),
|
||||
turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo))
|
||||
.version,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])),
|
||||
});
|
||||
if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha)
|
||||
throw new Error(
|
||||
"Static context source does not match the workflow revision",
|
||||
);
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
join(outputDirectory, "static-task-context.json"),
|
||||
JSON.stringify(context, null, 2) + "\n",
|
||||
);
|
||||
return context;
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
)
|
||||
await recordStaticTaskContext();
|
||||
@@ -0,0 +1,46 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import {
|
||||
BUILT_TEST_PROJECTS,
|
||||
SOURCE_TEST_PROJECTS,
|
||||
} from "../lib/test-projects.mjs";
|
||||
|
||||
const [group, ...arguments_] = process.argv.slice(2);
|
||||
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
|
||||
group
|
||||
];
|
||||
if (!projects) throw new Error("Expected source or built test group");
|
||||
// The group owns project selection; callers can request reporters or shards.
|
||||
if (
|
||||
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
|
||||
)
|
||||
throw new Error("Test groups cannot override their project inventory");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
resolve("node_modules/vitest/vitest.mjs"),
|
||||
"run",
|
||||
...projects.flatMap(({ name }) => ["--project", name]),
|
||||
...arguments_,
|
||||
],
|
||||
{ stdio: "inherit" },
|
||||
);
|
||||
let shutdownSignal;
|
||||
for (const signal of ["SIGINT", "SIGTERM"])
|
||||
process.on(signal, () => {
|
||||
shutdownSignal ??= signal;
|
||||
child.kill(signal);
|
||||
});
|
||||
child.once("error", (error) => {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
child.once("close", (code) => {
|
||||
process.exitCode =
|
||||
shutdownSignal === "SIGINT"
|
||||
? 130
|
||||
: shutdownSignal === "SIGTERM"
|
||||
? 143
|
||||
: (code ?? 1);
|
||||
});
|
||||
@@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
||||
|
||||
const ARTIFACT_NAME = "test-timings";
|
||||
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
@@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
|
||||
.slice(0, MAX_CANDIDATES);
|
||||
|
||||
for (const artifact of candidates) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
const run = normalizeWorkflowRun(
|
||||
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
|
||||
);
|
||||
const runAge = now - Date.parse(run?.created_at);
|
||||
if (
|
||||
@@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
|
||||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_branch === branch &&
|
||||
run.head_repository?.full_name === repository
|
||||
|
||||
@@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => {
|
||||
throw new Error(`Test report path is outside the checkout: ${name}`);
|
||||
return { name: path.split(sep).join("/"), durationMs, state };
|
||||
});
|
||||
const complete = process.env.SHARD_RESULT === "success";
|
||||
// Keep group outcomes intact: an unavailable group is not a completed failure.
|
||||
const groupResults = [
|
||||
process.env.SHARD_RESULT,
|
||||
...(process.env.SOURCE_REQUIRED === "true"
|
||||
? [process.env.SOURCE_RESULT]
|
||||
: []),
|
||||
];
|
||||
const groupsCompleted = groupResults.every((state) =>
|
||||
["success", "failure"].includes(state),
|
||||
);
|
||||
const passed =
|
||||
complete && process.env.MERGE_RESULT === "success" && report.success;
|
||||
groupResults.every((state) => state === "success") &&
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
|
||||
@@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) {
|
||||
comparison.complete &&
|
||||
report.unhandledErrors === 0 &&
|
||||
["passed", "failed"].includes(report.reason) &&
|
||||
["success", "failure"].includes(process.env.SHARD_RESULT) &&
|
||||
groupsCompleted &&
|
||||
["success", "failure"].includes(process.env.MERGE_RESULT),
|
||||
executionSha: process.env.GITHUB_SHA ?? null,
|
||||
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
|
||||
@@ -91,7 +102,9 @@ if (passed && verifyInventory) {
|
||||
);
|
||||
}
|
||||
const summary = [
|
||||
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
|
||||
groupsCompleted
|
||||
? "## Merged test diagnostics"
|
||||
: "## Partial test diagnostics",
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
@@ -99,11 +112,14 @@ const summary = [
|
||||
"",
|
||||
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
|
||||
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
|
||||
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
|
||||
...(process.env.SOURCE_REQUIRED === "true"
|
||||
? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`]
|
||||
: []),
|
||||
passed
|
||||
? "Successful timing history is available."
|
||||
: "This report is diagnostic only and is not a successful timing baseline.",
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
export interface WorkflowPath {
|
||||
readonly file: string;
|
||||
readonly ref: string | null;
|
||||
}
|
||||
export interface NormalizedWorkflowRun {
|
||||
readonly workflowPath: WorkflowPath | null;
|
||||
}
|
||||
export function normalizeWorkflowRun<T extends Record<string, unknown>>(
|
||||
value: T,
|
||||
): Omit<T, "workflowPath"> & NormalizedWorkflowRun;
|
||||
@@ -0,0 +1,15 @@
|
||||
/** Keep the API path as evidence and parse its optional ref at the boundary. */
|
||||
export function normalizeWorkflowRun(value) {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
throw new Error("workflow_run_malformed");
|
||||
const match =
|
||||
typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path)
|
||||
? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec(
|
||||
value.path,
|
||||
)
|
||||
: null;
|
||||
return {
|
||||
...value,
|
||||
workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null,
|
||||
};
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { TraceMap, eachMapping } from "@jridgewell/trace-mapping";
|
||||
import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";
|
||||
import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js";
|
||||
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
|
||||
import { parseEvidence } from "../../dist/domain/evidence.js";
|
||||
import { traceSourceMap } from "../../dist/javascript/sourceMaps/TraceSourceMap.js";
|
||||
@@ -89,10 +90,12 @@ async function verifyCase(executable, entrypoint, scenario) {
|
||||
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
|
||||
const endpoint = `http://127.0.0.1:${fixture.port}`;
|
||||
const provider = new CdpBrowserProvider();
|
||||
const targets = await provider.listTargets({
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [site.origin],
|
||||
});
|
||||
const targets = await provider.listTargets(
|
||||
listBrowserTargetsInputSchema.parse({
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [site.origin],
|
||||
}),
|
||||
);
|
||||
if (!targets.ok) throw targets.error;
|
||||
const target = targets.value.targets[0];
|
||||
assert.ok(target, "Owned fixture target missing");
|
||||
|
||||
@@ -8,6 +8,7 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { CdpBrowserProvider } from "../../dist/browser/CdpBrowserProvider.js";
|
||||
import { listBrowserTargetsInputSchema } from "../../dist/domain/browserObservation.js";
|
||||
import { CdpConnection } from "../../dist/browser/CdpConnection.js";
|
||||
import { parseEvidence } from "../../dist/domain/evidence.js";
|
||||
import { webExecutionSchema } from "../../dist/domain/webExecution.js";
|
||||
@@ -45,10 +46,12 @@ export async function verifyBrowserRuntime(
|
||||
try {
|
||||
fixture = await startRuntimeFixtureBrowser(executable, site.origin);
|
||||
const endpoint = `http://127.0.0.1:${fixture.port}`;
|
||||
const targets = await new CdpBrowserProvider().listTargets({
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [site.origin],
|
||||
});
|
||||
const targets = await new CdpBrowserProvider().listTargets(
|
||||
listBrowserTargetsInputSchema.parse({
|
||||
cdp_endpoint: endpoint,
|
||||
allowed_origins: [site.origin],
|
||||
}),
|
||||
);
|
||||
if (!targets.ok) throw targets.error;
|
||||
const target = targets.value.targets[0];
|
||||
assert.ok(target, "owned fixture page target missing");
|
||||
|
||||
@@ -1,17 +1,4 @@
|
||||
const sourceProjects = [
|
||||
"domain",
|
||||
"services",
|
||||
"adapters",
|
||||
"composition",
|
||||
"conformance",
|
||||
"evaluation",
|
||||
];
|
||||
|
||||
const sourceTest = (path) =>
|
||||
path.startsWith("src/") ||
|
||||
["composition", "conformance", "evaluation"].some((project) =>
|
||||
path.startsWith(`tests/${project}/`),
|
||||
);
|
||||
import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs";
|
||||
|
||||
/** Parse explicit test paths and an optional branch-comparison base. */
|
||||
export const parseDevelopmentTestRequest = (mode, arguments_) => {
|
||||
@@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => {
|
||||
path.split("/").some((part) => part === ".." || part === ".")
|
||||
)
|
||||
throw new Error(`Expected an exact repository test path: ${argument}`);
|
||||
if (mode !== "focused" && !sourceTest(path))
|
||||
throw new Error(
|
||||
`Use test:focused for compiled boundary tests: ${path}`,
|
||||
);
|
||||
const project = projectForTest(path);
|
||||
if (mode !== "focused" && project.group !== "source")
|
||||
throw new Error(`Use test:focused for built test projects: ${path}`);
|
||||
paths.push(path);
|
||||
}
|
||||
}
|
||||
@@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => {
|
||||
!/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "")
|
||||
)
|
||||
throw new Error("Changed tests require a resolved Git merge base");
|
||||
const projects = request.mode === "focused" ? [] : sourceProjects;
|
||||
const projects =
|
||||
request.mode === "focused"
|
||||
? []
|
||||
: SOURCE_TEST_PROJECTS.map(({ name }) => name);
|
||||
return {
|
||||
needsBuild: request.paths.some((path) => !sourceTest(path)),
|
||||
needsBuild: request.paths.some(
|
||||
(path) => projectForTest(path).group === "built",
|
||||
),
|
||||
artifactTasks: Object.entries(artifactConsumers).flatMap(
|
||||
([task, consumers]) =>
|
||||
request.paths.some((path) => consumers.includes(path)) ? [task] : [],
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
export interface TestProjectDefinition {
|
||||
readonly name: string;
|
||||
readonly group: "source" | "built";
|
||||
readonly include: readonly string[];
|
||||
readonly exclude?: readonly string[];
|
||||
}
|
||||
|
||||
export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[];
|
||||
export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[];
|
||||
export function projectForTest(testPath: string): TestProjectDefinition;
|
||||
@@ -0,0 +1,107 @@
|
||||
import { matchesGlob } from "node:path";
|
||||
|
||||
/** Canonical file ownership shared by Vitest and development-test planning. */
|
||||
export const SOURCE_TEST_PROJECTS = [
|
||||
{
|
||||
name: "domain",
|
||||
group: "source",
|
||||
include: ["src/{contracts,domain}/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "services",
|
||||
group: "source",
|
||||
include: ["src/application/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "adapters",
|
||||
group: "source",
|
||||
include: [
|
||||
"src/*.test.ts",
|
||||
"src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "composition",
|
||||
group: "source",
|
||||
include: ["tests/composition/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "conformance",
|
||||
group: "source",
|
||||
include: ["tests/conformance/**/*.test.ts"],
|
||||
exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "evaluation",
|
||||
group: "source",
|
||||
include: ["tests/evaluation/**/*.test.ts"],
|
||||
},
|
||||
];
|
||||
|
||||
export const BUILT_TEST_PROJECTS = [
|
||||
{
|
||||
name: "boundary",
|
||||
group: "built",
|
||||
include: [
|
||||
"tests/boundary/**/*.test.ts",
|
||||
"tests/conformance/providers/hopper/realHopperSearch.test.ts",
|
||||
],
|
||||
exclude: [
|
||||
"tests/boundary/cli/**/*.test.ts",
|
||||
"tests/boundary/mcp/**/*.test.ts",
|
||||
"tests/boundary/process/**/*.test.ts",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "cli-boundary",
|
||||
group: "built",
|
||||
include: ["tests/boundary/cli/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "process-boundary",
|
||||
group: "built",
|
||||
include: ["tests/boundary/process/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "mcp-boundary",
|
||||
group: "built",
|
||||
include: ["tests/boundary/mcp/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "acceptance",
|
||||
group: "built",
|
||||
include: ["tests/acceptance/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "process-global",
|
||||
group: "built",
|
||||
include: ["tests/process-global/**/*.test.ts"],
|
||||
},
|
||||
];
|
||||
|
||||
const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
|
||||
|
||||
/** Return the unique Vitest owner for an exact repository-relative test path. */
|
||||
export const projectForTest = (testPath) => {
|
||||
if (
|
||||
typeof testPath !== "string" ||
|
||||
testPath.startsWith("/") ||
|
||||
testPath.includes("\\") ||
|
||||
testPath.split("/").some((part) => part === ".." || part === ".") ||
|
||||
!testPath.endsWith(".test.ts")
|
||||
)
|
||||
throw new Error(`Expected a normalized repository test path: ${testPath}`);
|
||||
|
||||
const owners = ALL_TEST_PROJECTS.filter(
|
||||
({ include, exclude = [] }) =>
|
||||
include.some((pattern) => matchesGlob(testPath, pattern)) &&
|
||||
!exclude.some((pattern) => matchesGlob(testPath, pattern)),
|
||||
);
|
||||
if (owners.length !== 1)
|
||||
throw new Error(
|
||||
owners.length === 0
|
||||
? `No test project owns ${testPath}`
|
||||
: `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`,
|
||||
);
|
||||
return owners[0];
|
||||
};
|
||||
Reference in New Issue
Block a user