chore: merge upstream CI baseline fixes
This commit is contained in:
+21
-2
@@ -2,12 +2,15 @@ import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
import { planTestSelection } from "./test-selection.mjs";
|
||||
|
||||
// Test files select the host/runtime lanes that execute their focused suites.
|
||||
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
|
||||
// Actionlint validates workflow syntax only. The semantic suite asserts these
|
||||
// workflows' release, publication and self-hosted runner safeguards.
|
||||
const GUARDED_WORKFLOWS = new Set([
|
||||
".github/workflows/pr-title.yml",
|
||||
".github/workflows/ci-full-pr.yml",
|
||||
".github/workflows/release.yml",
|
||||
".github/workflows/real-ghidra-windows.yml",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
@@ -23,6 +26,15 @@ const FIXTURE_SOURCES = [
|
||||
"tests/conformance/versions/**",
|
||||
];
|
||||
const READINESS_SOURCES = ["tests/conformance/readiness/**"];
|
||||
// These reporters affect CI output only; they do not select a host or provider
|
||||
// verification lane when changed on their own.
|
||||
const PORTABLE_CI_REPORTERS = new Set([
|
||||
"scripts/ci/job-timings.mjs",
|
||||
"scripts/ci/job-timings-batch.mjs",
|
||||
"scripts/ci/test-report.mjs",
|
||||
"scripts/ci/test-timing-reporter.mjs",
|
||||
"scripts/ci/test-history.mjs",
|
||||
]);
|
||||
const matchesAny = (file, patterns) =>
|
||||
patterns.some((pattern) => matchesGlob(file, pattern));
|
||||
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
|
||||
@@ -198,6 +210,7 @@ if (tier === "reused") {
|
||||
}
|
||||
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
|
||||
if (file.startsWith("tests/")) continue;
|
||||
if (PORTABLE_CI_REPORTERS.has(file)) continue;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
@@ -238,11 +251,17 @@ const matrix = {
|
||||
]
|
||||
: [linux],
|
||||
};
|
||||
const plan = { ...selected, tier, matrix, files, reasons };
|
||||
const testSelection = planTestSelection(files, event, tier);
|
||||
const plan = { ...selected, tier, matrix, files, reasons, testSelection };
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) })
|
||||
Object.entries({
|
||||
...selected,
|
||||
tier,
|
||||
matrix: JSON.stringify(matrix),
|
||||
test_selection: JSON.stringify(testSelection),
|
||||
})
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
|
||||
+16
-9
@@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs";
|
||||
import { recordQualification } from "./qualification.mjs";
|
||||
|
||||
const jobs = JSON.parse(process.env.CI_JOBS);
|
||||
const outputs = jobs.changes.outputs;
|
||||
const outputs = jobs?.changes?.outputs;
|
||||
if (!["fast", "full", "reused"].includes(outputs?.tier))
|
||||
throw new Error(
|
||||
"CI classification requires a valid tier: fast, full, or reused",
|
||||
);
|
||||
const invalidFlags = CI_FLAGS.filter(
|
||||
(flag) => outputs[flag] !== "true" && outputs[flag] !== "false",
|
||||
);
|
||||
if (invalidFlags.length)
|
||||
throw new Error(
|
||||
`CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`,
|
||||
);
|
||||
const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true");
|
||||
const reused = outputs.tier === "reused";
|
||||
if (
|
||||
reused &&
|
||||
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
|
||||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
|
||||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
|
||||
CI_FLAGS.some((flag) => outputs[flag] !== "false"))
|
||||
selectedScopes.length !== 0)
|
||||
)
|
||||
throw new Error(
|
||||
"Reused qualification requires a scheduled admission receipt and no selected lanes",
|
||||
);
|
||||
if (
|
||||
outputs.tier === "full" &&
|
||||
CI_FLAGS.some((flag) => outputs[flag] !== "true")
|
||||
)
|
||||
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
const lanes = {
|
||||
@@ -31,9 +40,7 @@ const lanes = {
|
||||
docs: ["docs"],
|
||||
workflows: ["static"],
|
||||
};
|
||||
for (const [scope, value] of Object.entries(outputs)) {
|
||||
if (value === "true") required.push(...(lanes[scope] ?? [scope]));
|
||||
}
|
||||
for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope]));
|
||||
const failed = [...new Set(required)].filter(
|
||||
(name) => jobs[name]?.result !== "success",
|
||||
);
|
||||
|
||||
+69
-25
@@ -4,26 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
import { compareTestSelection } from "./test-selection.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const historyDirectory = join(root, ".cache/ci-test-history");
|
||||
const historyPath = join(historyDirectory, "test-results.json");
|
||||
const selectionDirectory = join(root, ".cache/ci-test-selection");
|
||||
const selectionPath = join(selectionDirectory, "report.json");
|
||||
// Never let a failed or partial rerun leave a previous successful history file.
|
||||
await rm(historyPath, { force: true });
|
||||
await rm(selectionPath, { force: true });
|
||||
const report = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
success: z.boolean(),
|
||||
numFailedTests: z.number().int().nonnegative(),
|
||||
testResults: z.array(
|
||||
reason: z.string().optional(),
|
||||
unhandledErrors: z.number().int().nonnegative().optional(),
|
||||
files: z.array(
|
||||
z.object({
|
||||
name: z.string().min(1),
|
||||
startTime: z.number().finite(),
|
||||
endTime: z.number().finite(),
|
||||
durationMs: z.number().finite().nonnegative(),
|
||||
state: z.string().optional(),
|
||||
}),
|
||||
),
|
||||
})
|
||||
.parse(JSON.parse(await readFile("test-results.json", "utf8")));
|
||||
.parse(JSON.parse(await readFile("test-timings.json", "utf8")));
|
||||
|
||||
const files = report.testResults.map(({ name, startTime, endTime }) => {
|
||||
const files = report.files.map(({ name, durationMs, state }) => {
|
||||
const path = relative(root, resolve(root, name));
|
||||
if (
|
||||
path === "" ||
|
||||
@@ -32,27 +39,47 @@ const files = report.testResults.map(({ name, startTime, endTime }) => {
|
||||
isAbsolute(path)
|
||||
)
|
||||
throw new Error(`Test report path is outside the checkout: ${name}`);
|
||||
const duration = endTime - startTime;
|
||||
if (!Number.isFinite(duration) || duration < 0)
|
||||
throw new Error(`Test report has an invalid duration: ${name}`);
|
||||
return { name: path.split(sep).join("/"), startTime: 0, endTime: duration };
|
||||
return { name: path.split(sep).join("/"), durationMs, state };
|
||||
});
|
||||
const complete = process.env.SHARD_RESULT === "success";
|
||||
const passed =
|
||||
complete &&
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success &&
|
||||
report.numFailedTests === 0;
|
||||
complete && process.env.MERGE_RESULT === "success" && report.success;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
|
||||
const expected = execFileSync(
|
||||
"git",
|
||||
["ls-files", "-z", "--", "src", "tests"],
|
||||
{ encoding: "utf8", maxBuffer: 8 * 1024 * 1024 },
|
||||
)
|
||||
.split("\0")
|
||||
.filter((path) => path.endsWith(".test.ts"));
|
||||
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
|
||||
const expected =
|
||||
verifyInventory || process.env.CI_TEST_SELECTION
|
||||
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
})
|
||||
.split("\0")
|
||||
.filter((path) => path.endsWith(".test.ts"))
|
||||
: [];
|
||||
let selection;
|
||||
if (process.env.CI_TEST_SELECTION) {
|
||||
const comparison = compareTestSelection(
|
||||
JSON.parse(process.env.CI_TEST_SELECTION),
|
||||
expected,
|
||||
files,
|
||||
);
|
||||
selection = {
|
||||
...comparison,
|
||||
complete:
|
||||
comparison.complete &&
|
||||
report.unhandledErrors === 0 &&
|
||||
["passed", "failed"].includes(report.reason) &&
|
||||
["success", "failure"].includes(process.env.SHARD_RESULT) &&
|
||||
["success", "failure"].includes(process.env.MERGE_RESULT),
|
||||
executionSha: process.env.GITHUB_SHA ?? null,
|
||||
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
|
||||
runId: process.env.GITHUB_RUN_ID ?? null,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
|
||||
};
|
||||
await mkdir(selectionDirectory, { recursive: true });
|
||||
await writeFile(selectionPath, JSON.stringify(selection));
|
||||
}
|
||||
if (passed && verifyInventory) {
|
||||
const actual = files.map(({ name }) => name);
|
||||
const expectedSet = new Set(expected);
|
||||
const actualSet = new Set(actual);
|
||||
@@ -66,18 +93,31 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
|
||||
const summary = [
|
||||
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
"### Observational PR test selection",
|
||||
"",
|
||||
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
|
||||
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
|
||||
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
|
||||
passed
|
||||
? "Successful timing history is available."
|
||||
: "This report is diagnostic only and is not a successful timing baseline.",
|
||||
"",
|
||||
"Durations include tests and hooks, worker preparation, environment, imports, and setup. They are scheduling weights, not summed workflow wall time.",
|
||||
"",
|
||||
"| Slowest file | Duration (ms) |",
|
||||
"| --- | ---: |",
|
||||
...files
|
||||
.toSorted((left, right) => right.endTime - left.endTime)
|
||||
.toSorted((left, right) => right.durationMs - left.durationMs)
|
||||
.slice(0, 20)
|
||||
.map(
|
||||
({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`,
|
||||
({ name, durationMs }) =>
|
||||
`| ${name.replaceAll("|", "\\|")} | ${durationMs} |`,
|
||||
),
|
||||
"",
|
||||
].join("\n");
|
||||
@@ -89,6 +129,10 @@ if (passed) {
|
||||
await mkdir(historyDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
historyPath,
|
||||
JSON.stringify({ success: true, numFailedTests: 0, testResults: files }),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
success: true,
|
||||
files: files.map(({ name, durationMs }) => ({ name, durationMs })),
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
export interface TestSelectionPlan {
|
||||
readonly version: 1;
|
||||
readonly mode: "full" | "candidate";
|
||||
readonly owners: readonly string[];
|
||||
readonly changedTests: readonly string[];
|
||||
readonly changedPaths: readonly string[];
|
||||
readonly reason: string;
|
||||
}
|
||||
export function planTestSelection(
|
||||
files: readonly string[],
|
||||
event: string,
|
||||
tier: string,
|
||||
): TestSelectionPlan;
|
||||
export function compareTestSelection(
|
||||
plan: unknown,
|
||||
inventory: readonly string[],
|
||||
results: readonly {
|
||||
readonly name: string;
|
||||
readonly durationMs: number;
|
||||
readonly state?: string;
|
||||
}[],
|
||||
): {
|
||||
readonly version: 1;
|
||||
readonly observational: true;
|
||||
readonly plan: TestSelectionPlan;
|
||||
readonly selected: readonly string[];
|
||||
readonly omitted: readonly string[];
|
||||
readonly missing: readonly string[];
|
||||
readonly unexpected: readonly string[];
|
||||
readonly duplicateFiles: number;
|
||||
readonly unobserved: readonly string[];
|
||||
readonly missedFailures: readonly string[];
|
||||
readonly complete: boolean;
|
||||
readonly omittedDurationMs: number;
|
||||
};
|
||||
@@ -0,0 +1,151 @@
|
||||
import { matchesGlob } from "node:path";
|
||||
|
||||
// Candidate ownership only: these leaf adapters and their producer tests can be
|
||||
// compared with full-suite outcomes. This table never changes test execution.
|
||||
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
|
||||
const OWNERS = {
|
||||
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
|
||||
apktool: ["src/apktool/**"],
|
||||
dotnet: ["src/dotnet/**"],
|
||||
evm: ["src/evm/**", "tests/boundary/evm/**"],
|
||||
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
|
||||
flutter: ["src/flutter/**"],
|
||||
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
|
||||
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
|
||||
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
|
||||
jeb: ["src/jeb/**"],
|
||||
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
|
||||
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
|
||||
wasm: ["src/wasm/**"],
|
||||
};
|
||||
const ownersOf = (path) =>
|
||||
Object.entries(OWNERS)
|
||||
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
|
||||
.map(([owner]) => owner);
|
||||
const validPath = (path) =>
|
||||
typeof path === "string" &&
|
||||
/^(src|tests)\//.test(path) &&
|
||||
!/[\\\0\r\n]/.test(path) &&
|
||||
!path.split("/").some((part) => ["", ".", ".."].includes(part));
|
||||
const full = (reason, changedPaths = []) => ({
|
||||
version: 1,
|
||||
mode: "full",
|
||||
owners: [],
|
||||
changedTests: [],
|
||||
changedPaths,
|
||||
reason,
|
||||
});
|
||||
|
||||
/** Plan an observational PR subset from the classifier's existing Git diff. */
|
||||
export function planTestSelection(files, event, tier) {
|
||||
if (event !== "pull_request" || tier !== "fast")
|
||||
return full("Only ordinary PRs are candidates", files);
|
||||
if (!files.length) return full("No changed paths available");
|
||||
const owners = new Set();
|
||||
const changedTests = [];
|
||||
for (const path of files) {
|
||||
if (!validPath(path))
|
||||
return full(`Shared or unknown input: ${path}`, files);
|
||||
const matches = ownersOf(path);
|
||||
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
|
||||
for (const owner of matches) owners.add(owner);
|
||||
if (path.endsWith(".test.ts")) changedTests.push(path);
|
||||
}
|
||||
return {
|
||||
version: 1,
|
||||
mode: "candidate",
|
||||
owners: [...owners].sort(),
|
||||
changedTests: [...new Set(changedTests)].sort(),
|
||||
changedPaths: files,
|
||||
reason:
|
||||
"Observe affected leaf owners; retain integration and unowned tests",
|
||||
};
|
||||
}
|
||||
|
||||
/** Compare a proposal with the full run; missing observations never prove parity. */
|
||||
export function compareTestSelection(plan, inventory, results) {
|
||||
if (
|
||||
plan?.version !== 1 ||
|
||||
!["full", "candidate"].includes(plan.mode) ||
|
||||
!Array.isArray(plan.owners) ||
|
||||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
|
||||
!Array.isArray(plan.changedTests) ||
|
||||
!Array.isArray(plan.changedPaths) ||
|
||||
plan.changedPaths.some((path) => typeof path !== "string") ||
|
||||
plan.changedTests.some(
|
||||
(path) => !validPath(path) || !path.endsWith(".test.ts"),
|
||||
) ||
|
||||
typeof plan.reason !== "string" ||
|
||||
(plan.mode === "candidate" && !plan.owners.length) ||
|
||||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
|
||||
)
|
||||
throw new Error("Invalid observational test selection plan");
|
||||
const expected = new Set(inventory);
|
||||
if (!expected.size || expected.size !== inventory.length)
|
||||
throw new Error("Test selection requires a nonempty unique inventory");
|
||||
const ownersByFile = new Map(
|
||||
plan.mode === "candidate"
|
||||
? inventory.map((path) => [path, ownersOf(path)])
|
||||
: [],
|
||||
);
|
||||
// Removed/renamed tests have no current executable identity. Keep all tests.
|
||||
const removed = plan.changedTests.filter((path) => !expected.has(path));
|
||||
const missingOwners = plan.owners.filter(
|
||||
(owner) =>
|
||||
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
|
||||
);
|
||||
const effective = removed.length
|
||||
? full(
|
||||
`Changed tests absent from current inventory: ${removed.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: missingOwners.length
|
||||
? full(
|
||||
`Owners without executable tests: ${missingOwners.join(", ")}`,
|
||||
plan.changedPaths,
|
||||
)
|
||||
: plan;
|
||||
const selected = [];
|
||||
const omitted = [];
|
||||
for (const path of inventory.toSorted()) {
|
||||
const owners = ownersByFile.get(path) ?? [];
|
||||
const keep =
|
||||
effective.mode === "full" ||
|
||||
!owners.length ||
|
||||
effective.changedTests.includes(path) ||
|
||||
owners.some((owner) => effective.owners.includes(owner));
|
||||
(keep ? selected : omitted).push(path);
|
||||
}
|
||||
const actual = new Set(results.map((file) => file.name));
|
||||
const missing = inventory.filter((path) => !actual.has(path));
|
||||
const unexpected = [...actual].filter((path) => !expected.has(path));
|
||||
const duplicateFiles = results.length - actual.size;
|
||||
const unobserved = results
|
||||
.filter((file) => !["passed", "failed"].includes(file.state))
|
||||
.map((file) => file.name);
|
||||
const omittedSet = new Set(omitted);
|
||||
const missedFailures = results
|
||||
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
|
||||
.map((file) => file.name);
|
||||
return {
|
||||
version: 1,
|
||||
observational: true,
|
||||
plan: effective,
|
||||
selected,
|
||||
omitted,
|
||||
missing,
|
||||
unexpected,
|
||||
duplicateFiles,
|
||||
unobserved,
|
||||
missedFailures,
|
||||
complete:
|
||||
!missing.length &&
|
||||
!unexpected.length &&
|
||||
!duplicateFiles &&
|
||||
!unobserved.length,
|
||||
// Phase totals are scheduling weights, not an estimate of wall-time savings.
|
||||
omittedDurationMs: results
|
||||
.filter((file) => omittedSet.has(file.name))
|
||||
.reduce((sum, file) => sum + file.durationMs, 0),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
|
||||
/** @implements {import("vitest/node").Reporter} */
|
||||
export default class TestTimingReporter {
|
||||
/**
|
||||
* @param {ReadonlyArray<import("vitest/node").TestModule>} modules
|
||||
* @param {ReadonlyArray<import("vitest/node").SerializedError>} errors
|
||||
* @param {import("vitest/node").TestRunEndReason} reason
|
||||
*/
|
||||
async onTestRunEnd(modules, errors, reason) {
|
||||
const files = modules.map((module) => {
|
||||
const diagnostic = module.diagnostic();
|
||||
// These disjoint phases match Vitest's per-file wall-time accounting.
|
||||
// Import timings already include transform waits; do not add them again.
|
||||
const phases = {
|
||||
execution: diagnostic.duration,
|
||||
preparation: diagnostic.prepareDuration,
|
||||
environment: diagnostic.environmentSetupDuration,
|
||||
imports: diagnostic.collectDuration,
|
||||
setup: diagnostic.setupDuration,
|
||||
};
|
||||
return {
|
||||
name: module.moduleId,
|
||||
state: module.state(),
|
||||
durationMs: Object.values(phases).reduce(
|
||||
(total, value) => total + value,
|
||||
0,
|
||||
),
|
||||
phases,
|
||||
};
|
||||
});
|
||||
await writeFile(
|
||||
"test-timings.json",
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
success: reason === "passed" && errors.length === 0,
|
||||
reason,
|
||||
unhandledErrors: errors.length,
|
||||
files,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user