ci: reduce reporter fan-out and preserve shard history

This commit is contained in:
morluto
2026-10-13 18:16:10 +08:00
parent 9928f357fb
commit ab02f3c8d6
11 changed files with 968 additions and 82 deletions
@@ -0,0 +1,288 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
const batchScript = fileURLToPath(
new URL("../../../scripts/ci/job-timings-batch.mjs", import.meta.url),
);
it("collects the latest attempt for recent completed CI and Release runs into a bounded manifest", async () => {
const recent = new Date().toISOString();
const fixture = await createFixture([
{
workflow_runs: [
{
id: 42,
run_attempt: 3,
status: "completed",
conclusion: "failure",
created_at: new Date(Date.parse(recent) + 1_000).toISOString(),
},
{
id: 41,
run_attempt: 1,
status: "completed",
conclusion: "cancelled",
created_at: recent,
},
],
},
{
workflow_runs: [
{
id: 43,
run_attempt: 2,
status: "completed",
conclusion: "success",
created_at: new Date(Date.parse(recent) + 2_000).toISOString(),
},
],
},
[
{
jobs: [
{
name: "Release lane",
status: "completed",
conclusion: "success",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 50_000).toISOString(),
steps: [],
},
],
},
],
[
{
jobs: [
{
name: "Failed lane",
status: "completed",
conclusion: "failure",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 55_000).toISOString(),
steps: [],
},
],
},
],
[
{
jobs: [
{
name: "Cancelled lane",
status: "completed",
conclusion: "cancelled",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 30_000).toISOString(),
steps: [],
},
],
},
],
]);
const result = await runBatch(fixture);
const manifest = JSON.parse(
await readFile(
join(fixture.outputDirectory, "batch-manifest.json"),
"utf8",
),
);
const calls = JSON.parse(await readFile(fixture.calls, "utf8"));
expect(result.stdout).toContain("3 collected runs");
expect(
await readFile(join(fixture.directory, "summary.md"), "utf8"),
).toContain("collected 3 of 3");
expect(manifest.selection).toMatchObject({
lookbackHours: 24,
maxRunsPerWorkflow: 100,
exhaustive: false,
});
expect(manifest.coverage).toMatchObject({
workflowsQueried: ["CI", "Release"],
selectedRuns: 3,
collectedRuns: 3,
});
expect(manifest.truncation).toEqual([]);
expect(manifest.errors).toEqual([]);
expect(manifest.runs).toMatchObject([
{ id: 43, runAttempt: 2, conclusion: "success" },
{ id: 42, runAttempt: 3, conclusion: "failure" },
{ id: 41, runAttempt: 1, conclusion: "cancelled" },
]);
expect(manifest.runs).toEqual(
expect.arrayContaining([
expect.objectContaining({
status: "collected",
rawPath: expect.any(String),
reportPath: expect.any(String),
}),
]),
);
expect(calls).toContainEqual([
"api",
"--paginate",
"--slurp",
"repos/example/rea/actions/runs/42/attempts/3/jobs?per_page=100",
]);
expect(calls).toContainEqual([
"api",
"--paginate",
"--slurp",
"repos/example/rea/actions/runs/41/attempts/1/jobs?per_page=100",
]);
const failedReport = JSON.parse(
await readFile(
join(fixture.outputDirectory, "runs/42-3/report.json"),
"utf8",
),
);
expect(failedReport.jobs[0].conclusion).toBe("failure");
expect(failedReport.jobs[0].execution.seconds).toBe(55);
});
it("records listing and per-run errors without claiming complete coverage", async () => {
const recent = new Date().toISOString();
const fixture = await createFixture([
{
workflow_runs: [
{
id: 7,
run_attempt: 2,
status: "completed",
conclusion: "success",
created_at: recent,
},
],
},
{ error: true },
{ jobs: "malformed" },
]);
await runBatch(fixture);
const manifest = JSON.parse(
await readFile(
join(fixture.outputDirectory, "batch-manifest.json"),
"utf8",
),
);
expect(manifest.selection.exhaustive).toBe(false);
expect(manifest.errors).toMatchObject([
{ scope: "workflow:Release" },
{ scope: "run" },
]);
expect(manifest.runs[0]).toMatchObject({
id: 7,
runAttempt: 2,
status: "error",
});
});
it("retains a partial manifest when its collection budget is exhausted", async () => {
const fixture = await createFixture([
{
workflow_runs: [8, 9].map((id) => ({
id,
run_attempt: 1,
status: "completed",
created_at: new Date().toISOString(),
})),
},
{ workflow_runs: [] },
]);
await executeWorkflowFixture(
process.execPath,
[
"--input-type=module",
"-e",
`
const { collectJobTimingsBatch } = await import(${JSON.stringify(pathToFileURL(batchScript).href)});
let clock = Date.now();
await collectJobTimingsBatch({
repository: "example/rea",
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY,
now: () => clock,
collect: async () => {
clock += 240001;
throw new Error("fixture request exhausted the remaining budget");
},
});
`,
],
{ cwd: fixture.directory, env: fixtureEnvironment(fixture) },
);
const manifest = JSON.parse(
await readFile(
join(fixture.outputDirectory, "batch-manifest.json"),
"utf8",
),
);
expect(manifest.coverage).toMatchObject({
selectedRuns: 2,
collectedRuns: 0,
});
expect(manifest.runs).toHaveLength(1);
expect(manifest.runs[0]).toMatchObject({ status: "error" });
expect(manifest.truncation.join(" ")).toContain("Time budget");
});
async function createFixture(responses: unknown[]) {
const directory = await createTestTempDirectory("rea-ci-job-timings-batch-");
const fakeBin = join(directory, "bin");
await mkdir(fakeBin);
const responsePath = join(directory, "responses.json");
const calls = join(directory, "calls.json");
await writeFile(responsePath, JSON.stringify(responses));
await writeFile(calls, "[]");
const fakeGh = join(fakeBin, "gh");
await writeFile(
fakeGh,
`#!/usr/bin/env node
const fs = require("node:fs");
const calls = JSON.parse(fs.readFileSync(process.env.CI_GH_CALLS, "utf8"));
calls.push(process.argv.slice(2));
fs.writeFileSync(process.env.CI_GH_CALLS, JSON.stringify(calls));
const responses = JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSES, "utf8"));
const response = responses[calls.length - 1];
if (response && response.error) { process.stderr.write("fixture API error"); process.exitCode = 1; }
else process.stdout.write(JSON.stringify(response));
`,
);
await chmod(fakeGh, 0o755);
return {
directory,
fakeBin,
responses: responsePath,
calls,
outputDirectory: join(directory, ".cache", "ci-job-timings"),
};
}
function runBatch(fixture: Awaited<ReturnType<typeof createFixture>>) {
return executeWorkflowFixture(process.execPath, [batchScript], {
cwd: fixture.directory,
env: fixtureEnvironment(fixture),
});
}
function fixtureEnvironment(
fixture: Awaited<ReturnType<typeof createFixture>>,
) {
return {
...process.env,
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
CI_REPOSITORY: "example/rea",
CI_JOB_TIMINGS_DIRECTORY: fixture.outputDirectory,
CI_GH_CALLS: fixture.calls,
CI_GH_RESPONSES: fixture.responses,
GITHUB_STEP_SUMMARY: join(fixture.directory, "summary.md"),
};
}
@@ -90,10 +90,9 @@ it("exports every canonical scope with read authority for admission and keeps te
const timings = z
.object({
on: z.object({
workflow_run: z.object({
workflows: z.array(z.string()),
types: z.array(z.string()),
}),
schedule: z.array(z.object({ cron: z.string() })),
workflow_dispatch: z.unknown(),
workflow_run: z.unknown().optional(),
}),
permissions: z.record(z.string(), z.string()),
jobs: z.object({
@@ -117,10 +116,8 @@ it("exports every canonical scope with read authority for admission and keeps te
),
),
);
expect(timings.on.workflow_run).toEqual({
workflows: ["CI", "Release"],
types: ["completed"],
});
expect(timings.on.workflow_run).toBeUndefined();
expect(timings.on.schedule).toHaveLength(1);
expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
const checkout = timings.jobs.report.steps.find((step) =>
step.uses?.startsWith("actions/checkout@"),
@@ -130,14 +127,9 @@ it("exports every canonical scope with read authority for admission and keeps te
"persist-credentials": false,
});
const report = timings.jobs.report.steps.find((step) =>
step.run?.includes("scripts/ci/job-timings.mjs"),
);
expect(report?.env?.CI_TIMING_RUN_ID).toBe(
"${{ github.event.workflow_run.id }}",
);
expect(report?.env?.CI_TIMING_RUN_ATTEMPT).toBe(
"${{ github.event.workflow_run.run_attempt }}",
step.run?.includes("scripts/ci/job-timings-batch.mjs"),
);
expect(report?.env?.CI_REPOSITORY).toBe("${{ github.repository }}");
});
const requiredNeeds = async () => {
@@ -152,6 +144,17 @@ it("makes the required gate wait for every CI lane, including future additions",
);
});
it("runs workflow validation inside required static checks when selected", async () => {
const { jobs } = await workflow();
const steps = z
.array(z.object({ run: z.string().optional(), if: z.string().optional() }))
.parse(jobs.static?.steps);
const validation = steps.filter((step) => step.run?.includes("actionlint"));
expect(validation).toHaveLength(1);
expect(validation[0]?.if).toBe("needs.changes.outputs.workflows == 'true'");
expect(jobs.workflows).toBeUndefined();
});
it("leaves cancellation ownership with CI so child lanes cannot cancel another tier", async () => {
const { jobs } = await workflow();
const children = Object.values(jobs).flatMap((job) =>
@@ -257,33 +260,44 @@ it.each([
["false", "skipped", true],
["false", "failure", false],
] as const)(
"enforces the SQLite required gate for sqlite=%s and result=%s",
async (sqlite, result, passes) => {
"enforces mapped required scopes for selected=%s and result=%s",
async (selected, result, passes) => {
const workflow = await requiredNeeds();
// GitHub exposes only direct dependencies in the required job's needs context.
const needs: Record<
string,
{ result: string; outputs?: Record<string, string> }
> = Object.fromEntries(
workflow.needs.map((name) => [name, { result: "success" }]),
);
needs.changes = { result: "success", outputs: { code: "true", sqlite } };
if (result === "missing") delete needs["sqlite-runtime"];
else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result };
const execution = executeWorkflowFixture(
process.execPath,
[
fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
),
],
{ env: { ...process.env, CI_JOBS: JSON.stringify(needs) } },
);
if (passes) await expect(execution).resolves.toMatchObject({ stderr: "" });
else
await expect(execution).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("sqlite-runtime"),
});
for (const [scope, lane] of [
["sqlite", "sqlite-runtime"],
["workflows", "static"],
] as const) {
// GitHub exposes only direct dependencies in the required job's needs context.
const needs: Record<
string,
{ result: string; outputs?: Record<string, string> }
> = Object.fromEntries(
workflow.needs.map((name) => [name, { result: "success" }]),
);
needs.changes = {
result: "success",
outputs: { code: "true", [scope]: selected },
};
if (result === "missing") delete needs[lane];
else needs[lane] = { result };
const execution = executeWorkflowFixture(
process.execPath,
[
fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
),
],
{ env: { ...process.env, CI_JOBS: JSON.stringify(needs) } },
);
// Static checks remain required even without workflow edits.
const expected = passes && !(lane === "static" && result !== "success");
if (expected)
await expect(execution).resolves.toMatchObject({ stderr: "" });
else
await expect(execution).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining(lane),
});
}
},
);
@@ -0,0 +1,188 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
const script = fileURLToPath(
new URL("../../../scripts/ci/test-history.mjs", import.meta.url),
);
const repository = "example/rea";
const repositoryId = 1;
const now = Date.now();
function artifact(id: number, overrides: Record<string, unknown> = {}) {
return {
id: id + 100,
name: "test-timings",
expired: false,
created_at: new Date(now - 60_000).toISOString(),
workflow_run: {
id,
repository_id: repositoryId,
head_repository_id: repositoryId,
head_branch: "main",
},
...overrides,
};
}
function run(id: number, overrides: Record<string, unknown> = {}) {
return {
id,
status: "completed",
conclusion: "success",
path: ".github/workflows/ci.yml",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_branch: "main",
head_repository: { full_name: repository },
...overrides,
};
}
async function invoke(
artifacts: unknown[],
runs: Record<number, unknown>,
options: {
apiError?: boolean;
repository?: string;
repositoryId?: string;
} = {},
) {
const directory = await createTestTempDirectory("rea-ci-test-history-");
const fakeBin = join(directory, "bin");
await mkdir(fakeBin);
const fixture = join(directory, "api.json");
const calls = join(directory, "api-calls");
const output = join(directory, "github-output");
await writeFile(fixture, JSON.stringify({ artifacts, runs }));
await writeFile(output, "");
await writeFile(calls, "");
const fakeGh = join(fakeBin, "gh");
await writeFile(
fakeGh,
[
"#!/bin/sh",
'printf \'%s\\n\' "$*" >> "$GH_API_CALLS"',
'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi',
'if [ "$4" = "repos/example/rea/actions/artifacts" ]; then',
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify({artifacts:x.artifacts}));\'',
"else",
' run_id="${4##*/}"',
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify(x.runs[Number(process.argv[1])] ?? {}));\' "$run_id"',
"fi",
"",
].join("\n"),
);
await chmod(fakeGh, 0o755);
const node24 = dirname(process.execPath);
const result = await executeWorkflowFixture(process.execPath, [script], {
cwd: directory,
env: {
...process.env,
PATH: [fakeBin, node24, process.env.PATH ?? ""].join(delimiter),
GH_FIXTURE_RESPONSE: fixture,
GH_API_CALLS: calls,
GH_API_ERROR: String(options.apiError ?? false),
GH_TOKEN: "fixture-token",
CI_REPOSITORY: options.repository ?? repository,
CI_REPOSITORY_ID: options.repositoryId ?? String(repositoryId),
CI_DEFAULT_BRANCH: "main",
GITHUB_OUTPUT: output,
},
});
return {
result,
output: await readFile(output, "utf8"),
calls: await readFile(calls, "utf8"),
};
}
it("selects the latest trusted full run that actually published timing history", async () => {
const { result, output, calls } = await invoke(
[
artifact(15),
artifact(14),
artifact(13, {
workflow_run: { ...artifact(13).workflow_run, head_repository_id: 2 },
}),
artifact(12, {
workflow_run: { ...artifact(12).workflow_run, head_branch: "feature" },
}),
artifact(11),
],
{
15: run(15, { conclusion: "failure" }),
14: run(14, { path: ".github/workflows/release.yml" }),
11: run(11, { event: "workflow_dispatch" }),
},
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "11" });
expect(output).toBe("run-id=11\n");
expect(calls).toContain("name=test-timings");
expect(calls).toContain("per_page=100");
expect(calls.match(/actions\/runs\//gu)).toHaveLength(3);
expect(calls).not.toContain("actions/workflows");
});
it("rejects expired, foreign, old, and non-full run candidates", async () => {
const { result, output } = await invoke(
[
artifact(15, { expired: true }),
artifact(14, {
workflow_run: { ...artifact(14).workflow_run, head_repository_id: 2 },
}),
artifact(13, {
created_at: new Date(now - 31 * 24 * 60 * 60 * 1000).toISOString(),
}),
artifact(12),
],
{
12: run(12, { event: "pull_request" }),
},
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(output).toBe("run-id=\n");
});
it("writes an empty run id when GitHub history is unavailable", async () => {
const { result, output } = await invoke([], {}, { apiError: true });
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(output).toBe("run-id=\n");
});
it("bounds run lookups to five artifacts even when inventory has more", async () => {
const ids = [25, 24, 23, 22, 21, 20, 19];
const { result, calls } = await invoke(
ids.map((id) => artifact(id)),
Object.fromEntries(
ids.map((id, index) => [
id,
run(id, index < 5 ? { conclusion: "failure" } : {}),
]),
),
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(calls.match(/actions\/runs\//gu)).toHaveLength(5);
});
it("returns empty history without querying for an invalid repository identity", async () => {
const { result, output, calls } = await invoke(
[],
{},
{
repository: "owner/repo?per_page=100",
repositoryId: "not-an-id",
},
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(output).toBe("run-id=\n");
expect(calls).toBe("");
});
@@ -80,10 +80,22 @@ it("keeps diagnosis available after merge failure and publishes only complete su
expect(history?.if).toContain("needs.test-shard.result == 'success'");
expect(history?.if).toContain("needs.changes.outputs.tier == 'full'");
expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json");
const download = (await workflow()).build!.steps.find(
const buildSteps = (await workflow()).build!.steps;
const lookup = buildSteps.find((step) =>
step.run?.includes("test-history.mjs"),
);
expect(lookup?.env).toMatchObject({
CI_REPOSITORY: "${{ github.repository }}",
CI_REPOSITORY_ID: "${{ github.repository_id }}",
CI_DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}",
});
const download = buildSteps.find(
(step) => step.name === "Download previous timing report",
);
expect(download?.with?.name).toBe(history?.with?.name);
expect(download?.with?.["run-id"]).toBe(
"${{ steps.history.outputs.run-id }}",
);
});
it("rejects successful shard reports with missing or duplicate portable files", async () => {