From 3af8f847f0e35843f51622a136165dd6ee12cdd0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:38:38 +0800 Subject: [PATCH 1/8] ci: fix PR metadata triggers and defer expensive runner work --- .github/workflows/ci-full-pr.yml | 18 ++++ .github/workflows/ci.yml | 59 +++++------ .github/workflows/pr-title.yml | 29 ++++++ docs/testing.md | 25 ++++- scripts/ci/plan.mjs | 2 + tests/boundary/filesystem/ciPrEvents.test.ts | 98 +++++++++++++++++++ .../filesystem/ciRequiredLanes.test.ts | 21 ++++ 7 files changed, 211 insertions(+), 41 deletions(-) create mode 100644 .github/workflows/ci-full-pr.yml create mode 100644 .github/workflows/pr-title.yml create mode 100644 tests/boundary/filesystem/ciPrEvents.test.ts diff --git a/.github/workflows/ci-full-pr.yml b/.github/workflows/ci-full-pr.yml new file mode 100644 index 000000000..8f76a2ea2 --- /dev/null +++ b/.github/workflows/ci-full-pr.yml @@ -0,0 +1,18 @@ +name: Full PR qualification + +on: + pull_request: + branches: [main, "release/*"] + types: [labeled] + +permissions: + contents: read + actions: read + +jobs: + qualification: + name: Qualify labeled PR revision + if: github.event.label.name == 'ci:full' + uses: ./.github/workflows/ci.yml + with: + source_sha: ${{ github.sha }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a9977f0d..cf5cd0a11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ on: workflow_call: inputs: source_sha: - description: Exact release source; must match the caller revision. + description: Exact qualification source; must match the caller revision. required: true type: string @@ -22,7 +22,7 @@ permissions: concurrency: # Complete qualification finishes even while main receives more pushes. # Scheduled/manual main runs coalesce to one active and the latest pending; - # release qualification has its own exact-revision group. + # release and label-triggered qualification use exact-revision groups. group: ci-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && 'qualification' || 'feedback' }}-${{ inputs.source_sha || github.event.pull_request.number || github.ref }} cancel-in-progress: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }} @@ -30,22 +30,6 @@ env: HUSKY: "0" jobs: - pr-title: - name: Conventional PR title - if: github.event_name == 'pull_request' && inputs.source_sha == '' - runs-on: ubuntu-latest - steps: - - name: Validate title for Release Please - env: - PR_TITLE: ${{ github.event.pull_request.title }} - run: >- - node -e 'const title = process.env.PR_TITLE ?? ""; - const conventional = /^(?:feat|fix|docs|refactor|perf|test|build|ci|chore|revert)(?:\([a-z0-9._/-]+\))?!?: .+/u; - if (!conventional.test(title)) { - console.error("PR title must use Conventional Commits, for example: feat: add historical source import"); - process.exit(1); - }' - changes: name: Classify changes runs-on: ubuntu-latest @@ -83,7 +67,7 @@ jobs: - uses: actions/setup-node@v4 with: node-version-file: .nvmrc - - name: Bind release qualification to the caller revision + - name: Bind qualification to the caller revision if: inputs.source_sha != '' env: SOURCE_SHA: ${{ inputs.source_sha }} @@ -262,7 +246,9 @@ jobs: retention-days: 7 sqlite-runtime: - needs: [changes, build] + # Preserve all selected coverage, but admit expensive host/provider work + # only after static validation. Portable tests still provide early feedback. + needs: [changes, static, build] if: needs.changes.outputs.sqlite == 'true' uses: ./.github/workflows/sqlite-runtime.yml @@ -350,7 +336,7 @@ jobs: conformance-fixtures: name: Source-built conformance fixtures - needs: changes + needs: [changes, static] if: needs.changes.outputs.fixtures == 'true' runs-on: ubuntu-latest timeout-minutes: 10 @@ -361,7 +347,7 @@ jobs: windows-native-build: name: Build bundled Windows x64 native controls - needs: changes + needs: [changes, static] if: needs.changes.outputs.runtime == 'true' runs-on: ubuntu-latest timeout-minutes: 10 @@ -397,7 +383,7 @@ jobs: package-e2e: name: Native package baseline (${{ matrix.platform }} ${{ matrix.architecture }}) - needs: [changes, build, windows-native-build] + needs: [changes, static, build, windows-native-build] if: needs.changes.outputs.runtime == 'true' strategy: fail-fast: false @@ -463,7 +449,7 @@ jobs: apple-artifacts: name: Real Apple artifact CLI/MCP E2E - needs: [changes, build] + needs: [changes, static, build] if: needs.changes.outputs.apple == 'true' runs-on: macos-15 timeout-minutes: 15 @@ -511,7 +497,7 @@ jobs: reconstruction-readiness: name: Reconstruction readiness - needs: changes + needs: [changes, static] if: needs.changes.outputs.readiness == 'true' runs-on: ubuntu-latest timeout-minutes: 15 @@ -540,7 +526,7 @@ jobs: windows-curated: name: Windows capability and package lane - needs: [changes, build, windows-native-build] + needs: [changes, static, build, windows-native-build] if: needs.changes.outputs.package == 'true' runs-on: windows-latest timeout-minutes: 25 @@ -672,52 +658,52 @@ jobs: if-no-files-found: error browser: - needs: changes + needs: [changes, static] if: needs.changes.outputs.browser == 'true' uses: ./.github/workflows/real-browser.yml binary_layout: - needs: changes + needs: [changes, static] if: needs.changes.outputs.binary_layout == 'true' uses: ./.github/workflows/real-binary-layout.yml evm: - needs: changes + needs: [changes, static] if: needs.changes.outputs.evm == 'true' uses: ./.github/workflows/real-evm-interface.yml javascript_recovery: - needs: changes + needs: [changes, static] if: needs.changes.outputs.javascript_recovery == 'true' uses: ./.github/workflows/real-javascript-recovery.yml recorded_crash: - needs: changes + needs: [changes, static] if: needs.changes.outputs.recorded_crash == 'true' uses: ./.github/workflows/real-recorded-crash.yml web_network: - needs: changes + needs: [changes, static] if: needs.changes.outputs.web_network == 'true' uses: ./.github/workflows/real-web-network-captures.yml web_runtime: - needs: changes + needs: [changes, static] if: needs.changes.outputs.web_runtime == 'true' uses: ./.github/workflows/real-web-runtime.yml web_source_map: - needs: changes + needs: [changes, static] if: needs.changes.outputs.web_source_map == 'true' uses: ./.github/workflows/real-web-source-map.yml nativeaot: - needs: changes + needs: [changes, static] if: needs.changes.outputs.nativeaot == 'true' uses: ./.github/workflows/nativeaot-fixtures.yml website: - needs: changes + needs: [changes, static] if: needs.changes.outputs.website == 'true' uses: ./.github/workflows/website-check.yml @@ -727,7 +713,6 @@ jobs: needs: [ changes, - pr-title, static, build, test-shard, diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml new file mode 100644 index 000000000..abe98bb48 --- /dev/null +++ b/.github/workflows/pr-title.yml @@ -0,0 +1,29 @@ +name: PR title + +on: + pull_request: + branches: [main, "release/*"] + types: [opened, synchronize, reopened, edited] + +permissions: {} + +concurrency: + group: pr-title-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + pr-title: + name: Conventional PR title + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Validate title for Release Please + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: >- + node -e 'const title = process.env.PR_TITLE ?? ""; + const conventional = /^(?:feat|fix|docs|refactor|perf|test|build|ci|chore|revert)(?:\([a-z0-9._/-]+\))?!?: .+/u; + if (!conventional.test(title)) { + console.error("PR title must use Conventional Commits, for example: feat: add historical source import"); + process.exit(1); + }' diff --git a/docs/testing.md b/docs/testing.md index 5c43a5c78..cf1b8febc 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -276,6 +276,15 @@ sharding. Static checks, documentation, build, package verification, and real-system lanes remain separate jobs so one kind of evidence cannot stand in for another. Workflow syntax and expression validation runs inside the required static-check job when workflow changes select it, avoiding a separate runner. +Native, provider, fixture, SQLite compatibility, and website lanes wait for +static checks to succeed before allocating their runners. This avoids spending +those runner slots on revisions that already fail static validation. Portable +test shards and documentation checks remain independent of static results, so +they can still provide diagnostic feedback. The required gate rejects skipped +selected lanes when static checks fail; passing revisions retain all selected +coverage. This ordering reduces peak demand on invalid revisions but can add +static-check waiting to otherwise passing runs; it is not a measured reduction +in end-to-end latency. The separate `CI timings` workflow batches recent completed CI and Release runs hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and @@ -841,9 +850,12 @@ observation. Manual dispatch, `ci:full`, and exact release qualification always run full validation. Full qualification includes aggregate coverage, -every portable provider lane, and all native hosts below. A `ci:full` PR label -selects full qualification on the next PR run; manual dispatch can request it -immediately. Fast-tier broad fallbacks retain all owner lanes and the three-host +every portable provider lane, and all native hosts below. Adding `ci:full` +immediately starts the separate `Full PR qualification` workflow against the +label event's merge revision; subsequent PR revisions also select the full tier +in normal CI while that label remains. Unrelated labels do not start validation +jobs or replace the main CI checks. Manual dispatch also requests full validation. +Fast-tier broad fallbacks retain all owner lanes and the three-host package matrix; Intel macOS and coverage remain full-tier obligations. Each classification job records its selection and broad-fallback reasons in the run summary. Representative Git-diff cases live in @@ -851,7 +863,12 @@ summary. Representative Git-diff cases live in `CI required` always runs and checks classification and every selected lane, rejecting failed, cancelled, or unexpectedly skipped work. Branch protection -should require this aggregate, `Static checks`, and `Portable tests`. Fast runs +should require this aggregate, `Static checks`, `Portable tests`, and +`Conventional PR title`. Title validation runs separately on PR opening, +reopening, commits, and edits; correcting a title refreshes only this lightweight +check. It reads title text through an environment variable, executes no PR code, +and has no token permissions. Its separate concurrency group cannot cancel +test runs. Fast runs cancel superseded feedback for the same PR or main branch. Scheduled/manual qualification has a separate concurrency group: active work finishes and only the latest pending run is retained. Release qualification is grouped by its diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index e9505b903..9384d9b05 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -8,6 +8,8 @@ const HOST_TEST_OWNERS = ["apple", "package", "sqlite"]; // Actionlint validates workflow syntax only. The semantic suite asserts these // workflows' release, publication and self-hosted runner safeguards. const GUARDED_WORKFLOWS = new Set([ + ".github/workflows/pr-title.yml", + ".github/workflows/ci-full-pr.yml", ".github/workflows/release.yml", ".github/workflows/real-ghidra-windows.yml", ".github/workflows/sqlite-runtime.yml", diff --git a/tests/boundary/filesystem/ciPrEvents.test.ts b/tests/boundary/filesystem/ciPrEvents.test.ts new file mode 100644 index 000000000..3562b5df9 --- /dev/null +++ b/tests/boundary/filesystem/ciPrEvents.test.ts @@ -0,0 +1,98 @@ +import { readFile } from "node:fs/promises"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +async function workflow(name: string) { + return z + .object({ + on: z.record(z.string(), z.unknown()), + permissions: z.record(z.string(), z.string()), + concurrency: z.object({ group: z.string() }).optional(), + jobs: z.record( + z.string(), + z.object({ + name: z.string().optional(), + if: z.string().optional(), + uses: z.string().optional(), + with: z.record(z.string(), z.string()).optional(), + steps: z + .array( + z.object({ + run: z.string().optional(), + uses: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + }), + ) + .optional(), + }), + ), + }) + .parse( + parse( + await readFile( + new URL(`../../../.github/workflows/${name}`, import.meta.url), + "utf8", + ), + ), + ); +} + +it("refreshes title validation on edits without checking out PR code or cancelling CI", async () => { + const title = await workflow("pr-title.yml"); + expect(title.on.pull_request).toMatchObject({ + types: ["opened", "synchronize", "reopened", "edited"], + }); + expect(title.permissions).toEqual({}); + expect(title.concurrency?.group).not.toBe( + (await workflow("ci.yml")).concurrency?.group, + ); + const jobs = Object.values(title.jobs); + expect(jobs).toHaveLength(1); + expect(jobs[0]?.name).toBe("Conventional PR title"); + expect(jobs[0]?.if).toBeUndefined(); // A skipped check must not green-light an invalid title on body edits. + const steps = jobs[0]?.steps ?? []; + expect(steps).toHaveLength(1); + expect(steps[0]?.uses).toBeUndefined(); + expect(steps[0]?.env).toEqual({ + PR_TITLE: "${{ github.event.pull_request.title }}", + }); + const command = z.string().parse(steps[0]?.run); + for (const title of [ + "fix(ci): repair workflow", + "feat!: changed contract", + "docs: literal $(exit 91) `exit 92` text", + ]) { + await expect( + executeWorkflowFixture("bash", ["-c", command], { + env: { ...process.env, PR_TITLE: title }, + }), + ).resolves.toMatchObject({ stderr: "" }); + } + for (const title of ["", "Fix CI", "feat: "]) { + await expect( + executeWorkflowFixture("bash", ["-c", command], { + env: { ...process.env, PR_TITLE: title }, + }), + ).rejects.toMatchObject({ code: 1 }); + } + expect((await workflow("ci.yml")).jobs["pr-title"]).toBeUndefined(); +}); + +it("qualifies only ci:full label additions at the caller's merge revision", async () => { + const full = await workflow("ci-full-pr.yml"); + expect(full.on.pull_request).toMatchObject({ types: ["labeled"] }); + expect(full.permissions).toEqual({ contents: "read", actions: "read" }); + expect(Object.values(full.jobs)).toEqual([ + { + name: "Qualify labeled PR revision", + if: "github.event.label.name == 'ci:full'", + uses: "./.github/workflows/ci.yml", + with: { source_sha: "${{ github.sha }}" }, + }, + ]); + // Metadata events must not create skipped replacements for the main CI checks. + expect(full.jobs.required).toBeUndefined(); + expect(full.concurrency).toBeUndefined(); +}); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index bc2e5eb06..b3388cf80 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -144,6 +144,27 @@ it("makes the required gate wait for every CI lane, including future additions", ); }); +it("holds native and provider fan-out behind static validation while portable feedback stays independent", async () => { + const { jobs } = await workflow(); + const earlyJobs = new Set([ + "changes", + "static", + "build", + "test-shard", + "test", + "docs", + "required", + ]); + for (const [name, job] of Object.entries(jobs)) { + if (earlyJobs.has(name)) continue; + expect(job.needs, `${name} must wait for static validation`).toContain( + "static", + ); + expect(job.if).not.toMatch(/always\(|cancelled\(/u); + } + expect(jobs["test-shard"]?.needs).not.toContain("static"); +}); + it("runs workflow validation inside required static checks when selected", async () => { const { jobs } = await workflow(); const steps = z From f0f09950f34d96e3b1c62fd04821af5e301f80c7 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:54:05 +0800 Subject: [PATCH 2/8] ci: finish main validation and balance complete test costs --- .github/workflows/ci.yml | 8 +- .gitignore | 1 + docs/testing.md | 22 ++- scripts/ci/test-report.mjs | 30 ++-- scripts/ci/test-timing-reporter.mjs | 40 +++++ .../filesystem/ciRequiredLanes.test.ts | 9 +- .../boundary/filesystem/ciTestReports.test.ts | 158 ++++++++++++++++-- .../filesystem/ciTestSequencer.test.ts | 31 +++- tests/ciTestSequencer.ts | 16 +- 9 files changed, 249 insertions(+), 66 deletions(-) create mode 100644 scripts/ci/test-timing-reporter.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cf5cd0a11..800574bde 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,11 @@ permissions: concurrency: # Complete qualification finishes even while main receives more pushes. + # Main feedback also finishes, coalescing newer pushes to one pending run. # Scheduled/manual main runs coalesce to one active and the latest pending; # release and label-triggered qualification use exact-revision groups. group: ci-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && 'qualification' || 'feedback' }}-${{ inputs.source_sha || github.event.pull_request.number || github.ref }} - cancel-in-progress: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }} + cancel-in-progress: ${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }} env: HUSKY: "0" @@ -289,6 +290,7 @@ jobs: --reporter=default --reporter=junit --reporter=json + --reporter=./scripts/ci/test-timing-reporter.mjs --outputFile.junit=test-results.xml --outputFile.json=test-results.json - name: Merge fast-suite JUnit and timing reports @@ -299,10 +301,11 @@ jobs: --reporter=default --reporter=junit --reporter=json + --reporter=./scripts/ci/test-timing-reporter.mjs --outputFile.junit=test-results.xml --outputFile.json=test-results.json - name: Summarize slowest test files - if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' + if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != '' env: SHARD_RESULT: ${{ needs.test-shard.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} @@ -315,6 +318,7 @@ jobs: path: | test-results.xml test-results.json + test-timings.json coverage/ if-no-files-found: error retention-days: 30 diff --git a/.gitignore b/.gitignore index 14f9ac533..0d006e13f 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ coverage/ .vitest/ .vitest-reports/ test-results.xml +test-timings.json .turbo/ .cache/ docs/.vitepress/cache/ diff --git a/docs/testing.md b/docs/testing.md index cf1b8febc..5cd561b2b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -263,15 +263,21 @@ CI owns coverage; aggregate and domain/contract thresholds are maintained in Coverage does not replace named boundary or real-provider scenarios. CI runs six native Vitest shards without retries. Each shard emits a blob -report; the merge job produces JUnit and JSON timing reports, plus aggregate -coverage during full qualification. It retains available +report; the merge job produces JUnit and JSON diagnostics, a module timing +report, plus aggregate coverage during full qualification. It retains available reports even when a shard fails, uploads available diagnostics, and writes the slowest files to the workflow summary. Partial reports are marked diagnostic only. Failed, cancelled, or missing selected shards still fail the required gate, and aggregate coverage thresholds remain enforced in full qualification. Only successful complete full-tier merges publish the small `test-timings` artifact, with checkout-relative paths; the build job downloads that artifact for the sequencer -instead of the coverage/JUnit bundle. Missing history falls back to native +instead of the coverage/JUnit bundle. Timing weights come from Vitest module +diagnostics and include tests and hooks, worker preparation, environment, +imports, and setup. Import timings already include transform waits, which are +not added again. The weights describe file costs under the measured worker and +coverage settings; their sum is not workflow elapsed time. The diagnostic +artifact retains the phase breakdown, while shard history keeps only each +file's total. Legacy test-body-only history and missing history fall back to native sharding. Static checks, documentation, build, package verification, and real-system lanes remain separate jobs so one kind of evidence cannot stand in for another. Workflow syntax and expression validation runs inside the required @@ -868,11 +874,11 @@ should require this aggregate, `Static checks`, `Portable tests`, and reopening, commits, and edits; correcting a title refreshes only this lightweight check. It reads title text through an environment variable, executes no PR code, and has no token permissions. Its separate concurrency group cannot cancel -test runs. Fast runs -cancel superseded feedback for the same PR or main branch. Scheduled/manual -qualification has a separate concurrency group: active work finishes and only -the latest pending run is retained. Release qualification is grouped by its -exact source SHA and cannot be cancelled by newer main pushes. Provider and +test runs. Fast PR runs cancel superseded feedback for the same PR. Main push +runs keep one active run and the latest pending run for that branch. +Scheduled/manual qualification has a separate concurrency group: active work +finishes and only the latest pending run is retained. Release qualification uses +its exact source SHA and cannot be cancelled by newer main pushes. Provider and website lanes inherit their caller's scheduling; reusable workflows have no independent cancellation group. Direct manual lane dispatches run independently on disposable hosted runners. Publication diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 9f2d979aa..86419ae9f 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -11,19 +11,18 @@ const historyPath = join(historyDirectory, "test-results.json"); await rm(historyPath, { force: true }); const report = z .object({ + version: z.literal(1), success: z.boolean(), - numFailedTests: z.number().int().nonnegative(), - testResults: z.array( + files: z.array( z.object({ name: z.string().min(1), - startTime: z.number().finite(), - endTime: z.number().finite(), + durationMs: z.number().finite().nonnegative(), }), ), }) - .parse(JSON.parse(await readFile("test-results.json", "utf8"))); + .parse(JSON.parse(await readFile("test-timings.json", "utf8"))); -const files = report.testResults.map(({ name, startTime, endTime }) => { +const files = report.files.map(({ name, durationMs }) => { const path = relative(root, resolve(root, name)); if ( path === "" || @@ -32,17 +31,11 @@ const files = report.testResults.map(({ name, startTime, endTime }) => { isAbsolute(path) ) throw new Error(`Test report path is outside the checkout: ${name}`); - const duration = endTime - startTime; - if (!Number.isFinite(duration) || duration < 0) - throw new Error(`Test report has an invalid duration: ${name}`); - return { name: path.split(sep).join("/"), startTime: 0, endTime: duration }; + return { name: path.split(sep).join("/"), durationMs }; }); const complete = process.env.SHARD_RESULT === "success"; const passed = - complete && - process.env.MERGE_RESULT === "success" && - report.success && - report.numFailedTests === 0; + complete && process.env.MERGE_RESULT === "success" && report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { @@ -71,13 +64,16 @@ const summary = [ ? "Successful timing history is available." : "This report is diagnostic only and is not a successful timing baseline.", "", + "Durations include tests and hooks, worker preparation, environment, imports, and setup. They are scheduling weights, not summed workflow wall time.", + "", "| Slowest file | Duration (ms) |", "| --- | ---: |", ...files - .toSorted((left, right) => right.endTime - left.endTime) + .toSorted((left, right) => right.durationMs - left.durationMs) .slice(0, 20) .map( - ({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`, + ({ name, durationMs }) => + `| ${name.replaceAll("|", "\\|")} | ${durationMs} |`, ), "", ].join("\n"); @@ -89,6 +85,6 @@ if (passed) { await mkdir(historyDirectory, { recursive: true }); await writeFile( historyPath, - JSON.stringify({ success: true, numFailedTests: 0, testResults: files }), + JSON.stringify({ version: 1, success: true, files }), ); } diff --git a/scripts/ci/test-timing-reporter.mjs b/scripts/ci/test-timing-reporter.mjs new file mode 100644 index 000000000..29f0cc582 --- /dev/null +++ b/scripts/ci/test-timing-reporter.mjs @@ -0,0 +1,40 @@ +import { writeFile } from "node:fs/promises"; + +/** @implements {import("vitest/node").Reporter} */ +export default class TestTimingReporter { + /** + * @param {ReadonlyArray} modules + * @param {ReadonlyArray} errors + * @param {import("vitest/node").TestRunEndReason} reason + */ + async onTestRunEnd(modules, errors, reason) { + const files = modules.map((module) => { + const diagnostic = module.diagnostic(); + // These disjoint phases match Vitest's per-file wall-time accounting. + // Import timings already include transform waits; do not add them again. + const phases = { + execution: diagnostic.duration, + preparation: diagnostic.prepareDuration, + environment: diagnostic.environmentSetupDuration, + imports: diagnostic.collectDuration, + setup: diagnostic.setupDuration, + }; + return { + name: module.moduleId, + durationMs: Object.values(phases).reduce( + (total, value) => total + value, + 0, + ), + phases, + }; + }); + await writeFile( + "test-timings.json", + JSON.stringify({ + version: 1, + success: reason === "passed" && errors.length === 0, + files, + }), + ); + } +} diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index b3388cf80..1bac771a2 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -249,12 +249,9 @@ it("keeps fast feedback separate from complete qualification and release proof", ]); expect(ci.concurrency.group).toContain("'qualification' || 'feedback'"); expect(ci.concurrency.group).toContain("inputs.source_sha ||"); - for (const fullEvent of ["schedule", "workflow_dispatch"]) - expect(ci.concurrency["cancel-in-progress"]).toContain( - `github.event_name != '${fullEvent}'`, - ); - expect(ci.concurrency["cancel-in-progress"]).toContain( - "inputs.source_sha == ''", + expect(ci.concurrency.group).toContain("github.ref"); + expect(ci.concurrency["cancel-in-progress"]).toBe( + "${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}", ); expect(ci.jobs.required?.if).toBe("${{ always() }}"); const shards = ci.jobs["test-shard"]?.steps ?? []; diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 2c44878de..333e93f08 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -54,6 +54,13 @@ it("collects failed-shard diagnostics before enforcing the shard result", async expect(upload).toBeGreaterThan(merge); expect(gate).toBeGreaterThan(upload); expect(steps[merge]?.if).not.toContain("needs.test-shard.result"); + const merges = steps.filter((step) => step.run?.includes("--reporter=json")); + expect(merges).toHaveLength(2); + for (const step of merges) + expect(step.run).toContain( + "--reporter=./scripts/ci/test-timing-reporter.mjs", + ); + expect(steps[upload]?.with?.path).toContain("test-timings.json"); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); for (const result of ["failure", "cancelled", "skipped", ""]) { @@ -107,11 +114,11 @@ it("rejects successful shard reports with missing or duplicate portable files", await workflowGit(root, ["add", "tests"]); const run = async (names: readonly string[]) => { await writeFile( - join(root, "test-results.json"), + join(root, "test-timings.json"), JSON.stringify({ + version: 1, success: true, - numFailedTests: 0, - testResults: names.map((name) => ({ name, startTime: 0, endTime: 1 })), + files: names.map((name) => ({ name, durationMs: 1 })), }), ); return execute(process.execPath, [fileURLToPath(script)], { @@ -173,20 +180,25 @@ it("exports portable timing history and removes it on a failed, partial, or inva const summary = join(root, "summary.md"); const history = join(root, ".cache/ci-test-history/test-results.json"); const report = { + version: 1, success: true, - numFailedTests: 0, - testResults: [ + files: [ { name: join(root, "tests/slow.test.ts"), - startTime: 10, - endTime: 110, - assertionResults: [{ large: "not timing data" }], + durationMs: 100, + phases: { + execution: 20, + preparation: 20, + environment: 10, + imports: 40, + setup: 10, + }, }, - { name: join(root, "tests/fast.test.ts"), startTime: 20, endTime: 30 }, + { name: join(root, "tests/fast.test.ts"), durationMs: 10 }, ], }; const run = async (value: unknown, shards = "success", merge = "success") => { - await writeFile(join(root, "test-results.json"), JSON.stringify(value)); + await writeFile(join(root, "test-timings.json"), JSON.stringify(value)); await writeFile(summary, ""); return execute(process.execPath, [fileURLToPath(script)], { cwd: root, @@ -200,7 +212,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva }; await run(report); const contents = await readFile(history, "utf8"); - expect(contents).not.toContain("assertionResults"); + expect(contents).not.toContain("phases"); + expect(contents).not.toContain("startTime"); expect(contents).not.toContain(root); expect( parseTestDurations(contents, resolve(root, "different-checkout")), @@ -211,7 +224,7 @@ it("exports portable timing history and removes it on a failed, partial, or inva ]), ); for (const [value, shards, merge] of [ - [{ ...report, success: false, numFailedTests: 1 }, "failure", "failure"], + [{ ...report, success: false }, "failure", "failure"], [report, "cancelled", "success"], [report, "success", "failure"], ] as const) { @@ -224,8 +237,125 @@ it("exports portable timing history and removes it on a failed, partial, or inva await expect( run({ ...report, - testResults: [{ name: "../outside.test.ts", startTime: 1, endTime: 3 }], + files: [{ name: "../outside.test.ts", durationMs: 2 }], + }), + ).rejects.toMatchObject({ code: 1 }); + await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); + await run(report); + await expect( + run({ + ...report, + files: [{ name: "tests/invalid.test.ts", durationMs: -1 }], }), ).rejects.toMatchObject({ code: 1 }); await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); }); + +it("recovers complete module costs from real blob reports without executing tests again", async () => { + const root = await createTestTempDirectory("rea-ci-module-timing-"); + const config = join(root, "vitest.config.mjs"); + await writeFile( + config, + 'export default { test: { globals: true, include: ["sample.test.js"], maxWorkers: 1 } };', + ); + await writeFile( + join(root, "sample.test.js"), + [ + 'import { writeFileSync } from "node:fs";', + "beforeAll(() => new Promise(resolve => setTimeout(resolve, 80)));", + "afterAll(() => new Promise(resolve => setTimeout(resolve, 80)));", + 'it("has cheap assertions and expensive outer hooks", () => { writeFileSync("executed", "yes"); expect(1).toBe(1); });', + ].join("\n"), + ); + const vitest = fileURLToPath( + new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url), + ); + const reporter = fileURLToPath( + new URL("../../../scripts/ci/test-timing-reporter.mjs", import.meta.url), + ); + const run = (args: string[]) => + execute(process.execPath, [vitest, ...args, "--config", config], { + cwd: root, + env: process.env, + timeout: 20_000, + }); + await run(["run", "--reporter=blob"]); + expect(await readFile(join(root, "executed"), "utf8")).toBe("yes"); + await rm(join(root, "executed")); + await run([ + "--merge-reports", + `--reporter=${reporter}`, + "--reporter=json", + "--outputFile.json=body.json", + ]); + await expect(readFile(join(root, "executed"))).rejects.toMatchObject({ + code: "ENOENT", + }); + const report = z + .object({ + version: z.literal(1), + success: z.literal(true), + files: z.array( + z.object({ + name: z.string(), + durationMs: z.number(), + phases: z.record(z.string(), z.number()), + }), + ), + }) + .parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8"))); + expect(report.files).toHaveLength(1); + const file = report.files[0]!; + const json = z + .object({ + testResults: z.array( + z.object({ startTime: z.number(), endTime: z.number() }), + ), + }) + .parse(JSON.parse(await readFile(join(root, "body.json"), "utf8"))); + const body = json.testResults[0]!; + expect( + file.phases.execution! - (body.endTime - body.startTime), + ).toBeGreaterThan(100); + expect(file.phases.imports).toBeGreaterThan(0); + expect(file.phases.preparation).toBeGreaterThan(0); + expect(file.durationMs).toBeCloseTo( + Object.values(file.phases).reduce((sum, value) => sum + value, 0), + ); + await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" }, + }); + expect( + parseTestDurations( + await readFile( + join(root, ".cache/ci-test-history/test-results.json"), + "utf8", + ), + root, + )?.get("sample.test.js"), + ).toBe(file.durationMs); + + await writeFile( + join(root, "sample.test.js"), + 'it("fails", () => { throw new Error("fixture failure"); });', + ); + await expect(run(["run", "--reporter=blob"])).rejects.toMatchObject({ + code: 1, + }); + await expect( + run(["--merge-reports", `--reporter=${reporter}`]), + ).rejects.toMatchObject({ code: 1 }); + const failed = JSON.parse( + await readFile(join(root, "test-timings.json"), "utf8"), + ) as unknown; + expect(failed).toMatchObject({ version: 1, success: false }); + // Even a caller accidentally marking the merge successful cannot publish this report. + await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" }, + }); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/boundary/filesystem/ciTestSequencer.test.ts b/tests/boundary/filesystem/ciTestSequencer.test.ts index 6177d9481..fec32826d 100644 --- a/tests/boundary/filesystem/ciTestSequencer.test.ts +++ b/tests/boundary/filesystem/ciTestSequencer.test.ts @@ -17,24 +17,24 @@ describe("CI test history", () => { const root = resolve("/workspace/rea"); const durations = parseTestDurations( JSON.stringify({ + version: 1, success: true, - numFailedTests: 0, - testResults: [ + files: [ { name: join(root, "tests/slow.test.ts"), - startTime: 10, - endTime: 110, + durationMs: 100, }, { name: join(root, "tests/slow.test.ts"), - startTime: 20, - endTime: 80, + durationMs: 60, }, { name: resolve(root, "../outside.test.ts"), - startTime: 10, - endTime: 1000, + durationMs: 990, }, + { name: "tests/negative.test.ts", durationMs: -1 }, + { name: "tests/zero.test.ts", durationMs: 0 }, + { name: "tests/invalid.test.ts", durationMs: "100" }, null, ], }), @@ -43,7 +43,20 @@ describe("CI test history", () => { expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]])); expect( - parseTestDurations('{"success": false, "numFailedTests": 1}', root), + parseTestDurations('{"version": 1, "success": false, "files": []}', root), + ).toBeUndefined(); + // Legacy test-body spans must not be mixed with complete module costs. + expect( + parseTestDurations( + JSON.stringify({ + success: true, + numFailedTests: 0, + testResults: [ + { name: "tests/legacy.test.ts", startTime: 0, endTime: 100 }, + ], + }), + root, + ), ).toBeUndefined(); expect(parseTestDurations("not json", root)).toBeUndefined(); expect(parseTestDurations("null", root)).toBeUndefined(); diff --git a/tests/ciTestSequencer.ts b/tests/ciTestSequencer.ts index 5e8c4fe5f..996e8ce7a 100644 --- a/tests/ciTestSequencer.ts +++ b/tests/ciTestSequencer.ts @@ -5,14 +5,13 @@ import { BaseSequencer, type TestSpecification } from "vitest/node"; import { z } from "zod"; const reportSchema = z.object({ + version: z.literal(1), success: z.literal(true), - numFailedTests: z.literal(0), - testResults: z.array(z.unknown()), + files: z.array(z.unknown()), }); const fileSchema = z.object({ name: z.string(), - startTime: z.number(), - endTime: z.number(), + durationMs: z.number().finite().positive(), }); export function parseTestDurations( @@ -30,15 +29,12 @@ export function parseTestDurations( if (!report.success) return undefined; const durations = new Map(); - for (const value of report.data.testResults) { + for (const value of report.data.files) { const result = fileSchema.safeParse(value); if (!result.success) continue; - const { name, startTime, endTime } = result.data; - const duration = endTime - startTime; + const { name, durationMs } = result.data; const path = relative(root, resolve(root, name)); if ( - !Number.isFinite(duration) || - duration <= 0 || path === "" || path === ".." || path.startsWith(".." + sep) || @@ -46,7 +42,7 @@ export function parseTestDurations( ) continue; const key = path.split(sep).join("/"); - durations.set(key, Math.max(durations.get(key) ?? 0, duration)); + durations.set(key, Math.max(durations.get(key) ?? 0, durationMs)); } return durations.size ? durations : undefined; } From 9f3f392eb8fe81060b44941ed754521984def3bd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:00:47 +0800 Subject: [PATCH 3/8] ci: bound core job execution time Limit classification to five minutes, static/build/docs/report jobs to ten minutes, and each portable shard to thirty minutes. The saved successful-job sample peaked at 392 seconds per shard and 61 seconds for other core jobs, leaving generous setup and coverage headroom. Runner queue waiting is unaffected. --- .github/workflows/ci.yml | 6 ++++++ docs/testing.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 800574bde..b1af43269 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,6 +34,7 @@ jobs: changes: name: Classify changes runs-on: ubuntu-latest + timeout-minutes: 5 permissions: contents: read actions: read @@ -110,6 +111,7 @@ jobs: needs: changes if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }} runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Require successful change classification env: @@ -148,6 +150,7 @@ jobs: needs: [changes, build] if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest + timeout-minutes: 30 strategy: fail-fast: false matrix: @@ -188,6 +191,7 @@ jobs: needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: read actions: read @@ -258,6 +262,7 @@ jobs: needs: [changes, test-shard] if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }} runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Require successful change classification env: @@ -642,6 +647,7 @@ jobs: needs: changes if: needs.changes.outputs.docs == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 diff --git a/docs/testing.md b/docs/testing.md index 5cd561b2b..22b5c0c77 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -292,6 +292,12 @@ coverage. This ordering reduces peak demand on invalid revisions but can add static-check waiting to otherwise passing runs; it is not a measured reduction in end-to-end latency. +Core jobs have execution limits of five minutes for classification, ten minutes +for static checks, builds, docs, and report merging, and thirty minutes per +portable test shard. These limits leave headroom for cold setup and full coverage +while preventing a hung job from holding the active main run for hours. They do +not bound runner queue waiting. Timeouts still fail the required gate. + The separate `CI timings` workflow batches recent completed CI and Release runs hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and a normalized point-in-time report for each collected workflow attempt, plus a From bd2c202dd930548194cdbc55c27a797f22279db9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:06:31 +0800 Subject: [PATCH 4/8] fix(ci): reject incomplete verification plans --- docs/testing.md | 5 +- scripts/ci/required.mjs | 25 +++-- .../boundary/filesystem/ciChangeScope.test.ts | 10 +- .../filesystem/ciRequiredLanes.test.ts | 92 +++++++++++++++++-- 4 files changed, 113 insertions(+), 19 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 22b5c0c77..6dddda64b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -874,7 +874,10 @@ summary. Representative Git-diff cases live in `tests/boundary/filesystem/ciChangeScope.test.ts`. `CI required` always runs and checks classification and every selected lane, -rejecting failed, cancelled, or unexpectedly skipped work. Branch protection +rejecting failed, cancelled, or unexpectedly skipped work. It rejects missing or +unknown tiers and requires every canonical scope to be explicitly `true` or +`false`; incomplete classification cannot silently omit verification lanes. +Branch protection should require this aggregate, `Static checks`, `Portable tests`, and `Conventional PR title`. Title validation runs separately on PR opening, reopening, commits, and edits; correcting a title refreshes only this lightweight diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index afe52c2b8..01907de37 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs"; import { recordQualification } from "./qualification.mjs"; const jobs = JSON.parse(process.env.CI_JOBS); -const outputs = jobs.changes.outputs; +const outputs = jobs?.changes?.outputs; +if (!["fast", "full", "reused"].includes(outputs?.tier)) + throw new Error( + "CI classification requires a valid tier: fast, full, or reused", + ); +const invalidFlags = CI_FLAGS.filter( + (flag) => outputs[flag] !== "true" && outputs[flag] !== "false", +); +if (invalidFlags.length) + throw new Error( + `CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`, + ); +const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true"); const reused = outputs.tier === "reused"; if ( reused && (process.env.GITHUB_EVENT_NAME !== "schedule" || !/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") || !Number.isSafeInteger(Number(outputs.qualification_run_id)) || - CI_FLAGS.some((flag) => outputs[flag] !== "false")) + selectedScopes.length !== 0) ) throw new Error( "Reused qualification requires a scheduled admission receipt and no selected lanes", ); -if ( - outputs.tier === "full" && - CI_FLAGS.some((flag) => outputs[flag] !== "true") -) +if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { @@ -31,9 +40,7 @@ const lanes = { docs: ["docs"], workflows: ["static"], }; -for (const [scope, value] of Object.entries(outputs)) { - if (value === "true") required.push(...(lanes[scope] ?? [scope])); -} +for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope])); const failed = [...new Set(required)].filter( (name) => jobs[name]?.result !== "success", ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 8ed397e84..759f5c367 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -662,8 +662,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn const command = fileURLToPath( new URL("../../../scripts/ci/required.mjs", import.meta.url), ); + const { CI_FLAGS } = await loadScopes(); const jobs = { - changes: { result: "success", outputs: { evm: "true" } }, + changes: { + result: "success", + outputs: { + ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), + tier: "fast", + evm: "true", + }, + }, static: { result: "success" }, test: { result: "success" }, evm: { result: "success" }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 1bac771a2..e460bb52b 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -26,6 +26,15 @@ const workflow = async (filename = "ci.yml") => ), ); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; + it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => { expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe( "needs.changes.outputs.sqlite == 'true'", @@ -57,13 +66,7 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => { const { jobs } = await workflow(); - const { CI_FLAGS } = z - .object({ CI_FLAGS: z.array(z.string()) }) - .parse( - await import( - new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href - ), - ); + const CI_FLAGS = await scopeFlags(); const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs); expect(jobs.changes?.permissions).toEqual({ contents: "read", @@ -294,7 +297,14 @@ it.each([ ); needs.changes = { result: "success", - outputs: { code: "true", [scope]: selected }, + outputs: { + ...Object.fromEntries( + (await scopeFlags()).map((flag) => [flag, "false"]), + ), + tier: "fast", + code: "true", + [scope]: selected, + }, }; if (result === "missing") delete needs[lane]; else needs[lane] = { result }; @@ -319,3 +329,69 @@ it.each([ } }, ); + +it("rejects incomplete or malformed classification even when available jobs succeeded", async () => { + const flags = await scopeFlags(); + const valid = { + ...Object.fromEntries(flags.map((flag) => [flag, "false"])), + tier: "fast", + }; + const run = (outputs: unknown) => + executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + CI_JOBS: JSON.stringify({ + changes: { result: "success", outputs }, + static: { result: "success" }, + test: { result: "success" }, + }), + CI_QUALIFICATION_DIRECTORY: "", + }, + }, + ); + await expect(run(valid)).resolves.toMatchObject({ stderr: "" }); + // Metadata is not a scope, even when its string happens to be "true". + await expect( + run({ ...valid, qualification_run_id: "true" }), + ).resolves.toMatchObject({ stderr: "" }); + for (const outputs of [ + undefined, + null, + {}, + { ...valid, tier: undefined }, + { ...valid, tier: "" }, + { ...valid, tier: "unknown" }, + ]) { + await expect(run(outputs)).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("valid tier"), + }); + } + for (const flag of flags) { + await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(`scope outputs: ${flag}`), + }); + } + for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) { + await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } + for (const tier of ["full", "reused"]) { + await expect( + run({ ...valid, tier, sqlite: undefined }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } +}); From c8d861f9764f8c07f0925dae0c6451de85fffd6a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:12:26 +0800 Subject: [PATCH 5/8] ci: scope timing helpers to portable verification --- docs/testing.md | 8 +++- scripts/ci/plan.mjs | 10 +++++ .../boundary/filesystem/ciChangeScope.test.ts | 41 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 6dddda64b..ac1a01f84 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -826,8 +826,12 @@ scripts also select it. Unrelated code edits retain the complete portable suite on the pinned Node version without starting four additional compatibility jobs. Full qualification retains every SQLite version. Shared process/contract/CLI boundaries, dependencies, other packaging inputs, -unknown paths, and changes to the planner select the full baseline. Package -script-only edits select their owning verifier when its entrypoint is +unknown paths, and changes to the planner select the full baseline. +CI timing collection, timing history, and test-report helper edits retain static +checks and the complete portable suite without selecting unrelated host/provider +lanes. Classification, qualification, gate, release receipt, and unknown CI +scripts keep the broad fallback. +Package script-only edits select their owning verifier when its entrypoint is identifiable; unknown or shared script changes select everything. Android build-script edits retain the Linux suite and workflow validation; real Android verification stays manual. Renames account for both paths. diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 9384d9b05..3c711a1f9 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -25,6 +25,15 @@ const FIXTURE_SOURCES = [ "tests/conformance/versions/**", ]; const READINESS_SOURCES = ["tests/conformance/readiness/**"]; +// These reporters affect CI output only; they do not select a host or provider +// verification lane when changed on their own. +const PORTABLE_CI_REPORTERS = new Set([ + "scripts/ci/job-timings.mjs", + "scripts/ci/job-timings-batch.mjs", + "scripts/ci/test-report.mjs", + "scripts/ci/test-timing-reporter.mjs", + "scripts/ci/test-history.mjs", +]); const matchesAny = (file, patterns) => patterns.some((pattern) => matchesGlob(file, pattern)); const selected = Object.fromEntries(flags.map((flag) => [flag, false])); @@ -200,6 +209,7 @@ if (tier === "reused") { } if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true; if (file.startsWith("tests/")) continue; + if (PORTABLE_CI_REPORTERS.has(file)) continue; if ( /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( file, diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 759f5c367..714d2f980 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -137,6 +137,47 @@ it("keeps every translated root README and authored guide on the documentation l }); }); +it.each([ + "job-timings.mjs", + "job-timings-batch.mjs", + "test-report.mjs", + "test-timing-reporter.mjs", + "test-history.mjs", +])( + "keeps portable CI reporter %s on static checks and portable tests", + async (name) => { + const { directory, base } = await fixture(); + const head = await change(directory, [`scripts/ci/${name}`]); + const plan = await classify(directory, base, head); + const { CI_FLAGS } = await loadScopes(); + expect(plan.code).toBe(true); + for (const flag of CI_FLAGS) + if (flag !== "code") expect(plan[flag]).toBe(false); + }, +); + +it("preserves a provider lane when a portable CI reporter changes with provider source", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, [ + "scripts/ci/test-report.mjs", + "src/evm/Example.ts", + ]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + evm: true, + browser: false, + package: false, + }); +}); + +it("keeps unknown CI scripts on complete qualification", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["scripts/ci/new-qualification.mjs"]); + const plan = await classify(directory, base, head); + const { CI_FLAGS } = await loadScopes(); + for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); +}); + it("does not charge a documentation PR for unrelated implementation updates on main", async () => { const { directory, base } = await fixture(); await git(directory, ["switch", "-c", "docs/change"]); From fa0a9b2d328a07d8ec2924dc224d51c620981dcc Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:21:09 +0800 Subject: [PATCH 6/8] ci: observe affected-owner test plans against full results --- .github/workflows/ci.yml | 5 + docs/testing.md | 21 ++ scripts/ci/plan.mjs | 11 +- scripts/ci/test-report.mjs | 70 ++++- scripts/ci/test-selection.d.mts | 35 +++ scripts/ci/test-selection.mjs | 151 +++++++++++ scripts/ci/test-timing-reporter.mjs | 3 + .../boundary/filesystem/ciChangeScope.test.ts | 17 ++ .../boundary/filesystem/ciTestReports.test.ts | 100 +++++++- tests/evaluation/ciTestSelection.test.ts | 240 ++++++++++++++++++ 10 files changed, 639 insertions(+), 14 deletions(-) create mode 100644 scripts/ci/test-selection.d.mts create mode 100644 scripts/ci/test-selection.mjs create mode 100644 tests/evaluation/ciTestSelection.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1af43269..28373f89d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: nativeaot: ${{ steps.plan.outputs.nativeaot }} website: ${{ steps.plan.outputs.website }} matrix: ${{ steps.plan.outputs.matrix }} + test_selection: ${{ steps.plan.outputs.test_selection }} steps: - uses: actions/checkout@v4 with: @@ -315,6 +316,8 @@ jobs: SHARD_RESULT: ${{ needs.test-shard.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" + CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: node scripts/ci/test-report.mjs - uses: actions/upload-artifact@v4 if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' @@ -325,6 +328,8 @@ jobs: test-results.json test-timings.json coverage/ + .cache/ci-test-selection/report.json + include-hidden-files: true if-no-files-found: error retention-days: 30 - name: Publish successful timing history diff --git a/docs/testing.md b/docs/testing.md index ac1a01f84..03121e172 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -805,6 +805,27 @@ remain present and fail if classification fails. Every code change retains the complete six-shard Linux suite; individual test files are not selected by imports or filenames. +Ordinary PRs also produce an **observational** test-selection plan. The candidate +leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately +from provider-job scopes. It proposes keeping affected owners plus every unowned +test, including all CLI/MCP, acceptance, composition, conformance, and process +tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall +back to the complete inventory. Renames account for both paths. Main, scheduled, +manual, and `ci:full` runs always propose the full suite. + +All six shards still execute the complete suite. The existing report merge +compares the proposal with actual module outcomes and retains +`.cache/ci-test-selection/report.json` in `test-results`, including selected and +omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and +executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate, +skipped, interrupted, or unhandled results cannot establish a complete comparison; +even a complete passing comparison is not proof of ownership completeness. +Before enabling omissions, review representative full-run comparisons and +cross-owner consumers, including failures, and measure completed CI feedback and +runner demand. There is no automatic promotion or switch to skip tests. The +initial proposal measures only leaf producer-test omissions; it does not promise +the savings of a broader affected-test planner. + `scripts/ci/plan.mjs` selects lanes using the ownership table in `scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider verifiers and the Linux x64 installed-package/Inspector baseline. The `apple` diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 3c711a1f9..86879216d 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { matchesGlob } from "node:path"; import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs"; +import { planTestSelection } from "./test-selection.mjs"; // Test files select the host/runtime lanes that execute their focused suites. const HOST_TEST_OWNERS = ["apple", "package", "sqlite"]; @@ -250,11 +251,17 @@ const matrix = { ] : [linux], }; -const plan = { ...selected, tier, matrix, files, reasons }; +const testSelection = planTestSelection(files, event, tier); +const plan = { ...selected, tier, matrix, files, reasons, testSelection }; if (process.env.GITHUB_OUTPUT) appendFileSync( process.env.GITHUB_OUTPUT, - Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) }) + Object.entries({ + ...selected, + tier, + matrix: JSON.stringify(matrix), + test_selection: JSON.stringify(testSelection), + }) .map(([key, value]) => `${key}=${value}\n`) .join(""), ); diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 86419ae9f..035a01990 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -4,25 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { z } from "zod"; +import { compareTestSelection } from "./test-selection.mjs"; + const root = process.cwd(); const historyDirectory = join(root, ".cache/ci-test-history"); const historyPath = join(historyDirectory, "test-results.json"); +const selectionDirectory = join(root, ".cache/ci-test-selection"); +const selectionPath = join(selectionDirectory, "report.json"); // Never let a failed or partial rerun leave a previous successful history file. await rm(historyPath, { force: true }); +await rm(selectionPath, { force: true }); const report = z .object({ version: z.literal(1), success: z.boolean(), + reason: z.string().optional(), + unhandledErrors: z.number().int().nonnegative().optional(), files: z.array( z.object({ name: z.string().min(1), durationMs: z.number().finite().nonnegative(), + state: z.string().optional(), }), ), }) .parse(JSON.parse(await readFile("test-timings.json", "utf8"))); -const files = report.files.map(({ name, durationMs }) => { +const files = report.files.map(({ name, durationMs, state }) => { const path = relative(root, resolve(root, name)); if ( path === "" || @@ -31,21 +39,47 @@ const files = report.files.map(({ name, durationMs }) => { isAbsolute(path) ) throw new Error(`Test report path is outside the checkout: ${name}`); - return { name: path.split(sep).join("/"), durationMs }; + return { name: path.split(sep).join("/"), durationMs, state }; }); const complete = process.env.SHARD_RESULT === "success"; const passed = complete && process.env.MERGE_RESULT === "success" && report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. -if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { - const expected = execFileSync( - "git", - ["ls-files", "-z", "--", "src", "tests"], - { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }, - ) - .split("\0") - .filter((path) => path.endsWith(".test.ts")); +const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; +const expected = + verifyInventory || process.env.CI_TEST_SELECTION + ? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }) + .split("\0") + .filter((path) => path.endsWith(".test.ts")) + : []; +let selection; +if (process.env.CI_TEST_SELECTION) { + const comparison = compareTestSelection( + JSON.parse(process.env.CI_TEST_SELECTION), + expected, + files, + ); + selection = { + ...comparison, + complete: + comparison.complete && + report.unhandledErrors === 0 && + ["passed", "failed"].includes(report.reason) && + ["success", "failure"].includes(process.env.SHARD_RESULT) && + ["success", "failure"].includes(process.env.MERGE_RESULT), + executionSha: process.env.GITHUB_SHA ?? null, + prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, + runId: process.env.GITHUB_RUN_ID ?? null, + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null, + }; + await mkdir(selectionDirectory, { recursive: true }); + await writeFile(selectionPath, JSON.stringify(selection)); +} +if (passed && verifyInventory) { const actual = files.map(({ name }) => name); const expectedSet = new Set(expected); const actualSet = new Set(actual); @@ -59,6 +93,16 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { const summary = [ complete ? "## Merged test diagnostics" : "## Partial test diagnostics", "", + ...(selection + ? [ + "### Observational PR test selection", + "", + `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, + `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, + "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "", + ] + : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, passed ? "Successful timing history is available." @@ -85,6 +129,10 @@ if (passed) { await mkdir(historyDirectory, { recursive: true }); await writeFile( historyPath, - JSON.stringify({ version: 1, success: true, files }), + JSON.stringify({ + version: 1, + success: true, + files: files.map(({ name, durationMs }) => ({ name, durationMs })), + }), ); } diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts new file mode 100644 index 000000000..c19414398 --- /dev/null +++ b/scripts/ci/test-selection.d.mts @@ -0,0 +1,35 @@ +export interface TestSelectionPlan { + readonly version: 1; + readonly mode: "full" | "candidate"; + readonly owners: readonly string[]; + readonly changedTests: readonly string[]; + readonly changedPaths: readonly string[]; + readonly reason: string; +} +export function planTestSelection( + files: readonly string[], + event: string, + tier: string, +): TestSelectionPlan; +export function compareTestSelection( + plan: unknown, + inventory: readonly string[], + results: readonly { + readonly name: string; + readonly durationMs: number; + readonly state?: string; + }[], +): { + readonly version: 1; + readonly observational: true; + readonly plan: TestSelectionPlan; + readonly selected: readonly string[]; + readonly omitted: readonly string[]; + readonly missing: readonly string[]; + readonly unexpected: readonly string[]; + readonly duplicateFiles: number; + readonly unobserved: readonly string[]; + readonly missedFailures: readonly string[]; + readonly complete: boolean; + readonly omittedDurationMs: number; +}; diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs new file mode 100644 index 000000000..8165e3658 --- /dev/null +++ b/scripts/ci/test-selection.mjs @@ -0,0 +1,151 @@ +import { matchesGlob } from "node:path"; + +// Candidate ownership only: these leaf adapters and their producer tests can be +// compared with full-suite outcomes. This table never changes test execution. +// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners. +const OWNERS = { + adb: ["src/adb/**", "tests/boundary/providers/adb/**"], + apktool: ["src/apktool/**"], + dotnet: ["src/dotnet/**"], + evm: ["src/evm/**", "tests/boundary/evm/**"], + firmware: ["src/firmware/**", "tests/boundary/firmware/**"], + flutter: ["src/flutter/**"], + ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"], + hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"], + ida: ["src/ida/**", "tests/boundary/providers/ida/**"], + jeb: ["src/jeb/**"], + pcap: ["src/pcap/**", "tests/boundary/pcap/**"], + sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"], + wasm: ["src/wasm/**"], +}; +const ownersOf = (path) => + Object.entries(OWNERS) + .filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob))) + .map(([owner]) => owner); +const validPath = (path) => + typeof path === "string" && + /^(src|tests)\//.test(path) && + !/[\\\0\r\n]/.test(path) && + !path.split("/").some((part) => ["", ".", ".."].includes(part)); +const full = (reason, changedPaths = []) => ({ + version: 1, + mode: "full", + owners: [], + changedTests: [], + changedPaths, + reason, +}); + +/** Plan an observational PR subset from the classifier's existing Git diff. */ +export function planTestSelection(files, event, tier) { + if (event !== "pull_request" || tier !== "fast") + return full("Only ordinary PRs are candidates", files); + if (!files.length) return full("No changed paths available"); + const owners = new Set(); + const changedTests = []; + for (const path of files) { + if (!validPath(path)) + return full(`Shared or unknown input: ${path}`, files); + const matches = ownersOf(path); + if (!matches.length) return full(`Shared or unknown input: ${path}`, files); + for (const owner of matches) owners.add(owner); + if (path.endsWith(".test.ts")) changedTests.push(path); + } + return { + version: 1, + mode: "candidate", + owners: [...owners].sort(), + changedTests: [...new Set(changedTests)].sort(), + changedPaths: files, + reason: + "Observe affected leaf owners; retain integration and unowned tests", + }; +} + +/** Compare a proposal with the full run; missing observations never prove parity. */ +export function compareTestSelection(plan, inventory, results) { + if ( + plan?.version !== 1 || + !["full", "candidate"].includes(plan.mode) || + !Array.isArray(plan.owners) || + plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) || + !Array.isArray(plan.changedTests) || + !Array.isArray(plan.changedPaths) || + plan.changedPaths.some((path) => typeof path !== "string") || + plan.changedTests.some( + (path) => !validPath(path) || !path.endsWith(".test.ts"), + ) || + typeof plan.reason !== "string" || + (plan.mode === "candidate" && !plan.owners.length) || + (plan.mode === "full" && (plan.owners.length || plan.changedTests.length)) + ) + throw new Error("Invalid observational test selection plan"); + const expected = new Set(inventory); + if (!expected.size || expected.size !== inventory.length) + throw new Error("Test selection requires a nonempty unique inventory"); + const ownersByFile = new Map( + plan.mode === "candidate" + ? inventory.map((path) => [path, ownersOf(path)]) + : [], + ); + // Removed/renamed tests have no current executable identity. Keep all tests. + const removed = plan.changedTests.filter((path) => !expected.has(path)); + const missingOwners = plan.owners.filter( + (owner) => + ![...ownersByFile.values()].some((owners) => owners.includes(owner)), + ); + const effective = removed.length + ? full( + `Changed tests absent from current inventory: ${removed.join(", ")}`, + plan.changedPaths, + ) + : missingOwners.length + ? full( + `Owners without executable tests: ${missingOwners.join(", ")}`, + plan.changedPaths, + ) + : plan; + const selected = []; + const omitted = []; + for (const path of inventory.toSorted()) { + const owners = ownersByFile.get(path) ?? []; + const keep = + effective.mode === "full" || + !owners.length || + effective.changedTests.includes(path) || + owners.some((owner) => effective.owners.includes(owner)); + (keep ? selected : omitted).push(path); + } + const actual = new Set(results.map((file) => file.name)); + const missing = inventory.filter((path) => !actual.has(path)); + const unexpected = [...actual].filter((path) => !expected.has(path)); + const duplicateFiles = results.length - actual.size; + const unobserved = results + .filter((file) => !["passed", "failed"].includes(file.state)) + .map((file) => file.name); + const omittedSet = new Set(omitted); + const missedFailures = results + .filter((file) => omittedSet.has(file.name) && file.state === "failed") + .map((file) => file.name); + return { + version: 1, + observational: true, + plan: effective, + selected, + omitted, + missing, + unexpected, + duplicateFiles, + unobserved, + missedFailures, + complete: + !missing.length && + !unexpected.length && + !duplicateFiles && + !unobserved.length, + // Phase totals are scheduling weights, not an estimate of wall-time savings. + omittedDurationMs: results + .filter((file) => omittedSet.has(file.name)) + .reduce((sum, file) => sum + file.durationMs, 0), + }; +} diff --git a/scripts/ci/test-timing-reporter.mjs b/scripts/ci/test-timing-reporter.mjs index 29f0cc582..ce0b1c5dc 100644 --- a/scripts/ci/test-timing-reporter.mjs +++ b/scripts/ci/test-timing-reporter.mjs @@ -21,6 +21,7 @@ export default class TestTimingReporter { }; return { name: module.moduleId, + state: module.state(), durationMs: Object.values(phases).reduce( (total, value) => total + value, 0, @@ -33,6 +34,8 @@ export default class TestTimingReporter { JSON.stringify({ version: 1, success: reason === "passed" && errors.length === 0, + reason, + unhandledErrors: errors.length, files, }), ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 714d2f980..fa4b71ebf 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -178,6 +178,23 @@ it("keeps unknown CI scripts on complete qualification", async () => { for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); }); +it("emits an observational owner proposal without changing ordinary or full portable execution", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["src/evm/Example.ts"]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + testSelection: { mode: "candidate", owners: ["evm"] }, + }); + expect( + await classify(directory, base, head, "pull_request", true), + ).toMatchObject({ + code: true, + testSelection: { mode: "full", owners: [] }, + }); + const output = await readFile(join(directory, "scope-output"), "utf8"); + expect(output).toContain('test_selection={"version":1,"mode":"full"'); +}); + it("does not charge a documentation PR for unrelated implementation updates on main", async () => { const { directory, base } = await fixture(); await git(directory, ["switch", "-c", "docs/change"]); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 333e93f08..06212ad07 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -8,6 +8,8 @@ import { expect, it } from "vitest"; import { parse } from "yaml"; import { z } from "zod"; +import { planTestSelection } from "../../../scripts/ci/test-selection.mjs"; + import { parseTestDurations } from "../../ciTestSequencer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { workflowGit } from "../../support/workflowGit.js"; @@ -61,6 +63,16 @@ it("collects failed-shard diagnostics before enforcing the shard result", async "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(steps[upload]?.with?.path).toContain("test-timings.json"); + expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); + expect(steps[upload]?.with?.path).toContain( + ".cache/ci-test-selection/report.json", + ); + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.CI_TEST_SELECTION).toBe( + "${{ needs.changes.outputs.test_selection }}", + ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); for (const result of ["failure", "cancelled", "skipped", ""]) { @@ -304,6 +316,13 @@ it("recovers complete module costs from real blob reports without executing test ), }) .parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8"))); + expect( + JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")), + ).toMatchObject({ + reason: "passed", + unhandledErrors: 0, + files: [{ state: "passed" }], + }); expect(report.files).toHaveLength(1); const file = report.files[0]!; const json = z @@ -349,7 +368,13 @@ it("recovers complete module costs from real blob reports without executing test const failed = JSON.parse( await readFile(join(root, "test-timings.json"), "utf8"), ) as unknown; - expect(failed).toMatchObject({ version: 1, success: false }); + expect(failed).toMatchObject({ + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: [{ state: "failed" }], + }); // Even a caller accidentally marking the merge successful cannot publish this report. await execute(process.execPath, [fileURLToPath(script)], { cwd: root, @@ -359,3 +384,76 @@ it("recovers complete module costs from real blob reports without executing test readFile(join(root, ".cache/ci-test-history/test-results.json")), ).rejects.toMatchObject({ code: "ENOENT" }); }); + +it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => { + const root = await createTestTempDirectory("rea-ci-selection-report-"); + await workflowGit(root, ["init"]); + const paths = [ + "src/sqlite/provider.test.ts", + "src/evm/provider.test.ts", + "tests/boundary/mcp/workflow.test.ts", + ]; + for (const path of paths) { + await mkdir(join(root, path, ".."), { recursive: true }); + await writeFile(join(root, path), "// fixture\n"); + } + await workflowGit(root, ["add", "src", "tests"]); + const timing = { + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: paths.map((name, index) => ({ + name, + durationMs: 10, + state: index === 1 ? "failed" : "passed", + })), + }; + const run = async (report: unknown) => { + await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); + await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "failure", + MERGE_RESULT: "failure", + CI_VERIFY_TEST_INVENTORY: "true", + CI_TEST_SELECTION: JSON.stringify( + planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"), + ), + GITHUB_SHA: "merge-sha", + CI_PR_HEAD_SHA: "pr-head-sha", + }, + }); + return JSON.parse( + await readFile( + join(root, ".cache/ci-test-selection/report.json"), + "utf8", + ), + ) as unknown; + }; + expect(await run(timing)).toMatchObject({ + observational: true, + complete: true, + selected: [paths[0], paths[2]], + omitted: [paths[1]], + missedFailures: [paths[1]], + executionSha: "merge-sha", + prHeadSha: "pr-head-sha", + }); + for (const report of [ + { ...timing, files: timing.files.slice(0, 1) }, + { ...timing, unhandledErrors: 1 }, + { ...timing, reason: "interrupted" }, + ]) + expect(await run(report)).toMatchObject({ complete: false }); + await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ + code: 1, + }); + await expect( + readFile(join(root, ".cache/ci-test-selection/report.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts new file mode 100644 index 000000000..1dff7f9e3 --- /dev/null +++ b/tests/evaluation/ciTestSelection.test.ts @@ -0,0 +1,240 @@ +import { expect, it } from "vitest"; + +import { + compareTestSelection, + planTestSelection, +} from "../../scripts/ci/test-selection.mjs"; + +const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts"; +const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts"; +const adbSource = "src/adb/AdbProvider.test.ts"; +const adbBoundary = + "tests/boundary/providers/adb/adbBugreportPublication.test.ts"; +const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts"; +const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts"; +const processBoundary = + "tests/boundary/process/processCaptureLifecycle.test.ts"; +const acceptance = "tests/acceptance/tools/localToolContracts.test.ts"; +const composition = "tests/composition/directAnalysisCleanup.test.ts"; +const evaluation = "tests/evaluation/testDiscovery.test.ts"; + +const passed = (name: string) => ({ name, durationMs: 10, state: "passed" }); + +it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(plan).toMatchObject({ + mode: "candidate", + owners: ["adb", "sqlite"], + }); + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([]); +}); + +it("omits only unrelated owned tests while retaining dynamic boundary suites", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([adbSource, adbBoundary]); + expect(report.complete).toBe(true); +}); + +it.each([ + ["unowned shared source", "src/application/Workflow.ts"], + ["fixture input", "tests/conformance/c/fixture.c"], + ["configuration", "vitest.config.ts"], + ["documentation", "README.md"], + ["CLI implementation", "src/cli/apktoolCommands.ts"], + ["MCP implementation", "src/server/registerSqliteTools.ts"], + ["process ownership", "src/process/ProcessOwnership.ts"], + ["acceptance test", acceptance], + ["composition test", composition], +])("uses full fallback for %s changes", (_description, path) => { + expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({ + mode: "full", + owners: [], + changedTests: [], + }); +}); + +it("uses full fallback outside ordinary fast pull requests or without diff paths", () => { + expect( + planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast") + .mode, + ).toBe("full"); + expect( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "full", + ).mode, + ).toBe("full"); + expect(planTestSelection([], "pull_request", "fast").mode).toBe("full"); +}); + +it("keeps new changed tests and falls back when a changed test was removed", () => { + const addedTest = "src/sqlite/NewDatabaseRule.test.ts"; + const candidate = planTestSelection([addedTest], "pull_request", "fast"); + expect(candidate.changedTests).toEqual([addedTest]); + const inventory = [addedTest, adbSource]; + expect( + compareTestSelection(candidate, inventory, inventory.map(passed)).selected, + ).toContain(addedTest); + + const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts"; + const report = compareTestSelection( + planTestSelection([removedTest], "pull_request", "fast"), + [sqliteSource, adbSource], + [passed(sqliteSource), passed(adbSource)], + ); + expect(report.plan.mode).toBe("full"); + expect(report.selected).toEqual([adbSource, sqliteSource]); + expect(report.omitted).toEqual([]); +}); + +it("marks incomplete inventories and nonterminal outcomes as incomplete", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [sqliteSource, adbSource]; + const completeResults = inventory.map(passed); + + expect( + compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete, + ).toBe(false); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed("tests/evaluation/unexpected.test.ts"), + ]), + ).toMatchObject({ + complete: false, + unexpected: ["tests/evaluation/unexpected.test.ts"], + }); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed(adbSource), + ]), + ).toMatchObject({ complete: false, duplicateFiles: 1 }); + for (const state of ["pending", "skipped"]) + expect( + compareTestSelection(plan, inventory, [ + passed(sqliteSource), + { name: adbSource, durationMs: 0, state }, + ]), + ).toMatchObject({ complete: false, unobserved: [adbSource] }); +}); + +it("reports a failure from an omitted test, including collection or module failures", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const report = compareTestSelection( + plan, + [sqliteSource, adbSource], + [passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }], + ); + + expect(report.omitted).toContain(adbSource); + expect(report.missedFailures).toEqual([adbSource]); +}); + +it("rejects plans that name unowned owners or malformed changed tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + expect(() => + compareTestSelection( + { ...plan, owners: ["unknown"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); + expect(() => + compareTestSelection( + { ...plan, changedTests: ["src/sqlite/not-a-test.ts"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); +}); + +it("uses full fallback when a changed owner has no executable tests", () => { + const report = compareTestSelection( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ), + [adbSource], + [passed(adbSource)], + ); + + expect(report.plan.mode).toBe("full"); + expect(report.omitted).toEqual([]); +}); From 490a285070268aee0ecd50001b9a9a83c1e313f0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:30:50 +0800 Subject: [PATCH 7/8] ci: reuse current-run builds across provider lanes --- .../actions/portable-build-cache/action.yml | 16 ++- .github/workflows/ci.yml | 45 ++++++-- .github/workflows/real-binary-layout.yml | 8 ++ .github/workflows/real-browser.yml | 8 ++ .github/workflows/real-evm-interface.yml | 8 ++ .../workflows/real-javascript-recovery.yml | 8 ++ .github/workflows/real-recorded-crash.yml | 8 ++ .../workflows/real-web-network-captures.yml | 8 ++ .github/workflows/real-web-runtime.yml | 8 ++ .github/workflows/real-web-source-map.yml | 8 ++ docs/testing.md | 10 ++ .../boundary/filesystem/ciChangeScope.test.ts | 4 +- .../filesystem/ciRequiredLanes.test.ts | 109 ++++++++++++++++++ .../boundary/filesystem/testTaskCache.test.ts | 35 +++++- 14 files changed, 266 insertions(+), 17 deletions(-) diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml index ad127ce3d..cee775ec5 100644 --- a/.github/actions/portable-build-cache/action.yml +++ b/.github/actions/portable-build-cache/action.yml @@ -1,14 +1,24 @@ name: Portable build cache -description: Reuse Turbo build outputs without serializing verification jobs. +description: Restore portable Turbo build outputs from the current run or cross-run cache. inputs: save: description: Save newly built outputs after the job succeeds. default: "false" + build_cache_artifact: + description: Reuse portable Turbo outputs uploaded by the current workflow run. + default: "" runs: using: composite steps: + - name: Download portable build outputs from this workflow run + if: inputs.build_cache_artifact != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.build_cache_artifact }} + path: .turbo/cache/ - name: Resolve portable build task hashes id: plan + if: inputs.build_cache_artifact == '' shell: bash env: BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }} @@ -28,14 +38,14 @@ runs: console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`); ' >> "$GITHUB_OUTPUT" - name: Restore portable build outputs - if: inputs.save != 'true' + if: inputs.build_cache_artifact == '' && inputs.save != 'true' uses: actions/cache/restore@v4 with: path: .turbo/cache/ key: ${{ steps.plan.outputs.key }} restore-keys: ${{ steps.plan.outputs.prefix }} - name: Restore and save portable build outputs - if: inputs.save == 'true' + if: inputs.build_cache_artifact == '' && inputs.save == 'true' uses: actions/cache@v4 with: path: .turbo/cache/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28373f89d..e0516da85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,18 @@ jobs: build: name: Build package needs: changes - if: needs.changes.outputs.code == 'true' + # Reusable provider workflows consume this run's portable build artifact, + # even when a workflow-only edit selects the lane without portable tests. + if: >- + needs.changes.outputs.code == 'true' || + needs.changes.outputs.browser == 'true' || + needs.changes.outputs.binary_layout == 'true' || + needs.changes.outputs.evm == 'true' || + needs.changes.outputs.javascript_recovery == 'true' || + needs.changes.outputs.recorded_crash == 'true' || + needs.changes.outputs.web_network == 'true' || + needs.changes.outputs.web_runtime == 'true' || + needs.changes.outputs.web_source_map == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -673,44 +684,60 @@ jobs: if-no-files-found: error browser: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.browser == 'true' uses: ./.github/workflows/real-browser.yml + with: + build_cache_artifact: rea-build-cache binary_layout: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.binary_layout == 'true' uses: ./.github/workflows/real-binary-layout.yml + with: + build_cache_artifact: rea-build-cache evm: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.evm == 'true' uses: ./.github/workflows/real-evm-interface.yml + with: + build_cache_artifact: rea-build-cache javascript_recovery: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.javascript_recovery == 'true' uses: ./.github/workflows/real-javascript-recovery.yml + with: + build_cache_artifact: rea-build-cache recorded_crash: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.recorded_crash == 'true' uses: ./.github/workflows/real-recorded-crash.yml + with: + build_cache_artifact: rea-build-cache web_network: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_network == 'true' uses: ./.github/workflows/real-web-network-captures.yml + with: + build_cache_artifact: rea-build-cache web_runtime: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_runtime == 'true' uses: ./.github/workflows/real-web-runtime.yml + with: + build_cache_artifact: rea-build-cache web_source_map: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_source_map == 'true' uses: ./.github/workflows/real-web-source-map.yml + with: + build_cache_artifact: rea-build-cache nativeaot: needs: [changes, static] diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index 8dd6f9866..5cec952ce 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -8,6 +8,12 @@ on: type: boolean default: false workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -29,6 +35,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire exact upstream libraries in an isolated runner environment shell: bash run: | diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index 11a8b7a53..912fdafb1 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -2,6 +2,12 @@ name: Real browser verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: env: @@ -22,6 +28,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml index b335f1391..dfdd3970f 100644 --- a/.github/workflows/real-evm-interface.yml +++ b/.github/workflows/real-evm-interface.yml @@ -3,6 +3,12 @@ name: Real offline EVM interface on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -23,6 +29,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire unchanged fixture compiler in isolated runner storage shell: bash run: | diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml index d94127d63..817dc4dc0 100644 --- a/.github/workflows/real-javascript-recovery.yml +++ b/.github/workflows/real-javascript-recovery.yml @@ -2,6 +2,12 @@ name: Real JavaScript recovery on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: @@ -24,6 +30,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Fetch pinned Wakaru release into the job workspace shell: bash run: | diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml index dc56034c8..04467a608 100644 --- a/.github/workflows/real-recorded-crash.yml +++ b/.github/workflows/real-recorded-crash.yml @@ -3,6 +3,12 @@ name: Real recorded crash evidence on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -26,6 +32,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install GDB on the disposable CI runner shell: bash run: | diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml index df340520d..d41bb37a5 100644 --- a/.github/workflows/real-web-network-captures.yml +++ b/.github/workflows/real-web-network-captures.yml @@ -3,6 +3,12 @@ name: Real historical web network captures on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -25,6 +31,8 @@ jobs: run: npm install --global npm@11.16.0 - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire fixed upstream native profile in runner temporary storage shell: bash run: | diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml index 0ecdb92c6..2575c55f9 100644 --- a/.github/workflows/real-web-runtime.yml +++ b/.github/workflows/real-web-runtime.yml @@ -1,6 +1,12 @@ name: Real website runtime attribution on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser:runtime env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml index b94662129..8e0869eb7 100644 --- a/.github/workflows/real-web-source-map.yml +++ b/.github/workflows/real-web-source-map.yml @@ -1,6 +1,12 @@ name: Real website source-map verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install the pinned compiler into an isolated fixture directory run: npm install --prefix "$RUNNER_TEMP/rea-web-source-map-compiler" --no-audit --no-fund --package-lock=false esbuild@0.25.10 - run: npm run verify:browser:source-maps diff --git a/docs/testing.md b/docs/testing.md index 03121e172..1288a954a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -982,6 +982,16 @@ timing data is reused; live provider results are never cached. Turbo outputs are restored by task hash, with trusted default-branch runs as cache writers and PRs as restore-only consumers. npm caches hold downloads, not `node_modules`. +The eight automatic browser, binary-layout, EVM, JavaScript-recovery, +recorded-crash, and web provider lanes download `rea-build-cache` from their own +run after the build and static jobs succeed. Their existing build commands still +validate task hashes and rebuild mismatched inputs; provider observations always +run. Provider-workflow-only edits also admit the build without enabling source +test shards. Direct provider dispatches retain the cross-run restore-only cache. +This removes repeated compilation on fresh source revisions, but adds the build +as a prerequisite: a delayed build can delay provider admission. Compare queue +and execution times before claiming an end-to-end speedup. + These native baseline checks complement the Linux source-test shards and the separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA, browser and managed-tool claims require their corresponding verification lanes. diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index fa4b71ebf..163219c39 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -680,7 +680,9 @@ it.each([ async (scope, name) => { const workflow = z .object({ - on: z.object({ workflow_call: z.null() }).passthrough(), + on: z + .object({ workflow_call: z.object({}).passthrough().nullable() }) + .passthrough(), }) .parse( parse( diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index e460bb52b..b6e02b54a 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -168,6 +168,115 @@ it("holds native and provider fan-out behind static validation while portable fe expect(jobs["test-shard"]?.needs).not.toContain("static"); }); +it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => { + const { jobs } = await workflow(); + const consumers = [ + "browser", + "binary_layout", + "evm", + "javascript_recovery", + "recorded_crash", + "web_network", + "web_runtime", + "web_source_map", + ]; + const buildTerms = z + .string() + .parse(jobs.build?.if) + .split("||") + .map((term) => term.trim()); + // Provider workflow edits select their lane without code=true. The build + // producer must still run, without selecting the portable test shards. + expect(new Set(buildTerms)).toEqual( + new Set( + ["code", ...consumers].map( + (scope) => `needs.changes.outputs.${scope} == 'true'`, + ), + ), + ); + expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'"); + for (const name of consumers) { + const caller = jobs[name]; + expect(caller?.needs).toContain("build"); + expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" }); + const path = z.string().parse(caller?.uses); + const called = z + .object({ + on: z.object({ + workflow_call: z.object({ + inputs: z.record(z.string(), z.unknown()), + }), + workflow_dispatch: z.unknown(), + }), + jobs: z.record( + z.string(), + z.object({ + steps: z.array( + z.object({ + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + }), + ), + }), + ), + }) + .parse( + parse( + await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"), + ), + ); + expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({ + type: "string", + default: "", + }); + expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true); + for (const job of Object.values(called.jobs)) + expect( + job.steps.find( + (step) => step.uses === "./.github/actions/portable-build-cache", + )?.with, + ).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" }); + } + const action = z + .object({ + runs: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + uses: z.string().optional(), + if: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }) + .parse( + parse( + await readFile( + new URL( + "../../../.github/actions/portable-build-cache/action.yml", + import.meta.url, + ), + "utf8", + ), + ), + ); + const download = action.runs.steps.find( + (step) => step.uses === "actions/download-artifact@v4", + ); + expect(download?.with).toEqual({ + name: "${{ inputs.build_cache_artifact }}", + path: ".turbo/cache/", + }); + expect(download?.if).toBe("inputs.build_cache_artifact != ''"); + expect(download?.["continue-on-error"]).not.toBe(true); + for (const step of action.runs.steps.filter( + (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"), + )) + expect(step.if).toContain("inputs.build_cache_artifact == ''"); +}); + it("runs workflow validation inside required static checks when selected", async () => { const { jobs } = await workflow(); const steps = z diff --git a/tests/boundary/filesystem/testTaskCache.test.ts b/tests/boundary/filesystem/testTaskCache.test.ts index 7e06ce1a1..5d877e829 100644 --- a/tests/boundary/filesystem/testTaskCache.test.ts +++ b/tests/boundary/filesystem/testTaskCache.test.ts @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { cp, readFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; import { promisify } from "node:util"; import { expect, it } from "vitest"; @@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")( await workflowGit(root, ["init", "--initial-branch=main"]); await workflowGit(root, ["add", "."]); await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]); - const run = () => + const run = (cwd = root) => execute( process.execPath, [ @@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")( "test:run", "--output-logs=errors-only", ], - { cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } }, + { cwd, env: { ...process.env, TURBO_DAEMON: "false" } }, ); await run(); await writeFile(join(root, "README_zh.md"), "wrong command"); @@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")( "prepared\n", ); } + // Provider jobs receive the producer's cache in a fresh checkout, not its + // outputs. Turbo must restore identical inputs and rebuild changed ones. + const consumer = await createTestTempDirectory("rea-cache-consumer-"); + await cp(root, consumer, { + recursive: true, + filter: (path) => + basename(path) !== ".turbo" && + !/^prepared-\d+\.txt$/u.test(basename(path)), + }); + await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), { + recursive: true, + }); + const restored = await run(consumer); + expect(restored.stdout).toContain(`${dependencies.length} cached`); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\n"); + await writeFile( + join(consumer, "artifact-input.txt"), + "changed producer input", + ); + await run(consumer); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\nprepared\n"); }, ); From fb28ec1537e2d02aaca96325483c849c5c8ade6f Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:31:18 -0700 Subject: [PATCH 8/8] test(apple): keep the same-named slice dyld fixture openable on x64 hosts (#2755) Add a plain x86_64 slice to the dyld fixture while retaining both same-named arm64e slices and their independent environment assertions. The original fixture was rejected by real x64 CLI admission before tracing. The contributing PR's complete Linux CI run passes all 30 jobs, including the portable suite. Codex and Greptile reviews completed without findings; no unresolved threads remain. This restores the main test baseline needed for #2635. The change is limited to test data and preserves the original author's contribution. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- tests/boundary/cli/dylibResolutionCli.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/boundary/cli/dylibResolutionCli.test.ts b/tests/boundary/cli/dylibResolutionCli.test.ts index f6482a2f3..66fa21ba3 100644 --- a/tests/boundary/cli/dylibResolutionCli.test.ts +++ b/tests/boundary/cli/dylibResolutionCli.test.ts @@ -477,6 +477,7 @@ cliTest( "reports each same-named slice's own embedded dyld environment", async ({ cli }) => { // arm64e and arm64e.x1 slices both report the architecture name arm64e. + // A plain x86_64 slice keeps the bundle openable on x64 hosts. for (const [settings, expected] of [ [[null, "DYLD_PRINT_RPATHS=1"], ["DYLD_PRINT_RPATHS=1"]], [ @@ -486,8 +487,8 @@ cliTest( ] as const) { const app = await fixtureApp(); await writeFiles(app, { - "Contents/MacOS/App": fatImage( - [CPU.arm64e, CPU.arm64eX1].map((cpu, index) => { + "Contents/MacOS/App": fatImage([ + ...[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => { const setting = settings[index]; return { cpu, @@ -500,7 +501,8 @@ cliTest( }), }; }), - ), + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), }); const result = await cli.run({ arguments: [ @@ -516,7 +518,9 @@ cliTest( const trace = dylibResolutionResultSchema.parse( parseEvidence(result.json).normalized_result, ); - expect(trace.roots).toHaveLength(2); + expect( + trace.roots.filter(({ architecture }) => architecture === "arm64e"), + ).toHaveLength(2); const environments = trace.findings .filter(({ kind }) => kind === "dyld-environment-present") .map(({ explanation }) =>