chore: merge upstream CI baseline fixes

This commit is contained in:
morluto
2026-10-13 20:13:27 +08:00
29 changed files with 1542 additions and 150 deletions
@@ -1,14 +1,24 @@
name: Portable build cache
description: Reuse Turbo build outputs without serializing verification jobs.
description: Restore portable Turbo build outputs from the current run or cross-run cache.
inputs:
save:
description: Save newly built outputs after the job succeeds.
default: "false"
build_cache_artifact:
description: Reuse portable Turbo outputs uploaded by the current workflow run.
default: ""
runs:
using: composite
steps:
- name: Download portable build outputs from this workflow run
if: inputs.build_cache_artifact != ''
uses: actions/download-artifact@v4
with:
name: ${{ inputs.build_cache_artifact }}
path: .turbo/cache/
- name: Resolve portable build task hashes
id: plan
if: inputs.build_cache_artifact == ''
shell: bash
env:
BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }}
@@ -28,14 +38,14 @@ runs:
console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`);
' >> "$GITHUB_OUTPUT"
- name: Restore portable build outputs
if: inputs.save != 'true'
if: inputs.build_cache_artifact == '' && inputs.save != 'true'
uses: actions/cache/restore@v4
with:
path: .turbo/cache/
key: ${{ steps.plan.outputs.key }}
restore-keys: ${{ steps.plan.outputs.prefix }}
- name: Restore and save portable build outputs
if: inputs.save == 'true'
if: inputs.build_cache_artifact == '' && inputs.save == 'true'
uses: actions/cache@v4
with:
path: .turbo/cache/
+18
View File
@@ -0,0 +1,18 @@
name: Full PR qualification
on:
pull_request:
branches: [main, "release/*"]
types: [labeled]
permissions:
contents: read
actions: read
jobs:
qualification:
name: Qualify labeled PR revision
if: github.event.label.name == 'ci:full'
uses: ./.github/workflows/ci.yml
with:
source_sha: ${{ github.sha }}
+67 -40
View File
@@ -12,7 +12,7 @@ on:
workflow_call:
inputs:
source_sha:
description: Exact release source; must match the caller revision.
description: Exact qualification source; must match the caller revision.
required: true
type: string
@@ -21,34 +21,20 @@ permissions:
concurrency:
# Complete qualification finishes even while main receives more pushes.
# Main feedback also finishes, coalescing newer pushes to one pending run.
# Scheduled/manual main runs coalesce to one active and the latest pending;
# release qualification has its own exact-revision group.
# release and label-triggered qualification use exact-revision groups.
group: ci-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && 'qualification' || 'feedback' }}-${{ inputs.source_sha || github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}
cancel-in-progress: ${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}
env:
HUSKY: "0"
jobs:
pr-title:
name: Conventional PR title
if: github.event_name == 'pull_request' && inputs.source_sha == ''
runs-on: ubuntu-latest
steps:
- name: Validate title for Release Please
env:
PR_TITLE: ${{ github.event.pull_request.title }}
run: >-
node -e 'const title = process.env.PR_TITLE ?? "";
const conventional = /^(?:feat|fix|docs|refactor|perf|test|build|ci|chore|revert)(?:\([a-z0-9._/-]+\))?!?: .+/u;
if (!conventional.test(title)) {
console.error("PR title must use Conventional Commits, for example: feat: add historical source import");
process.exit(1);
}'
changes:
name: Classify changes
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
actions: read
@@ -75,6 +61,7 @@ jobs:
nativeaot: ${{ steps.plan.outputs.nativeaot }}
website: ${{ steps.plan.outputs.website }}
matrix: ${{ steps.plan.outputs.matrix }}
test_selection: ${{ steps.plan.outputs.test_selection }}
steps:
- uses: actions/checkout@v4
with:
@@ -83,7 +70,7 @@ jobs:
- uses: actions/setup-node@v4
with:
node-version-file: .nvmrc
- name: Bind release qualification to the caller revision
- name: Bind qualification to the caller revision
if: inputs.source_sha != ''
env:
SOURCE_SHA: ${{ inputs.source_sha }}
@@ -125,6 +112,7 @@ jobs:
needs: changes
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Require successful change classification
env:
@@ -163,6 +151,7 @@ jobs:
needs: [changes, build]
if: needs.changes.outputs.code == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
@@ -201,8 +190,20 @@ jobs:
build:
name: Build package
needs: changes
if: needs.changes.outputs.code == 'true'
# Reusable provider workflows consume this run's portable build artifact,
# even when a workflow-only edit selects the lane without portable tests.
if: >-
needs.changes.outputs.code == 'true' ||
needs.changes.outputs.browser == 'true' ||
needs.changes.outputs.binary_layout == 'true' ||
needs.changes.outputs.evm == 'true' ||
needs.changes.outputs.javascript_recovery == 'true' ||
needs.changes.outputs.recorded_crash == 'true' ||
needs.changes.outputs.web_network == 'true' ||
needs.changes.outputs.web_runtime == 'true' ||
needs.changes.outputs.web_source_map == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
actions: read
@@ -262,7 +263,9 @@ jobs:
retention-days: 7
sqlite-runtime:
needs: [changes, build]
# Preserve all selected coverage, but admit expensive host/provider work
# only after static validation. Portable tests still provide early feedback.
needs: [changes, static, build]
if: needs.changes.outputs.sqlite == 'true'
uses: ./.github/workflows/sqlite-runtime.yml
@@ -271,6 +274,7 @@ jobs:
needs: [changes, test-shard]
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: Require successful change classification
env:
@@ -303,6 +307,7 @@ jobs:
--reporter=default
--reporter=junit
--reporter=json
--reporter=./scripts/ci/test-timing-reporter.mjs
--outputFile.junit=test-results.xml
--outputFile.json=test-results.json
- name: Merge fast-suite JUnit and timing reports
@@ -313,14 +318,17 @@ jobs:
--reporter=default
--reporter=junit
--reporter=json
--reporter=./scripts/ci/test-timing-reporter.mjs
--outputFile.junit=test-results.xml
--outputFile.json=test-results.json
- name: Summarize slowest test files
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != ''
env:
SHARD_RESULT: ${{ needs.test-shard.result }}
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
CI_VERIFY_TEST_INVENTORY: "true"
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: node scripts/ci/test-report.mjs
- uses: actions/upload-artifact@v4
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
@@ -329,7 +337,10 @@ jobs:
path: |
test-results.xml
test-results.json
test-timings.json
coverage/
.cache/ci-test-selection/report.json
include-hidden-files: true
if-no-files-found: error
retention-days: 30
- name: Publish successful timing history
@@ -350,7 +361,7 @@ jobs:
conformance-fixtures:
name: Source-built conformance fixtures
needs: changes
needs: [changes, static]
if: needs.changes.outputs.fixtures == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -361,7 +372,7 @@ jobs:
windows-native-build:
name: Build bundled Windows x64 native controls
needs: changes
needs: [changes, static]
if: needs.changes.outputs.runtime == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
@@ -397,7 +408,7 @@ jobs:
package-e2e:
name: Native package baseline (${{ matrix.platform }} ${{ matrix.architecture }})
needs: [changes, build, windows-native-build]
needs: [changes, static, build, windows-native-build]
if: needs.changes.outputs.runtime == 'true'
strategy:
fail-fast: false
@@ -463,7 +474,7 @@ jobs:
apple-artifacts:
name: Real Apple artifact CLI/MCP E2E
needs: [changes, build]
needs: [changes, static, build]
if: needs.changes.outputs.apple == 'true'
runs-on: macos-15
timeout-minutes: 15
@@ -511,7 +522,7 @@ jobs:
reconstruction-readiness:
name: Reconstruction readiness
needs: changes
needs: [changes, static]
if: needs.changes.outputs.readiness == 'true'
runs-on: ubuntu-latest
timeout-minutes: 15
@@ -540,7 +551,7 @@ jobs:
windows-curated:
name: Windows capability and package lane
needs: [changes, build, windows-native-build]
needs: [changes, static, build, windows-native-build]
if: needs.changes.outputs.package == 'true'
runs-on: windows-latest
timeout-minutes: 25
@@ -652,6 +663,7 @@ jobs:
needs: changes
if: needs.changes.outputs.docs == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
@@ -672,52 +684,68 @@ jobs:
if-no-files-found: error
browser:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.browser == 'true'
uses: ./.github/workflows/real-browser.yml
with:
build_cache_artifact: rea-build-cache
binary_layout:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.binary_layout == 'true'
uses: ./.github/workflows/real-binary-layout.yml
with:
build_cache_artifact: rea-build-cache
evm:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.evm == 'true'
uses: ./.github/workflows/real-evm-interface.yml
with:
build_cache_artifact: rea-build-cache
javascript_recovery:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.javascript_recovery == 'true'
uses: ./.github/workflows/real-javascript-recovery.yml
with:
build_cache_artifact: rea-build-cache
recorded_crash:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.recorded_crash == 'true'
uses: ./.github/workflows/real-recorded-crash.yml
with:
build_cache_artifact: rea-build-cache
web_network:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.web_network == 'true'
uses: ./.github/workflows/real-web-network-captures.yml
with:
build_cache_artifact: rea-build-cache
web_runtime:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.web_runtime == 'true'
uses: ./.github/workflows/real-web-runtime.yml
with:
build_cache_artifact: rea-build-cache
web_source_map:
needs: changes
needs: [changes, static, build]
if: needs.changes.outputs.web_source_map == 'true'
uses: ./.github/workflows/real-web-source-map.yml
with:
build_cache_artifact: rea-build-cache
nativeaot:
needs: changes
needs: [changes, static]
if: needs.changes.outputs.nativeaot == 'true'
uses: ./.github/workflows/nativeaot-fixtures.yml
website:
needs: changes
needs: [changes, static]
if: needs.changes.outputs.website == 'true'
uses: ./.github/workflows/website-check.yml
@@ -727,7 +755,6 @@ jobs:
needs:
[
changes,
pr-title,
static,
build,
test-shard,
+29
View File
@@ -0,0 +1,29 @@
name: PR title
on:
pull_request:
branches: [main, "release/*"]
types: [opened, synchronize, reopened, edited]
permissions: {}
concurrency:
group: pr-title-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
pr-title:
name: Conventional PR title
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- name: Validate title for Release Please
env:
PR_TITLE: ${{ github.event.pull_request.title }}
run: >-
node -e 'const title = process.env.PR_TITLE ?? "";
const conventional = /^(?:feat|fix|docs|refactor|perf|test|build|ci|chore|revert)(?:\([a-z0-9._/-]+\))?!?: .+/u;
if (!conventional.test(title)) {
console.error("PR title must use Conventional Commits, for example: feat: add historical source import");
process.exit(1);
}'
+8
View File
@@ -8,6 +8,12 @@ on:
type: boolean
default: false
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
permissions:
contents: read
@@ -29,6 +35,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Acquire exact upstream libraries in an isolated runner environment
shell: bash
run: |
+8
View File
@@ -2,6 +2,12 @@ name: Real browser verification
on:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
workflow_dispatch:
env:
@@ -22,6 +28,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- run: npm run verify:browser
env:
REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome
+8
View File
@@ -3,6 +3,12 @@ name: Real offline EVM interface
on:
workflow_dispatch:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
permissions:
contents: read
@@ -23,6 +29,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Acquire unchanged fixture compiler in isolated runner storage
shell: bash
run: |
@@ -2,6 +2,12 @@ name: Real JavaScript recovery
on:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
workflow_dispatch:
permissions:
@@ -24,6 +30,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Fetch pinned Wakaru release into the job workspace
shell: bash
run: |
@@ -3,6 +3,12 @@ name: Real recorded crash evidence
on:
workflow_dispatch:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
permissions:
contents: read
@@ -26,6 +32,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Install GDB on the disposable CI runner
shell: bash
run: |
@@ -3,6 +3,12 @@ name: Real historical web network captures
on:
workflow_dispatch:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
permissions:
contents: read
@@ -25,6 +31,8 @@ jobs:
run: npm install --global npm@11.16.0
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Acquire fixed upstream native profile in runner temporary storage
shell: bash
run: |
+8
View File
@@ -1,6 +1,12 @@
name: Real website runtime attribution
on:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
workflow_dispatch:
permissions:
contents: read
@@ -18,6 +24,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- run: npm run verify:browser:runtime
env:
REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome
@@ -1,6 +1,12 @@
name: Real website source-map verification
on:
workflow_call:
inputs:
build_cache_artifact:
description: Portable build cache artifact from the calling CI run.
required: false
type: string
default: ""
workflow_dispatch:
permissions:
contents: read
@@ -18,6 +24,8 @@ jobs:
cache: npm
- run: npm ci
- uses: ./.github/actions/portable-build-cache
with:
build_cache_artifact: ${{ inputs.build_cache_artifact }}
- name: Install the pinned compiler into an isolated fixture directory
run: npm install --prefix "$RUNNER_TEMP/rea-web-source-map-compiler" --no-audit --no-fund --package-lock=false esbuild@0.25.10
- run: npm run verify:browser:source-maps
+1
View File
@@ -36,6 +36,7 @@ coverage/
.vitest/
.vitest-reports/
test-results.xml
test-timings.json
.turbo/
.cache/
docs/.vitepress/cache/
+81 -14
View File
@@ -263,19 +263,40 @@ CI owns coverage; aggregate and domain/contract thresholds are maintained in
Coverage does not replace named boundary or real-provider scenarios.
CI runs six native Vitest shards without retries. Each shard emits a blob
report; the merge job produces JUnit and JSON timing reports, plus aggregate
coverage during full qualification. It retains available
report; the merge job produces JUnit and JSON diagnostics, a module timing
report, plus aggregate coverage during full qualification. It retains available
reports even when a shard fails, uploads available diagnostics, and writes the
slowest files to the workflow summary. Partial reports are marked diagnostic
only. Failed, cancelled, or missing selected shards still fail the required gate,
and aggregate coverage thresholds remain enforced in full qualification.
Only successful complete full-tier merges publish the small `test-timings` artifact, with
checkout-relative paths; the build job downloads that artifact for the sequencer
instead of the coverage/JUnit bundle. Missing history falls back to native
instead of the coverage/JUnit bundle. Timing weights come from Vitest module
diagnostics and include tests and hooks, worker preparation, environment,
imports, and setup. Import timings already include transform waits, which are
not added again. The weights describe file costs under the measured worker and
coverage settings; their sum is not workflow elapsed time. The diagnostic
artifact retains the phase breakdown, while shard history keeps only each
file's total. Legacy test-body-only history and missing history fall back to native
sharding. Static checks, documentation, build, package verification, and
real-system lanes remain separate jobs so one kind of evidence cannot stand in
for another. Workflow syntax and expression validation runs inside the required
static-check job when workflow changes select it, avoiding a separate runner.
Native, provider, fixture, SQLite compatibility, and website lanes wait for
static checks to succeed before allocating their runners. This avoids spending
those runner slots on revisions that already fail static validation. Portable
test shards and documentation checks remain independent of static results, so
they can still provide diagnostic feedback. The required gate rejects skipped
selected lanes when static checks fail; passing revisions retain all selected
coverage. This ordering reduces peak demand on invalid revisions but can add
static-check waiting to otherwise passing runs; it is not a measured reduction
in end-to-end latency.
Core jobs have execution limits of five minutes for classification, ten minutes
for static checks, builds, docs, and report merging, and thirty minutes per
portable test shard. These limits leave headroom for cold setup and full coverage
while preventing a hung job from holding the active main run for hours. They do
not bound runner queue waiting. Timeouts still fail the required gate.
The separate `CI timings` workflow batches recent completed CI and Release runs
hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and
@@ -784,6 +805,27 @@ remain present and fail if classification fails. Every code change retains the
complete six-shard Linux suite; individual test files are not selected by
imports or filenames.
Ordinary PRs also produce an **observational** test-selection plan. The candidate
leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately
from provider-job scopes. It proposes keeping affected owners plus every unowned
test, including all CLI/MCP, acceptance, composition, conformance, and process
tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall
back to the complete inventory. Renames account for both paths. Main, scheduled,
manual, and `ci:full` runs always propose the full suite.
All six shards still execute the complete suite. The existing report merge
compares the proposal with actual module outcomes and retains
`.cache/ci-test-selection/report.json` in `test-results`, including selected and
omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and
executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate,
skipped, interrupted, or unhandled results cannot establish a complete comparison;
even a complete passing comparison is not proof of ownership completeness.
Before enabling omissions, review representative full-run comparisons and
cross-owner consumers, including failures, and measure completed CI feedback and
runner demand. There is no automatic promotion or switch to skip tests. The
initial proposal measures only leaf producer-test omissions; it does not promise
the savings of a broader affected-test planner.
`scripts/ci/plan.mjs` selects lanes using the ownership table in
`scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider
verifiers and the Linux x64 installed-package/Inspector baseline. The `apple`
@@ -805,8 +847,12 @@ scripts also select it. Unrelated code edits retain the complete portable suite
on the pinned Node version without starting four additional compatibility jobs.
Full qualification retains every SQLite version.
Shared process/contract/CLI boundaries, dependencies, other packaging inputs,
unknown paths, and changes to the planner select the full baseline. Package
script-only edits select their owning verifier when its entrypoint is
unknown paths, and changes to the planner select the full baseline.
CI timing collection, timing history, and test-report helper edits retain static
checks and the complete portable suite without selecting unrelated host/provider
lanes. Classification, qualification, gate, release receipt, and unknown CI
scripts keep the broad fallback.
Package script-only edits select their owning verifier when its entrypoint is
identifiable; unknown or shared script changes select everything. Android
build-script edits retain the Linux suite and workflow validation; real Android
verification stays manual. Renames account for both paths.
@@ -841,21 +887,32 @@ observation. Manual dispatch, `ci:full`, and exact release qualification always
run full validation.
Full qualification includes aggregate coverage,
every portable provider lane, and all native hosts below. A `ci:full` PR label
selects full qualification on the next PR run; manual dispatch can request it
immediately. Fast-tier broad fallbacks retain all owner lanes and the three-host
every portable provider lane, and all native hosts below. Adding `ci:full`
immediately starts the separate `Full PR qualification` workflow against the
label event's merge revision; subsequent PR revisions also select the full tier
in normal CI while that label remains. Unrelated labels do not start validation
jobs or replace the main CI checks. Manual dispatch also requests full validation.
Fast-tier broad fallbacks retain all owner lanes and the three-host
package matrix; Intel macOS and coverage remain full-tier obligations. Each
classification job records its selection and broad-fallback reasons in the run
summary. Representative Git-diff cases live in
`tests/boundary/filesystem/ciChangeScope.test.ts`.
`CI required` always runs and checks classification and every selected lane,
rejecting failed, cancelled, or unexpectedly skipped work. Branch protection
should require this aggregate, `Static checks`, and `Portable tests`. Fast runs
cancel superseded feedback for the same PR or main branch. Scheduled/manual
qualification has a separate concurrency group: active work finishes and only
the latest pending run is retained. Release qualification is grouped by its
exact source SHA and cannot be cancelled by newer main pushes. Provider and
rejecting failed, cancelled, or unexpectedly skipped work. It rejects missing or
unknown tiers and requires every canonical scope to be explicitly `true` or
`false`; incomplete classification cannot silently omit verification lanes.
Branch protection
should require this aggregate, `Static checks`, `Portable tests`, and
`Conventional PR title`. Title validation runs separately on PR opening,
reopening, commits, and edits; correcting a title refreshes only this lightweight
check. It reads title text through an environment variable, executes no PR code,
and has no token permissions. Its separate concurrency group cannot cancel
test runs. Fast PR runs cancel superseded feedback for the same PR. Main push
runs keep one active run and the latest pending run for that branch.
Scheduled/manual qualification has a separate concurrency group: active work
finishes and only the latest pending run is retained. Release qualification uses
its exact source SHA and cannot be cancelled by newer main pushes. Provider and
website lanes inherit their caller's scheduling; reusable
workflows have no independent cancellation group. Direct manual lane dispatches
run independently on disposable hosted runners. Publication
@@ -925,6 +982,16 @@ timing data is reused; live provider results are never cached. Turbo outputs are
restored by task hash, with trusted default-branch runs as cache writers and PRs
as restore-only consumers. npm caches hold downloads, not `node_modules`.
The eight automatic browser, binary-layout, EVM, JavaScript-recovery,
recorded-crash, and web provider lanes download `rea-build-cache` from their own
run after the build and static jobs succeed. Their existing build commands still
validate task hashes and rebuild mismatched inputs; provider observations always
run. Provider-workflow-only edits also admit the build without enabling source
test shards. Direct provider dispatches retain the cross-run restore-only cache.
This removes repeated compilation on fresh source revisions, but adds the build
as a prerequisite: a delayed build can delay provider admission. Compare queue
and execution times before claiming an end-to-end speedup.
These native baseline checks complement the Linux source-test shards and the
separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,
browser and managed-tool claims require their corresponding verification lanes.
+21 -2
View File
@@ -2,12 +2,15 @@ import { execFileSync } from "node:child_process";
import { appendFileSync } from "node:fs";
import { matchesGlob } from "node:path";
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
import { planTestSelection } from "./test-selection.mjs";
// Test files select the host/runtime lanes that execute their focused suites.
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
// Actionlint validates workflow syntax only. The semantic suite asserts these
// workflows' release, publication and self-hosted runner safeguards.
const GUARDED_WORKFLOWS = new Set([
".github/workflows/pr-title.yml",
".github/workflows/ci-full-pr.yml",
".github/workflows/release.yml",
".github/workflows/real-ghidra-windows.yml",
".github/workflows/sqlite-runtime.yml",
@@ -23,6 +26,15 @@ const FIXTURE_SOURCES = [
"tests/conformance/versions/**",
];
const READINESS_SOURCES = ["tests/conformance/readiness/**"];
// These reporters affect CI output only; they do not select a host or provider
// verification lane when changed on their own.
const PORTABLE_CI_REPORTERS = new Set([
"scripts/ci/job-timings.mjs",
"scripts/ci/job-timings-batch.mjs",
"scripts/ci/test-report.mjs",
"scripts/ci/test-timing-reporter.mjs",
"scripts/ci/test-history.mjs",
]);
const matchesAny = (file, patterns) =>
patterns.some((pattern) => matchesGlob(file, pattern));
const selected = Object.fromEntries(flags.map((flag) => [flag, false]));
@@ -198,6 +210,7 @@ if (tier === "reused") {
}
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
if (file.startsWith("tests/")) continue;
if (PORTABLE_CI_REPORTERS.has(file)) continue;
if (
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
file,
@@ -238,11 +251,17 @@ const matrix = {
]
: [linux],
};
const plan = { ...selected, tier, matrix, files, reasons };
const testSelection = planTestSelection(files, event, tier);
const plan = { ...selected, tier, matrix, files, reasons, testSelection };
if (process.env.GITHUB_OUTPUT)
appendFileSync(
process.env.GITHUB_OUTPUT,
Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) })
Object.entries({
...selected,
tier,
matrix: JSON.stringify(matrix),
test_selection: JSON.stringify(testSelection),
})
.map(([key, value]) => `${key}=${value}\n`)
.join(""),
);
+16 -9
View File
@@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs";
import { recordQualification } from "./qualification.mjs";
const jobs = JSON.parse(process.env.CI_JOBS);
const outputs = jobs.changes.outputs;
const outputs = jobs?.changes?.outputs;
if (!["fast", "full", "reused"].includes(outputs?.tier))
throw new Error(
"CI classification requires a valid tier: fast, full, or reused",
);
const invalidFlags = CI_FLAGS.filter(
(flag) => outputs[flag] !== "true" && outputs[flag] !== "false",
);
if (invalidFlags.length)
throw new Error(
`CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`,
);
const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true");
const reused = outputs.tier === "reused";
if (
reused &&
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
CI_FLAGS.some((flag) => outputs[flag] !== "false"))
selectedScopes.length !== 0)
)
throw new Error(
"Reused qualification requires a scheduled admission receipt and no selected lanes",
);
if (
outputs.tier === "full" &&
CI_FLAGS.some((flag) => outputs[flag] !== "true")
)
if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
throw new Error("Full qualification requires every canonical CI scope");
const required = reused ? ["changes"] : ["changes", "static", "test"];
const lanes = {
@@ -31,9 +40,7 @@ const lanes = {
docs: ["docs"],
workflows: ["static"],
};
for (const [scope, value] of Object.entries(outputs)) {
if (value === "true") required.push(...(lanes[scope] ?? [scope]));
}
for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope]));
const failed = [...new Set(required)].filter(
(name) => jobs[name]?.result !== "success",
);
+69 -25
View File
@@ -4,26 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path";
import { z } from "zod";
import { compareTestSelection } from "./test-selection.mjs";
const root = process.cwd();
const historyDirectory = join(root, ".cache/ci-test-history");
const historyPath = join(historyDirectory, "test-results.json");
const selectionDirectory = join(root, ".cache/ci-test-selection");
const selectionPath = join(selectionDirectory, "report.json");
// Never let a failed or partial rerun leave a previous successful history file.
await rm(historyPath, { force: true });
await rm(selectionPath, { force: true });
const report = z
.object({
version: z.literal(1),
success: z.boolean(),
numFailedTests: z.number().int().nonnegative(),
testResults: z.array(
reason: z.string().optional(),
unhandledErrors: z.number().int().nonnegative().optional(),
files: z.array(
z.object({
name: z.string().min(1),
startTime: z.number().finite(),
endTime: z.number().finite(),
durationMs: z.number().finite().nonnegative(),
state: z.string().optional(),
}),
),
})
.parse(JSON.parse(await readFile("test-results.json", "utf8")));
.parse(JSON.parse(await readFile("test-timings.json", "utf8")));
const files = report.testResults.map(({ name, startTime, endTime }) => {
const files = report.files.map(({ name, durationMs, state }) => {
const path = relative(root, resolve(root, name));
if (
path === "" ||
@@ -32,27 +39,47 @@ const files = report.testResults.map(({ name, startTime, endTime }) => {
isAbsolute(path)
)
throw new Error(`Test report path is outside the checkout: ${name}`);
const duration = endTime - startTime;
if (!Number.isFinite(duration) || duration < 0)
throw new Error(`Test report has an invalid duration: ${name}`);
return { name: path.split(sep).join("/"), startTime: 0, endTime: duration };
return { name: path.split(sep).join("/"), durationMs, state };
});
const complete = process.env.SHARD_RESULT === "success";
const passed =
complete &&
process.env.MERGE_RESULT === "success" &&
report.success &&
report.numFailedTests === 0;
complete && process.env.MERGE_RESULT === "success" && report.success;
// Successful shards and merged reports must account for the complete portable
// inventory. Coverage thresholds alone cannot prove that every file ran.
if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
const expected = execFileSync(
"git",
["ls-files", "-z", "--", "src", "tests"],
{ encoding: "utf8", maxBuffer: 8 * 1024 * 1024 },
)
.split("\0")
.filter((path) => path.endsWith(".test.ts"));
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
const expected =
verifyInventory || process.env.CI_TEST_SELECTION
? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], {
encoding: "utf8",
maxBuffer: 8 * 1024 * 1024,
})
.split("\0")
.filter((path) => path.endsWith(".test.ts"))
: [];
let selection;
if (process.env.CI_TEST_SELECTION) {
const comparison = compareTestSelection(
JSON.parse(process.env.CI_TEST_SELECTION),
expected,
files,
);
selection = {
...comparison,
complete:
comparison.complete &&
report.unhandledErrors === 0 &&
["passed", "failed"].includes(report.reason) &&
["success", "failure"].includes(process.env.SHARD_RESULT) &&
["success", "failure"].includes(process.env.MERGE_RESULT),
executionSha: process.env.GITHUB_SHA ?? null,
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
runId: process.env.GITHUB_RUN_ID ?? null,
runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
};
await mkdir(selectionDirectory, { recursive: true });
await writeFile(selectionPath, JSON.stringify(selection));
}
if (passed && verifyInventory) {
const actual = files.map(({ name }) => name);
const expectedSet = new Set(expected);
const actualSet = new Set(actual);
@@ -66,18 +93,31 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
const summary = [
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
"",
...(selection
? [
"### Observational PR test selection",
"",
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
"",
]
: []),
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
passed
? "Successful timing history is available."
: "This report is diagnostic only and is not a successful timing baseline.",
"",
"Durations include tests and hooks, worker preparation, environment, imports, and setup. They are scheduling weights, not summed workflow wall time.",
"",
"| Slowest file | Duration (ms) |",
"| --- | ---: |",
...files
.toSorted((left, right) => right.endTime - left.endTime)
.toSorted((left, right) => right.durationMs - left.durationMs)
.slice(0, 20)
.map(
({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`,
({ name, durationMs }) =>
`| ${name.replaceAll("|", "\\|")} | ${durationMs} |`,
),
"",
].join("\n");
@@ -89,6 +129,10 @@ if (passed) {
await mkdir(historyDirectory, { recursive: true });
await writeFile(
historyPath,
JSON.stringify({ success: true, numFailedTests: 0, testResults: files }),
JSON.stringify({
version: 1,
success: true,
files: files.map(({ name, durationMs }) => ({ name, durationMs })),
}),
);
}
+35
View File
@@ -0,0 +1,35 @@
export interface TestSelectionPlan {
readonly version: 1;
readonly mode: "full" | "candidate";
readonly owners: readonly string[];
readonly changedTests: readonly string[];
readonly changedPaths: readonly string[];
readonly reason: string;
}
export function planTestSelection(
files: readonly string[],
event: string,
tier: string,
): TestSelectionPlan;
export function compareTestSelection(
plan: unknown,
inventory: readonly string[],
results: readonly {
readonly name: string;
readonly durationMs: number;
readonly state?: string;
}[],
): {
readonly version: 1;
readonly observational: true;
readonly plan: TestSelectionPlan;
readonly selected: readonly string[];
readonly omitted: readonly string[];
readonly missing: readonly string[];
readonly unexpected: readonly string[];
readonly duplicateFiles: number;
readonly unobserved: readonly string[];
readonly missedFailures: readonly string[];
readonly complete: boolean;
readonly omittedDurationMs: number;
};
+151
View File
@@ -0,0 +1,151 @@
import { matchesGlob } from "node:path";
// Candidate ownership only: these leaf adapters and their producer tests can be
// compared with full-suite outcomes. This table never changes test execution.
// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners.
const OWNERS = {
adb: ["src/adb/**", "tests/boundary/providers/adb/**"],
apktool: ["src/apktool/**"],
dotnet: ["src/dotnet/**"],
evm: ["src/evm/**", "tests/boundary/evm/**"],
firmware: ["src/firmware/**", "tests/boundary/firmware/**"],
flutter: ["src/flutter/**"],
ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"],
hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"],
ida: ["src/ida/**", "tests/boundary/providers/ida/**"],
jeb: ["src/jeb/**"],
pcap: ["src/pcap/**", "tests/boundary/pcap/**"],
sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"],
wasm: ["src/wasm/**"],
};
const ownersOf = (path) =>
Object.entries(OWNERS)
.filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob)))
.map(([owner]) => owner);
const validPath = (path) =>
typeof path === "string" &&
/^(src|tests)\//.test(path) &&
!/[\\\0\r\n]/.test(path) &&
!path.split("/").some((part) => ["", ".", ".."].includes(part));
const full = (reason, changedPaths = []) => ({
version: 1,
mode: "full",
owners: [],
changedTests: [],
changedPaths,
reason,
});
/** Plan an observational PR subset from the classifier's existing Git diff. */
export function planTestSelection(files, event, tier) {
if (event !== "pull_request" || tier !== "fast")
return full("Only ordinary PRs are candidates", files);
if (!files.length) return full("No changed paths available");
const owners = new Set();
const changedTests = [];
for (const path of files) {
if (!validPath(path))
return full(`Shared or unknown input: ${path}`, files);
const matches = ownersOf(path);
if (!matches.length) return full(`Shared or unknown input: ${path}`, files);
for (const owner of matches) owners.add(owner);
if (path.endsWith(".test.ts")) changedTests.push(path);
}
return {
version: 1,
mode: "candidate",
owners: [...owners].sort(),
changedTests: [...new Set(changedTests)].sort(),
changedPaths: files,
reason:
"Observe affected leaf owners; retain integration and unowned tests",
};
}
/** Compare a proposal with the full run; missing observations never prove parity. */
export function compareTestSelection(plan, inventory, results) {
if (
plan?.version !== 1 ||
!["full", "candidate"].includes(plan.mode) ||
!Array.isArray(plan.owners) ||
plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) ||
!Array.isArray(plan.changedTests) ||
!Array.isArray(plan.changedPaths) ||
plan.changedPaths.some((path) => typeof path !== "string") ||
plan.changedTests.some(
(path) => !validPath(path) || !path.endsWith(".test.ts"),
) ||
typeof plan.reason !== "string" ||
(plan.mode === "candidate" && !plan.owners.length) ||
(plan.mode === "full" && (plan.owners.length || plan.changedTests.length))
)
throw new Error("Invalid observational test selection plan");
const expected = new Set(inventory);
if (!expected.size || expected.size !== inventory.length)
throw new Error("Test selection requires a nonempty unique inventory");
const ownersByFile = new Map(
plan.mode === "candidate"
? inventory.map((path) => [path, ownersOf(path)])
: [],
);
// Removed/renamed tests have no current executable identity. Keep all tests.
const removed = plan.changedTests.filter((path) => !expected.has(path));
const missingOwners = plan.owners.filter(
(owner) =>
![...ownersByFile.values()].some((owners) => owners.includes(owner)),
);
const effective = removed.length
? full(
`Changed tests absent from current inventory: ${removed.join(", ")}`,
plan.changedPaths,
)
: missingOwners.length
? full(
`Owners without executable tests: ${missingOwners.join(", ")}`,
plan.changedPaths,
)
: plan;
const selected = [];
const omitted = [];
for (const path of inventory.toSorted()) {
const owners = ownersByFile.get(path) ?? [];
const keep =
effective.mode === "full" ||
!owners.length ||
effective.changedTests.includes(path) ||
owners.some((owner) => effective.owners.includes(owner));
(keep ? selected : omitted).push(path);
}
const actual = new Set(results.map((file) => file.name));
const missing = inventory.filter((path) => !actual.has(path));
const unexpected = [...actual].filter((path) => !expected.has(path));
const duplicateFiles = results.length - actual.size;
const unobserved = results
.filter((file) => !["passed", "failed"].includes(file.state))
.map((file) => file.name);
const omittedSet = new Set(omitted);
const missedFailures = results
.filter((file) => omittedSet.has(file.name) && file.state === "failed")
.map((file) => file.name);
return {
version: 1,
observational: true,
plan: effective,
selected,
omitted,
missing,
unexpected,
duplicateFiles,
unobserved,
missedFailures,
complete:
!missing.length &&
!unexpected.length &&
!duplicateFiles &&
!unobserved.length,
// Phase totals are scheduling weights, not an estimate of wall-time savings.
omittedDurationMs: results
.filter((file) => omittedSet.has(file.name))
.reduce((sum, file) => sum + file.durationMs, 0),
};
}
+43
View File
@@ -0,0 +1,43 @@
import { writeFile } from "node:fs/promises";
/** @implements {import("vitest/node").Reporter} */
export default class TestTimingReporter {
/**
* @param {ReadonlyArray<import("vitest/node").TestModule>} modules
* @param {ReadonlyArray<import("vitest/node").SerializedError>} errors
* @param {import("vitest/node").TestRunEndReason} reason
*/
async onTestRunEnd(modules, errors, reason) {
const files = modules.map((module) => {
const diagnostic = module.diagnostic();
// These disjoint phases match Vitest's per-file wall-time accounting.
// Import timings already include transform waits; do not add them again.
const phases = {
execution: diagnostic.duration,
preparation: diagnostic.prepareDuration,
environment: diagnostic.environmentSetupDuration,
imports: diagnostic.collectDuration,
setup: diagnostic.setupDuration,
};
return {
name: module.moduleId,
state: module.state(),
durationMs: Object.values(phases).reduce(
(total, value) => total + value,
0,
),
phases,
};
});
await writeFile(
"test-timings.json",
JSON.stringify({
version: 1,
success: reason === "passed" && errors.length === 0,
reason,
unhandledErrors: errors.length,
files,
}),
);
}
}
@@ -477,6 +477,7 @@ cliTest(
"reports each same-named slice's own embedded dyld environment",
async ({ cli }) => {
// arm64e and arm64e.x1 slices both report the architecture name arm64e.
// A plain x86_64 slice keeps the bundle openable on x64 hosts.
for (const [settings, expected] of [
[[null, "DYLD_PRINT_RPATHS=1"], ["DYLD_PRINT_RPATHS=1"]],
[
@@ -486,8 +487,8 @@ cliTest(
] as const) {
const app = await fixtureApp();
await writeFiles(app, {
"Contents/MacOS/App": fatImage(
[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => {
"Contents/MacOS/App": fatImage([
...[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => {
const setting = settings[index];
return {
cpu,
@@ -500,7 +501,8 @@ cliTest(
}),
};
}),
),
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
]),
});
const result = await cli.run({
arguments: [
@@ -516,7 +518,9 @@ cliTest(
const trace = dylibResolutionResultSchema.parse(
parseEvidence(result.json).normalized_result,
);
expect(trace.roots).toHaveLength(2);
expect(
trace.roots.filter(({ architecture }) => architecture === "arm64e"),
).toHaveLength(2);
const environments = trace.findings
.filter(({ kind }) => kind === "dyld-environment-present")
.map(({ explanation }) =>
@@ -137,6 +137,64 @@ it("keeps every translated root README and authored guide on the documentation l
});
});
it.each([
"job-timings.mjs",
"job-timings-batch.mjs",
"test-report.mjs",
"test-timing-reporter.mjs",
"test-history.mjs",
])(
"keeps portable CI reporter %s on static checks and portable tests",
async (name) => {
const { directory, base } = await fixture();
const head = await change(directory, [`scripts/ci/${name}`]);
const plan = await classify(directory, base, head);
const { CI_FLAGS } = await loadScopes();
expect(plan.code).toBe(true);
for (const flag of CI_FLAGS)
if (flag !== "code") expect(plan[flag]).toBe(false);
},
);
it("preserves a provider lane when a portable CI reporter changes with provider source", async () => {
const { directory, base } = await fixture();
const head = await change(directory, [
"scripts/ci/test-report.mjs",
"src/evm/Example.ts",
]);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
evm: true,
browser: false,
package: false,
});
});
it("keeps unknown CI scripts on complete qualification", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["scripts/ci/new-qualification.mjs"]);
const plan = await classify(directory, base, head);
const { CI_FLAGS } = await loadScopes();
for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true);
});
it("emits an observational owner proposal without changing ordinary or full portable execution", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["src/evm/Example.ts"]);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
testSelection: { mode: "candidate", owners: ["evm"] },
});
expect(
await classify(directory, base, head, "pull_request", true),
).toMatchObject({
code: true,
testSelection: { mode: "full", owners: [] },
});
const output = await readFile(join(directory, "scope-output"), "utf8");
expect(output).toContain('test_selection={"version":1,"mode":"full"');
});
it("does not charge a documentation PR for unrelated implementation updates on main", async () => {
const { directory, base } = await fixture();
await git(directory, ["switch", "-c", "docs/change"]);
@@ -622,7 +680,9 @@ it.each([
async (scope, name) => {
const workflow = z
.object({
on: z.object({ workflow_call: z.null() }).passthrough(),
on: z
.object({ workflow_call: z.object({}).passthrough().nullable() })
.passthrough(),
})
.parse(
parse(
@@ -662,8 +722,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn
const command = fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
);
const { CI_FLAGS } = await loadScopes();
const jobs = {
changes: { result: "success", outputs: { evm: "true" } },
changes: {
result: "success",
outputs: {
...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])),
tier: "fast",
evm: "true",
},
},
static: { result: "success" },
test: { result: "success" },
evm: { result: "success" },
@@ -0,0 +1,98 @@
import { readFile } from "node:fs/promises";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
async function workflow(name: string) {
return z
.object({
on: z.record(z.string(), z.unknown()),
permissions: z.record(z.string(), z.string()),
concurrency: z.object({ group: z.string() }).optional(),
jobs: z.record(
z.string(),
z.object({
name: z.string().optional(),
if: z.string().optional(),
uses: z.string().optional(),
with: z.record(z.string(), z.string()).optional(),
steps: z
.array(
z.object({
run: z.string().optional(),
uses: z.string().optional(),
env: z.record(z.string(), z.string()).optional(),
}),
)
.optional(),
}),
),
})
.parse(
parse(
await readFile(
new URL(`../../../.github/workflows/${name}`, import.meta.url),
"utf8",
),
),
);
}
it("refreshes title validation on edits without checking out PR code or cancelling CI", async () => {
const title = await workflow("pr-title.yml");
expect(title.on.pull_request).toMatchObject({
types: ["opened", "synchronize", "reopened", "edited"],
});
expect(title.permissions).toEqual({});
expect(title.concurrency?.group).not.toBe(
(await workflow("ci.yml")).concurrency?.group,
);
const jobs = Object.values(title.jobs);
expect(jobs).toHaveLength(1);
expect(jobs[0]?.name).toBe("Conventional PR title");
expect(jobs[0]?.if).toBeUndefined(); // A skipped check must not green-light an invalid title on body edits.
const steps = jobs[0]?.steps ?? [];
expect(steps).toHaveLength(1);
expect(steps[0]?.uses).toBeUndefined();
expect(steps[0]?.env).toEqual({
PR_TITLE: "${{ github.event.pull_request.title }}",
});
const command = z.string().parse(steps[0]?.run);
for (const title of [
"fix(ci): repair workflow",
"feat!: changed contract",
"docs: literal $(exit 91) `exit 92` text",
]) {
await expect(
executeWorkflowFixture("bash", ["-c", command], {
env: { ...process.env, PR_TITLE: title },
}),
).resolves.toMatchObject({ stderr: "" });
}
for (const title of ["", "Fix CI", "feat: "]) {
await expect(
executeWorkflowFixture("bash", ["-c", command], {
env: { ...process.env, PR_TITLE: title },
}),
).rejects.toMatchObject({ code: 1 });
}
expect((await workflow("ci.yml")).jobs["pr-title"]).toBeUndefined();
});
it("qualifies only ci:full label additions at the caller's merge revision", async () => {
const full = await workflow("ci-full-pr.yml");
expect(full.on.pull_request).toMatchObject({ types: ["labeled"] });
expect(full.permissions).toEqual({ contents: "read", actions: "read" });
expect(Object.values(full.jobs)).toEqual([
{
name: "Qualify labeled PR revision",
if: "github.event.label.name == 'ci:full'",
uses: "./.github/workflows/ci.yml",
with: { source_sha: "${{ github.sha }}" },
},
]);
// Metadata events must not create skipped replacements for the main CI checks.
expect(full.jobs.required).toBeUndefined();
expect(full.concurrency).toBeUndefined();
});
+217 -14
View File
@@ -26,6 +26,15 @@ const workflow = async (filename = "ci.yml") =>
),
);
const scopeFlags = async () =>
z
.object({ CI_FLAGS: z.array(z.string()) })
.parse(
await import(
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
),
).CI_FLAGS;
it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => {
expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe(
"needs.changes.outputs.sqlite == 'true'",
@@ -57,13 +66,7 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and
it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => {
const { jobs } = await workflow();
const { CI_FLAGS } = z
.object({ CI_FLAGS: z.array(z.string()) })
.parse(
await import(
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
),
);
const CI_FLAGS = await scopeFlags();
const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs);
expect(jobs.changes?.permissions).toEqual({
contents: "read",
@@ -144,6 +147,136 @@ it("makes the required gate wait for every CI lane, including future additions",
);
});
it("holds native and provider fan-out behind static validation while portable feedback stays independent", async () => {
const { jobs } = await workflow();
const earlyJobs = new Set([
"changes",
"static",
"build",
"test-shard",
"test",
"docs",
"required",
]);
for (const [name, job] of Object.entries(jobs)) {
if (earlyJobs.has(name)) continue;
expect(job.needs, `${name} must wait for static validation`).toContain(
"static",
);
expect(job.if).not.toMatch(/always\(|cancelled\(/u);
}
expect(jobs["test-shard"]?.needs).not.toContain("static");
});
it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => {
const { jobs } = await workflow();
const consumers = [
"browser",
"binary_layout",
"evm",
"javascript_recovery",
"recorded_crash",
"web_network",
"web_runtime",
"web_source_map",
];
const buildTerms = z
.string()
.parse(jobs.build?.if)
.split("||")
.map((term) => term.trim());
// Provider workflow edits select their lane without code=true. The build
// producer must still run, without selecting the portable test shards.
expect(new Set(buildTerms)).toEqual(
new Set(
["code", ...consumers].map(
(scope) => `needs.changes.outputs.${scope} == 'true'`,
),
),
);
expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'");
for (const name of consumers) {
const caller = jobs[name];
expect(caller?.needs).toContain("build");
expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" });
const path = z.string().parse(caller?.uses);
const called = z
.object({
on: z.object({
workflow_call: z.object({
inputs: z.record(z.string(), z.unknown()),
}),
workflow_dispatch: z.unknown(),
}),
jobs: z.record(
z.string(),
z.object({
steps: z.array(
z.object({
uses: z.string().optional(),
with: z.record(z.string(), z.unknown()).optional(),
}),
),
}),
),
})
.parse(
parse(
await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"),
),
);
expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({
type: "string",
default: "",
});
expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true);
for (const job of Object.values(called.jobs))
expect(
job.steps.find(
(step) => step.uses === "./.github/actions/portable-build-cache",
)?.with,
).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" });
}
const action = z
.object({
runs: z.object({
steps: z.array(
z.object({
id: z.string().optional(),
uses: z.string().optional(),
if: z.string().optional(),
with: z.record(z.string(), z.unknown()).optional(),
"continue-on-error": z.boolean().optional(),
}),
),
}),
})
.parse(
parse(
await readFile(
new URL(
"../../../.github/actions/portable-build-cache/action.yml",
import.meta.url,
),
"utf8",
),
),
);
const download = action.runs.steps.find(
(step) => step.uses === "actions/download-artifact@v4",
);
expect(download?.with).toEqual({
name: "${{ inputs.build_cache_artifact }}",
path: ".turbo/cache/",
});
expect(download?.if).toBe("inputs.build_cache_artifact != ''");
expect(download?.["continue-on-error"]).not.toBe(true);
for (const step of action.runs.steps.filter(
(step) => step.id === "plan" || step.uses?.startsWith("actions/cache"),
))
expect(step.if).toContain("inputs.build_cache_artifact == ''");
});
it("runs workflow validation inside required static checks when selected", async () => {
const { jobs } = await workflow();
const steps = z
@@ -228,12 +361,9 @@ it("keeps fast feedback separate from complete qualification and release proof",
]);
expect(ci.concurrency.group).toContain("'qualification' || 'feedback'");
expect(ci.concurrency.group).toContain("inputs.source_sha ||");
for (const fullEvent of ["schedule", "workflow_dispatch"])
expect(ci.concurrency["cancel-in-progress"]).toContain(
`github.event_name != '${fullEvent}'`,
);
expect(ci.concurrency["cancel-in-progress"]).toContain(
"inputs.source_sha == ''",
expect(ci.concurrency.group).toContain("github.ref");
expect(ci.concurrency["cancel-in-progress"]).toBe(
"${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}",
);
expect(ci.jobs.required?.if).toBe("${{ always() }}");
const shards = ci.jobs["test-shard"]?.steps ?? [];
@@ -276,7 +406,14 @@ it.each([
);
needs.changes = {
result: "success",
outputs: { code: "true", [scope]: selected },
outputs: {
...Object.fromEntries(
(await scopeFlags()).map((flag) => [flag, "false"]),
),
tier: "fast",
code: "true",
[scope]: selected,
},
};
if (result === "missing") delete needs[lane];
else needs[lane] = { result };
@@ -301,3 +438,69 @@ it.each([
}
},
);
it("rejects incomplete or malformed classification even when available jobs succeeded", async () => {
const flags = await scopeFlags();
const valid = {
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
tier: "fast",
};
const run = (outputs: unknown) =>
executeWorkflowFixture(
process.execPath,
[
fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
),
],
{
env: {
...process.env,
CI_JOBS: JSON.stringify({
changes: { result: "success", outputs },
static: { result: "success" },
test: { result: "success" },
}),
CI_QUALIFICATION_DIRECTORY: "",
},
},
);
await expect(run(valid)).resolves.toMatchObject({ stderr: "" });
// Metadata is not a scope, even when its string happens to be "true".
await expect(
run({ ...valid, qualification_run_id: "true" }),
).resolves.toMatchObject({ stderr: "" });
for (const outputs of [
undefined,
null,
{},
{ ...valid, tier: undefined },
{ ...valid, tier: "" },
{ ...valid, tier: "unknown" },
]) {
await expect(run(outputs)).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("valid tier"),
});
}
for (const flag of flags) {
await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining(`scope outputs: ${flag}`),
});
}
for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) {
await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("scope outputs: sqlite"),
});
}
for (const tier of ["full", "reused"]) {
await expect(
run({ ...valid, tier, sqlite: undefined }),
).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("scope outputs: sqlite"),
});
}
});
+242 -14
View File
@@ -1,5 +1,5 @@
import { execFile } from "node:child_process";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
@@ -8,6 +8,8 @@ import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { planTestSelection } from "../../../scripts/ci/test-selection.mjs";
import { parseTestDurations } from "../../ciTestSequencer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { workflowGit } from "../../support/workflowGit.js";
@@ -54,6 +56,23 @@ it("collects failed-shard diagnostics before enforcing the shard result", async
expect(upload).toBeGreaterThan(merge);
expect(gate).toBeGreaterThan(upload);
expect(steps[merge]?.if).not.toContain("needs.test-shard.result");
const merges = steps.filter((step) => step.run?.includes("--reporter=json"));
expect(merges).toHaveLength(2);
for (const step of merges)
expect(step.run).toContain(
"--reporter=./scripts/ci/test-timing-reporter.mjs",
);
expect(steps[upload]?.with?.path).toContain("test-timings.json");
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
expect(steps[upload]?.with?.path).toContain(
".cache/ci-test-selection/report.json",
);
const summary = steps.find((step) =>
step.run?.includes("scripts/ci/test-report.mjs"),
);
expect(summary?.env?.CI_TEST_SELECTION).toBe(
"${{ needs.changes.outputs.test_selection }}",
);
const command = steps[gate]?.run;
if (command === undefined) throw new Error("Missing shard result gate");
for (const result of ["failure", "cancelled", "skipped", ""]) {
@@ -107,11 +126,11 @@ it("rejects successful shard reports with missing or duplicate portable files",
await workflowGit(root, ["add", "tests"]);
const run = async (names: readonly string[]) => {
await writeFile(
join(root, "test-results.json"),
join(root, "test-timings.json"),
JSON.stringify({
version: 1,
success: true,
numFailedTests: 0,
testResults: names.map((name) => ({ name, startTime: 0, endTime: 1 })),
files: names.map((name) => ({ name, durationMs: 1 })),
}),
);
return execute(process.execPath, [fileURLToPath(script)], {
@@ -173,20 +192,25 @@ it("exports portable timing history and removes it on a failed, partial, or inva
const summary = join(root, "summary.md");
const history = join(root, ".cache/ci-test-history/test-results.json");
const report = {
version: 1,
success: true,
numFailedTests: 0,
testResults: [
files: [
{
name: join(root, "tests/slow.test.ts"),
startTime: 10,
endTime: 110,
assertionResults: [{ large: "not timing data" }],
durationMs: 100,
phases: {
execution: 20,
preparation: 20,
environment: 10,
imports: 40,
setup: 10,
},
},
{ name: join(root, "tests/fast.test.ts"), startTime: 20, endTime: 30 },
{ name: join(root, "tests/fast.test.ts"), durationMs: 10 },
],
};
const run = async (value: unknown, shards = "success", merge = "success") => {
await writeFile(join(root, "test-results.json"), JSON.stringify(value));
await writeFile(join(root, "test-timings.json"), JSON.stringify(value));
await writeFile(summary, "");
return execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
@@ -200,7 +224,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva
};
await run(report);
const contents = await readFile(history, "utf8");
expect(contents).not.toContain("assertionResults");
expect(contents).not.toContain("phases");
expect(contents).not.toContain("startTime");
expect(contents).not.toContain(root);
expect(
parseTestDurations(contents, resolve(root, "different-checkout")),
@@ -211,7 +236,7 @@ it("exports portable timing history and removes it on a failed, partial, or inva
]),
);
for (const [value, shards, merge] of [
[{ ...report, success: false, numFailedTests: 1 }, "failure", "failure"],
[{ ...report, success: false }, "failure", "failure"],
[report, "cancelled", "success"],
[report, "success", "failure"],
] as const) {
@@ -224,8 +249,211 @@ it("exports portable timing history and removes it on a failed, partial, or inva
await expect(
run({
...report,
testResults: [{ name: "../outside.test.ts", startTime: 1, endTime: 3 }],
files: [{ name: "../outside.test.ts", durationMs: 2 }],
}),
).rejects.toMatchObject({ code: 1 });
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
await run(report);
await expect(
run({
...report,
files: [{ name: "tests/invalid.test.ts", durationMs: -1 }],
}),
).rejects.toMatchObject({ code: 1 });
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
});
it("recovers complete module costs from real blob reports without executing tests again", async () => {
const root = await createTestTempDirectory("rea-ci-module-timing-");
const config = join(root, "vitest.config.mjs");
await writeFile(
config,
'export default { test: { globals: true, include: ["sample.test.js"], maxWorkers: 1 } };',
);
await writeFile(
join(root, "sample.test.js"),
[
'import { writeFileSync } from "node:fs";',
"beforeAll(() => new Promise(resolve => setTimeout(resolve, 80)));",
"afterAll(() => new Promise(resolve => setTimeout(resolve, 80)));",
'it("has cheap assertions and expensive outer hooks", () => { writeFileSync("executed", "yes"); expect(1).toBe(1); });',
].join("\n"),
);
const vitest = fileURLToPath(
new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url),
);
const reporter = fileURLToPath(
new URL("../../../scripts/ci/test-timing-reporter.mjs", import.meta.url),
);
const run = (args: string[]) =>
execute(process.execPath, [vitest, ...args, "--config", config], {
cwd: root,
env: process.env,
timeout: 20_000,
});
await run(["run", "--reporter=blob"]);
expect(await readFile(join(root, "executed"), "utf8")).toBe("yes");
await rm(join(root, "executed"));
await run([
"--merge-reports",
`--reporter=${reporter}`,
"--reporter=json",
"--outputFile.json=body.json",
]);
await expect(readFile(join(root, "executed"))).rejects.toMatchObject({
code: "ENOENT",
});
const report = z
.object({
version: z.literal(1),
success: z.literal(true),
files: z.array(
z.object({
name: z.string(),
durationMs: z.number(),
phases: z.record(z.string(), z.number()),
}),
),
})
.parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")));
expect(
JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")),
).toMatchObject({
reason: "passed",
unhandledErrors: 0,
files: [{ state: "passed" }],
});
expect(report.files).toHaveLength(1);
const file = report.files[0]!;
const json = z
.object({
testResults: z.array(
z.object({ startTime: z.number(), endTime: z.number() }),
),
})
.parse(JSON.parse(await readFile(join(root, "body.json"), "utf8")));
const body = json.testResults[0]!;
expect(
file.phases.execution! - (body.endTime - body.startTime),
).toBeGreaterThan(100);
expect(file.phases.imports).toBeGreaterThan(0);
expect(file.phases.preparation).toBeGreaterThan(0);
expect(file.durationMs).toBeCloseTo(
Object.values(file.phases).reduce((sum, value) => sum + value, 0),
);
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
});
expect(
parseTestDurations(
await readFile(
join(root, ".cache/ci-test-history/test-results.json"),
"utf8",
),
root,
)?.get("sample.test.js"),
).toBe(file.durationMs);
await writeFile(
join(root, "sample.test.js"),
'it("fails", () => { throw new Error("fixture failure"); });',
);
await expect(run(["run", "--reporter=blob"])).rejects.toMatchObject({
code: 1,
});
await expect(
run(["--merge-reports", `--reporter=${reporter}`]),
).rejects.toMatchObject({ code: 1 });
const failed = JSON.parse(
await readFile(join(root, "test-timings.json"), "utf8"),
) as unknown;
expect(failed).toMatchObject({
version: 1,
success: false,
reason: "failed",
unhandledErrors: 0,
files: [{ state: "failed" }],
});
// Even a caller accidentally marking the merge successful cannot publish this report.
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
});
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
});
it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => {
const root = await createTestTempDirectory("rea-ci-selection-report-");
await workflowGit(root, ["init"]);
const paths = [
"src/sqlite/provider.test.ts",
"src/evm/provider.test.ts",
"tests/boundary/mcp/workflow.test.ts",
];
for (const path of paths) {
await mkdir(join(root, path, ".."), { recursive: true });
await writeFile(join(root, path), "// fixture\n");
}
await workflowGit(root, ["add", "src", "tests"]);
const timing = {
version: 1,
success: false,
reason: "failed",
unhandledErrors: 0,
files: paths.map((name, index) => ({
name,
durationMs: 10,
state: index === 1 ? "failed" : "passed",
})),
};
const run = async (report: unknown) => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
SHARD_RESULT: "failure",
MERGE_RESULT: "failure",
CI_VERIFY_TEST_INVENTORY: "true",
CI_TEST_SELECTION: JSON.stringify(
planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"),
),
GITHUB_SHA: "merge-sha",
CI_PR_HEAD_SHA: "pr-head-sha",
},
});
return JSON.parse(
await readFile(
join(root, ".cache/ci-test-selection/report.json"),
"utf8",
),
) as unknown;
};
expect(await run(timing)).toMatchObject({
observational: true,
complete: true,
selected: [paths[0], paths[2]],
omitted: [paths[1]],
missedFailures: [paths[1]],
executionSha: "merge-sha",
prHeadSha: "pr-head-sha",
});
for (const report of [
{ ...timing, files: timing.files.slice(0, 1) },
{ ...timing, unhandledErrors: 1 },
{ ...timing, reason: "interrupted" },
])
expect(await run(report)).toMatchObject({ complete: false });
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
code: 1,
});
await expect(
readFile(join(root, ".cache/ci-test-selection/report.json")),
).rejects.toMatchObject({ code: "ENOENT" });
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
});
@@ -17,24 +17,24 @@ describe("CI test history", () => {
const root = resolve("/workspace/rea");
const durations = parseTestDurations(
JSON.stringify({
version: 1,
success: true,
numFailedTests: 0,
testResults: [
files: [
{
name: join(root, "tests/slow.test.ts"),
startTime: 10,
endTime: 110,
durationMs: 100,
},
{
name: join(root, "tests/slow.test.ts"),
startTime: 20,
endTime: 80,
durationMs: 60,
},
{
name: resolve(root, "../outside.test.ts"),
startTime: 10,
endTime: 1000,
durationMs: 990,
},
{ name: "tests/negative.test.ts", durationMs: -1 },
{ name: "tests/zero.test.ts", durationMs: 0 },
{ name: "tests/invalid.test.ts", durationMs: "100" },
null,
],
}),
@@ -43,7 +43,20 @@ describe("CI test history", () => {
expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]]));
expect(
parseTestDurations('{"success": false, "numFailedTests": 1}', root),
parseTestDurations('{"version": 1, "success": false, "files": []}', root),
).toBeUndefined();
// Legacy test-body spans must not be mixed with complete module costs.
expect(
parseTestDurations(
JSON.stringify({
success: true,
numFailedTests: 0,
testResults: [
{ name: "tests/legacy.test.ts", startTime: 0, endTime: 100 },
],
}),
root,
),
).toBeUndefined();
expect(parseTestDurations("not json", root)).toBeUndefined();
expect(parseTestDurations("null", root)).toBeUndefined();
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { cp, readFile, writeFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
@@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")(
await workflowGit(root, ["init", "--initial-branch=main"]);
await workflowGit(root, ["add", "."]);
await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]);
const run = () =>
const run = (cwd = root) =>
execute(
process.execPath,
[
@@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")(
"test:run",
"--output-logs=errors-only",
],
{ cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } },
{ cwd, env: { ...process.env, TURBO_DAEMON: "false" } },
);
await run();
await writeFile(join(root, "README_zh.md"), "wrong command");
@@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")(
"prepared\n",
);
}
// Provider jobs receive the producer's cache in a fresh checkout, not its
// outputs. Turbo must restore identical inputs and rebuild changed ones.
const consumer = await createTestTempDirectory("rea-cache-consumer-");
await cp(root, consumer, {
recursive: true,
filter: (path) =>
basename(path) !== ".turbo" &&
!/^prepared-\d+\.txt$/u.test(basename(path)),
});
await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), {
recursive: true,
});
const restored = await run(consumer);
expect(restored.stdout).toContain(`${dependencies.length} cached`);
for (const index of dependencies.keys())
expect(
await readFile(join(consumer, `prepared-${index}.txt`), "utf8"),
).toBe("prepared\n");
await writeFile(
join(consumer, "artifact-input.txt"),
"changed producer input",
);
await run(consumer);
for (const index of dependencies.keys())
expect(
await readFile(join(consumer, `prepared-${index}.txt`), "utf8"),
).toBe("prepared\nprepared\n");
},
);
+6 -10
View File
@@ -5,14 +5,13 @@ import { BaseSequencer, type TestSpecification } from "vitest/node";
import { z } from "zod";
const reportSchema = z.object({
version: z.literal(1),
success: z.literal(true),
numFailedTests: z.literal(0),
testResults: z.array(z.unknown()),
files: z.array(z.unknown()),
});
const fileSchema = z.object({
name: z.string(),
startTime: z.number(),
endTime: z.number(),
durationMs: z.number().finite().positive(),
});
export function parseTestDurations(
@@ -30,15 +29,12 @@ export function parseTestDurations(
if (!report.success) return undefined;
const durations = new Map<string, number>();
for (const value of report.data.testResults) {
for (const value of report.data.files) {
const result = fileSchema.safeParse(value);
if (!result.success) continue;
const { name, startTime, endTime } = result.data;
const duration = endTime - startTime;
const { name, durationMs } = result.data;
const path = relative(root, resolve(root, name));
if (
!Number.isFinite(duration) ||
duration <= 0 ||
path === "" ||
path === ".." ||
path.startsWith(".." + sep) ||
@@ -46,7 +42,7 @@ export function parseTestDurations(
)
continue;
const key = path.split(sep).join("/");
durations.set(key, Math.max(durations.get(key) ?? 0, duration));
durations.set(key, Math.max(durations.get(key) ?? 0, durationMs));
}
return durations.size ? durations : undefined;
}
+240
View File
@@ -0,0 +1,240 @@
import { expect, it } from "vitest";
import {
compareTestSelection,
planTestSelection,
} from "../../scripts/ci/test-selection.mjs";
const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts";
const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts";
const adbSource = "src/adb/AdbProvider.test.ts";
const adbBoundary =
"tests/boundary/providers/adb/adbBugreportPublication.test.ts";
const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts";
const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts";
const processBoundary =
"tests/boundary/process/processCaptureLifecycle.test.ts";
const acceptance = "tests/acceptance/tools/localToolContracts.test.ts";
const composition = "tests/composition/directAnalysisCleanup.test.ts";
const evaluation = "tests/evaluation/testDiscovery.test.ts";
const passed = (name: string) => ({ name, durationMs: 10, state: "passed" });
it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary],
"pull_request",
"fast",
);
const inventory = [
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
];
const report = compareTestSelection(plan, inventory, inventory.map(passed));
expect(plan).toMatchObject({
mode: "candidate",
owners: ["adb", "sqlite"],
});
expect(report.selected).toEqual(
expect.arrayContaining([
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
]),
);
expect(report.omitted).toEqual([]);
});
it("omits only unrelated owned tests while retaining dynamic boundary suites", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const inventory = [
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
];
const report = compareTestSelection(plan, inventory, inventory.map(passed));
expect(report.selected).toEqual(
expect.arrayContaining([
sqliteSource,
sqliteBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
]),
);
expect(report.omitted).toEqual([adbSource, adbBoundary]);
expect(report.complete).toBe(true);
});
it.each([
["unowned shared source", "src/application/Workflow.ts"],
["fixture input", "tests/conformance/c/fixture.c"],
["configuration", "vitest.config.ts"],
["documentation", "README.md"],
["CLI implementation", "src/cli/apktoolCommands.ts"],
["MCP implementation", "src/server/registerSqliteTools.ts"],
["process ownership", "src/process/ProcessOwnership.ts"],
["acceptance test", acceptance],
["composition test", composition],
])("uses full fallback for %s changes", (_description, path) => {
expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({
mode: "full",
owners: [],
changedTests: [],
});
});
it("uses full fallback outside ordinary fast pull requests or without diff paths", () => {
expect(
planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast")
.mode,
).toBe("full");
expect(
planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"full",
).mode,
).toBe("full");
expect(planTestSelection([], "pull_request", "fast").mode).toBe("full");
});
it("keeps new changed tests and falls back when a changed test was removed", () => {
const addedTest = "src/sqlite/NewDatabaseRule.test.ts";
const candidate = planTestSelection([addedTest], "pull_request", "fast");
expect(candidate.changedTests).toEqual([addedTest]);
const inventory = [addedTest, adbSource];
expect(
compareTestSelection(candidate, inventory, inventory.map(passed)).selected,
).toContain(addedTest);
const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts";
const report = compareTestSelection(
planTestSelection([removedTest], "pull_request", "fast"),
[sqliteSource, adbSource],
[passed(sqliteSource), passed(adbSource)],
);
expect(report.plan.mode).toBe("full");
expect(report.selected).toEqual([adbSource, sqliteSource]);
expect(report.omitted).toEqual([]);
});
it("marks incomplete inventories and nonterminal outcomes as incomplete", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const inventory = [sqliteSource, adbSource];
const completeResults = inventory.map(passed);
expect(
compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete,
).toBe(false);
expect(
compareTestSelection(plan, inventory, [
...completeResults,
passed("tests/evaluation/unexpected.test.ts"),
]),
).toMatchObject({
complete: false,
unexpected: ["tests/evaluation/unexpected.test.ts"],
});
expect(
compareTestSelection(plan, inventory, [
...completeResults,
passed(adbSource),
]),
).toMatchObject({ complete: false, duplicateFiles: 1 });
for (const state of ["pending", "skipped"])
expect(
compareTestSelection(plan, inventory, [
passed(sqliteSource),
{ name: adbSource, durationMs: 0, state },
]),
).toMatchObject({ complete: false, unobserved: [adbSource] });
});
it("reports a failure from an omitted test, including collection or module failures", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const report = compareTestSelection(
plan,
[sqliteSource, adbSource],
[passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }],
);
expect(report.omitted).toContain(adbSource);
expect(report.missedFailures).toEqual([adbSource]);
});
it("rejects plans that name unowned owners or malformed changed tests", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
expect(() =>
compareTestSelection(
{ ...plan, owners: ["unknown"] },
[sqliteSource],
[passed(sqliteSource)],
),
).toThrow("Invalid observational test selection plan");
expect(() =>
compareTestSelection(
{ ...plan, changedTests: ["src/sqlite/not-a-test.ts"] },
[sqliteSource],
[passed(sqliteSource)],
),
).toThrow("Invalid observational test selection plan");
});
it("uses full fallback when a changed owner has no executable tests", () => {
const report = compareTestSelection(
planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
),
[adbSource],
[passed(adbSource)],
);
expect(report.plan.mode).toBe("full");
expect(report.omitted).toEqual([]);
});