Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c0cc77766e | ||
|
|
d13c9f840d | ||
|
|
0cf224e762 | ||
|
|
52d4ce6ced | ||
|
|
e05d3399f1 | ||
|
|
db3c9f084b | ||
|
|
0c7f4dfbd9 |
@@ -0,0 +1,45 @@
|
||||
name: CI timings
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI, Release]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
report:
|
||||
name: Report completed workflow attempt
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
# Completed PR runs can contain untrusted changes. Execute only the
|
||||
# reporter from the trusted default branch; consume jobs as API data.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Record queue, execution, and matched step timing
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_TIMING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
CI_TIMING_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
|
||||
run: node scripts/ci/job-timings.mjs
|
||||
- name: Upload CI timing artifacts
|
||||
if: always() && hashFiles('.cache/ci-job-timings/*') != ''
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ci-job-timings
|
||||
path: .cache/ci-job-timings/
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
retention-days: 14
|
||||
+106
-19
@@ -5,16 +5,26 @@ on:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
branches: [main, "release/*"]
|
||||
schedule:
|
||||
- cron: "23 0-2,4-23 * * *"
|
||||
- cron: "23 3 * * *"
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
inputs:
|
||||
source_sha:
|
||||
description: Exact release source; must match the caller revision.
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ci-${{ github.event.pull_request.number || github.ref }}
|
||||
# The latest main push verifies the full tree, including earlier changes.
|
||||
# Publication uses its own release workflow and concurrency group.
|
||||
cancel-in-progress: true
|
||||
# Complete qualification finishes even while main receives more pushes.
|
||||
# Scheduled/manual main runs coalesce to one active and the latest pending;
|
||||
# release qualification has its own exact-revision group.
|
||||
group: ci-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && 'qualification' || 'feedback' }}-${{ inputs.source_sha || github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}
|
||||
|
||||
env:
|
||||
HUSKY: "0"
|
||||
@@ -22,7 +32,7 @@ env:
|
||||
jobs:
|
||||
pr-title:
|
||||
name: Conventional PR title
|
||||
if: github.event_name == 'pull_request'
|
||||
if: github.event_name == 'pull_request' && inputs.source_sha == ''
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Validate title for Release Please
|
||||
@@ -39,8 +49,14 @@ jobs:
|
||||
changes:
|
||||
name: Classify changes
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
tier: ${{ steps.plan.outputs.tier }}
|
||||
qualification_run_id: ${{ steps.qualification.outputs.run_id }}
|
||||
code: ${{ steps.plan.outputs.code }}
|
||||
sqlite: ${{ steps.plan.outputs.sqlite }}
|
||||
runtime: ${{ steps.plan.outputs.runtime }}
|
||||
package: ${{ steps.plan.outputs.package }}
|
||||
apple: ${{ steps.plan.outputs.apple }}
|
||||
@@ -62,22 +78,52 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs.source_sha || github.sha }}
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Bind release qualification to the caller revision
|
||||
if: inputs.source_sha != ''
|
||||
env:
|
||||
SOURCE_SHA: ${{ inputs.source_sha }}
|
||||
run: |
|
||||
test "${SOURCE_SHA}" = "${GITHUB_SHA}"
|
||||
test "${SOURCE_SHA}" = "$(git rev-parse HEAD)"
|
||||
- name: Find a verified ancestor for push selection
|
||||
id: baseline
|
||||
if: github.event_name == 'push' && inputs.source_sha == ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: node scripts/ci/baseline.mjs
|
||||
- name: Admit scheduled full qualification
|
||||
id: qualification
|
||||
if: github.event_name == 'schedule' && inputs.source_sha == ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_REPOSITORY_ID: ${{ github.repository_id }}
|
||||
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
CI_FORCE_FULL: ${{ github.event.schedule == '23 3 * * *' }}
|
||||
run: node scripts/ci/qualification.mjs
|
||||
- name: Select verification lanes
|
||||
id: plan
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
||||
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
CI_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }}
|
||||
BASE_SHA: ${{ steps.baseline.outputs.base || github.event.pull_request.base.sha }}
|
||||
BASELINE_REASON: ${{ steps.baseline.outputs.reason }}
|
||||
HEAD_SHA: ${{ inputs.source_sha || github.event.pull_request.head.sha || github.sha }}
|
||||
CI_FULL: ${{ inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full') }}
|
||||
CI_SKIP_FULL: ${{ steps.qualification.outputs.run_full == 'false' }}
|
||||
run: node scripts/ci/plan.mjs
|
||||
|
||||
static:
|
||||
name: Static checks
|
||||
needs: changes
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Require successful change classification
|
||||
@@ -129,7 +175,14 @@ jobs:
|
||||
with:
|
||||
name: rea-test-runtime
|
||||
path: .
|
||||
- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
- name: Run complete portable suite with coverage
|
||||
if: needs.changes.outputs.tier == 'full'
|
||||
run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
env:
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- name: Run complete portable suite for fast feedback
|
||||
if: needs.changes.outputs.tier == 'fast'
|
||||
run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
|
||||
env:
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- uses: actions/upload-artifact@v4
|
||||
@@ -160,7 +213,7 @@ jobs:
|
||||
- name: Reuse portable build outputs across runs
|
||||
uses: ./.github/actions/portable-build-cache
|
||||
with:
|
||||
save: "true"
|
||||
save: ${{ github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
|
||||
- run: npm run test:prepare
|
||||
- name: Find successful default-branch timing history
|
||||
id: history
|
||||
@@ -172,7 +225,7 @@ jobs:
|
||||
run: |
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${CI_REPOSITORY}/actions/workflows/ci.yml/runs" \
|
||||
-f branch="${CI_DEFAULT_BRANCH}" -f event=push -f status=success -f per_page=1 \
|
||||
-f branch="${CI_DEFAULT_BRANCH}" -f event=schedule -f status=success -f per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty')"
|
||||
echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}"
|
||||
- name: Download previous timing report
|
||||
@@ -211,13 +264,13 @@ jobs:
|
||||
|
||||
sqlite-runtime:
|
||||
needs: [changes, build]
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
if: needs.changes.outputs.sqlite == 'true'
|
||||
uses: ./.github/workflows/sqlite-runtime.yml
|
||||
|
||||
test:
|
||||
name: Test (coverage + timing)
|
||||
name: Portable tests
|
||||
needs: [changes, test-shard]
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Require successful change classification
|
||||
@@ -241,7 +294,7 @@ jobs:
|
||||
merge-multiple: true
|
||||
- name: Merge coverage, JUnit, and timing reports
|
||||
id: merge
|
||||
if: needs.changes.outputs.code == 'true' && success()
|
||||
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success()
|
||||
run: >-
|
||||
npm run test:ci:merge --
|
||||
--coverage.reportsDirectory=coverage
|
||||
@@ -253,11 +306,22 @@ jobs:
|
||||
--reporter=json
|
||||
--outputFile.junit=test-results.xml
|
||||
--outputFile.json=test-results.json
|
||||
- name: Merge fast-suite JUnit and timing reports
|
||||
id: merge-fast
|
||||
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && success()
|
||||
run: >-
|
||||
npm exec -- vitest --merge-reports
|
||||
--reporter=default
|
||||
--reporter=junit
|
||||
--reporter=json
|
||||
--outputFile.junit=test-results.xml
|
||||
--outputFile.json=test-results.json
|
||||
- name: Summarize slowest test files
|
||||
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
|
||||
env:
|
||||
SHARD_RESULT: ${{ needs.test-shard.result }}
|
||||
MERGE_RESULT: ${{ steps.merge.outcome }}
|
||||
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
|
||||
CI_VERIFY_TEST_INVENTORY: "true"
|
||||
run: node scripts/ci/test-report.mjs
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
|
||||
@@ -270,7 +334,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
- name: Publish successful timing history
|
||||
if: needs.changes.outputs.code == 'true' && success() && needs.test-shard.result == 'success'
|
||||
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success() && needs.test-shard.result == 'success'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: test-timings
|
||||
@@ -381,9 +445,20 @@ jobs:
|
||||
- run: npm run verify:package
|
||||
env:
|
||||
npm_config_cache: ${{ steps.npm-cache.outputs.path }}
|
||||
REA_VERIFIED_PACKAGE_DIRECTORY: ${{ needs.changes.outputs.tier == 'full' && matrix.platform == 'linux' && matrix.architecture == 'x64' && '.cache/verified-package' || '' }}
|
||||
- name: Verify real Node Inspector discovery and observation
|
||||
timeout-minutes: 10
|
||||
run: npm run verify:inspector
|
||||
- name: Retain the exact qualified release package
|
||||
if: needs.changes.outputs.tier == 'full' && matrix.platform == 'linux' && matrix.architecture == 'x64'
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: verified-release-package
|
||||
path: .cache/verified-package/
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
retention-days: 7
|
||||
|
||||
apple-artifacts:
|
||||
name: Real Apple artifact CLI/MCP E2E
|
||||
@@ -657,7 +732,7 @@ jobs:
|
||||
|
||||
required:
|
||||
name: CI required
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ always() }}
|
||||
needs:
|
||||
[
|
||||
changes,
|
||||
@@ -687,12 +762,24 @@ jobs:
|
||||
website,
|
||||
]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Require every selected lane to pass
|
||||
id: gate
|
||||
env:
|
||||
CI_JOBS: ${{ toJSON(needs) }}
|
||||
CI_QUALIFICATION_DIRECTORY: .cache/ci-qualification
|
||||
run: node scripts/ci/required.mjs
|
||||
- name: Upload full qualification receipt artifact
|
||||
if: needs.changes.outputs.tier == 'full' && success()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.gate.outputs.receipt_name }}
|
||||
path: ${{ steps.gate.outputs.receipt_path }}
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
retention-days: 7
|
||||
|
||||
@@ -12,10 +12,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: real-binary-layout-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=768
|
||||
HUSKY: "0"
|
||||
|
||||
@@ -4,10 +4,6 @@ on:
|
||||
workflow_call:
|
||||
workflow_dispatch:
|
||||
|
||||
concurrency:
|
||||
group: real-browser-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
HUSKY: "0"
|
||||
|
||||
|
||||
@@ -7,10 +7,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: real-evm-interface-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=768
|
||||
HUSKY: "0"
|
||||
|
||||
@@ -7,10 +7,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: javascript-recovery-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
env:
|
||||
HUSKY: "0"
|
||||
NODE_OPTIONS: "--max-old-space-size=768"
|
||||
|
||||
@@ -7,10 +7,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: real-recorded-crash-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
NODE_OPTIONS: --max-old-space-size=768
|
||||
HUSKY: "0"
|
||||
|
||||
@@ -7,10 +7,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: real-web-network-captures-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HUSKY: "0"
|
||||
NODE_OPTIONS: --max-old-space-size=768
|
||||
|
||||
@@ -4,9 +4,6 @@ on:
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: real-web-runtime-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
env:
|
||||
HUSKY: "0"
|
||||
jobs:
|
||||
|
||||
@@ -4,9 +4,6 @@ on:
|
||||
workflow_dispatch:
|
||||
permissions:
|
||||
contents: read
|
||||
concurrency:
|
||||
group: real-web-source-map-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
env:
|
||||
HUSKY: "0"
|
||||
jobs:
|
||||
|
||||
@@ -290,13 +290,25 @@ jobs:
|
||||
if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true'
|
||||
run: npm run docs:check && git diff --exit-code
|
||||
|
||||
publish:
|
||||
name: Publish to npm
|
||||
qualification:
|
||||
name: Qualify exact release revision
|
||||
needs: [merged-release, release-please]
|
||||
if: >-
|
||||
always() && !cancelled() &&
|
||||
((needs.merged-release.result == 'success' && needs.merged-release.outputs.release_created == 'true') ||
|
||||
(needs.release-please.result == 'success' && inputs.phase == 'publish' && needs.release-please.outputs.release_created == 'true'))
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
uses: ./.github/workflows/ci.yml
|
||||
with:
|
||||
source_sha: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }}
|
||||
|
||||
publish:
|
||||
name: Publish to npm
|
||||
needs: [merged-release, release-please, qualification]
|
||||
if: >-
|
||||
always() && !cancelled() && needs.qualification.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -315,17 +327,20 @@ jobs:
|
||||
|
||||
- run: npm ci
|
||||
|
||||
- name: Prepare Windows native build toolchain
|
||||
run: sudo apt-get update && sudo apt-get install --no-install-recommends -y g++-mingw-w64-x86-64
|
||||
- name: Download the exact qualified package
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: verified-release-package
|
||||
path: .cache/verified-package/
|
||||
|
||||
- name: Build and verify bundled Windows native controls
|
||||
run: npm run build:windows-native && npm run verify:windows-native:artifact
|
||||
|
||||
- name: Verify packaged release before publishing
|
||||
run: npm run verify:package
|
||||
- name: Verify qualified package provenance and digest
|
||||
id: package
|
||||
run: node scripts/ci/verified-package.mjs validate .cache/verified-package
|
||||
|
||||
- name: Publish
|
||||
shell: bash
|
||||
env:
|
||||
VERIFIED_PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }}
|
||||
run: |
|
||||
version="$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)')"
|
||||
published_version="$(npm view "rea-agents@${version}" version --json 2>/dev/null || true)"
|
||||
@@ -333,7 +348,7 @@ jobs:
|
||||
echo "rea-agents@${version} is already published; verifying the existing package."
|
||||
else
|
||||
tag="$(node scripts/release-npm-tag.mjs "${version}")"
|
||||
npm publish --access public --tag "${tag}"
|
||||
npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}"
|
||||
fi
|
||||
|
||||
- name: Verify published CLI from npm
|
||||
|
||||
@@ -7,10 +7,6 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: website-check-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Website checks
|
||||
|
||||
+12
-3
@@ -6,7 +6,10 @@ checks to pass. Merging that PR into main starts publication automatically.
|
||||
|
||||
The Release workflow validates the merged release metadata and ancestry before
|
||||
creating a tag. npm and MCP Registry publication use the exact reviewed merge
|
||||
SHA returned by Release Please. Later main commits cannot change that source.
|
||||
SHA returned by Release Please. Before either registry is published, the release
|
||||
run calls full CI qualification at that exact revision, including coverage,
|
||||
portable providers, and every native package host. Later main commits cannot
|
||||
change or cancel that proof.
|
||||
Ordinary main pushes and closed, unmerged PRs cannot publish packages. Check
|
||||
both publication jobs and the published-package canary; a merged PR or GitHub
|
||||
tag alone does not establish that publication finished. Retry failed jobs in
|
||||
@@ -196,8 +199,14 @@ both registry publishes.
|
||||
See [npm's provenance implementation](https://github.com/npm/cli/blob/v11.16.0/workspaces/libnpmpublish/lib/provenance.js)
|
||||
for the use of GitHub's workflow ref and commit SHA.
|
||||
|
||||
The workflow builds the bundled Windows controls and verifies the packaged
|
||||
artifact before npm publication. It then verifies the published CLI, the
|
||||
Full qualification builds the bundled Windows controls and verifies the packaged
|
||||
artifact on its native hosts. The Linux x64 verifier retains the exact tarball
|
||||
it installed and tested, after its workspace cleanup completes. The npm job
|
||||
downloads that archive from the same workflow run and checks its source SHA,
|
||||
run identity/attempt, package and Node versions, host identity, and SHA-256.
|
||||
It publishes that archive without rebuilding or repacking it. Re-running failed
|
||||
publisher jobs can reuse a prior successful attempt from the same run.
|
||||
The workflow then verifies the published CLI, the
|
||||
capability-scoped MCP catalog, and the isolated package update path before
|
||||
publishing MCP Registry metadata.
|
||||
|
||||
|
||||
+90
-29
@@ -263,18 +263,34 @@ CI owns coverage; aggregate and domain/contract thresholds are maintained in
|
||||
Coverage does not replace named boundary or real-provider scenarios.
|
||||
|
||||
CI runs six native Vitest shards without retries. Each shard emits a blob
|
||||
report; the merge job produces aggregate coverage plus JUnit and JSON timing
|
||||
report; the merge job produces JUnit and JSON timing reports, plus aggregate
|
||||
coverage during full qualification. It retains available
|
||||
reports even when a shard fails, uploads available diagnostics, and writes the
|
||||
slowest files to the workflow summary. Partial reports are marked diagnostic
|
||||
only. Failed, cancelled, or missing selected shards still fail the required gate,
|
||||
and aggregate coverage thresholds remain enforced by the merge command.
|
||||
Only successful complete merges publish the small `test-timings` artifact, with
|
||||
and aggregate coverage thresholds remain enforced in full qualification.
|
||||
Only successful complete full-tier merges publish the small `test-timings` artifact, with
|
||||
checkout-relative paths; the build job downloads that artifact for the sequencer
|
||||
instead of the coverage/JUnit bundle. Missing history falls back to native
|
||||
sharding. Static checks, documentation, build, package verification, and
|
||||
real-system lanes remain separate jobs so one kind of evidence cannot stand in
|
||||
for another.
|
||||
|
||||
The separate `CI timings` workflow runs after CI or Release completes and uploads
|
||||
`ci-job-timings` with raw API pages and a normalized point-in-time report for that
|
||||
workflow attempt. It runs the reporter from the trusted default branch with
|
||||
read-only permissions, treating the triggering run as API data. Its summary separates queue
|
||||
waiting (`created_at` to `started_at`) from completed execution and reports
|
||||
percentiles plus matched `npm ci` and artifact-step durations.
|
||||
Queue excludes dependency waiting before job creation and does not establish
|
||||
whether account concurrency or runner availability caused the wait. Queued jobs with
|
||||
placeholder start times, skipped jobs, and missing or invalid timestamps remain
|
||||
unknown. The reporter targets the triggering run and attempt explicitly.
|
||||
Summed job elapsed time is parallel work, not workflow wall time, CPU time,
|
||||
memory usage, or billing. Step names identify samples, not complete transfer
|
||||
costs. Its queue, execution, and failures are independent of the required gate
|
||||
and publication. CPU time and peak process-family memory are not measured by this API.
|
||||
|
||||
The build job saves portable Turbo outputs across runs. Eight Linux real-system
|
||||
lanes restore this cache without waiting for the build job. The shared action
|
||||
derives its key from `test:prepare --dry=json` task hashes, runner OS and
|
||||
@@ -717,7 +733,7 @@ or the GitHub issue and pull-request templates run formatting and
|
||||
generated-document validation, including the small onboarding example consistency
|
||||
suite, without the source-test shards or native package
|
||||
lanes. Classification compares the PR head with its merge base, so later
|
||||
base-branch changes do not expand that scope. Static and coverage aggregate jobs
|
||||
base-branch changes do not expand that scope. Static and portable-test aggregate jobs
|
||||
remain present and fail if classification fails. Every code change retains the
|
||||
complete six-shard Linux suite; individual test files are not selected by
|
||||
imports or filenames.
|
||||
@@ -736,6 +752,12 @@ lane. Edits to verifier entrypoints run only by manual, licensed or self-hosted
|
||||
workflows run static checks and the Linux suite. Website changes run website
|
||||
checks. Provider workflow changes validate the workflow and run that provider's
|
||||
verifier. Manual-provider/Android workflow changes run workflow validation only.
|
||||
The four-version SQLite compatibility matrix has its own `sqlite` scope for
|
||||
SQLite implementation, contracts, CLI/MCP registration, fixtures, and shared
|
||||
runtime boundaries. SQLite and server-lifecycle test edits and the SQLite npm
|
||||
scripts also select it. Unrelated code edits retain the complete portable suite
|
||||
on the pinned Node version without starting four additional compatibility jobs.
|
||||
Full qualification retains every SQLite version.
|
||||
Shared process/contract/CLI boundaries, dependencies, other packaging inputs,
|
||||
unknown paths, and changes to the planner select the full baseline. Package
|
||||
script-only edits select their owning verifier when its entrypoint is
|
||||
@@ -746,38 +768,73 @@ verification stays manual. Renames account for both paths.
|
||||
This narrower selection trades PR-time coverage for runner capacity. A change to
|
||||
unowned shared runtime code that fails only on arm64, macOS, Windows, the Apple
|
||||
artifact lane, or a provider lane that does not own the file is not caught on
|
||||
the PR; the main-push full baseline reports it after merge. Ownership globs do
|
||||
the PR; scheduled full qualification reports it after merge. Ownership globs do
|
||||
not follow imports. Use the `ci:full` label when a change may have native-host
|
||||
or provider effects outside its owners.
|
||||
|
||||
Main pushes and manual dispatch retain the full baseline, including all portable
|
||||
provider lanes and the native matrix below. A `ci:full` PR label selects the full
|
||||
baseline on the next PR run; manual dispatch can request it immediately. Each
|
||||
Ordinary main pushes use the same owner scopes as PRs. Their diff starts at a
|
||||
successful ancestor push run of this CI workflow, rather than the preceding
|
||||
push: changes from failed or cancelled intermediate runs remain in scope.
|
||||
Missing, untrusted, or non-ancestor history selects every lane. The lookup is
|
||||
bounded to the last 100 successful push runs; lookup failures broaden selection.
|
||||
Every code change still runs all six portable shards, without coverage
|
||||
instrumentation in this fast tier. This deliberately avoids relying on an
|
||||
import graph for generated files, filesystem inputs, subprocesses, or bridges.
|
||||
|
||||
Scheduled qualification is admitted hourly at minute 23. A forced full run at
|
||||
03:23 UTC refreshes runner and dependency drift every day. Other hourly runs
|
||||
can reuse a successful full-qualification receipt for the exact source SHA
|
||||
from the same repository's trusted CI workflow on its default branch. Both the
|
||||
receipt and original workflow run must be less than 24 hours old, so retrying an
|
||||
old gate cannot renew stale proof. Missing, expired, failed, foreign, or uncertain proof runs full
|
||||
validation; API lookup failure also runs full. Lookups examine at most 100 named
|
||||
receipts and three candidate runs within a 20-second budget. Reused runs select
|
||||
no validation lanes, identify the original run, and produce no new receipt, so
|
||||
reuse cannot indefinitely extend proof freshness. They make no fresh provider
|
||||
observation. Manual dispatch, `ci:full`, and exact release qualification always
|
||||
run full validation.
|
||||
|
||||
Full qualification includes aggregate coverage,
|
||||
every portable provider lane, and all native hosts below. A `ci:full` PR label
|
||||
selects full qualification on the next PR run; manual dispatch can request it
|
||||
immediately. Fast-tier broad fallbacks retain all owner lanes and the three-host
|
||||
package matrix; Intel macOS and coverage remain full-tier obligations. Each
|
||||
classification job records its selection and broad-fallback reasons in the run
|
||||
summary. Representative Git-diff cases live in
|
||||
`tests/boundary/filesystem/ciChangeScope.test.ts`.
|
||||
|
||||
`CI required` checks classification and every selected lane, rejecting failed,
|
||||
cancelled, or unexpectedly skipped work. Existing required check names are
|
||||
preserved; changing branch protection to require the new aggregate is a separate
|
||||
repository-settings operation. Newer commits cancel superseded CI for the same
|
||||
PR or main branch. Release publication has its own concurrency group and retains
|
||||
its package verification and public-registry canary. Licensed/self-hosted
|
||||
`CI required` always runs and checks classification and every selected lane,
|
||||
rejecting failed, cancelled, or unexpectedly skipped work. Branch protection
|
||||
should require this aggregate, `Static checks`, and `Portable tests`. Fast runs
|
||||
cancel superseded feedback for the same PR or main branch. Scheduled/manual
|
||||
qualification has a separate concurrency group: active work finishes and only
|
||||
the latest pending run is retained. Release qualification is grouped by its
|
||||
exact source SHA and cannot be cancelled by newer main pushes. Provider and
|
||||
website lanes inherit their caller's scheduling; reusable
|
||||
workflows have no independent cancellation group. Direct manual lane dispatches
|
||||
run independently on disposable hosted runners. Publication
|
||||
requires that qualification to succeed and consumes its exact verified tarball,
|
||||
checking source SHA, workflow run, run attempt, package/Node version and SHA-256
|
||||
before npm publication. Earlier attempts from the same workflow run are accepted
|
||||
when rerunning failed publisher jobs; future attempts and other runs are rejected.
|
||||
The public-registry canary still verifies the published package. Licensed/self-hosted
|
||||
provider workflows and the Android smoke test retain their explicit triggers.
|
||||
|
||||
CI exercises the pinned Node.js runtime on native hosted runners:
|
||||
|
||||
| Host | Runner | Baseline checks |
|
||||
| --------------------- | ------------------ | ---------------------------------------------------------------------------------------- |
|
||||
| Linux x64 | `ubuntu-latest` | Installed package and real Node Inspector CLI/MCP |
|
||||
| Linux arm64 (aarch64) | `ubuntu-24.04-arm` | Installed package and real Node Inspector CLI/MCP |
|
||||
| macOS 15 arm64 | `macos-15` | Installed package and real Node Inspector CLI/MCP |
|
||||
| macOS 15 x64 | `macos-15-intel` | Installed package and real Node Inspector CLI/MCP (main, dispatch and `ci:full` only) |
|
||||
| Windows x64 | `windows-latest` | Curated capabilities, native controls, installed package and real Node Inspector CLI/MCP |
|
||||
| Host | Runner | Baseline checks |
|
||||
| --------------------- | ------------------ | ------------------------------------------------------------------------------------------- |
|
||||
| Linux x64 | `ubuntu-latest` | Installed package and real Node Inspector CLI/MCP |
|
||||
| Linux arm64 (aarch64) | `ubuntu-24.04-arm` | Installed package and real Node Inspector CLI/MCP |
|
||||
| macOS 15 arm64 | `macos-15` | Installed package and real Node Inspector CLI/MCP |
|
||||
| macOS 15 x64 | `macos-15-intel` | Installed package and real Node Inspector CLI/MCP (hourly, dispatch, release and `ci:full`) |
|
||||
| Windows x64 | `windows-latest` | Curated capabilities, native controls, installed package and real Node Inspector CLI/MCP |
|
||||
|
||||
macOS x64 is the slowest package host. Pull requests without the `ci:full` label
|
||||
omit it, so darwin x64 installed-package and Inspector regressions surface on
|
||||
the main-push full baseline rather than on the PR.
|
||||
macOS x64 is the slowest package host. Ordinary PRs and main pushes omit it,
|
||||
so darwin x64 installed-package and Inspector regressions surface in full
|
||||
qualification. Schedule admission can be delayed by GitHub; the hourly trigger
|
||||
does not guarantee an hourly completed result. Release publication waits for
|
||||
its own complete result at the exact released revision.
|
||||
|
||||
Package lanes assert the actual Node platform/architecture before verification
|
||||
and record those values with the Node version. The package matrix runs up to
|
||||
@@ -803,17 +860,21 @@ The Linux source-test shards download a shared runtime and generated test
|
||||
metadata snapshot produced by the build job with `npm run test:prepare`. The
|
||||
snapshot includes `dist/`, generated skills, the MCP tool catalog, product
|
||||
catalog, and managed-conformance metadata. Each shard still installs the locked
|
||||
dependencies and runs its assigned coverage-enabled tests; it no longer
|
||||
dependencies and runs its assigned tests; full qualification enables coverage. It no longer
|
||||
regenerates the same metadata independently.
|
||||
|
||||
The build job also retrieves the merged timing report from the last successful
|
||||
default-branch push and includes that same report in each shard's snapshot.
|
||||
default-branch scheduled qualification and includes that same report in each shard's snapshot.
|
||||
CI assigns discovered files by historical duration, with a deterministic
|
||||
fallback weight for new files, rather than dividing only by file count. Every
|
||||
discovered file still belongs to exactly one shard; worker limits, project
|
||||
ordering, coverage instrumentation, and aggregate thresholds remain unchanged.
|
||||
discovered file still belongs to exactly one shard; worker limits and project
|
||||
ordering remain unchanged. The merge job rejects successful reports with missing,
|
||||
unexpected, or duplicate portable files. Coverage instrumentation and aggregate
|
||||
thresholds remain enforced in full qualification.
|
||||
Missing or unusable history falls back to Vitest's default assignment. Only
|
||||
timing data is reused; persistent compile caches remain disabled.
|
||||
timing data is reused; live provider results are never cached. Turbo outputs are
|
||||
restored by task hash, with trusted default-branch runs as cache writers and PRs
|
||||
as restore-only consumers. npm caches hold downloads, not `node_modules`.
|
||||
|
||||
These native baseline checks complement the Linux source-test shards and the
|
||||
separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA,
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const branch = process.env.CI_DEFAULT_BRANCH;
|
||||
const head = process.env.HEAD_SHA;
|
||||
const shaPattern = /^[0-9a-f]{40}$/i;
|
||||
|
||||
function ancestor(base) {
|
||||
try {
|
||||
execFileSync("git", ["merge-base", "--is-ancestor", base, head], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function write(base, reason) {
|
||||
const output = { base, reason };
|
||||
if (process.env.GITHUB_OUTPUT) {
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`base=${base}\nreason=${reason.replace(/[\r\n]/g, " ")}\n`,
|
||||
);
|
||||
}
|
||||
console.log(JSON.stringify(output));
|
||||
}
|
||||
|
||||
if (!repository || !branch || !head || !shaPattern.test(head)) {
|
||||
write("", "Repository, default branch, or checked-out head is invalid");
|
||||
} else {
|
||||
try {
|
||||
const response = execFileSync(
|
||||
"gh",
|
||||
[
|
||||
"api",
|
||||
"--method",
|
||||
"GET",
|
||||
`repos/${repository}/actions/workflows/ci.yml/runs`,
|
||||
"-f",
|
||||
`branch=${branch}`,
|
||||
"-f",
|
||||
"event=push",
|
||||
"-f",
|
||||
"status=success",
|
||||
"-F",
|
||||
"per_page=100",
|
||||
],
|
||||
{ encoding: "utf8", timeout: 15_000, maxBuffer: 4 * 1024 * 1024 },
|
||||
);
|
||||
const runs = JSON.parse(response).workflow_runs;
|
||||
if (!Array.isArray(runs))
|
||||
throw new Error("GitHub returned no workflow run list");
|
||||
const base = runs.find(
|
||||
(run) =>
|
||||
run?.event === "push" &&
|
||||
run?.conclusion === "success" &&
|
||||
run?.head_branch === branch &&
|
||||
run?.head_repository?.full_name === repository &&
|
||||
typeof run?.head_sha === "string" &&
|
||||
shaPattern.test(run.head_sha) &&
|
||||
run.head_sha.toLowerCase() !== head.toLowerCase() &&
|
||||
ancestor(run.head_sha),
|
||||
)?.head_sha;
|
||||
if (base) write(base, "");
|
||||
else
|
||||
write(
|
||||
"",
|
||||
"No trusted successful CI push baseline is an ancestor of this head",
|
||||
);
|
||||
} catch {
|
||||
write("", "Could not read a successful CI push baseline from GitHub");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,353 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MAX_BUFFER = 16 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings");
|
||||
const STEP_COSTS = [
|
||||
{ key: "npmCi", pattern: /\bnpm ci\b/iu },
|
||||
{ key: "artifactUpload", pattern: /upload.*artifact/iu },
|
||||
{ key: "artifactDownload", pattern: /download.*artifact/iu },
|
||||
];
|
||||
|
||||
/** Fetch and summarize one workflow attempt without turning telemetry into a gate. */
|
||||
async function collectJobTimings({
|
||||
repository,
|
||||
runId,
|
||||
runAttempt,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
observedAt = new Date().toISOString(),
|
||||
env = process.env,
|
||||
}) {
|
||||
const identity = validateIdentity({ repository, runId, runAttempt });
|
||||
const endpoint = `repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100`;
|
||||
const { stdout } = await execFileAsync(
|
||||
"gh",
|
||||
["api", "--paginate", "--slurp", endpoint],
|
||||
{ env, encoding: "utf8", maxBuffer: MAX_BUFFER, timeout: TIMEOUT_MS },
|
||||
);
|
||||
const rawPath = join(outputDirectory, "jobs.json");
|
||||
const reportPath = join(outputDirectory, "report.json");
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(rawPath, stdout, "utf8");
|
||||
const pages = parsePages(stdout);
|
||||
const jobs = pages.flatMap(({ jobs: pageJobs }, index) =>
|
||||
validateJobs(pageJobs, index),
|
||||
);
|
||||
const report = normalizeReport({ identity, jobs, observedAt });
|
||||
|
||||
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
if (summaryPath !== undefined)
|
||||
await appendSummary(summaryPath, report, rawPath, reportPath);
|
||||
return { rawPath, reportPath, report };
|
||||
}
|
||||
|
||||
function validateIdentity({ repository, runId, runAttempt }) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
return {
|
||||
repository,
|
||||
runId: parsePositiveInteger(runId, "CI_TIMING_RUN_ID"),
|
||||
runAttempt: parsePositiveInteger(runAttempt, "CI_TIMING_RUN_ATTEMPT"),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
if (typeof value === "string" && /^[1-9]\d*$/u.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
|
||||
function parsePages(stdout) {
|
||||
let pages;
|
||||
try {
|
||||
pages = JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("GitHub jobs API returned malformed JSON", { cause });
|
||||
}
|
||||
if (!Array.isArray(pages) || pages.length === 0)
|
||||
throw new Error("GitHub jobs API slurp must contain a nonempty page array");
|
||||
for (const page of pages)
|
||||
if (
|
||||
typeof page !== "object" ||
|
||||
page === null ||
|
||||
Array.isArray(page) ||
|
||||
!Array.isArray(page.jobs)
|
||||
)
|
||||
throw new Error("GitHub jobs API page must contain a jobs array");
|
||||
return pages;
|
||||
}
|
||||
|
||||
function validateJobs(jobs, pageIndex) {
|
||||
return jobs.map((job, jobIndex) => {
|
||||
if (
|
||||
typeof job !== "object" ||
|
||||
job === null ||
|
||||
Array.isArray(job) ||
|
||||
typeof job.name !== "string" ||
|
||||
job.name.length === 0 ||
|
||||
typeof job.status !== "string"
|
||||
)
|
||||
throw new Error(
|
||||
`GitHub jobs API page ${String(pageIndex + 1)} job ${String(jobIndex + 1)} is malformed`,
|
||||
);
|
||||
return job;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeReport({ identity, jobs, observedAt }) {
|
||||
const normalizedJobs = jobs.map(normalizeJob);
|
||||
const queues = normalizedJobs
|
||||
.map(({ queue }) => queue.seconds)
|
||||
.filter((value) => value !== null);
|
||||
const completedExecutions = normalizedJobs
|
||||
.filter(
|
||||
({ status, conclusion }) =>
|
||||
status === "completed" && conclusion !== "skipped",
|
||||
)
|
||||
.map(({ execution }) => execution.seconds)
|
||||
.filter((value) => value !== null);
|
||||
const completedDurationSumSeconds = completedExecutions.reduce(
|
||||
(sum, seconds) => sum + seconds,
|
||||
0,
|
||||
);
|
||||
const completedExecutionUnknown = normalizedJobs.filter(
|
||||
({ status, conclusion, execution }) =>
|
||||
status === "completed" &&
|
||||
conclusion !== "skipped" &&
|
||||
execution.seconds === null,
|
||||
).length;
|
||||
const skipped = normalizedJobs.filter(
|
||||
({ conclusion }) => conclusion === "skipped",
|
||||
).length;
|
||||
const costs = Object.fromEntries(STEP_COSTS.map(({ key }) => [key, []]));
|
||||
for (const job of normalizedJobs)
|
||||
for (const step of job.steps)
|
||||
if (step.costKey !== null)
|
||||
costs[step.costKey].push({ job: job.name, ...step });
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
run: {
|
||||
...identity,
|
||||
observedAt,
|
||||
snapshot:
|
||||
normalizedJobs.length > 0 &&
|
||||
normalizedJobs.every(({ status }) => status === "completed")
|
||||
? "completed job status snapshot; unknown timestamps remain unmeasured"
|
||||
: "partial; job statuses may still be queued or in progress",
|
||||
},
|
||||
counts: {
|
||||
jobs: normalizedJobs.length,
|
||||
completed: normalizedJobs.filter(({ status }) => status === "completed")
|
||||
.length,
|
||||
skipped,
|
||||
queueUnknown: normalizedJobs.filter(({ queue }) => queue.seconds === null)
|
||||
.length,
|
||||
executionUnknown: normalizedJobs.filter(
|
||||
({ execution }) => execution.seconds === null,
|
||||
).length,
|
||||
completedExecutionUnknown,
|
||||
},
|
||||
aggregates: {
|
||||
queueSeconds: percentileSummary(queues),
|
||||
completedExecutionSeconds: percentileSummary(completedExecutions),
|
||||
summedCompletedJobElapsedSeconds: {
|
||||
value:
|
||||
completedExecutions.length === 0
|
||||
? null
|
||||
: round(completedDurationSumSeconds),
|
||||
knownCount: completedExecutions.length,
|
||||
unknownCount: completedExecutionUnknown,
|
||||
label:
|
||||
"sum of known completed job elapsed durations; incomplete when timestamps are unknown; not CPU time, billing, or workflow wall time",
|
||||
},
|
||||
},
|
||||
jobs: normalizedJobs,
|
||||
stepCostInterpretation:
|
||||
"Matched install/artifact steps by step name; these are not total artifact transfer costs.",
|
||||
stepCosts: costs,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeJob(job) {
|
||||
const skipped = job.conclusion === "skipped";
|
||||
const hasStarted = ["in_progress", "completed"].includes(job.status);
|
||||
const queue = skipped
|
||||
? unknownDuration("job was skipped")
|
||||
: !hasStarted
|
||||
? unknownDuration("job start is not established by its status")
|
||||
: duration(job.created_at, job.started_at);
|
||||
const execution = skipped
|
||||
? unknownDuration("job was skipped")
|
||||
: job.status === "completed"
|
||||
? duration(job.started_at, job.completed_at)
|
||||
: unknownDuration("job is not completed");
|
||||
const steps = Array.isArray(job.steps) ? job.steps.map(normalizeStep) : [];
|
||||
return {
|
||||
id: Number.isSafeInteger(job.id) && job.id > 0 ? job.id : null,
|
||||
url: stringOrNull(job.html_url),
|
||||
name: job.name,
|
||||
workflowName: stringOrNull(job.workflow_name),
|
||||
status: job.status,
|
||||
conclusion: stringOrNull(job.conclusion),
|
||||
createdAt: stringOrNull(job.created_at),
|
||||
startedAt: stringOrNull(job.started_at),
|
||||
completedAt: stringOrNull(job.completed_at),
|
||||
queue,
|
||||
execution,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeStep(step) {
|
||||
if (
|
||||
typeof step !== "object" ||
|
||||
step === null ||
|
||||
Array.isArray(step) ||
|
||||
typeof step.name !== "string"
|
||||
)
|
||||
throw new Error("GitHub jobs API contains a malformed job step");
|
||||
const cost = STEP_COSTS.find(({ pattern }) => pattern.test(step.name));
|
||||
return {
|
||||
name: step.name,
|
||||
costKey: cost?.key ?? null,
|
||||
duration: duration(step.started_at, step.completed_at),
|
||||
};
|
||||
}
|
||||
|
||||
function duration(startValue, endValue) {
|
||||
if (typeof startValue !== "string" || typeof endValue !== "string")
|
||||
return unknownDuration("timestamp unavailable");
|
||||
const start = Date.parse(startValue);
|
||||
const end = Date.parse(endValue);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end))
|
||||
return unknownDuration("invalid timestamp");
|
||||
const milliseconds = end - start;
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0)
|
||||
return unknownDuration("timestamps out of order");
|
||||
return { seconds: round(milliseconds / 1_000), reason: null };
|
||||
}
|
||||
|
||||
function unknownDuration(reason) {
|
||||
return { seconds: null, reason };
|
||||
}
|
||||
|
||||
function percentileSummary(values) {
|
||||
return {
|
||||
count: values.length,
|
||||
p50: nearestRank(values, 0.5),
|
||||
p95: nearestRank(values, 0.95),
|
||||
unit: "seconds",
|
||||
};
|
||||
}
|
||||
|
||||
function nearestRank(values, percentile) {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
return round(sorted[Math.ceil(percentile * sorted.length) - 1]);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 1_000) / 1_000;
|
||||
}
|
||||
|
||||
function stringOrNull(value) {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
async function appendSummary(summaryPath, report, rawPath, reportPath) {
|
||||
await mkdir(dirname(summaryPath), { recursive: true });
|
||||
const rows = [
|
||||
["Queue", report.aggregates.queueSeconds],
|
||||
["Completed execution", report.aggregates.completedExecutionSeconds],
|
||||
]
|
||||
.map(
|
||||
([label, item]) =>
|
||||
`| ${label} | ${display(item.p50)} | ${display(item.p95)} | ${String(item.count)} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const jobRows = report.jobs
|
||||
.map(
|
||||
(job) =>
|
||||
`| ${escapeCell(job.name)} | ${job.status} | ${job.conclusion ?? "unknown"} | ${display(job.queue.seconds)} | ${job.queue.seconds === null ? job.queue.reason : ""} | ${display(job.execution.seconds)} | ${job.execution.seconds === null ? job.execution.reason : ""} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const costRows = STEP_COSTS.map(({ key }) => {
|
||||
const samples = report.stepCosts[key];
|
||||
const values = samples
|
||||
.map(({ duration: item }) => item.seconds)
|
||||
.filter((value) => value !== null);
|
||||
return `| ${key} | ${String(samples.length)} | ${display(nearestRank(values, 0.5))} | ${samples.filter(({ duration: item }) => item.seconds === null).length} |`;
|
||||
}).join("\n");
|
||||
const text = [
|
||||
"### CI job timing snapshot",
|
||||
"",
|
||||
`Observed at ${report.run.observedAt} for run ${String(report.run.runId)}, attempt ${String(report.run.runAttempt)}. This is a point-in-time snapshot of fetched jobs. Queue starts at job creation and excludes preceding dependency waiting. Missing, queued, in-progress, invalid, or out-of-order timestamps are reported as unknown. Elapsed durations are not CPU time or billing data.`,
|
||||
"",
|
||||
`Fetched ${String(report.counts.jobs)} jobs; ${String(report.counts.completed)} are completed (${String(report.counts.skipped)} skipped); ${String(report.counts.queueUnknown)} queue durations and ${String(report.counts.executionUnknown)} execution durations (${String(report.counts.completedExecutionUnknown)} for non-skipped completed jobs) are unknown.`,
|
||||
"",
|
||||
"| Metric | p50 (s) | p95 (s) | Samples |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
rows,
|
||||
"",
|
||||
"| Job | Status | Conclusion | Queue (s) | Queue detail | Execution (s) | Execution detail |",
|
||||
"| --- | --- | --- | ---: | --- | ---: | --- |",
|
||||
jobRows,
|
||||
"",
|
||||
`Sum of known completed job elapsed durations: **${display(report.aggregates.summedCompletedJobElapsedSeconds.value)} s** across ${String(report.aggregates.summedCompletedJobElapsedSeconds.knownCount)} known and ${String(report.aggregates.summedCompletedJobElapsedSeconds.unknownCount)} unknown jobs; this is not CPU time, billing, or workflow wall time.`,
|
||||
"",
|
||||
"Matched install/artifact steps by name; these are not total artifact transfer costs.",
|
||||
"",
|
||||
"| Matched step group | Samples | p50 (s) | Unknown |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
costRows,
|
||||
"",
|
||||
`Reports: \`${rawPath}\`, \`${reportPath}\`.`,
|
||||
"",
|
||||
].join("\n");
|
||||
await writeFile(summaryPath, text, { encoding: "utf8", flag: "a" });
|
||||
}
|
||||
|
||||
function display(value) {
|
||||
return value === null ? "unknown" : String(value);
|
||||
}
|
||||
|
||||
function escapeCell(value) {
|
||||
return value.replaceAll("|", "\\|").replaceAll("\n", " ");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
const result = await collectJobTimings({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
runId: process.env.CI_TIMING_RUN_ID,
|
||||
runAttempt: process.env.CI_TIMING_RUN_ATTEMPT,
|
||||
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
summaryPath: process.env.GITHUB_STEP_SUMMARY,
|
||||
});
|
||||
process.stdout.write(
|
||||
`Wrote ${result.report.jobs.length} job timing records to ${result.reportPath}\n`,
|
||||
);
|
||||
} catch (cause) {
|
||||
process.stderr.write(
|
||||
`CI timing telemetry failed: ${cause instanceof Error ? cause.message : String(cause)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
+129
-103
@@ -1,30 +1,17 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
|
||||
const flags = [
|
||||
...new Set([
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"apple",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
]),
|
||||
];
|
||||
// Test files select the native-host lanes that run them explicitly. Provider
|
||||
// scopes remain implementation-owned, so test-only edits stay off those lanes.
|
||||
const NATIVE_TEST_OWNERS = ["apple", "package"];
|
||||
// Test files select the host/runtime lanes that execute their focused suites.
|
||||
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
|
||||
// Actionlint validates workflow syntax only. The semantic suite asserts these
|
||||
// workflows' release, publication and self-hosted runner safeguards.
|
||||
const GUARDED_WORKFLOWS = new Set([
|
||||
".github/workflows/release.yml",
|
||||
".github/workflows/real-ghidra-windows.yml",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
".github/workflows/ci-timings.yml",
|
||||
]);
|
||||
// Sources compiled by build-conformance-fixtures.mjs. The readiness journey
|
||||
// analyzes the built C fixture as well as its own applications.
|
||||
@@ -63,6 +50,10 @@ function classifyPackage(base, head) {
|
||||
]),
|
||||
].filter((key) => beforeScripts?.[key] !== afterScripts?.[key]);
|
||||
for (const script of changed) {
|
||||
if (script === "test:sqlite" || script === "test:sqlite:run") {
|
||||
selected.sqlite = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
/^(test(?::|$)|check(?::|$)|lint(?::|$)|format(?::|$)|knip$|jscpd$)/.test(
|
||||
script,
|
||||
@@ -108,98 +99,133 @@ function classifyPackage(base, head) {
|
||||
}
|
||||
|
||||
const event = process.env.GITHUB_EVENT_NAME;
|
||||
let files = [];
|
||||
if (
|
||||
event === "push" ||
|
||||
if (!["pull_request", "push", "schedule", "workflow_dispatch"].includes(event))
|
||||
throw new Error(`Unsupported CI event: ${event}`);
|
||||
const requestedFull =
|
||||
event === "schedule" ||
|
||||
event === "workflow_dispatch" ||
|
||||
process.env.CI_FULL === "true"
|
||||
) {
|
||||
process.env.CI_FULL === "true";
|
||||
const tier =
|
||||
event === "schedule" &&
|
||||
process.env.CI_SKIP_FULL === "true" &&
|
||||
process.env.CI_FULL !== "true"
|
||||
? "reused"
|
||||
: requestedFull
|
||||
? "full"
|
||||
: "fast";
|
||||
let files = [];
|
||||
if (tier === "reused") {
|
||||
reasons.push(
|
||||
"Recent exact-SHA full qualification reused; no fresh provider observation",
|
||||
);
|
||||
} else if (tier === "full") {
|
||||
full("Full baseline requested by event or ci:full label");
|
||||
} else if (event === "pull_request") {
|
||||
const base = git("merge-base", process.env.BASE_SHA, process.env.HEAD_SHA);
|
||||
} else if (event === "pull_request" || event === "push") {
|
||||
const head = process.env.HEAD_SHA;
|
||||
// Disable rename detection so both the removed and added paths participate.
|
||||
files = git("diff", "--name-only", "--no-renames", "-z", base, head)
|
||||
.split("\0")
|
||||
.filter(Boolean);
|
||||
for (const file of files) {
|
||||
if (
|
||||
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
|
||||
file,
|
||||
) ||
|
||||
file.startsWith("docs/") ||
|
||||
file.startsWith(".github/ISSUE_TEMPLATE/")
|
||||
) {
|
||||
selected.docs = true;
|
||||
selected.website ||= matchesAny(file, SCOPES.website);
|
||||
continue;
|
||||
}
|
||||
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
|
||||
selected.code = true;
|
||||
for (const owner of NATIVE_TEST_OWNERS)
|
||||
selected[owner] ||= matchesAny(file, SCOPES[owner]);
|
||||
continue;
|
||||
}
|
||||
if (matchesAny(file, MANUAL_VERIFIERS)) {
|
||||
selected.code = true;
|
||||
continue;
|
||||
}
|
||||
const owners = Object.entries(SCOPES)
|
||||
.filter(([, patterns]) =>
|
||||
patterns.some((pattern) => matchesGlob(file, pattern)),
|
||||
)
|
||||
.map(([owner]) => owner);
|
||||
for (const owner of owners) selected[owner] = true;
|
||||
if (
|
||||
file.startsWith(".github/workflows/") ||
|
||||
file === ".github/actionlint.yaml"
|
||||
) {
|
||||
selected.workflows = true;
|
||||
if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
|
||||
if (file !== ".github/workflows/ci.yml") continue;
|
||||
}
|
||||
if (owners.length === 1 && owners[0] === "website") continue;
|
||||
if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
|
||||
if (file === "package.json") {
|
||||
classifyPackage(base, head);
|
||||
continue;
|
||||
}
|
||||
selected.code = true;
|
||||
if (matchesAny(file, FIXTURE_SOURCES)) {
|
||||
selected.fixtures = true;
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
|
||||
if (file.startsWith("tests/")) continue;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
)
|
||||
) {
|
||||
full(`Shared boundary: ${file}`);
|
||||
} else if (owners.length) {
|
||||
if (file.startsWith("src/") || file.startsWith("bridge/"))
|
||||
selected.runtime = true;
|
||||
} else if (/^(src|bridge)\//.test(file)) {
|
||||
// Unowned runtime sources keep the Linux x64 installed-package baseline;
|
||||
// ci:full, main pushes and manual dispatch retain every native host.
|
||||
selected.runtime = true;
|
||||
reasons.push(`Unowned runtime source: ${file}`);
|
||||
} else if (file.startsWith(".agents/skills/")) {
|
||||
selected.runtime = true;
|
||||
selected.docs = true;
|
||||
} else {
|
||||
full(`Unclassified input: ${file}`);
|
||||
let base;
|
||||
if (event === "pull_request") {
|
||||
base = git("merge-base", process.env.BASE_SHA, head);
|
||||
} else {
|
||||
const candidate = process.env.BASE_SHA;
|
||||
const baselineReason = process.env.BASELINE_REASON;
|
||||
try {
|
||||
if (!candidate || !/^[0-9a-f]{40}$/i.test(candidate))
|
||||
throw new Error(
|
||||
baselineReason || "No valid successful push baseline was found",
|
||||
);
|
||||
execFileSync("git", ["merge-base", "--is-ancestor", candidate, head], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
base = candidate;
|
||||
} catch {
|
||||
full(
|
||||
baselineReason ||
|
||||
"No valid successful push baseline ancestor; selecting all lanes",
|
||||
);
|
||||
}
|
||||
}
|
||||
} else throw new Error(`Unsupported CI event: ${event}`);
|
||||
if (base) {
|
||||
// Disable rename detection so both the removed and added paths participate.
|
||||
files = git("diff", "--name-only", "--no-renames", "-z", base, head)
|
||||
.split("\0")
|
||||
.filter(Boolean);
|
||||
for (const file of files) {
|
||||
if (
|
||||
/^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test(
|
||||
file,
|
||||
) ||
|
||||
file.startsWith("docs/") ||
|
||||
file.startsWith(".github/ISSUE_TEMPLATE/")
|
||||
) {
|
||||
selected.docs = true;
|
||||
selected.website ||= matchesAny(file, SCOPES.website);
|
||||
continue;
|
||||
}
|
||||
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
|
||||
selected.code = true;
|
||||
for (const owner of HOST_TEST_OWNERS)
|
||||
selected[owner] ||= matchesAny(file, SCOPES[owner]);
|
||||
continue;
|
||||
}
|
||||
if (matchesAny(file, MANUAL_VERIFIERS)) {
|
||||
selected.code = true;
|
||||
continue;
|
||||
}
|
||||
const owners = Object.entries(SCOPES)
|
||||
.filter(([, patterns]) =>
|
||||
patterns.some((pattern) => matchesGlob(file, pattern)),
|
||||
)
|
||||
.map(([owner]) => owner);
|
||||
for (const owner of owners) selected[owner] = true;
|
||||
if (
|
||||
file.startsWith(".github/workflows/") ||
|
||||
file === ".github/actionlint.yaml"
|
||||
) {
|
||||
selected.workflows = true;
|
||||
if (GUARDED_WORKFLOWS.has(file)) selected.code = true;
|
||||
if (file !== ".github/workflows/ci.yml") continue;
|
||||
}
|
||||
if (owners.length === 1 && owners[0] === "website") continue;
|
||||
if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true;
|
||||
if (file === "package.json") {
|
||||
classifyPackage(base, head);
|
||||
continue;
|
||||
}
|
||||
selected.code = true;
|
||||
if (matchesAny(file, FIXTURE_SOURCES)) {
|
||||
selected.fixtures = true;
|
||||
selected.readiness = true;
|
||||
}
|
||||
if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true;
|
||||
if (file.startsWith("tests/")) continue;
|
||||
if (
|
||||
/^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test(
|
||||
file,
|
||||
)
|
||||
) {
|
||||
full(`Shared boundary: ${file}`);
|
||||
} else if (owners.length) {
|
||||
if (file.startsWith("src/") || file.startsWith("bridge/"))
|
||||
selected.runtime = true;
|
||||
} else if (/^(src|bridge)\//.test(file)) {
|
||||
// Unowned runtime sources keep the Linux x64 installed-package baseline;
|
||||
// full qualification retains every native host.
|
||||
selected.runtime = true;
|
||||
reasons.push(`Unowned runtime source: ${file}`);
|
||||
} else if (file.startsWith(".agents/skills/")) {
|
||||
selected.runtime = true;
|
||||
selected.docs = true;
|
||||
} else {
|
||||
full(`Unclassified input: ${file}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
selected.runtime ||= selected.package;
|
||||
const linux = { os: "ubuntu-latest", platform: "linux", architecture: "x64" };
|
||||
// macOS x64 is the slowest package host. Pull requests run it only with ci:full;
|
||||
// main pushes, schedules and manual dispatch always include it.
|
||||
// macOS x64 is the slowest package host. Fast qualification omits it.
|
||||
const intel =
|
||||
event !== "pull_request" || process.env.CI_FULL === "true"
|
||||
tier === "full"
|
||||
? [{ os: "macos-15-intel", platform: "darwin", architecture: "x64" }]
|
||||
: [];
|
||||
const matrix = {
|
||||
@@ -212,17 +238,17 @@ const matrix = {
|
||||
]
|
||||
: [linux],
|
||||
};
|
||||
const plan = { ...selected, matrix, files, reasons };
|
||||
const plan = { ...selected, tier, matrix, files, reasons };
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries({ ...selected, matrix: JSON.stringify(matrix) })
|
||||
Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) })
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Selected CI lanes\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
|
||||
`### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`,
|
||||
);
|
||||
console.log(JSON.stringify(plan));
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ARTIFACT_NAME = "ci-full-qualification-v1";
|
||||
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
const positiveId = (value) =>
|
||||
(typeof value === "string" || typeof value === "number") &&
|
||||
/^[1-9]\d*$/.test(String(value)) &&
|
||||
Number.isSafeInteger(Number(value));
|
||||
const commitSha = (value) =>
|
||||
typeof value === "string" && /^[0-9a-f]{40}$/.test(value);
|
||||
|
||||
/** Record only after the required gate has verified every full-tier obligation. */
|
||||
export function recordQualification(directory) {
|
||||
const sourceSha = process.env.GITHUB_SHA;
|
||||
const runId = process.env.GITHUB_RUN_ID;
|
||||
const runAttempt = process.env.GITHUB_RUN_ATTEMPT;
|
||||
const repository = process.env.GITHUB_REPOSITORY;
|
||||
if (
|
||||
!commitSha(sourceSha) ||
|
||||
!positiveId(runId) ||
|
||||
!positiveId(runAttempt) ||
|
||||
!repository
|
||||
)
|
||||
throw new Error(
|
||||
"Full qualification receipt requires source and workflow identity",
|
||||
);
|
||||
const checkoutSha = execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
if (checkoutSha !== sourceSha)
|
||||
throw new Error("Qualification checkout differs from workflow source");
|
||||
mkdirSync(directory, { recursive: true });
|
||||
const path = resolve(directory, "receipt.json");
|
||||
writeFileSync(
|
||||
path,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
tier: "full",
|
||||
sourceSha,
|
||||
repository,
|
||||
runId: Number(runId),
|
||||
runAttempt: Number(runAttempt),
|
||||
qualifiedAt: new Date().toISOString(),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`receipt_name=${ARTIFACT_NAME}\nreceipt_path=${path}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
function admit() {
|
||||
if (process.env.CI_FORCE_FULL === "true")
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Forced daily runner and dependency refresh",
|
||||
};
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const repositoryId = Number(process.env.CI_REPOSITORY_ID);
|
||||
const branch = process.env.CI_DEFAULT_BRANCH;
|
||||
const head = process.env.HEAD_SHA;
|
||||
if (!repository || !positiveId(repositoryId) || !branch || !commitSha(head))
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Qualification lookup identity is incomplete",
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const deadline = now + 20_000;
|
||||
const api = (...arguments_) => {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0)
|
||||
throw new Error("Qualification lookup budget exhausted");
|
||||
return JSON.parse(
|
||||
execFileSync("gh", ["api", "--method", "GET", ...arguments_], {
|
||||
encoding: "utf8",
|
||||
timeout: Math.min(15_000, remaining),
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}),
|
||||
);
|
||||
};
|
||||
try {
|
||||
const { artifacts } = api(
|
||||
`repos/${repository}/actions/artifacts`,
|
||||
"-f",
|
||||
`name=${ARTIFACT_NAME}`,
|
||||
"-F",
|
||||
"per_page=100",
|
||||
);
|
||||
if (!Array.isArray(artifacts))
|
||||
throw new Error("Missing qualification artifact inventory");
|
||||
const candidates = artifacts.filter((artifact) => {
|
||||
const lineage = artifact?.workflow_run;
|
||||
const age = now - Date.parse(artifact?.created_at);
|
||||
return (
|
||||
artifact?.name === ARTIFACT_NAME &&
|
||||
artifact?.expired === false &&
|
||||
typeof artifact?.created_at === "string" &&
|
||||
Number.isFinite(age) &&
|
||||
age >= 0 &&
|
||||
age < MAX_AGE_MS &&
|
||||
positiveId(lineage?.id) &&
|
||||
String(lineage.id) !== process.env.GITHUB_RUN_ID &&
|
||||
lineage?.repository_id === repositoryId &&
|
||||
lineage?.head_repository_id === repositoryId &&
|
||||
lineage?.head_branch === branch &&
|
||||
lineage?.head_sha === head
|
||||
);
|
||||
});
|
||||
// Artifact metadata narrows the search before run lookups. Never scan an
|
||||
// unbounded history or treat a missing receipt as a successful qualification.
|
||||
for (const artifact of candidates.slice(0, 3)) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
);
|
||||
// An old run can upload a new artifact when only its gate is rerun.
|
||||
// Bound the original run age as well so retries cannot renew stale proof.
|
||||
const runAge = Date.now() - Date.parse(run.created_at);
|
||||
if (
|
||||
typeof run.created_at === "string" &&
|
||||
Number.isFinite(runAge) &&
|
||||
runAge >= 0 &&
|
||||
runAge < MAX_AGE_MS &&
|
||||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_sha === head &&
|
||||
run.head_branch === branch &&
|
||||
run.head_repository?.full_name === repository
|
||||
) {
|
||||
return {
|
||||
run_full: false,
|
||||
run_id: run.id,
|
||||
reason: `Reusing full qualification from run ${run.id}; no fresh provider observation`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "No trusted full qualification for this SHA within 24 hours",
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Qualification lookup unavailable; running full validation",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) {
|
||||
const decision = admit();
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries(decision)
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Scheduled qualification\n\n${decision.reason}.\n`,
|
||||
);
|
||||
console.log(JSON.stringify(decision));
|
||||
}
|
||||
+28
-2
@@ -1,10 +1,30 @@
|
||||
import { CI_FLAGS } from "./scopes.mjs";
|
||||
import { recordQualification } from "./qualification.mjs";
|
||||
|
||||
const jobs = JSON.parse(process.env.CI_JOBS);
|
||||
const outputs = jobs.changes.outputs;
|
||||
const required = ["changes", "static", "test"];
|
||||
const reused = outputs.tier === "reused";
|
||||
if (
|
||||
reused &&
|
||||
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
|
||||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
|
||||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
|
||||
CI_FLAGS.some((flag) => outputs[flag] !== "false"))
|
||||
)
|
||||
throw new Error(
|
||||
"Reused qualification requires a scheduled admission receipt and no selected lanes",
|
||||
);
|
||||
if (
|
||||
outputs.tier === "full" &&
|
||||
CI_FLAGS.some((flag) => outputs[flag] !== "true")
|
||||
)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
const lanes = {
|
||||
code: ["build", "test-shard", "sqlite-runtime"],
|
||||
code: ["build", "test-shard"],
|
||||
runtime: ["windows-native-build", "package-e2e"],
|
||||
package: ["windows-curated"],
|
||||
sqlite: ["sqlite-runtime"],
|
||||
apple: ["apple-artifacts"],
|
||||
fixtures: ["conformance-fixtures"],
|
||||
readiness: ["reconstruction-readiness"],
|
||||
@@ -23,3 +43,9 @@ for (const [name, job] of Object.entries(jobs)) {
|
||||
if (failed.length)
|
||||
throw new Error(`CI lanes did not pass: ${failed.join(", ")}`);
|
||||
console.log("Every selected CI lane passed");
|
||||
if (reused)
|
||||
console.log(
|
||||
`Qualification reused from run ${outputs.qualification_run_id}; no fresh provider observation`,
|
||||
);
|
||||
if (outputs.tier === "full" && process.env.CI_QUALIFICATION_DIRECTORY)
|
||||
recordQualification(process.env.CI_QUALIFICATION_DIRECTORY);
|
||||
|
||||
@@ -85,6 +85,55 @@ export const SCOPES = {
|
||||
"tests/boundary/filesystem/javascriptArtifactContentIdentity.test.ts",
|
||||
"tests/boundary/process/sourceMapUncHosts.test.ts",
|
||||
],
|
||||
// Native SQLite runtime compatibility for the producer, public adapters,
|
||||
// fixtures, shared lifecycle, and the package script that defines its suite.
|
||||
sqlite: [
|
||||
"src/sqlite/**",
|
||||
"src/domain/sqlite/**",
|
||||
"src/contracts/sqlite/**",
|
||||
"src/application/sqlite/**",
|
||||
"src/composition/sqlite.ts",
|
||||
"src/cli/sqliteCommands.ts",
|
||||
"src/server/registerSqliteTools.ts",
|
||||
"src/application/DirectAnalysisStatus*.ts",
|
||||
"src/application/CapabilityInventory*.ts",
|
||||
"src/application/AnalysisProvider*.ts",
|
||||
"src/contracts/toolContracts.ts",
|
||||
"src/contracts/toolContractTypes.ts",
|
||||
"src/contracts/toolEffects*.ts",
|
||||
"src/contracts/toolOutputSchemaPrimitives.ts",
|
||||
"src/domain/analysisError*.ts",
|
||||
"src/domain/evidence*.ts",
|
||||
"src/domain/inputIssueProjection*.ts",
|
||||
"src/domain/jsonSerialization*.ts",
|
||||
"src/domain/jsonValue*.ts",
|
||||
"src/domain/localPath*.ts",
|
||||
"src/domain/providerAdapterError*.ts",
|
||||
"src/domain/providerCleanupError*.ts",
|
||||
"src/domain/result*.ts",
|
||||
"src/process/OwnedCommand*.ts",
|
||||
"src/process/ProviderProcess*.ts",
|
||||
"src/process/PrivateRuntimeRoot*.ts",
|
||||
"src/artifacts/ArtifactReader*.ts",
|
||||
"src/artifacts/ArtifactResourceScope*.ts",
|
||||
"src/artifacts/ArtifactPaths*.ts",
|
||||
"src/artifacts/readStableArtifact*.ts",
|
||||
"src/filesystem/OwnedFileHandle*.ts",
|
||||
"src/server/EvidenceMcpServer*.ts",
|
||||
"src/server/analysisAdmission*.ts",
|
||||
"src/server/contractToolHandler*.ts",
|
||||
"src/server/toolRegistrationOptions*.ts",
|
||||
"src/server/createServer*.ts",
|
||||
"src/cli.ts",
|
||||
"src/cliCommandNames.ts",
|
||||
"src/cliLogging.ts",
|
||||
"src/cli/commandCancellation.ts",
|
||||
"tests/boundary/sqlite/**",
|
||||
"tests/fixtures/sqlite/**",
|
||||
"tests/acceptance/sqliteDatabaseCli.test.ts",
|
||||
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
],
|
||||
browser: [
|
||||
".github/workflows/real-browser.yml",
|
||||
"scripts/verify-real-browser.mjs",
|
||||
@@ -312,6 +361,20 @@ export const SCOPES = {
|
||||
],
|
||||
};
|
||||
|
||||
export const CI_FLAGS = [
|
||||
...new Set([
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"apple",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
]),
|
||||
];
|
||||
|
||||
// Verifier entrypoints that only manual, licensed, or self-hosted workflows
|
||||
// run. No selected CI lane executes them, so their edits need only the static
|
||||
// checks and the source-test suite. Shared helpers under scripts/lib remain
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
||||
|
||||
@@ -42,6 +43,26 @@ const passed =
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success &&
|
||||
report.numFailedTests === 0;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
|
||||
const expected = execFileSync(
|
||||
"git",
|
||||
["ls-files", "-z", "--", "src", "tests"],
|
||||
{ encoding: "utf8", maxBuffer: 8 * 1024 * 1024 },
|
||||
)
|
||||
.split("\0")
|
||||
.filter((path) => path.endsWith(".test.ts"));
|
||||
const actual = files.map(({ name }) => name);
|
||||
const expectedSet = new Set(expected);
|
||||
const actualSet = new Set(actual);
|
||||
const missing = expected.filter((path) => !actualSet.has(path));
|
||||
const unexpected = actual.filter((path) => !expectedSet.has(path));
|
||||
if (missing.length || unexpected.length || actualSet.size !== actual.length)
|
||||
throw new Error(
|
||||
`Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`,
|
||||
);
|
||||
}
|
||||
const summary = [
|
||||
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
|
||||
"",
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { execFile } from "node:child_process";
|
||||
import { copyFile, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const MANIFEST_NAME = "manifest.json";
|
||||
const TARBALL_NAME = "package.tgz";
|
||||
const MANIFEST_KEYS = [
|
||||
"schemaVersion",
|
||||
"sourceSha",
|
||||
"runId",
|
||||
"runAttempt",
|
||||
"platform",
|
||||
"architecture",
|
||||
"packageVersion",
|
||||
"nodeVersion",
|
||||
"sha256",
|
||||
].sort();
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/** Normalize workflow identity and host facts into the manifest-owned fields. */
|
||||
export function createVerifiedPackageMetadata({
|
||||
sourceSha,
|
||||
runId,
|
||||
runAttempt,
|
||||
platform,
|
||||
architecture,
|
||||
packageVersion,
|
||||
nodeVersion,
|
||||
}) {
|
||||
if (!/^[a-f0-9]{40}$/u.test(sourceSha))
|
||||
throw new Error("Source SHA must be a full commit SHA");
|
||||
return {
|
||||
sourceSha,
|
||||
runId: parsePositiveInteger(runId, "GITHUB_RUN_ID"),
|
||||
runAttempt: parsePositiveInteger(runAttempt, "GITHUB_RUN_ATTEMPT"),
|
||||
platform,
|
||||
architecture,
|
||||
packageVersion,
|
||||
nodeVersion,
|
||||
};
|
||||
}
|
||||
|
||||
/** Build the single supported manifest representation for a verified tarball. */
|
||||
function createVerifiedPackageManifest(metadata) {
|
||||
const manifest = { schemaVersion: 1, ...metadata };
|
||||
validateManifestShape(manifest);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/** Write a manifest after the caller has completed qualification and cleanup. */
|
||||
export async function writeVerifiedPackageManifest(directory, metadata) {
|
||||
const manifest = createVerifiedPackageManifest(metadata);
|
||||
await writeFile(
|
||||
join(directory, MANIFEST_NAME),
|
||||
`${JSON.stringify(manifest, null, 2)}\n`,
|
||||
{ encoding: "utf8" },
|
||||
);
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/** Copy the qualified archive under its fixed name and prepare its manifest. */
|
||||
export async function retainVerifiedPackage({
|
||||
sourceTarball,
|
||||
directory,
|
||||
metadata,
|
||||
}) {
|
||||
const tarball = join(directory, TARBALL_NAME);
|
||||
await copyFile(sourceTarball, tarball);
|
||||
const sha256 = await sha256File(tarball);
|
||||
return {
|
||||
directory,
|
||||
metadata: createVerifiedPackageManifest({ ...metadata, sha256 }),
|
||||
};
|
||||
}
|
||||
|
||||
/** Validate a retained package against the current checkout and workflow attempt. */
|
||||
async function validateVerifiedPackage({
|
||||
directory,
|
||||
sourceSha,
|
||||
runId,
|
||||
runAttempt,
|
||||
packageVersion,
|
||||
nodeVersion,
|
||||
}) {
|
||||
if (!/^[a-f0-9]{40}$/u.test(sourceSha))
|
||||
throw new Error("Expected source SHA must be a full commit SHA");
|
||||
if (!Number.isSafeInteger(runId) || runId <= 0)
|
||||
throw new Error("Expected workflow run ID must be a positive integer");
|
||||
if (!Number.isSafeInteger(runAttempt) || runAttempt <= 0)
|
||||
throw new Error("Expected workflow run attempt must be a positive integer");
|
||||
const manifestPath = join(directory, MANIFEST_NAME);
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
||||
} catch (cause) {
|
||||
throw new Error(`Could not read verified package manifest: ${cause}`, {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
validateManifestShape(manifest);
|
||||
assertEqual(manifest.sourceSha, sourceSha, "source SHA");
|
||||
assertEqual(manifest.runId, runId, "workflow run ID");
|
||||
// A successful producer may belong to an earlier attempt when only failed
|
||||
// publisher jobs are rerun. The workflow still requires same-run qualification.
|
||||
if (manifest.runAttempt > runAttempt)
|
||||
throw new Error("Verified package workflow run attempt is from the future");
|
||||
assertEqual(manifest.platform, "linux", "verifier platform");
|
||||
assertEqual(manifest.architecture, "x64", "verifier architecture");
|
||||
assertEqual(manifest.packageVersion, packageVersion, "package version");
|
||||
assertEqual(manifest.nodeVersion, nodeVersion, "Node.js version");
|
||||
|
||||
// The artifact format deliberately fixes the name; no manifest value can
|
||||
// redirect validation to another path.
|
||||
const tarball = resolve(directory, TARBALL_NAME);
|
||||
const digest = await sha256File(tarball);
|
||||
assertEqual(manifest.sha256, digest, "tarball SHA-256");
|
||||
return tarball;
|
||||
}
|
||||
|
||||
function validateManifestShape(manifest) {
|
||||
if (
|
||||
typeof manifest !== "object" ||
|
||||
manifest === null ||
|
||||
Array.isArray(manifest)
|
||||
)
|
||||
throw new Error("Verified package manifest must be a JSON object");
|
||||
const keys = Object.keys(manifest).sort();
|
||||
if (
|
||||
keys.length !== MANIFEST_KEYS.length ||
|
||||
keys.some((key, index) => key !== MANIFEST_KEYS[index])
|
||||
)
|
||||
throw new Error("Verified package manifest has missing or unknown fields");
|
||||
if (manifest.schemaVersion !== 1)
|
||||
throw new Error("Unsupported verified package manifest schema version");
|
||||
if (
|
||||
typeof manifest.sourceSha !== "string" ||
|
||||
!/^[a-f0-9]{40}$/u.test(manifest.sourceSha)
|
||||
)
|
||||
throw new Error("Verified package manifest has an invalid source SHA");
|
||||
if (!Number.isSafeInteger(manifest.runId) || manifest.runId <= 0)
|
||||
throw new Error("Verified package manifest has an invalid workflow run ID");
|
||||
if (!Number.isSafeInteger(manifest.runAttempt) || manifest.runAttempt <= 0)
|
||||
throw new Error(
|
||||
"Verified package manifest has an invalid workflow run attempt",
|
||||
);
|
||||
for (const key of [
|
||||
"platform",
|
||||
"architecture",
|
||||
"packageVersion",
|
||||
"nodeVersion",
|
||||
])
|
||||
if (typeof manifest[key] !== "string" || manifest[key].length === 0)
|
||||
throw new Error(`Verified package manifest has an invalid ${key}`);
|
||||
if (
|
||||
typeof manifest.sha256 !== "string" ||
|
||||
!/^[a-f0-9]{64}$/u.test(manifest.sha256)
|
||||
)
|
||||
throw new Error("Verified package manifest has an invalid tarball SHA-256");
|
||||
}
|
||||
|
||||
function assertEqual(actual, expected, label) {
|
||||
if (actual !== expected)
|
||||
throw new Error(`Verified package ${label} mismatch`);
|
||||
}
|
||||
|
||||
async function sha256File(path) {
|
||||
const hash = createHash("sha256");
|
||||
for await (const chunk of createReadStream(path)) hash.update(chunk);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function validateFromCli(directory) {
|
||||
const sourceSha = process.env.GITHUB_SHA;
|
||||
const runId = parsePositiveInteger(
|
||||
process.env.GITHUB_RUN_ID,
|
||||
"GITHUB_RUN_ID",
|
||||
);
|
||||
const runAttempt = parsePositiveInteger(
|
||||
process.env.GITHUB_RUN_ATTEMPT,
|
||||
"GITHUB_RUN_ATTEMPT",
|
||||
);
|
||||
if (sourceSha === undefined || !/^[a-f0-9]{40}$/u.test(sourceSha))
|
||||
throw new Error("GITHUB_SHA must be a full commit SHA");
|
||||
const checkoutSha = (
|
||||
await execFileAsync("git", ["rev-parse", "HEAD"])
|
||||
).stdout.trim();
|
||||
assertEqual(checkoutSha, sourceSha, "checked out source SHA");
|
||||
const packageJson = JSON.parse(await readFile("package.json", "utf8"));
|
||||
const tarball = await validateVerifiedPackage({
|
||||
directory,
|
||||
sourceSha,
|
||||
runId,
|
||||
runAttempt,
|
||||
packageVersion: packageJson.version,
|
||||
nodeVersion: process.version,
|
||||
});
|
||||
if (process.env.GITHUB_OUTPUT === undefined)
|
||||
throw new Error(
|
||||
"GITHUB_OUTPUT is required to publish the validated tarball path",
|
||||
);
|
||||
await writeFile(process.env.GITHUB_OUTPUT, `tarball=${tarball}\n`, {
|
||||
encoding: "utf8",
|
||||
flag: "a",
|
||||
});
|
||||
process.stdout.write(`${tarball}\n`);
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
if (value === undefined || !/^[1-9]\d*$/u.test(value))
|
||||
throw new Error(`${name} must be a positive integer`);
|
||||
const parsed = Number(value);
|
||||
if (!Number.isSafeInteger(parsed))
|
||||
throw new Error(`${name} must be a safe positive integer`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
const invokedPath = process.argv[1];
|
||||
if (
|
||||
invokedPath !== undefined &&
|
||||
resolve(invokedPath) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
const [command, directory, ...extra] = process.argv.slice(2);
|
||||
if (command !== "validate" || directory === undefined || extra.length > 0) {
|
||||
process.stderr.write(
|
||||
"Usage: node scripts/ci/verified-package.mjs validate <artifact-directory>\n",
|
||||
);
|
||||
process.exitCode = 2;
|
||||
} else {
|
||||
await validateFromCli(directory);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,8 @@
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, rm, unlink } from "node:fs/promises";
|
||||
import { execFile } from "node:child_process";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { PRODUCT_IDENTITY } from "../dist/identity.js";
|
||||
@@ -21,12 +23,19 @@ import { verifyRepositorySkill } from "./verify-repository-skill.mjs";
|
||||
import { verifyManaged } from "./verify-package-managed.mjs";
|
||||
import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import {
|
||||
createVerifiedPackageMetadata,
|
||||
retainVerifiedPackage,
|
||||
writeVerifiedPackageManifest,
|
||||
} from "./ci/verified-package.mjs";
|
||||
import {
|
||||
measureMcpStartup,
|
||||
profileMcpModuleLoading,
|
||||
} from "./lib/mcp-startup-probe.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const execFileAsync = promisify(execFile);
|
||||
const retentionDirectory = process.env.REA_VERIFIED_PACKAGE_DIRECTORY;
|
||||
const verifierRun = createVerifierRun();
|
||||
const workspace = await mkdtemp(join(tmpdir(), "rea-package-"));
|
||||
const evidenceRoot = join(workspace, "evidence");
|
||||
@@ -37,6 +46,8 @@ let mcpModuleLoading;
|
||||
let update;
|
||||
let repositorySkill;
|
||||
let report;
|
||||
let retainedPackage;
|
||||
let retentionMetadata;
|
||||
|
||||
const runStage = async (name, action) => {
|
||||
const started = performance.now();
|
||||
@@ -49,6 +60,27 @@ const runStage = async (name, action) => {
|
||||
};
|
||||
|
||||
try {
|
||||
if (retentionDirectory !== undefined && retentionDirectory !== "") {
|
||||
const sourceSha = (
|
||||
await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root })
|
||||
).stdout.trim();
|
||||
retentionMetadata = createVerifiedPackageMetadata({
|
||||
sourceSha,
|
||||
runId: process.env.GITHUB_RUN_ID,
|
||||
runAttempt: process.env.GITHUB_RUN_ATTEMPT,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
packageVersion: PRODUCT_IDENTITY.packageVersion,
|
||||
nodeVersion: process.version,
|
||||
});
|
||||
await mkdir(retentionDirectory, { recursive: true });
|
||||
await unlink(join(retentionDirectory, "manifest.json")).catch((cause) => {
|
||||
if (
|
||||
!(cause instanceof Error && "code" in cause && cause.code === "ENOENT")
|
||||
)
|
||||
throw cause;
|
||||
});
|
||||
}
|
||||
repositorySkill = await runStage("repository skill", () =>
|
||||
verifyRepositorySkill({ root, workspace }),
|
||||
);
|
||||
@@ -215,6 +247,13 @@ try {
|
||||
mcpStartup,
|
||||
mcpModuleLoading,
|
||||
};
|
||||
if (retentionMetadata !== undefined) {
|
||||
retainedPackage = await retainVerifiedPackage({
|
||||
sourceTarball: tarball,
|
||||
directory: retentionDirectory,
|
||||
metadata: retentionMetadata,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await runStage("workspace cleanup", () =>
|
||||
rm(workspace, { recursive: true, force: true }),
|
||||
@@ -223,6 +262,11 @@ try {
|
||||
const verifierLineage = await runStage("verifier lineage", () =>
|
||||
completeVerifierRun(verifierRun),
|
||||
);
|
||||
if (retainedPackage !== undefined)
|
||||
await writeVerifiedPackageManifest(
|
||||
retainedPackage.directory,
|
||||
retainedPackage.metadata,
|
||||
);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ verifier_run: verifierLineage, ...report })}\n`,
|
||||
);
|
||||
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
import {
|
||||
DYLIB_RESOLUTION_LIMITATIONS,
|
||||
deriveFindings,
|
||||
type TracedRootSlice,
|
||||
} from "./dylibResolutionFindings.js";
|
||||
|
||||
/**
|
||||
@@ -568,6 +569,7 @@ export const traceDylibLoading = async (
|
||||
for (const path of request.unclassified ?? [])
|
||||
images.set(path, await view.image(path));
|
||||
const roots: DylibTrace["roots"][number][] = [];
|
||||
const rootSlices: TracedRootSlice[] = [];
|
||||
const edges: Edge[] = [];
|
||||
const withoutArchitecture: string[] = [];
|
||||
const environmentRoots = new Set<string>();
|
||||
@@ -597,6 +599,7 @@ export const traceDylibLoading = async (
|
||||
)
|
||||
environmentRoots.add(root);
|
||||
roots.push({ image: root, architecture: slice.architecture });
|
||||
rootSlices.push({ image: root, slice });
|
||||
edges.push(
|
||||
...(await traceProcess(
|
||||
{
|
||||
@@ -641,7 +644,7 @@ export const traceDylibLoading = async (
|
||||
facts.status === "parsed" ? facts.slices.map(withoutDependencies) : [],
|
||||
})),
|
||||
edges,
|
||||
findings: deriveFindings(edges, roots, images),
|
||||
findings: deriveFindings(edges, rootSlices),
|
||||
coverage: {
|
||||
status:
|
||||
unparsed.length === 0 &&
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
import type {
|
||||
DylibEdge as Edge,
|
||||
DylibFinding as Finding,
|
||||
DylibTrace,
|
||||
MachoImageFacts,
|
||||
MachoSlice,
|
||||
} from "./dylibResolution.js";
|
||||
|
||||
/** One traced process root with the exact slice it was traced from. */
|
||||
export interface TracedRootSlice {
|
||||
readonly image: string;
|
||||
readonly slice: MachoSlice;
|
||||
}
|
||||
|
||||
export const deriveFindings = (
|
||||
edges: readonly Edge[],
|
||||
roots: DylibTrace["roots"],
|
||||
images: ReadonlyMap<string, MachoImageFacts>,
|
||||
roots: readonly TracedRootSlice[],
|
||||
): Finding[] => {
|
||||
const findings: Finding[] = [];
|
||||
edges.forEach((edge, index) => {
|
||||
@@ -32,15 +36,11 @@ export const deriveFindings = (
|
||||
explanation: `The modeled dyld search checks ${earlier.join(", ")} before ${edge.resolution.image ?? edge.install_name}. A compatible Mach-O placed at an earlier modeled path could take precedence over that fallback, subject to unmodeled search inputs and code-signing library validation; library validation is not evaluated here (see inspect_signature).${edge.loader_conditional ? " This loader loads only conditionally; if it never loads, this search never occurs." : ""}${edge.resolution.status === "conditional" ? " Unmodeled environment overrides or earlier unknown candidates may take precedence; placing an image at this rpath does not establish which image dyld will load." : ""}`,
|
||||
});
|
||||
});
|
||||
for (const { image, architecture } of roots) {
|
||||
const facts = images.get(image);
|
||||
const slice =
|
||||
facts?.status === "parsed"
|
||||
? facts.slices.find(
|
||||
(candidate) => candidate.architecture === architecture,
|
||||
)
|
||||
: undefined;
|
||||
const environment = slice?.dyld_environment ?? [];
|
||||
// Slices can share an architecture name (arm64e and arm64e.x1), so each
|
||||
// finding uses the slice its root was traced from, not a name lookup.
|
||||
for (const { image, slice } of roots) {
|
||||
const { architecture } = slice;
|
||||
const environment = slice.dyld_environment;
|
||||
if (environment.length > 0)
|
||||
findings.push({
|
||||
kind: "dyld-environment-present",
|
||||
@@ -48,7 +48,7 @@ export const deriveFindings = (
|
||||
image,
|
||||
basis: "derived",
|
||||
explanation:
|
||||
slice?.file_type === "execute"
|
||||
slice.file_type === "execute"
|
||||
? `${image} (${architecture}) sets dyld environment variables through LC_DYLD_ENVIRONMENT (${environment.join(", ")}); any image-selection overrides they request are not modeled. Diagnostic settings alone do not alter resolution.`
|
||||
: `${image} (${architecture}) contains LC_DYLD_ENVIRONMENT (${environment.join(", ")}). Dyld reads these commands only from the main executable; entries in this non-executable root are observations and do not alter resolution.`,
|
||||
});
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { analyzeJavaScriptStaticSource } from "./javascriptStaticAnalysis.js";
|
||||
|
||||
const registration = (runtime: string, callback: string, chunk = 1) =>
|
||||
"(globalThis." +
|
||||
runtime +
|
||||
" = globalThis." +
|
||||
runtime +
|
||||
" || []).push([[" +
|
||||
chunk +
|
||||
"], {a: function() {}}, " +
|
||||
callback +
|
||||
"]);";
|
||||
|
||||
it.each(["webpackChunkFixture", "rspackChunkFixture"])(
|
||||
"keeps differing callback facts for %s independent of source order",
|
||||
(runtime) => {
|
||||
const first = registration(
|
||||
runtime,
|
||||
'function(r) { r(7); r.e("chunk-a"); }',
|
||||
);
|
||||
const second = registration(
|
||||
runtime,
|
||||
'function(r) { r(8); r.e("chunk-b"); }',
|
||||
);
|
||||
for (const source of [first + "\n" + second, second + "\n" + first]) {
|
||||
const result = analyzeJavaScriptStaticSource(source);
|
||||
expect(result.parse_status).toBe("complete");
|
||||
expect(result.bundler_registrations).toHaveLength(2);
|
||||
expect(
|
||||
result.bundler_registrations.flatMap((r) => r.entry_module_keys).sort(),
|
||||
).toEqual(["7", "8"]);
|
||||
expect(
|
||||
result.bundler_registrations.flatMap((r) => r.async_chunk_keys).sort(),
|
||||
).toEqual(["chunk-a", "chunk-b"]);
|
||||
}
|
||||
expect(
|
||||
analyzeJavaScriptStaticSource(first + "\n" + first).bundler_registrations,
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
analyzeJavaScriptStaticSource(first).bundler_registrations[0],
|
||||
).toMatchObject({
|
||||
entry_module_keys: ["7"],
|
||||
async_chunk_keys: ["chunk-a"],
|
||||
});
|
||||
expect(
|
||||
analyzeJavaScriptStaticSource(second).bundler_registrations[0],
|
||||
).toMatchObject({
|
||||
entry_module_keys: ["8"],
|
||||
async_chunk_keys: ["chunk-b"],
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["function(r) { r(7); }", "function(r) { r(8); }"],
|
||||
['function(r) { r.e("a"); }', 'function(r) { r.e("b"); }'],
|
||||
["function(r) {}", "function(s) {}"],
|
||||
["function(r) {}", "function(r) { r(dynamic); }"],
|
||||
["function(r) {}", "function(r) { r.e(dynamic); }"],
|
||||
])("retains callback metadata differences: %s / %s", (left, right) => {
|
||||
const result = analyzeJavaScriptStaticSource(
|
||||
registration("webpackChunkFixture", left) +
|
||||
"\n" +
|
||||
registration("webpackChunkFixture", right),
|
||||
);
|
||||
expect(result.bundler_registrations).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("retains distinct chunk identities with identical callbacks", () => {
|
||||
const callback = "function(r) { r(7); }";
|
||||
const result = analyzeJavaScriptStaticSource(
|
||||
registration("webpackChunkFixture", callback, 1) +
|
||||
"\n" +
|
||||
registration("webpackChunkFixture", callback, 2),
|
||||
);
|
||||
expect(result.bundler_registrations).toHaveLength(2);
|
||||
});
|
||||
@@ -72,9 +72,9 @@ describe("JavaScript semantic analysis: structure 1", () => {
|
||||
});
|
||||
const shadow = onlyBinding(ir, "ipcRenderer");
|
||||
expect(shadow.provenance).toEqual({
|
||||
status: "local",
|
||||
status: "unknown",
|
||||
origins: [],
|
||||
reason: null,
|
||||
reason: "Parameter provenance depends on caller arguments.",
|
||||
});
|
||||
const shadowReference = ir.references.find(
|
||||
({ name, bindingId, role }) =>
|
||||
|
||||
@@ -691,6 +691,14 @@ export const registrationKey = (
|
||||
registration.runtime,
|
||||
registration.chunk_keys,
|
||||
String(registration.unknown_chunk_keys),
|
||||
// Equal factory tables can be registered with different runtime callbacks.
|
||||
// Both early collection and finalization must retain those recovered facts.
|
||||
registration.runtime_require_name,
|
||||
registration.runtime_module_cache_status,
|
||||
registration.entry_module_keys,
|
||||
String(registration.unknown_entry_module_keys),
|
||||
registration.async_chunk_keys,
|
||||
String(registration.unknown_async_chunk_keys),
|
||||
registration.modules.map(({ module_key: key, source_sha256: digest }) => [
|
||||
key,
|
||||
digest,
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect } from "vitest";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js";
|
||||
|
||||
cliTest.for(["CLI", "stdio MCP"] as const)(
|
||||
"retains both repeated chunk callback observations through %s",
|
||||
async (surface, { cli }) => {
|
||||
const root = await createTestTempDirectory("rea-bundler-callbacks-");
|
||||
const source = [
|
||||
'(globalThis.webpackChunkFixture = globalThis.webpackChunkFixture || []).push([[1], {a: function() {}}, function(r) { r(7); r.e("chunk-a"); }]);',
|
||||
'(globalThis.webpackChunkFixture = globalThis.webpackChunkFixture || []).push([[1], {a: function() {}}, function(r) { r(8); r.e("chunk-b"); }]);',
|
||||
].join("\n");
|
||||
await writeFile(join(root, "bundle.js"), source);
|
||||
const { graph } = await analyzeJavaScriptDirectory(surface, cli, root);
|
||||
const chunk = graph.nodes.find(({ kind }) => kind === "javascript-chunk");
|
||||
for (const [index, entry, asyncChunk] of [
|
||||
[0, "7", "chunk-a"],
|
||||
[1, "8", "chunk-b"],
|
||||
] as const) {
|
||||
expect(chunk?.observations).toContainEqual(
|
||||
expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
entry_module_keys: [entry],
|
||||
async_chunk_keys: [asyncChunk],
|
||||
}),
|
||||
evidence: expect.objectContaining({
|
||||
location: expect.objectContaining({
|
||||
value: expect.objectContaining({
|
||||
source: "bundle.js",
|
||||
start: { line: index + 1, column: 0 },
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(graph.edges).toContainEqual(
|
||||
expect.objectContaining({
|
||||
source_node_id: chunk?.node_id,
|
||||
properties: expect.objectContaining({
|
||||
kind: "bundler-entry-module",
|
||||
module_key: entry,
|
||||
resolution_status: "not-found",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(graph.edges).toContainEqual(
|
||||
expect.objectContaining({
|
||||
source_node_id: chunk?.node_id,
|
||||
properties: expect.objectContaining({
|
||||
kind: "bundler-async-chunk",
|
||||
chunk_key: asyncChunk,
|
||||
resolution_status: "not-found",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -4,11 +4,13 @@ import { dirname, join } from "node:path";
|
||||
import { describe, expect } from "vitest";
|
||||
|
||||
import {
|
||||
CPU,
|
||||
FILE_TYPE,
|
||||
LC,
|
||||
buildVersionCommand,
|
||||
dylibCommand,
|
||||
dyldEnvironmentCommand,
|
||||
fatImage,
|
||||
machoImage,
|
||||
rpathCommand,
|
||||
} from "../../../src/artifacts/apple/MachoImage.fixture.js";
|
||||
@@ -471,6 +473,60 @@ cliTest(
|
||||
},
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"reports each same-named slice's own embedded dyld environment",
|
||||
async ({ cli }) => {
|
||||
// arm64e and arm64e.x1 slices both report the architecture name arm64e.
|
||||
for (const [settings, expected] of [
|
||||
[[null, "DYLD_PRINT_RPATHS=1"], ["DYLD_PRINT_RPATHS=1"]],
|
||||
[
|
||||
["DYLD_PRINT_LIBRARIES=1", "DYLD_PRINT_RPATHS=1"],
|
||||
["DYLD_PRINT_LIBRARIES=1", "DYLD_PRINT_RPATHS=1"],
|
||||
],
|
||||
] as const) {
|
||||
const app = await fixtureApp();
|
||||
await writeFiles(app, {
|
||||
"Contents/MacOS/App": fatImage(
|
||||
[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => {
|
||||
const setting = settings[index];
|
||||
return {
|
||||
cpu,
|
||||
bytes: machoImage({
|
||||
cpu,
|
||||
commands:
|
||||
setting === null || setting === undefined
|
||||
? []
|
||||
: [dyldEnvironmentCommand(setting)],
|
||||
}),
|
||||
};
|
||||
}),
|
||||
),
|
||||
});
|
||||
const result = await cli.run({
|
||||
arguments: [
|
||||
"trace-dylib-resolution",
|
||||
app,
|
||||
"--root",
|
||||
"Contents/MacOS/App",
|
||||
"--json",
|
||||
],
|
||||
environment: ENVIRONMENT,
|
||||
});
|
||||
expect(result.exitCode, JSON.stringify(result.json)).toBe(0);
|
||||
const trace = dylibResolutionResultSchema.parse(
|
||||
parseEvidence(result.json).normalized_result,
|
||||
);
|
||||
expect(trace.roots).toHaveLength(2);
|
||||
const environments = trace.findings
|
||||
.filter(({ kind }) => kind === "dyld-environment-present")
|
||||
.map(({ explanation }) =>
|
||||
expected.filter((setting) => explanation.includes(setting)),
|
||||
);
|
||||
expect(environments).toEqual(expected.map((setting) => [setting]));
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
cliTest(
|
||||
"retains definitive load findings for an explicitly selected app library root",
|
||||
async ({ cli }) => {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
chmod,
|
||||
cp,
|
||||
mkdir,
|
||||
readFile,
|
||||
@@ -78,6 +79,7 @@ async function classify(
|
||||
head: string,
|
||||
event = "pull_request",
|
||||
full = false,
|
||||
skipFull = false,
|
||||
) {
|
||||
const output = join(directory, "scope-output");
|
||||
await writeFile(output, "");
|
||||
@@ -92,6 +94,7 @@ async function classify(
|
||||
HEAD_SHA: head,
|
||||
GITHUB_EVENT_NAME: event,
|
||||
CI_FULL: String(full),
|
||||
CI_SKIP_FULL: String(skipFull),
|
||||
GITHUB_OUTPUT: output,
|
||||
},
|
||||
},
|
||||
@@ -99,6 +102,21 @@ async function classify(
|
||||
return z.record(z.string(), z.unknown()).parse(JSON.parse(result.stdout));
|
||||
}
|
||||
|
||||
it("reuses only scheduled qualification and preserves every explicit full request", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const reused = await classify(directory, base, base, "schedule", false, true);
|
||||
expect(reused.tier).toBe("reused");
|
||||
const { CI_FLAGS } = await loadScopes();
|
||||
for (const flag of CI_FLAGS) expect(reused[flag]).toBe(false);
|
||||
for (const event of ["workflow_dispatch", "pull_request", "schedule"])
|
||||
expect(
|
||||
(await classify(directory, base, base, event, true, true)).tier,
|
||||
).toBe("full");
|
||||
expect(
|
||||
(await classify(directory, base, base, "push", false, true)).tier,
|
||||
).toBe("fast");
|
||||
});
|
||||
|
||||
it("keeps every translated root README and authored guide on the documentation lane", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const readmes = (await readdir(new URL("../../../", import.meta.url))).filter(
|
||||
@@ -161,13 +179,148 @@ it("retains complete checks when implementation is renamed into documentation",
|
||||
});
|
||||
});
|
||||
|
||||
it("retains the complete main-push baseline even for documentation", async () => {
|
||||
it("keeps main pushes scoped when a successful ancestor baseline is available", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, ["README_ja.md"]);
|
||||
expect(await classify(directory, base, head, "push")).toMatchObject({
|
||||
tier: "fast",
|
||||
code: false,
|
||||
package: false,
|
||||
docs: true,
|
||||
matrix: {
|
||||
include: [
|
||||
{ os: "ubuntu-latest", platform: "linux", architecture: "x64" },
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("includes changes from earlier cancelled pushes since the last successful baseline", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
await change(directory, ["src/evm/client.ts"]);
|
||||
const head = await change(directory, ["README.md"]);
|
||||
expect(await classify(directory, base, head, "push")).toMatchObject({
|
||||
tier: "fast",
|
||||
code: true,
|
||||
evm: true,
|
||||
runtime: true,
|
||||
package: false,
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["", "not-a-commit"])(
|
||||
"fails closed to all lanes for a missing or invalid push baseline (%s)",
|
||||
async (base) => {
|
||||
const { directory } = await fixture();
|
||||
const head = await change(directory, ["README.md"]);
|
||||
const plan = await classify(directory, base, head, "push");
|
||||
expect(plan).toMatchObject({
|
||||
tier: "fast",
|
||||
code: true,
|
||||
package: true,
|
||||
evm: true,
|
||||
browser: true,
|
||||
reasons: expect.arrayContaining([expect.stringContaining("baseline")]),
|
||||
});
|
||||
const { CI_FLAGS } = await loadScopes();
|
||||
for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it("selects only a trusted successful ancestor from recent CI push runs", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, ["src/evm/client.ts"]);
|
||||
const bin = join(directory, "fake-bin");
|
||||
await mkdir(bin);
|
||||
const gh = join(bin, "gh");
|
||||
await writeFile(gh, '#!/bin/sh\ncat "$GH_FIXTURE_RESPONSE"\n');
|
||||
await chmod(gh, 0o755);
|
||||
const response = join(directory, "runs.json");
|
||||
await writeFile(
|
||||
response,
|
||||
JSON.stringify({
|
||||
workflow_runs: [
|
||||
{
|
||||
event: "push",
|
||||
conclusion: "success",
|
||||
head_branch: "main",
|
||||
head_repository: { full_name: "untrusted/fork" },
|
||||
head_sha: base,
|
||||
},
|
||||
{
|
||||
event: "push",
|
||||
conclusion: "success",
|
||||
head_branch: "main",
|
||||
head_repository: { full_name: "owner/repo" },
|
||||
head_sha: base,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
const output = join(directory, "baseline-output");
|
||||
const result = await execFileAsync(
|
||||
process.execPath,
|
||||
[new URL("../../../scripts/ci/baseline.mjs", import.meta.url).pathname],
|
||||
{
|
||||
cwd: directory,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${bin}:${process.env.PATH}`,
|
||||
GH_FIXTURE_RESPONSE: response,
|
||||
CI_REPOSITORY: "owner/repo",
|
||||
CI_DEFAULT_BRANCH: "main",
|
||||
HEAD_SHA: head,
|
||||
GITHUB_OUTPUT: output,
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(JSON.parse(result.stdout)).toEqual({ base, reason: "" });
|
||||
expect(await readFile(output, "utf8")).toBe(`base=${base}\nreason=\n`);
|
||||
});
|
||||
|
||||
it("returns an empty baseline when GitHub lookup fails", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, ["src/evm/client.ts"]);
|
||||
const bin = join(directory, "fake-bin");
|
||||
await mkdir(bin);
|
||||
const gh = join(bin, "gh");
|
||||
await writeFile(gh, "#!/bin/sh\nexit 1\n");
|
||||
await chmod(gh, 0o755);
|
||||
const result = await execFileAsync(
|
||||
process.execPath,
|
||||
[new URL("../../../scripts/ci/baseline.mjs", import.meta.url).pathname],
|
||||
{
|
||||
cwd: directory,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${bin}:${process.env.PATH}`,
|
||||
CI_REPOSITORY: "owner/repo",
|
||||
CI_DEFAULT_BRANCH: "main",
|
||||
HEAD_SHA: head,
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
base: "",
|
||||
reason: expect.stringContaining("Could not read"),
|
||||
});
|
||||
expect(head).not.toBe(base);
|
||||
});
|
||||
|
||||
it("fails closed when the successful push baseline is not an ancestor", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
await git(directory, ["switch", "-c", "unrelated"]);
|
||||
const unrelated = await change(directory, ["other.txt"]);
|
||||
await git(directory, ["switch", "main"]);
|
||||
const head = await change(directory, ["README.md"]);
|
||||
expect(await classify(directory, unrelated, head, "push")).toMatchObject({
|
||||
tier: "fast",
|
||||
code: true,
|
||||
package: true,
|
||||
evm: true,
|
||||
browser: true,
|
||||
});
|
||||
expect(head).not.toBe(base);
|
||||
});
|
||||
|
||||
it("fails classification rather than treating an invalid Git comparison as a scope", async () => {
|
||||
@@ -207,7 +360,8 @@ it("retains native Inspector verification on every documented package host", asy
|
||||
);
|
||||
expect(packageInspector.if).toBeUndefined();
|
||||
expect(
|
||||
(await classify((await fixture()).directory, "", "", "push")).matrix,
|
||||
(await classify((await fixture()).directory, "", "", "workflow_dispatch"))
|
||||
.matrix,
|
||||
).toEqual({
|
||||
include: [
|
||||
{ os: "ubuntu-latest", platform: "linux", architecture: "x64" },
|
||||
@@ -227,7 +381,14 @@ it("retains native Inspector verification on every documented package host", asy
|
||||
it.each([
|
||||
[
|
||||
"src/evm/client.ts",
|
||||
{ code: true, runtime: true, evm: true, browser: false, package: false },
|
||||
{
|
||||
code: true,
|
||||
runtime: true,
|
||||
evm: true,
|
||||
browser: false,
|
||||
package: false,
|
||||
sqlite: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
"src/browser/execution/evaluate.ts",
|
||||
@@ -324,6 +485,38 @@ it.each([
|
||||
"src/process/ProcessOwnership.test.ts",
|
||||
{ code: true, package: true, apple: false, evm: false, browser: false },
|
||||
],
|
||||
[
|
||||
"src/sqlite/SqliteDatabaseProvider.ts",
|
||||
{ code: true, sqlite: true, runtime: true, package: false },
|
||||
],
|
||||
[
|
||||
"src/application/sqlite/SqliteDatabaseService.test.ts",
|
||||
{ code: true, sqlite: true, runtime: false },
|
||||
],
|
||||
[
|
||||
"tests/boundary/sqlite/sqliteDatabaseProvider.test.ts",
|
||||
{ code: true, sqlite: true, runtime: false },
|
||||
],
|
||||
[
|
||||
"tests/fixtures/sqlite/database.ts",
|
||||
{ code: true, sqlite: true, runtime: false },
|
||||
],
|
||||
["tests/acceptance/sqliteDatabaseCli.test.ts", { code: true, sqlite: true }],
|
||||
[
|
||||
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
|
||||
{ code: true, sqlite: true },
|
||||
],
|
||||
["src/process/OwnedCommand.ts", { code: true, sqlite: true, runtime: true }],
|
||||
["src/server/createServer.ts", { code: true, sqlite: true, runtime: true }],
|
||||
[
|
||||
"src/contracts/toolContracts.ts",
|
||||
{ code: true, sqlite: true, runtime: true },
|
||||
],
|
||||
[
|
||||
"src/domain/analysisErrorProjection.ts",
|
||||
{ code: true, sqlite: true, runtime: true },
|
||||
],
|
||||
["src/domain/javascript/example.ts", { code: true, sqlite: false }],
|
||||
])("selects the appropriate lanes for %s", async (path, expected) => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, [path]);
|
||||
@@ -366,6 +559,52 @@ it("scopes package script edits while retaining broad dependency checks", async
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["test:sqlite", "test:sqlite:run"])(
|
||||
"routes %s script changes to the SQLite runtime matrix",
|
||||
async (script) => {
|
||||
const { directory } = await fixture();
|
||||
const base = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({ scripts: { [script]: "vitest run" } }),
|
||||
);
|
||||
const head = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({
|
||||
scripts: { [script]: "vitest run --reporter=default" },
|
||||
}),
|
||||
);
|
||||
expect(await classify(directory, base, head)).toMatchObject({
|
||||
code: true,
|
||||
sqlite: true,
|
||||
evm: false,
|
||||
package: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps unrelated test script changes off the SQLite runtime matrix", async () => {
|
||||
const { directory } = await fixture();
|
||||
const base = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({ scripts: { "test:domain": "vitest run src/domain" } }),
|
||||
);
|
||||
const head = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({
|
||||
scripts: { "test:domain": "vitest run src/domain --reporter=default" },
|
||||
}),
|
||||
);
|
||||
expect(await classify(directory, base, head)).toMatchObject({
|
||||
code: true,
|
||||
sqlite: false,
|
||||
evm: false,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["browser", "real-browser"],
|
||||
["binary_layout", "real-binary-layout"],
|
||||
@@ -458,7 +697,10 @@ it.each([
|
||||
"src/process/capture/example.test.ts",
|
||||
{ code: true, runtime: false, browser: false, package: false },
|
||||
],
|
||||
[".nvmrc", { code: true, package: true, browser: true, evm: true }],
|
||||
[
|
||||
".nvmrc",
|
||||
{ code: true, package: true, browser: true, evm: true, sqlite: true },
|
||||
],
|
||||
[
|
||||
"tsconfig.build.json",
|
||||
{ code: true, package: true, browser: true, evm: true },
|
||||
@@ -595,8 +837,10 @@ it.each(["workflow_dispatch", "schedule"])(
|
||||
async (event) => {
|
||||
const { directory, base } = await fixture();
|
||||
expect(await classify(directory, base, base, event)).toMatchObject({
|
||||
tier: "full",
|
||||
code: true,
|
||||
package: true,
|
||||
sqlite: true,
|
||||
evm: true,
|
||||
browser: true,
|
||||
});
|
||||
@@ -610,7 +854,14 @@ it("honors ci:full for successive PR revisions", async () => {
|
||||
for (const head of [first, second]) {
|
||||
expect(
|
||||
await classify(directory, base, head, "pull_request", true),
|
||||
).toMatchObject({ code: true, package: true, evm: true, browser: true });
|
||||
).toMatchObject({
|
||||
tier: "full",
|
||||
code: true,
|
||||
package: true,
|
||||
sqlite: true,
|
||||
evm: true,
|
||||
browser: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -650,7 +901,7 @@ const INTEL_HOST = {
|
||||
architecture: "x64",
|
||||
};
|
||||
|
||||
it("keeps macOS x64 package verification for ci:full and non-PR events", async () => {
|
||||
it("keeps macOS x64 package verification only for full qualification", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, ["src/windows/Example.ts"]);
|
||||
const matrix = z.object({ include: z.array(z.unknown()) });
|
||||
@@ -661,7 +912,8 @@ it("keeps macOS x64 package verification for ci:full and non-PR events", async (
|
||||
expect(pullRequest).toHaveLength(3);
|
||||
expect(pullRequest).not.toContainEqual(INTEL_HOST);
|
||||
expect(await hosts("pull_request", true)).toContainEqual(INTEL_HOST);
|
||||
for (const event of ["push", "schedule", "workflow_dispatch"])
|
||||
expect(await hosts("push", false)).not.toContainEqual(INTEL_HOST);
|
||||
for (const event of ["schedule", "workflow_dispatch"])
|
||||
expect(await hosts(event, false)).toContainEqual(INTEL_HOST);
|
||||
});
|
||||
|
||||
@@ -674,6 +926,7 @@ const workflowFile = async (name: string): Promise<unknown> =>
|
||||
);
|
||||
|
||||
const scopesSchema = z.object({
|
||||
CI_FLAGS: z.array(z.string()),
|
||||
SCOPES: z.record(z.string(), z.array(z.string())),
|
||||
MANUAL_VERIFIERS: z.array(z.string()),
|
||||
});
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
executeWorkflowFixture,
|
||||
workflowGit,
|
||||
} from "../../support/workflowGit.js";
|
||||
|
||||
const reporter = fileURLToPath(
|
||||
new URL("../../../scripts/ci/job-timings.mjs", import.meta.url),
|
||||
);
|
||||
|
||||
it("merges paginated jobs, reports elapsed metrics, and preserves raw responses", async () => {
|
||||
const fixture = await createFixture([
|
||||
{
|
||||
total_count: 4,
|
||||
jobs: [
|
||||
{
|
||||
name: "Build package",
|
||||
workflow_name: "CI",
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:10Z",
|
||||
completed_at: "2026-10-13T00:01:10Z",
|
||||
steps: [
|
||||
{
|
||||
name: "Install dependencies (npm ci)",
|
||||
started_at: "2026-10-13T00:00:10Z",
|
||||
completed_at: "2026-10-13T00:00:40Z",
|
||||
},
|
||||
{
|
||||
name: "Upload artifact",
|
||||
started_at: "2026-10-13T00:00:40Z",
|
||||
completed_at: "2026-10-13T00:00:41Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
total_count: 4,
|
||||
jobs: [
|
||||
{
|
||||
name: "Queued lane",
|
||||
status: "queued",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:00Z",
|
||||
completed_at: null,
|
||||
steps: [],
|
||||
},
|
||||
{
|
||||
name: "Running lane",
|
||||
status: "in_progress",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:15Z",
|
||||
completed_at: null,
|
||||
steps: [
|
||||
{
|
||||
name: "Download artifacts",
|
||||
started_at: "2026-10-13T00:00:20Z",
|
||||
completed_at: "2026-10-13T00:00:22Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Skipped lane",
|
||||
status: "completed",
|
||||
conclusion: "skipped",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:00Z",
|
||||
completed_at: "2026-10-13T00:00:00Z",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
const result = await runReporter(fixture);
|
||||
const report = JSON.parse(
|
||||
await readFile(join(fixture.outputDirectory, "report.json"), "utf8"),
|
||||
);
|
||||
|
||||
expect(report.run.snapshot).toContain("partial");
|
||||
expect(report.counts).toMatchObject({
|
||||
jobs: 4,
|
||||
completed: 2,
|
||||
skipped: 1,
|
||||
queueUnknown: 2,
|
||||
executionUnknown: 3,
|
||||
completedExecutionUnknown: 0,
|
||||
});
|
||||
expect(report.aggregates.queueSeconds).toMatchObject({
|
||||
count: 2,
|
||||
p50: 10,
|
||||
p95: 15,
|
||||
});
|
||||
expect(report.aggregates.completedExecutionSeconds).toMatchObject({
|
||||
count: 1,
|
||||
p50: 60,
|
||||
p95: 60,
|
||||
});
|
||||
expect(report.aggregates.summedCompletedJobElapsedSeconds).toMatchObject({
|
||||
value: 60,
|
||||
knownCount: 1,
|
||||
unknownCount: 0,
|
||||
label: expect.stringContaining("not CPU time"),
|
||||
});
|
||||
expect(report.jobs[1].queue).toEqual({
|
||||
seconds: null,
|
||||
reason: "job start is not established by its status",
|
||||
});
|
||||
expect(report.jobs[3].queue).toEqual({
|
||||
seconds: null,
|
||||
reason: "job was skipped",
|
||||
});
|
||||
expect(report.jobs[3].execution).toEqual({
|
||||
seconds: null,
|
||||
reason: "job was skipped",
|
||||
});
|
||||
expect(report.stepCosts.npmCi[0].duration.seconds).toBe(30);
|
||||
expect(report.stepCosts.artifactUpload[0].duration.seconds).toBe(1);
|
||||
expect(report.stepCosts.artifactDownload[0].duration.seconds).toBe(2);
|
||||
expect(report.stepCostInterpretation).toContain(
|
||||
"not total artifact transfer costs",
|
||||
);
|
||||
expect(report.cpuSeconds).toBeUndefined();
|
||||
expect(result.stdout).toContain("4 job timing records");
|
||||
expect(
|
||||
await readFile(join(fixture.outputDirectory, "jobs.json"), "utf8"),
|
||||
).toBe(fixture.raw);
|
||||
const summary = await readFile(fixture.summary, "utf8");
|
||||
expect(summary).toContain("point-in-time snapshot");
|
||||
expect(summary).toContain("unknown");
|
||||
expect(summary).toContain("not CPU time");
|
||||
expect(summary).toContain("60 s");
|
||||
expect(summary).toContain("not total artifact transfer costs");
|
||||
const args = JSON.parse(await readFile(fixture.args, "utf8"));
|
||||
expect(args).toEqual([
|
||||
"api",
|
||||
"--paginate",
|
||||
"--slurp",
|
||||
"repos/example/rea/actions/runs/42/attempts/2/jobs?per_page=100",
|
||||
]);
|
||||
});
|
||||
|
||||
it("marks invalid and out-of-order timestamps unknown instead of emitting negative durations", async () => {
|
||||
const fixture = await createFixture([
|
||||
{
|
||||
jobs: [
|
||||
{
|
||||
name: "Invalid dates",
|
||||
status: "completed",
|
||||
created_at: "not-a-date",
|
||||
started_at: "2026-10-13T00:00:10Z",
|
||||
completed_at: "2026-10-13T00:00:05Z",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await runReporter(fixture);
|
||||
const report = JSON.parse(
|
||||
await readFile(join(fixture.outputDirectory, "report.json"), "utf8"),
|
||||
);
|
||||
|
||||
expect(report.jobs[0].queue).toEqual({
|
||||
seconds: null,
|
||||
reason: "invalid timestamp",
|
||||
});
|
||||
expect(report.jobs[0].execution).toEqual({
|
||||
seconds: null,
|
||||
reason: "timestamps out of order",
|
||||
});
|
||||
expect(report.counts.completedExecutionUnknown).toBe(1);
|
||||
expect(report.aggregates.summedCompletedJobElapsedSeconds).toMatchObject({
|
||||
value: null,
|
||||
unknownCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails on malformed pages or a failed GitHub API request", async () => {
|
||||
const malformed = await createFixture([{ jobs: "not-an-array" }]);
|
||||
await expect(runReporter(malformed)).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(join(malformed.outputDirectory, "jobs.json"), "utf8"),
|
||||
).resolves.toBe(malformed.raw);
|
||||
|
||||
const failedApi = await createFixture([], { exitCode: 1 });
|
||||
await expect(runReporter(failedApi)).rejects.toThrow();
|
||||
});
|
||||
|
||||
async function createFixture(
|
||||
pages: unknown[],
|
||||
{ exitCode = 0 }: { exitCode?: number } = {},
|
||||
) {
|
||||
const directory = await createTestTempDirectory("rea-ci-job-timings-");
|
||||
await workflowGit(directory, ["init", "--initial-branch=main"]);
|
||||
const fakeBin = join(directory, "bin");
|
||||
await mkdir(fakeBin);
|
||||
const raw = JSON.stringify(pages);
|
||||
const rawPath = join(directory, "gh-response.json");
|
||||
const args = join(directory, "gh-args.json");
|
||||
await writeFile(rawPath, raw);
|
||||
const fakeGh = join(fakeBin, "gh");
|
||||
await writeFile(
|
||||
fakeGh,
|
||||
`#!/usr/bin/env node\nconst fs = require("node:fs");\nfs.writeFileSync(process.env.CI_GH_ARGS, JSON.stringify(process.argv.slice(2)));\nprocess.stdout.write(fs.readFileSync(process.env.CI_GH_RESPONSE, "utf8"));\nprocess.exitCode = ${String(exitCode)};\n`,
|
||||
);
|
||||
await chmod(fakeGh, 0o755);
|
||||
const outputDirectory = join(directory, ".cache", "ci-job-timings");
|
||||
const summary = join(directory, "summary.md");
|
||||
await writeFile(summary, "");
|
||||
return {
|
||||
directory,
|
||||
fakeBin,
|
||||
raw,
|
||||
rawPath,
|
||||
args,
|
||||
outputDirectory,
|
||||
summary,
|
||||
};
|
||||
}
|
||||
|
||||
function runReporter(fixture: Awaited<ReturnType<typeof createFixture>>) {
|
||||
return executeWorkflowFixture(process.execPath, [reporter], {
|
||||
cwd: fixture.directory,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
|
||||
CI_REPOSITORY: "example/rea",
|
||||
CI_JOB_TIMINGS_DIRECTORY: fixture.outputDirectory,
|
||||
CI_GH_ARGS: fixture.args,
|
||||
CI_GH_RESPONSE: fixture.rawPath,
|
||||
CI_TIMING_RUN_ID: "42",
|
||||
CI_TIMING_RUN_ATTEMPT: "2",
|
||||
GITHUB_STEP_SUMMARY: fixture.summary,
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
executeWorkflowFixture,
|
||||
workflowGit,
|
||||
} from "../../support/workflowGit.js";
|
||||
|
||||
const qualificationScript = fileURLToPath(
|
||||
new URL("../../../scripts/ci/qualification.mjs", import.meta.url),
|
||||
);
|
||||
const requiredScript = fileURLToPath(
|
||||
new URL("../../../scripts/ci/required.mjs", import.meta.url),
|
||||
);
|
||||
const now = Date.now();
|
||||
const validSha = "a".repeat(40);
|
||||
const repository = "example/rea";
|
||||
|
||||
function artifact(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 42,
|
||||
name: "ci-full-qualification-v1",
|
||||
expired: false,
|
||||
created_at: new Date(now - 60_000).toISOString(),
|
||||
workflow_run: {
|
||||
id: 7,
|
||||
repository_id: 1,
|
||||
head_repository_id: 1,
|
||||
head_branch: "main",
|
||||
head_sha: validSha,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function run(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 7,
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
path: ".github/workflows/ci.yml",
|
||||
event: "schedule",
|
||||
created_at: new Date(now - 120_000).toISOString(),
|
||||
head_sha: validSha,
|
||||
head_branch: "main",
|
||||
head_repository: { full_name: repository },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function admissionFixture(
|
||||
candidate: Record<string, unknown> = artifact(),
|
||||
candidateRun: Record<string, unknown> = run(),
|
||||
options: { apiError?: boolean; malformed?: boolean; force?: boolean } = {},
|
||||
) {
|
||||
const directory = await createTestTempDirectory("rea-ci-qualification-");
|
||||
const bin = join(directory, "bin");
|
||||
await mkdir(bin);
|
||||
const fixturePath = join(directory, "api.json");
|
||||
await writeFile(
|
||||
fixturePath,
|
||||
options.malformed
|
||||
? "not-json"
|
||||
: JSON.stringify({ artifacts: [candidate], run: candidateRun }),
|
||||
);
|
||||
const gh = join(bin, "gh");
|
||||
await writeFile(
|
||||
gh,
|
||||
[
|
||||
"#!/bin/sh",
|
||||
'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi',
|
||||
'case "$*" in',
|
||||
' *actions/artifacts*) cat "$GH_FIXTURE_RESPONSE" ;;',
|
||||
" *actions/runs/7*) node -e 'const x=require(process.env.GH_FIXTURE_RESPONSE); process.stdout.write(JSON.stringify(x.run))' ;;",
|
||||
" *) exit 2 ;;",
|
||||
"esac",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
await chmod(gh, 0o755);
|
||||
const output = join(directory, "github-output");
|
||||
await writeFile(output, "");
|
||||
const env = {
|
||||
...process.env,
|
||||
PATH: `${bin}:${process.env.PATH ?? ""}`,
|
||||
GH_FIXTURE_RESPONSE: fixturePath,
|
||||
GH_API_ERROR: String(options.apiError ?? false),
|
||||
CI_REPOSITORY: repository,
|
||||
CI_REPOSITORY_ID: "1",
|
||||
CI_DEFAULT_BRANCH: "main",
|
||||
HEAD_SHA: validSha,
|
||||
GITHUB_RUN_ID: "100",
|
||||
GITHUB_OUTPUT: output,
|
||||
...(options.force ? { CI_FORCE_FULL: "true" } : {}),
|
||||
};
|
||||
return { directory, output, env };
|
||||
}
|
||||
|
||||
async function decide(
|
||||
candidate?: Record<string, unknown>,
|
||||
candidateRun?: Record<string, unknown>,
|
||||
options?: { apiError?: boolean; malformed?: boolean; force?: boolean },
|
||||
) {
|
||||
const fixture = await admissionFixture(candidate, candidateRun, options);
|
||||
const result = await executeWorkflowFixture(
|
||||
process.execPath,
|
||||
[qualificationScript],
|
||||
{
|
||||
cwd: fixture.directory,
|
||||
env: fixture.env,
|
||||
},
|
||||
);
|
||||
return {
|
||||
...fixture,
|
||||
decision: z
|
||||
.record(z.string(), z.unknown())
|
||||
.parse(JSON.parse(result.stdout)),
|
||||
outputText: await readFile(fixture.output, "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
it("reuses only a recent trusted full qualification for the exact default-branch SHA", async () => {
|
||||
const { decision, outputText } = await decide();
|
||||
expect(decision).toMatchObject({ run_full: false, run_id: 7 });
|
||||
expect(Object.keys(decision).sort()).toEqual([
|
||||
"reason",
|
||||
"run_full",
|
||||
"run_id",
|
||||
]);
|
||||
expect(outputText).toContain("run_full=false\n");
|
||||
expect(outputText).toContain("run_id=7\n");
|
||||
});
|
||||
|
||||
const rejectedCandidates: Array<
|
||||
[string, () => Record<string, unknown>, () => Record<string, unknown>]
|
||||
> = [
|
||||
[
|
||||
"old",
|
||||
() =>
|
||||
artifact({
|
||||
created_at: new Date(now - 24 * 60 * 60 * 1000 - 1).toISOString(),
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"future",
|
||||
() => artifact({ created_at: new Date(now + 60_000).toISOString() }),
|
||||
run,
|
||||
],
|
||||
["expired", () => artifact({ expired: true }), run],
|
||||
[
|
||||
"foreign repository",
|
||||
() =>
|
||||
artifact({
|
||||
workflow_run: { ...artifact().workflow_run, repository_id: 2 },
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"foreign branch",
|
||||
() =>
|
||||
artifact({
|
||||
workflow_run: { ...artifact().workflow_run, head_branch: "other" },
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"foreign SHA",
|
||||
() =>
|
||||
artifact({
|
||||
workflow_run: { ...artifact().workflow_run, head_sha: "b".repeat(40) },
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"untrusted workflow",
|
||||
artifact,
|
||||
() => run({ path: ".github/workflows/release.yml" }),
|
||||
],
|
||||
["failed run", artifact, () => run({ conclusion: "failure" })],
|
||||
[
|
||||
"old producer rerun with a fresh artifact",
|
||||
artifact,
|
||||
() =>
|
||||
run({ created_at: new Date(now - 25 * 60 * 60 * 1000).toISOString() }),
|
||||
],
|
||||
["wrong workflow event", artifact, () => run({ event: "push" })],
|
||||
];
|
||||
|
||||
it.each(rejectedCandidates)(
|
||||
"runs full qualification for a %s candidate",
|
||||
async (_name, makeArtifact, makeRun) => {
|
||||
const { decision } = await decide(makeArtifact(), makeRun());
|
||||
expect(decision.run_full).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["API error", { apiError: true }],
|
||||
["malformed query response", { malformed: true }],
|
||||
] as const)("fails closed on %s", async (_name, options) => {
|
||||
const { decision } = await decide(undefined, undefined, options);
|
||||
expect(decision.run_full).toBe(true);
|
||||
});
|
||||
|
||||
it("lets the daily force-full setting bypass the GitHub API", async () => {
|
||||
const { decision } = await decide(undefined, undefined, {
|
||||
force: true,
|
||||
apiError: true,
|
||||
});
|
||||
expect(decision).toMatchObject({ run_full: true });
|
||||
expect(decision.reason).toContain("Forced");
|
||||
});
|
||||
|
||||
async function workflowContract() {
|
||||
const workflowPath = new URL(
|
||||
"../../../.github/workflows/ci.yml",
|
||||
import.meta.url,
|
||||
);
|
||||
const parsed = parse(await readFile(workflowPath, "utf8"));
|
||||
return z
|
||||
.object({
|
||||
jobs: z.record(
|
||||
z.string(),
|
||||
z.object({ needs: z.unknown().optional() }).passthrough(),
|
||||
),
|
||||
})
|
||||
.parse(parsed).jobs;
|
||||
}
|
||||
|
||||
async function requiredRun(
|
||||
changes: Record<string, string>,
|
||||
overrides: { failed?: string; runAttempt?: string; event?: string } = {},
|
||||
) {
|
||||
const directory = await createTestTempDirectory("rea-ci-receipt-");
|
||||
await workflowGit(directory, ["init", "--initial-branch=main"]);
|
||||
await writeFile(join(directory, "fixture.txt"), "qualified source\n");
|
||||
await workflowGit(directory, ["add", "."]);
|
||||
await workflowGit(directory, ["commit", "-m", "fixture source"]);
|
||||
const head = await workflowGit(directory, ["rev-parse", "HEAD"]);
|
||||
const jobs = await workflowContract();
|
||||
const results: Record<
|
||||
string,
|
||||
{ result: string; outputs?: Record<string, string> }
|
||||
> = {};
|
||||
for (const name of z.array(z.string()).parse(jobs.required?.needs)) {
|
||||
results[name] = {
|
||||
result:
|
||||
name === overrides.failed
|
||||
? "failure"
|
||||
: changes.tier === "reused"
|
||||
? "skipped"
|
||||
: "success",
|
||||
};
|
||||
}
|
||||
results.changes = { result: "success", outputs: changes };
|
||||
const receiptDirectory = join(directory, ".cache", "receipt");
|
||||
const output = join(directory, "github-output");
|
||||
await writeFile(output, "");
|
||||
const env = {
|
||||
...process.env,
|
||||
CI_JOBS: JSON.stringify(results),
|
||||
CI_QUALIFICATION_DIRECTORY: receiptDirectory,
|
||||
GITHUB_EVENT_NAME: overrides.event ?? "schedule",
|
||||
GITHUB_SHA: head,
|
||||
GITHUB_RUN_ID: "100",
|
||||
GITHUB_RUN_ATTEMPT: overrides.runAttempt ?? "1",
|
||||
GITHUB_REPOSITORY: repository,
|
||||
GITHUB_OUTPUT: output,
|
||||
};
|
||||
const result = executeWorkflowFixture(process.execPath, [requiredScript], {
|
||||
cwd: directory,
|
||||
env,
|
||||
});
|
||||
return { directory, head, output, result };
|
||||
}
|
||||
|
||||
it("writes a full receipt only after every canonical flag and workflow need pass", async () => {
|
||||
const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url);
|
||||
const { CI_FLAGS } = await import(scopesPath.href);
|
||||
const changes = Object.fromEntries(
|
||||
CI_FLAGS.map((flag: string) => [flag, "true"]),
|
||||
);
|
||||
changes.tier = "full";
|
||||
const { directory, head, output, result } = await requiredRun(changes);
|
||||
await expect(result).resolves.toMatchObject({ stderr: "" });
|
||||
const receiptPath = join(directory, ".cache", "receipt", "receipt.json");
|
||||
const receipt = z
|
||||
.object({
|
||||
sourceSha: z.string(),
|
||||
runId: z.number(),
|
||||
runAttempt: z.number(),
|
||||
})
|
||||
.parse(JSON.parse(await readFile(receiptPath, "utf8")));
|
||||
expect(receipt).toMatchObject({ sourceSha: head, runId: 100, runAttempt: 1 });
|
||||
expect(await readFile(output, "utf8")).toContain(
|
||||
`receipt_path=${receiptPath}\n`,
|
||||
);
|
||||
expect(await readFile(output, "utf8")).toContain(
|
||||
"receipt_name=ci-full-qualification-v1\n",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["failed selected lane", { failed: "sqlite-runtime" }],
|
||||
["partial full scope", {}],
|
||||
] as const)(
|
||||
"does not issue a receipt after %s",
|
||||
async (caseName, overrides) => {
|
||||
const scopesPath = new URL(
|
||||
"../../../scripts/ci/scopes.mjs",
|
||||
import.meta.url,
|
||||
);
|
||||
const { CI_FLAGS } = await import(scopesPath.href);
|
||||
const changes = Object.fromEntries(
|
||||
CI_FLAGS.map((flag: string) => [flag, "true"]),
|
||||
);
|
||||
changes.tier = "full";
|
||||
if (caseName === "partial full scope") changes.sqlite = "false";
|
||||
const outcome = await requiredRun(changes, overrides);
|
||||
await expect(outcome.result).rejects.toMatchObject({ code: 1 });
|
||||
await expect(
|
||||
readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
|
||||
},
|
||||
);
|
||||
|
||||
it("never writes a fresh receipt for a reused scheduled qualification", async () => {
|
||||
const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url);
|
||||
const { CI_FLAGS } = await import(scopesPath.href);
|
||||
const changes = Object.fromEntries(
|
||||
CI_FLAGS.map((flag: string) => [flag, "false"]),
|
||||
);
|
||||
changes.tier = "reused";
|
||||
changes.qualification_run_id = "7";
|
||||
const outcome = await requiredRun(changes);
|
||||
const result = await outcome.result;
|
||||
await expect(
|
||||
readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(result.stdout).toContain("Qualification reused from run 7");
|
||||
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["missing receipt run", { qualification_run_id: "" }, "schedule"],
|
||||
["selected validation lane", { sqlite: "true" }, "schedule"],
|
||||
["manual dispatch", {}, "workflow_dispatch"],
|
||||
] as const)(
|
||||
"rejects qualification reuse with %s",
|
||||
async (_name, override, event) => {
|
||||
const { CI_FLAGS } = await import(
|
||||
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
|
||||
);
|
||||
const changes: Record<string, string> = {
|
||||
...Object.fromEntries(CI_FLAGS.map((flag: string) => [flag, "false"])),
|
||||
tier: "reused",
|
||||
qualification_run_id: "7",
|
||||
...override,
|
||||
};
|
||||
const outcome = await requiredRun(changes, { event });
|
||||
await expect(outcome.result).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stderr: expect.stringContaining("scheduled admission receipt"),
|
||||
});
|
||||
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
|
||||
},
|
||||
);
|
||||
@@ -27,6 +27,9 @@ const workflow = async (filename = "ci.yml") =>
|
||||
);
|
||||
|
||||
it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => {
|
||||
expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe(
|
||||
"needs.changes.outputs.sqlite == 'true'",
|
||||
);
|
||||
const steps = z.array(
|
||||
z.object({
|
||||
uses: z.string().optional(),
|
||||
@@ -52,6 +55,91 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and
|
||||
expect(downloads[0]?.with).toEqual({ name, path: "." });
|
||||
});
|
||||
|
||||
it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => {
|
||||
const { jobs } = await workflow();
|
||||
const { CI_FLAGS } = z
|
||||
.object({ CI_FLAGS: z.array(z.string()) })
|
||||
.parse(
|
||||
await import(
|
||||
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
|
||||
),
|
||||
);
|
||||
const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs);
|
||||
expect(jobs.changes?.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "read",
|
||||
});
|
||||
expect(jobs.build?.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "read",
|
||||
});
|
||||
for (const flag of CI_FLAGS)
|
||||
expect(outputs[flag]).toBe(`\${{ steps.plan.outputs.${flag} }}`);
|
||||
const steps = z
|
||||
.array(
|
||||
z.object({
|
||||
run: z.string().optional(),
|
||||
if: z.string().optional(),
|
||||
}),
|
||||
)
|
||||
.parse(jobs.required?.steps);
|
||||
const telemetry = steps.find((step) =>
|
||||
step.run?.includes("scripts/ci/job-timings.mjs"),
|
||||
);
|
||||
expect(telemetry).toBeUndefined();
|
||||
const timings = z
|
||||
.object({
|
||||
on: z.object({
|
||||
workflow_run: z.object({
|
||||
workflows: z.array(z.string()),
|
||||
types: z.array(z.string()),
|
||||
}),
|
||||
}),
|
||||
permissions: z.record(z.string(), z.string()),
|
||||
jobs: z.object({
|
||||
report: z.object({
|
||||
steps: z.array(
|
||||
z.object({
|
||||
uses: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
with: z.record(z.string(), z.unknown()).optional(),
|
||||
env: z.record(z.string(), z.unknown()).optional(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
}),
|
||||
})
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
new URL("../../../.github/workflows/ci-timings.yml", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(timings.on.workflow_run).toEqual({
|
||||
workflows: ["CI", "Release"],
|
||||
types: ["completed"],
|
||||
});
|
||||
expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
|
||||
const checkout = timings.jobs.report.steps.find((step) =>
|
||||
step.uses?.startsWith("actions/checkout@"),
|
||||
);
|
||||
expect(checkout?.with).toEqual({
|
||||
ref: "${{ github.event.repository.default_branch }}",
|
||||
"persist-credentials": false,
|
||||
});
|
||||
const report = timings.jobs.report.steps.find((step) =>
|
||||
step.run?.includes("scripts/ci/job-timings.mjs"),
|
||||
);
|
||||
expect(report?.env?.CI_TIMING_RUN_ID).toBe(
|
||||
"${{ github.event.workflow_run.id }}",
|
||||
);
|
||||
expect(report?.env?.CI_TIMING_RUN_ATTEMPT).toBe(
|
||||
"${{ github.event.workflow_run.run_attempt }}",
|
||||
);
|
||||
});
|
||||
|
||||
const requiredNeeds = async () => {
|
||||
const { jobs } = await workflow();
|
||||
return { jobs, needs: z.array(z.string()).parse(jobs.required?.needs) };
|
||||
@@ -64,6 +152,102 @@ it("makes the required gate wait for every CI lane, including future additions",
|
||||
);
|
||||
});
|
||||
|
||||
it("leaves cancellation ownership with CI so child lanes cannot cancel another tier", async () => {
|
||||
const { jobs } = await workflow();
|
||||
const children = Object.values(jobs).flatMap((job) =>
|
||||
typeof job.uses === "string" && job.uses.startsWith("./.github/workflows/")
|
||||
? [job.uses]
|
||||
: [],
|
||||
);
|
||||
expect(children.length).toBeGreaterThan(0);
|
||||
for (const path of children) {
|
||||
const child = z
|
||||
.object({ concurrency: z.unknown().optional() })
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
child.concurrency,
|
||||
`${path} must use caller scheduling`,
|
||||
).toBeUndefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps fast feedback separate from complete qualification and release proof", async () => {
|
||||
const ci = z
|
||||
.object({
|
||||
on: z.object({
|
||||
push: z.unknown(),
|
||||
pull_request: z.unknown(),
|
||||
schedule: z.array(z.object({ cron: z.string() })),
|
||||
workflow_call: z.object({
|
||||
inputs: z.object({
|
||||
source_sha: z.object({
|
||||
required: z.literal(true),
|
||||
type: z.literal("string"),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
concurrency: z.object({
|
||||
group: z.string(),
|
||||
"cancel-in-progress": z.string(),
|
||||
}),
|
||||
jobs: z.record(
|
||||
z.string(),
|
||||
z.object({
|
||||
if: z.string().optional(),
|
||||
steps: z
|
||||
.array(
|
||||
z.object({
|
||||
run: z.string().optional(),
|
||||
if: z.string().optional(),
|
||||
with: z.record(z.string(), z.unknown()).optional(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
}),
|
||||
),
|
||||
})
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
new URL("../../../.github/workflows/ci.yml", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(ci.on.schedule).toEqual([
|
||||
{ cron: "23 0-2,4-23 * * *" },
|
||||
{ cron: "23 3 * * *" },
|
||||
]);
|
||||
expect(ci.concurrency.group).toContain("'qualification' || 'feedback'");
|
||||
expect(ci.concurrency.group).toContain("inputs.source_sha ||");
|
||||
for (const fullEvent of ["schedule", "workflow_dispatch"])
|
||||
expect(ci.concurrency["cancel-in-progress"]).toContain(
|
||||
`github.event_name != '${fullEvent}'`,
|
||||
);
|
||||
expect(ci.concurrency["cancel-in-progress"]).toContain(
|
||||
"inputs.source_sha == ''",
|
||||
);
|
||||
expect(ci.jobs.required?.if).toBe("${{ always() }}");
|
||||
const shards = ci.jobs["test-shard"]?.steps ?? [];
|
||||
const fast = shards.find((step) => step.run?.includes("npm run test:run"));
|
||||
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
|
||||
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
|
||||
expect(fast?.run).toContain("--reporter=blob");
|
||||
expect(fast?.run).not.toContain("--coverage");
|
||||
expect(full?.if).toBe("needs.changes.outputs.tier == 'full'");
|
||||
const proof = ci.jobs["package-e2e"]?.steps?.find(
|
||||
(step) => step.with?.name === "verified-release-package",
|
||||
);
|
||||
expect(proof?.if).toContain("needs.changes.outputs.tier == 'full'");
|
||||
expect(proof?.if).toContain("matrix.platform == 'linux'");
|
||||
expect(proof?.if).toContain("matrix.architecture == 'x64'");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["true", "success", true],
|
||||
["true", "failure", false],
|
||||
@@ -71,9 +255,10 @@ it.each([
|
||||
["true", "skipped", false],
|
||||
["true", "missing", false],
|
||||
["false", "skipped", true],
|
||||
["false", "failure", false],
|
||||
] as const)(
|
||||
"enforces the SQLite required gate for code=%s and result=%s",
|
||||
async (code, result, passes) => {
|
||||
"enforces the SQLite required gate for sqlite=%s and result=%s",
|
||||
async (sqlite, result, passes) => {
|
||||
const workflow = await requiredNeeds();
|
||||
// GitHub exposes only direct dependencies in the required job's needs context.
|
||||
const needs: Record<
|
||||
@@ -82,7 +267,7 @@ it.each([
|
||||
> = Object.fromEntries(
|
||||
workflow.needs.map((name) => [name, { result: "success" }]),
|
||||
);
|
||||
needs.changes = { result: "success", outputs: { code } };
|
||||
needs.changes = { result: "success", outputs: { code: "true", sqlite } };
|
||||
if (result === "missing") delete needs["sqlite-runtime"];
|
||||
else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result };
|
||||
const execution = executeWorkflowFixture(
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
@@ -10,6 +10,7 @@ import { z } from "zod";
|
||||
|
||||
import { parseTestDurations } from "../../ciTestSequencer.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { workflowGit } from "../../support/workflowGit.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const script = new URL("../../../scripts/ci/test-report.mjs", import.meta.url);
|
||||
@@ -77,6 +78,7 @@ it("keeps diagnosis available after merge failure and publishes only complete su
|
||||
const history = steps.find((step) => step.with?.name === "test-timings");
|
||||
expect(history?.if).toContain("success()");
|
||||
expect(history?.if).toContain("needs.test-shard.result == 'success'");
|
||||
expect(history?.if).toContain("needs.changes.outputs.tier == 'full'");
|
||||
expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json");
|
||||
const download = (await workflow()).build!.steps.find(
|
||||
(step) => step.name === "Download previous timing report",
|
||||
@@ -84,6 +86,49 @@ it("keeps diagnosis available after merge failure and publishes only complete su
|
||||
expect(download?.with?.name).toBe(history?.with?.name);
|
||||
});
|
||||
|
||||
it("rejects successful shard reports with missing or duplicate portable files", async () => {
|
||||
const root = await createTestTempDirectory("rea-ci-report-inventory-");
|
||||
await workflowGit(root, ["init"]);
|
||||
await mkdir(join(root, "tests"));
|
||||
for (const name of ["first", "second"])
|
||||
await writeFile(join(root, `tests/${name}.test.ts`), "// fixture\n");
|
||||
await workflowGit(root, ["add", "tests"]);
|
||||
const run = async (names: readonly string[]) => {
|
||||
await writeFile(
|
||||
join(root, "test-results.json"),
|
||||
JSON.stringify({
|
||||
success: true,
|
||||
numFailedTests: 0,
|
||||
testResults: names.map((name) => ({ name, startTime: 0, endTime: 1 })),
|
||||
}),
|
||||
);
|
||||
return execute(process.execPath, [fileURLToPath(script)], {
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
SHARD_RESULT: "success",
|
||||
MERGE_RESULT: "success",
|
||||
CI_VERIFY_TEST_INVENTORY: "true",
|
||||
},
|
||||
});
|
||||
};
|
||||
await expect(
|
||||
run(["tests/first.test.ts", "tests/second.test.ts"]),
|
||||
).resolves.toMatchObject({ stderr: "" });
|
||||
for (const names of [
|
||||
["tests/first.test.ts"],
|
||||
["tests/first.test.ts", "tests/second.test.ts", "tests/unplanned.test.ts"],
|
||||
["tests/first.test.ts", "tests/first.test.ts", "tests/second.test.ts"],
|
||||
]) {
|
||||
await expect(run(names)).rejects.toMatchObject({
|
||||
stderr: expect.stringContaining("Incomplete portable test inventory"),
|
||||
});
|
||||
await expect(
|
||||
readFile(join(root, ".cache/ci-test-history/test-results.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
}
|
||||
});
|
||||
|
||||
it("runs the existing onboarding and Python producer guards from their selected CI owners", async () => {
|
||||
const jobs = await workflow();
|
||||
const scripts = z
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
executeWorkflowFixture,
|
||||
workflowGit,
|
||||
} from "../../support/workflowGit.js";
|
||||
|
||||
const validator = fileURLToPath(
|
||||
new URL("../../../scripts/ci/verified-package.mjs", import.meta.url),
|
||||
);
|
||||
type Fixture = Awaited<ReturnType<typeof createFixture>>;
|
||||
type FixtureOptions = {
|
||||
override?: Record<string, string | number>;
|
||||
manifestContents?: string;
|
||||
writeManifest?: boolean;
|
||||
};
|
||||
|
||||
it("validates a tarball against the exact checkout and current workflow attempt", async () => {
|
||||
const fixture = await createFixture();
|
||||
const result = await validate(fixture);
|
||||
|
||||
expect(result.stdout.trim()).toBe(
|
||||
join(fixture.artifactDirectory, "package.tgz"),
|
||||
);
|
||||
expect(await readFile(fixture.output, "utf8")).toBe(
|
||||
`tarball=${join(fixture.artifactDirectory, "package.tgz")}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a workflow SHA that differs from the checked out HEAD", async () => {
|
||||
const fixture = await createFixture();
|
||||
|
||||
await expect(validate(fixture, "c".repeat(40))).rejects.toThrow();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["wrong checkout SHA", { sourceSha: "c".repeat(40) }],
|
||||
["wrong workflow run", { runId: 124 }],
|
||||
["future workflow attempt", { runAttempt: 3 }],
|
||||
["wrong verifier platform", { platform: "darwin" }],
|
||||
["wrong verifier architecture", { architecture: "arm64" }],
|
||||
["wrong package digest", { sha256: "c".repeat(64) }],
|
||||
])("rejects %s", async (_label, override) => {
|
||||
const fixture = await createFixture({ override });
|
||||
|
||||
await expect(validate(fixture)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("accepts an earlier successful attempt from the same workflow run", async () => {
|
||||
const fixture = await createFixture({ override: { runAttempt: 1 } });
|
||||
|
||||
await expect(validate(fixture)).resolves.toMatchObject({
|
||||
stdout: expect.any(String),
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["malformed manifest", "{"],
|
||||
["manifest with missing fields", JSON.stringify({ schemaVersion: 1 })],
|
||||
])("rejects %s", async (_label, contents) => {
|
||||
const fixture = await createFixture({ manifestContents: contents });
|
||||
|
||||
await expect(validate(fixture)).rejects.toThrow();
|
||||
});
|
||||
|
||||
it("rejects an absent manifest", async () => {
|
||||
const fixture = await createFixture({ writeManifest: false });
|
||||
|
||||
await expect(validate(fixture)).rejects.toThrow();
|
||||
});
|
||||
|
||||
async function createFixture({
|
||||
override = {},
|
||||
manifestContents,
|
||||
writeManifest = true,
|
||||
}: FixtureOptions = {}) {
|
||||
const directory = await createTestTempDirectory("rea-verified-package-");
|
||||
await workflowGit(directory, ["init", "--initial-branch=main"]);
|
||||
await writeFile(
|
||||
join(directory, "package.json"),
|
||||
'{"name":"fixture","version":"1.2.3"}\n',
|
||||
);
|
||||
await workflowGit(directory, ["add", "package.json"]);
|
||||
await workflowGit(directory, [
|
||||
"commit",
|
||||
"-m",
|
||||
"test: verified package fixture",
|
||||
]);
|
||||
const sourceSha = await workflowGit(directory, ["rev-parse", "HEAD"]);
|
||||
const artifactDirectory = join(directory, "artifact");
|
||||
await mkdir(artifactDirectory);
|
||||
const archive = "verified tarball fixture";
|
||||
await writeFile(join(artifactDirectory, "package.tgz"), archive);
|
||||
const manifest = {
|
||||
schemaVersion: 1,
|
||||
sourceSha,
|
||||
runId: 123,
|
||||
runAttempt: 2,
|
||||
platform: "linux",
|
||||
architecture: "x64",
|
||||
packageVersion: "1.2.3",
|
||||
nodeVersion: process.version,
|
||||
sha256: createHash("sha256").update(archive).digest("hex"),
|
||||
...override,
|
||||
};
|
||||
if (writeManifest)
|
||||
await writeFile(
|
||||
join(artifactDirectory, "manifest.json"),
|
||||
manifestContents ?? JSON.stringify(manifest),
|
||||
);
|
||||
const output = join(directory, "github-output");
|
||||
await writeFile(output, "");
|
||||
return { directory, sourceSha, artifactDirectory, output };
|
||||
}
|
||||
|
||||
function validate(fixture: Fixture, sourceSha = fixture.sourceSha) {
|
||||
return executeWorkflowFixture(
|
||||
process.execPath,
|
||||
[validator, "validate", fixture.artifactDirectory],
|
||||
{
|
||||
cwd: fixture.directory,
|
||||
env: {
|
||||
...process.env,
|
||||
GITHUB_SHA: sourceSha,
|
||||
GITHUB_RUN_ID: "123",
|
||||
GITHUB_RUN_ATTEMPT: "2",
|
||||
GITHUB_OUTPUT: fixture.output,
|
||||
},
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -651,13 +651,16 @@ it.skipIf(process.platform === "win32").each([
|
||||
.object({
|
||||
jobs: z.record(
|
||||
z.string(),
|
||||
// Jobs that call a reusable workflow declare `uses` instead of steps.
|
||||
z.object({
|
||||
steps: z.array(
|
||||
z.object({
|
||||
name: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
}),
|
||||
),
|
||||
steps: z
|
||||
.array(
|
||||
z.object({
|
||||
name: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
}),
|
||||
),
|
||||
})
|
||||
@@ -672,7 +675,7 @@ it.skipIf(process.platform === "win32").each([
|
||||
const command = z
|
||||
.string()
|
||||
.parse(
|
||||
workflow.jobs[job]?.steps.find(
|
||||
workflow.jobs[job]?.steps?.find(
|
||||
(step) =>
|
||||
step.name ===
|
||||
(job === "merged-release"
|
||||
|
||||
@@ -34,6 +34,12 @@ async function readReleaseWorkflow() {
|
||||
"release-proposal": jobSchema,
|
||||
"release-please": jobSchema,
|
||||
"merged-release": jobSchema,
|
||||
qualification: z.object({
|
||||
if: z.string(),
|
||||
needs: z.array(z.string()),
|
||||
uses: z.string(),
|
||||
with: z.record(z.string(), z.string()),
|
||||
}),
|
||||
publish: jobSchema,
|
||||
"publish-mcp": jobSchema,
|
||||
}),
|
||||
@@ -271,10 +277,19 @@ it("binds npm and MCP publication to the same immutable release SHA", async () =
|
||||
release_created: "${{ steps.release.outputs.release_created }}",
|
||||
sha: "${{ steps.release.outputs.sha }}",
|
||||
});
|
||||
expect(workflow.jobs.publish.if).toContain("inputs.phase == 'publish'");
|
||||
expect(workflow.jobs.publish.if).toContain(
|
||||
expect(workflow.jobs.qualification.if).toContain("inputs.phase == 'publish'");
|
||||
expect(workflow.jobs.qualification.if).toContain(
|
||||
"needs.release-please.outputs.release_created",
|
||||
);
|
||||
expect(workflow.jobs.qualification.uses).toBe("./.github/workflows/ci.yml");
|
||||
expect(workflow.jobs.qualification.with).toEqual({
|
||||
source_sha:
|
||||
"${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }}",
|
||||
});
|
||||
expect(workflow.jobs.publish.needs).toContain("qualification");
|
||||
expect(workflow.jobs.publish.if).toContain(
|
||||
"needs.qualification.result == 'success'",
|
||||
);
|
||||
expect(workflow.jobs["publish-mcp"].if).toContain(
|
||||
"needs.publish.result == 'success'",
|
||||
);
|
||||
@@ -309,7 +324,7 @@ it("binds npm and MCP publication to the same immutable release SHA", async () =
|
||||
)?.run;
|
||||
expect(publishCommand).toContain("scripts/release-npm-tag.mjs");
|
||||
expect(publishCommand).toContain(
|
||||
'npm publish --access public --tag "${tag}"',
|
||||
'npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}"',
|
||||
);
|
||||
for (const [version, tag] of [
|
||||
["6.1.0", "latest"],
|
||||
@@ -644,7 +659,7 @@ it("routes website verification through the central PR policy", async () => {
|
||||
|
||||
// Publishing is irreversible. Keep the release authority invariant as a static
|
||||
// boundary check; it is not proof that package installation or publishing works.
|
||||
it("requires package verification before publishing and retains the registry canary", async () => {
|
||||
it("requires qualified package validation before publishing and retains the registry canary", async () => {
|
||||
const workflow = z
|
||||
.object({
|
||||
jobs: z.object({
|
||||
@@ -663,7 +678,7 @@ it("requires package verification before publishing and retains the registry can
|
||||
);
|
||||
const commands = workflow.jobs.publish.steps.map(({ run }) => run ?? "");
|
||||
const verify = commands.findIndex((command) =>
|
||||
command.includes("npm run verify:package"),
|
||||
command.includes("scripts/ci/verified-package.mjs validate"),
|
||||
);
|
||||
const publish = commands.findIndex((command) =>
|
||||
command.includes("npm publish"),
|
||||
|
||||
Reference in New Issue
Block a user