merge: retain main CI planning and artifact checks

This commit is contained in:
64579525
2026-10-13 19:37:50 +08:00
136 changed files with 34298 additions and 798 deletions
@@ -0,0 +1,61 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { cliTest } from "../../support/cli/cliFixture.js";
import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js";
cliTest.for(["CLI", "stdio MCP"] as const)(
"retains both repeated chunk callback observations through %s",
async (surface, { cli }) => {
const root = await createTestTempDirectory("rea-bundler-callbacks-");
const source = [
'(globalThis.webpackChunkFixture = globalThis.webpackChunkFixture || []).push([[1], {a: function() {}}, function(r) { r(7); r.e("chunk-a"); }]);',
'(globalThis.webpackChunkFixture = globalThis.webpackChunkFixture || []).push([[1], {a: function() {}}, function(r) { r(8); r.e("chunk-b"); }]);',
].join("\n");
await writeFile(join(root, "bundle.js"), source);
const { graph } = await analyzeJavaScriptDirectory(surface, cli, root);
const chunk = graph.nodes.find(({ kind }) => kind === "javascript-chunk");
for (const [index, entry, asyncChunk] of [
[0, "7", "chunk-a"],
[1, "8", "chunk-b"],
] as const) {
expect(chunk?.observations).toContainEqual(
expect.objectContaining({
properties: expect.objectContaining({
entry_module_keys: [entry],
async_chunk_keys: [asyncChunk],
}),
evidence: expect.objectContaining({
location: expect.objectContaining({
value: expect.objectContaining({
source: "bundle.js",
start: { line: index + 1, column: 0 },
}),
}),
}),
}),
);
expect(graph.edges).toContainEqual(
expect.objectContaining({
source_node_id: chunk?.node_id,
properties: expect.objectContaining({
kind: "bundler-entry-module",
module_key: entry,
resolution_status: "not-found",
}),
}),
);
expect(graph.edges).toContainEqual(
expect.objectContaining({
source_node_id: chunk?.node_id,
properties: expect.objectContaining({
kind: "bundler-async-chunk",
chunk_key: asyncChunk,
resolution_status: "not-found",
}),
}),
);
}
},
);
@@ -0,0 +1,110 @@
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { expect } from "vitest";
import { encodeNibArchiveFixture } from "../../../src/artifacts/apple/NibArchive.fixture.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { interfaceBuilderAnalysisSchema } from "../../../src/domain/apple/interfaceBuilderGraph.js";
import { thinMach } from "../../../src/domain/binaryTarget.fixture.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { cliTest } from "../../support/cli/cliFixture.js";
cliTest(
"preserves literal NIB fields and UTF-8 names through CLI and stdio MCP",
async ({ cli }) => {
const root = await createTestTempDirectory("rea-nib-field-identity-");
const app = join(root, "Fixture.app");
const contents = join(app, "Contents");
await mkdir(join(contents, "Resources"), { recursive: true });
await mkdir(join(contents, "MacOS"));
await writeFile(
join(contents, "MacOS/App"),
thinMach(0xfeedfacf, 0x0100000c),
);
await writeFile(
join(contents, "Info.plist"),
"<plist><dict><key>CFBundleExecutable</key><string>App</string><key>CFBundleIdentifier</key><string>dev.rea.nib-fields</string></dict></plist>",
);
const archive = encodeNibArchiveFixture({
classes: ["UIView", "\uFEFFCustomView"],
objects: [
{
classIndex: 0,
values: {
["__proto__"]: { ref: 1 },
constructor: 7,
enabled: true,
"\uFEFFenabled": false,
},
},
{ classIndex: 1, values: { ["__proto__"]: 23 } },
],
});
const path = join(contents, "Resources/Panel.nib");
await writeFile(path, archive);
const result = await cli.run({
arguments: ["decode-interface-builder", app, "--json"],
environment: { REA_LOG_LEVEL: "silent", REA_ANALYSIS_PROVIDER: "auto" },
});
expect(result.exitCode, result.stdout + result.stderr).toBe(0);
const contract = toolContract("decode_interface_builder");
const evidence = contract.outputSchema.parse(result.json);
const transport = new StdioClientTransport({
command: process.execPath,
args: [resolve("scripts/rea.mjs"), "mcp"],
env: {
PATH: process.env.PATH ?? "",
REA_LOG_LEVEL: "silent",
REA_ANALYSIS_PROVIDER: "auto",
},
stderr: "pipe",
});
const client = new Client({ name: "nib-field-identity", version: "1" });
try {
await client.connect(transport);
const opened = await client.callTool({
name: "open_binary",
arguments: { path: app },
});
expect(opened.isError, JSON.stringify(opened)).not.toBe(true);
const response = await client.callTool({
name: contract.name,
arguments: {},
});
expect(response.isError, JSON.stringify(response)).not.toBe(true);
const mcp = contract.outputSchema.parse(response.structuredContent);
expect(mcp.normalized_result).toEqual(evidence.normalized_result);
} finally {
try {
await client.close();
} finally {
await transport.close();
}
}
const report = interfaceBuilderAnalysisSchema.parse(
evidence.normalized_result,
);
expect(report.documents[0]?.archive_sha256).toBe(
createHash("sha256").update(archive).digest("hex"),
);
const objects = report.graph.nodes.filter(
({ attributes }) => attributes.interface_builder_object_id !== undefined,
);
expect(objects).toHaveLength(2);
expect(objects[0]?.attributes.nibValues).toEqual({
["__proto__"]: { objectID: "1" },
constructor: 7,
enabled: true,
"\uFEFFenabled": false,
});
expect(objects[1]?.attributes).toMatchObject({
class_name: "\uFEFFCustomView",
nibValues: { ["__proto__"]: 23 },
});
expect(await readFile(path)).toEqual(archive);
},
60_000,
);
@@ -12,6 +12,38 @@ const selectors = [
] as const;
describe("CLI analysis selector boundaries", () => {
cliTest(
"accepts file offsets and rejects mixed coordinate selectors before opening a target",
async ({ cli }) => {
const selected = await cli.run({
arguments: [
"decompile",
"does-not-exist",
"--file-offset=0x210",
"--json",
],
environment: { REA_LOG_LEVEL: "silent" },
});
expect(selected.json).toMatchObject({ code: "target_unavailable" });
const conflicting = await cli.run({
arguments: [
"decompile",
"does-not-exist",
"0x140001010",
"--file-offset=0x210",
"--json",
],
environment: { REA_LOG_LEVEL: "silent" },
});
expect(conflicting.json).toMatchObject({
code: "invalid_request",
details: {
operation: "procedure_pseudo_code",
issues: [{ path: ["file-offset"], reason: "invalid_value" }],
},
});
},
);
cliTest(
"rejects missing and conflicting selectors before opening a target",
async ({ cli }) => {
@@ -0,0 +1,85 @@
import { expect, it } from "vitest";
import { Cli } from "incur";
import pino from "pino";
import type { ApktoolResourceAnalysisPort } from "../../../src/application/apktool/ApktoolResourceAnalysisPort.js";
import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js";
import { registerApktoolCommands } from "../../../src/cli/apktoolCommands.js";
import { presentOmittableDefaults } from "../../../src/cli/omittableDefaults.js";
import type { ApktoolRequest } from "../../../src/domain/apktool/apktoolResourceAnalysis.js";
import { ok } from "../../../src/domain/result.js";
const invoke = async (argv: readonly string[]) => {
const requests: ApktoolRequest[] = [];
const provider: ApktoolResourceAnalysisPort = {
async inspectAvailability() {
throw new Error("unexpected provider probe");
},
async close() {},
async execute(request) {
requests.push(request);
return ok(
createAnalysisExecution(
{ strings: [] },
{ id: "apktool", name: "Apktool", version: null },
),
);
},
};
const cli = Cli.create("rea", { sync: false });
registerApktoolCommands(cli, pino({ level: "silent" }), {}, provider);
presentOmittableDefaults(cli);
let stdout = "";
let exitCode = 0;
await cli.serve([...argv], {
env: {},
exit: (code) => {
exitCode = code;
},
stdout: (text) => {
stdout += text;
},
});
return { exitCode, stdout, requests };
};
it("dispatches --no-strings as include_strings false", async () => {
const result = await invoke([
"decode-android-resources",
"/targets/app.apk",
"--no-strings",
"--json",
]);
expect(result.exitCode).toBe(0);
expect(result.requests).toEqual([
{
operation: "decode_android_resources",
input: { path: "/targets/app.apk", include_strings: false },
},
]);
});
it("keeps string projection enabled when the flag is omitted", async () => {
const result = await invoke([
"decode-android-resources",
"/targets/app.apk",
"--json",
]);
expect(result.exitCode).toBe(0);
expect(result.requests).toEqual([
{
operation: "decode_android_resources",
input: { path: "/targets/app.apk", include_strings: true },
},
]);
});
it("documents the supported negative flag in command help", async () => {
const result = await invoke(["decode-android-resources", "--help"]);
expect(result.exitCode).toBe(0);
expect(result.stdout).toContain("--no-strings");
});
@@ -4,11 +4,13 @@ import { dirname, join } from "node:path";
import { describe, expect } from "vitest";
import {
CPU,
FILE_TYPE,
LC,
buildVersionCommand,
dylibCommand,
dyldEnvironmentCommand,
fatImage,
machoImage,
rpathCommand,
} from "../../../src/artifacts/apple/MachoImage.fixture.js";
@@ -471,6 +473,64 @@ cliTest(
},
);
cliTest(
"reports each same-named slice's own embedded dyld environment",
async ({ cli }) => {
// arm64e and arm64e.x1 slices both report the architecture name arm64e.
// A plain x86_64 slice keeps the bundle openable on x64 hosts.
for (const [settings, expected] of [
[[null, "DYLD_PRINT_RPATHS=1"], ["DYLD_PRINT_RPATHS=1"]],
[
["DYLD_PRINT_LIBRARIES=1", "DYLD_PRINT_RPATHS=1"],
["DYLD_PRINT_LIBRARIES=1", "DYLD_PRINT_RPATHS=1"],
],
] as const) {
const app = await fixtureApp();
await writeFiles(app, {
"Contents/MacOS/App": fatImage([
...[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => {
const setting = settings[index];
return {
cpu,
bytes: machoImage({
cpu,
commands:
setting === null || setting === undefined
? []
: [dyldEnvironmentCommand(setting)],
}),
};
}),
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
]),
});
const result = await cli.run({
arguments: [
"trace-dylib-resolution",
app,
"--root",
"Contents/MacOS/App",
"--json",
],
environment: ENVIRONMENT,
});
expect(result.exitCode, JSON.stringify(result.json)).toBe(0);
const trace = dylibResolutionResultSchema.parse(
parseEvidence(result.json).normalized_result,
);
expect(
trace.roots.filter(({ architecture }) => architecture === "arm64e"),
).toHaveLength(2);
const environments = trace.findings
.filter(({ kind }) => kind === "dyld-environment-present")
.map(({ explanation }) =>
expected.filter((setting) => explanation.includes(setting)),
);
expect(environments).toEqual(expected.map((setting) => [setting]));
}
},
);
cliTest(
"retains definitive load findings for an explicitly selected app library root",
async ({ cli }) => {
@@ -21,14 +21,14 @@ const adHocProgram = async (directory: string, name: string) => {
describe.skipIf(process.platform !== "darwin")(
"native signature CLI with echoed path text",
() => {
cliTest(
"does not read signature fields from the file name codesign echoes",
async ({ cli }) => {
cliTest.for(["\n", "\r\n"])(
"does not read signature fields from the file name codesign echoes (%j)",
async (newline, { cli }) => {
const directory = await createTestTempDirectory("rea-signature-");
const forged = "Developer ID Application: Example (ABCDE12345)";
const program = await adHocProgram(
directory,
`tool\nAuthority=${forged}\nTeamIdentifier=ABCDE12345`,
`tool${newline}Authority=${forged}${newline}Identifier=forged${newline}TeamIdentifier=ABCDE12345`,
);
const unsignedName = await adHocProgram(
directory,
@@ -75,7 +75,7 @@ describe.skipIf(process.platform !== "darwin")(
normalized_result: {
identifier: expect.stringMatching(
new RegExp(
`^tool\\nAuthority=${forged.replace(/[()]/gu, "\\$&")}\\nTeamIdentifier=ABCDE12345-`,
`^tool${newline}Authority=${forged.replace(/[()]/gu, "\\$&")}${newline}Identifier=forged${newline}TeamIdentifier=ABCDE12345-`,
"u",
),
),
@@ -38,8 +38,19 @@ it("completes an empty ZIP extraction with an empty identity", async () => {
expect(await readdir(output)).toEqual([]);
});
const legacyControlNames = [
["legacy ASCII control", Buffer.from("Icon\r"), false, "Icon\r"],
[
"legacy CP437 control",
Buffer.from([0x82, 0x09, 0x7f, 0x2e, 0x61]),
false,
"é\t\u007F.a",
],
] as const;
it.each([
["legacy CP437", Buffer.from([0x82, 0x2e, 0x74, 0x78, 0x74]), false, "é.txt"],
...legacyControlNames,
[
"EFS with leading U+FEFF",
Buffer.from([0xef, 0xbb, 0xbf, 0x61, 0x62]),
@@ -68,6 +79,38 @@ it.each([
},
);
it.skipIf(process.platform === "win32").each(legacyControlNames)(
"extracts %s ZIP members under their stored names",
async (_label, rawName, utf8Flag, logicalPath) => {
const root = await createTestTempDirectory("rea-zip-extracted-name-");
const path = join(root, "names.zip");
await writeSingleEntryZip(path, "x.txt");
const archive = await readFile(path);
await writeFile(
path,
rewriteSingleEntryName(archive, "x.txt", rawName, utf8Flag),
);
const target = await parseBinaryTarget(path);
if (!target.ok) throw target.error;
const output = join(root, "output");
const result = await withArtifactClient(target.value, (client) =>
client.execute("extract_artifact", { output_root: output }),
);
if (!result.ok) throw result.error;
const extraction = artifactExtractionResultSchema.parse(
result.value.result,
);
expect(
extraction.artifacts.map(({ relative_path }) => relative_path),
).toEqual([logicalPath]);
expect(await readdir(output)).toEqual([logicalPath]);
expect(await readFile(join(output, logicalPath), "utf8")).toBe(
"member bytes",
);
},
);
it("preserves valid Info-ZIP Unicode Path overrides", async () => {
const root = await createTestTempDirectory("rea-zip-unicode-path-");
const path = join(root, "unicode-path.zip");
+330 -9
View File
@@ -1,4 +1,5 @@
import {
chmod,
cp,
mkdir,
readFile,
@@ -79,6 +80,7 @@ async function classify(
head: string,
event = "pull_request",
full = false,
skipFull = false,
) {
const output = join(directory, "scope-output");
await writeFile(output, "");
@@ -93,6 +95,7 @@ async function classify(
HEAD_SHA: head,
GITHUB_EVENT_NAME: event,
CI_FULL: String(full),
CI_SKIP_FULL: String(skipFull),
GITHUB_OUTPUT: output,
},
},
@@ -100,6 +103,21 @@ async function classify(
return z.record(z.string(), z.unknown()).parse(JSON.parse(result.stdout));
}
it("reuses only scheduled qualification and preserves every explicit full request", async () => {
const { directory, base } = await fixture();
const reused = await classify(directory, base, base, "schedule", false, true);
expect(reused.tier).toBe("reused");
const { CI_FLAGS } = await loadScopes();
for (const flag of CI_FLAGS) expect(reused[flag]).toBe(false);
for (const event of ["workflow_dispatch", "pull_request", "schedule"])
expect(
(await classify(directory, base, base, event, true, true)).tier,
).toBe("full");
expect(
(await classify(directory, base, base, "push", false, true)).tier,
).toBe("fast");
});
it("keeps every translated root README and authored guide on the documentation lane", async () => {
const { directory, base } = await fixture();
const readmes = (await readdir(new URL("../../../", import.meta.url))).filter(
@@ -119,6 +137,64 @@ it("keeps every translated root README and authored guide on the documentation l
});
});
it.each([
"job-timings.mjs",
"job-timings-batch.mjs",
"test-report.mjs",
"test-timing-reporter.mjs",
"test-history.mjs",
])(
"keeps portable CI reporter %s on static checks and portable tests",
async (name) => {
const { directory, base } = await fixture();
const head = await change(directory, [`scripts/ci/${name}`]);
const plan = await classify(directory, base, head);
const { CI_FLAGS } = await loadScopes();
expect(plan.code).toBe(true);
for (const flag of CI_FLAGS)
if (flag !== "code") expect(plan[flag]).toBe(false);
},
);
it("preserves a provider lane when a portable CI reporter changes with provider source", async () => {
const { directory, base } = await fixture();
const head = await change(directory, [
"scripts/ci/test-report.mjs",
"src/evm/Example.ts",
]);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
evm: true,
browser: false,
package: false,
});
});
it("keeps unknown CI scripts on complete qualification", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["scripts/ci/new-qualification.mjs"]);
const plan = await classify(directory, base, head);
const { CI_FLAGS } = await loadScopes();
for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true);
});
it("emits an observational owner proposal without changing ordinary or full portable execution", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["src/evm/Example.ts"]);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
testSelection: { mode: "candidate", owners: ["evm"] },
});
expect(
await classify(directory, base, head, "pull_request", true),
).toMatchObject({
code: true,
testSelection: { mode: "full", owners: [] },
});
const output = await readFile(join(directory, "scope-output"), "utf8");
expect(output).toContain('test_selection={"version":1,"mode":"full"');
});
it("does not charge a documentation PR for unrelated implementation updates on main", async () => {
const { directory, base } = await fixture();
await git(directory, ["switch", "-c", "docs/change"]);
@@ -162,13 +238,148 @@ it("retains complete checks when implementation is renamed into documentation",
});
});
it("retains the complete main-push baseline even for documentation", async () => {
it("keeps main pushes scoped when a successful ancestor baseline is available", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["README_ja.md"]);
expect(await classify(directory, base, head, "push")).toMatchObject({
tier: "fast",
code: false,
package: false,
docs: true,
matrix: {
include: [
{ os: "ubuntu-latest", platform: "linux", architecture: "x64" },
],
},
});
});
it("includes changes from earlier cancelled pushes since the last successful baseline", async () => {
const { directory, base } = await fixture();
await change(directory, ["src/evm/client.ts"]);
const head = await change(directory, ["README.md"]);
expect(await classify(directory, base, head, "push")).toMatchObject({
tier: "fast",
code: true,
evm: true,
runtime: true,
package: false,
});
});
it.each(["", "not-a-commit"])(
"fails closed to all lanes for a missing or invalid push baseline (%s)",
async (base) => {
const { directory } = await fixture();
const head = await change(directory, ["README.md"]);
const plan = await classify(directory, base, head, "push");
expect(plan).toMatchObject({
tier: "fast",
code: true,
package: true,
evm: true,
browser: true,
reasons: expect.arrayContaining([expect.stringContaining("baseline")]),
});
const { CI_FLAGS } = await loadScopes();
for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true);
},
);
it("selects only a trusted successful ancestor from recent CI push runs", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["src/evm/client.ts"]);
const bin = join(directory, "fake-bin");
await mkdir(bin);
const gh = join(bin, "gh");
await writeFile(gh, '#!/bin/sh\ncat "$GH_FIXTURE_RESPONSE"\n');
await chmod(gh, 0o755);
const response = join(directory, "runs.json");
await writeFile(
response,
JSON.stringify({
workflow_runs: [
{
event: "push",
conclusion: "success",
head_branch: "main",
head_repository: { full_name: "untrusted/fork" },
head_sha: base,
},
{
event: "push",
conclusion: "success",
head_branch: "main",
head_repository: { full_name: "owner/repo" },
head_sha: base,
},
],
}),
);
const output = join(directory, "baseline-output");
const result = await execFileAsync(
process.execPath,
[new URL("../../../scripts/ci/baseline.mjs", import.meta.url).pathname],
{
cwd: directory,
env: {
...process.env,
PATH: `${bin}:${process.env.PATH}`,
GH_FIXTURE_RESPONSE: response,
CI_REPOSITORY: "owner/repo",
CI_DEFAULT_BRANCH: "main",
HEAD_SHA: head,
GITHUB_OUTPUT: output,
},
},
);
expect(JSON.parse(result.stdout)).toEqual({ base, reason: "" });
expect(await readFile(output, "utf8")).toBe(`base=${base}\nreason=\n`);
});
it("returns an empty baseline when GitHub lookup fails", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["src/evm/client.ts"]);
const bin = join(directory, "fake-bin");
await mkdir(bin);
const gh = join(bin, "gh");
await writeFile(gh, "#!/bin/sh\nexit 1\n");
await chmod(gh, 0o755);
const result = await execFileAsync(
process.execPath,
[new URL("../../../scripts/ci/baseline.mjs", import.meta.url).pathname],
{
cwd: directory,
env: {
...process.env,
PATH: `${bin}:${process.env.PATH}`,
CI_REPOSITORY: "owner/repo",
CI_DEFAULT_BRANCH: "main",
HEAD_SHA: head,
},
},
);
expect(JSON.parse(result.stdout)).toMatchObject({
base: "",
reason: expect.stringContaining("Could not read"),
});
expect(head).not.toBe(base);
});
it("fails closed when the successful push baseline is not an ancestor", async () => {
const { directory, base } = await fixture();
await git(directory, ["switch", "-c", "unrelated"]);
const unrelated = await change(directory, ["other.txt"]);
await git(directory, ["switch", "main"]);
const head = await change(directory, ["README.md"]);
expect(await classify(directory, unrelated, head, "push")).toMatchObject({
tier: "fast",
code: true,
package: true,
evm: true,
browser: true,
});
expect(head).not.toBe(base);
});
it("fails classification rather than treating an invalid Git comparison as a scope", async () => {
@@ -208,7 +419,8 @@ it("retains native Inspector verification on every documented package host", asy
);
expect(packageInspector.if).toBeUndefined();
expect(
(await classify((await fixture()).directory, "", "", "push")).matrix,
(await classify((await fixture()).directory, "", "", "workflow_dispatch"))
.matrix,
).toEqual({
include: [
{ os: "ubuntu-latest", platform: "linux", architecture: "x64" },
@@ -228,7 +440,14 @@ it("retains native Inspector verification on every documented package host", asy
it.each([
[
"src/evm/client.ts",
{ code: true, runtime: true, evm: true, browser: false, package: false },
{
code: true,
runtime: true,
evm: true,
browser: false,
package: false,
sqlite: false,
},
],
[
"src/browser/execution/evaluate.ts",
@@ -349,6 +568,38 @@ it.each([
"src/process/ProcessOwnership.test.ts",
{ code: true, package: true, apple: false, evm: false, browser: false },
],
[
"src/sqlite/SqliteDatabaseProvider.ts",
{ code: true, sqlite: true, runtime: true, package: false },
],
[
"src/application/sqlite/SqliteDatabaseService.test.ts",
{ code: true, sqlite: true, runtime: false },
],
[
"tests/boundary/sqlite/sqliteDatabaseProvider.test.ts",
{ code: true, sqlite: true, runtime: false },
],
[
"tests/fixtures/sqlite/database.ts",
{ code: true, sqlite: true, runtime: false },
],
["tests/acceptance/sqliteDatabaseCli.test.ts", { code: true, sqlite: true }],
[
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
{ code: true, sqlite: true },
],
["src/process/OwnedCommand.ts", { code: true, sqlite: true, runtime: true }],
["src/server/createServer.ts", { code: true, sqlite: true, runtime: true }],
[
"src/contracts/toolContracts.ts",
{ code: true, sqlite: true, runtime: true },
],
[
"src/domain/analysisErrorProjection.ts",
{ code: true, sqlite: true, runtime: true },
],
["src/domain/javascript/example.ts", { code: true, sqlite: false }],
])("selects the appropriate lanes for %s", async (path, expected) => {
const { directory, base } = await fixture();
const head = await change(directory, [path]);
@@ -391,6 +642,52 @@ it("scopes package script edits while retaining broad dependency checks", async
});
});
it.each(["test:sqlite", "test:sqlite:run"])(
"routes %s script changes to the SQLite runtime matrix",
async (script) => {
const { directory } = await fixture();
const base = await change(
directory,
["package.json"],
JSON.stringify({ scripts: { [script]: "vitest run" } }),
);
const head = await change(
directory,
["package.json"],
JSON.stringify({
scripts: { [script]: "vitest run --reporter=default" },
}),
);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
sqlite: true,
evm: false,
package: false,
});
},
);
it("keeps unrelated test script changes off the SQLite runtime matrix", async () => {
const { directory } = await fixture();
const base = await change(
directory,
["package.json"],
JSON.stringify({ scripts: { "test:domain": "vitest run src/domain" } }),
);
const head = await change(
directory,
["package.json"],
JSON.stringify({
scripts: { "test:domain": "vitest run src/domain --reporter=default" },
}),
);
expect(await classify(directory, base, head)).toMatchObject({
code: true,
sqlite: false,
evm: false,
});
});
it.each([
["browser", "real-browser"],
["binary_layout", "real-binary-layout"],
@@ -407,7 +704,9 @@ it.each([
async (scope, name) => {
const workflow = z
.object({
on: z.object({ workflow_call: z.null() }).passthrough(),
on: z
.object({ workflow_call: z.object({}).passthrough().nullable() })
.passthrough(),
})
.parse(
parse(
@@ -447,8 +746,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn
const command = fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
);
const { CI_FLAGS } = await loadScopes();
const jobs = {
changes: { result: "success", outputs: { evm: "true" } },
changes: {
result: "success",
outputs: {
...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])),
tier: "fast",
evm: "true",
},
},
static: { result: "success" },
test: { result: "success" },
evm: { result: "success" },
@@ -484,7 +791,10 @@ it.each([
"src/process/capture/example.test.ts",
{ code: true, runtime: false, browser: false, package: false },
],
[".nvmrc", { code: true, package: true, browser: true, evm: true }],
[
".nvmrc",
{ code: true, package: true, browser: true, evm: true, sqlite: true },
],
[
"tsconfig.build.json",
{ code: true, package: true, browser: true, evm: true },
@@ -621,8 +931,10 @@ it.each(["workflow_dispatch", "schedule"])(
async (event) => {
const { directory, base } = await fixture();
expect(await classify(directory, base, base, event)).toMatchObject({
tier: "full",
code: true,
package: true,
sqlite: true,
evm: true,
browser: true,
});
@@ -636,7 +948,14 @@ it("honors ci:full for successive PR revisions", async () => {
for (const head of [first, second]) {
expect(
await classify(directory, base, head, "pull_request", true),
).toMatchObject({ code: true, package: true, evm: true, browser: true });
).toMatchObject({
tier: "full",
code: true,
package: true,
sqlite: true,
evm: true,
browser: true,
});
}
});
@@ -676,7 +995,7 @@ const INTEL_HOST = {
architecture: "x64",
};
it("keeps macOS x64 package verification for ci:full and non-PR events", async () => {
it("keeps macOS x64 package verification only for full qualification", async () => {
const { directory, base } = await fixture();
const head = await change(directory, ["src/windows/Example.ts"]);
const matrix = z.object({ include: z.array(z.unknown()) });
@@ -687,7 +1006,8 @@ it("keeps macOS x64 package verification for ci:full and non-PR events", async (
expect(pullRequest).toHaveLength(3);
expect(pullRequest).not.toContainEqual(INTEL_HOST);
expect(await hosts("pull_request", true)).toContainEqual(INTEL_HOST);
for (const event of ["push", "schedule", "workflow_dispatch"])
expect(await hosts("push", false)).not.toContainEqual(INTEL_HOST);
for (const event of ["schedule", "workflow_dispatch"])
expect(await hosts(event, false)).toContainEqual(INTEL_HOST);
});
@@ -700,6 +1020,7 @@ const workflowFile = async (name: string): Promise<unknown> =>
);
const scopesSchema = z.object({
CI_FLAGS: z.array(z.string()),
SCOPES: z.record(z.string(), z.array(z.string())),
MANUAL_VERIFIERS: z.array(z.string()),
});
@@ -0,0 +1,243 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
executeWorkflowFixture,
workflowGit,
} from "../../support/workflowGit.js";
const reporter = fileURLToPath(
new URL("../../../scripts/ci/job-timings.mjs", import.meta.url),
);
it("merges paginated jobs, reports elapsed metrics, and preserves raw responses", async () => {
const fixture = await createFixture([
{
total_count: 4,
jobs: [
{
name: "Build package",
workflow_name: "CI",
status: "completed",
conclusion: "success",
created_at: "2026-10-13T00:00:00Z",
started_at: "2026-10-13T00:00:10Z",
completed_at: "2026-10-13T00:01:10Z",
steps: [
{
name: "Install dependencies (npm ci)",
started_at: "2026-10-13T00:00:10Z",
completed_at: "2026-10-13T00:00:40Z",
},
{
name: "Upload artifact",
started_at: "2026-10-13T00:00:40Z",
completed_at: "2026-10-13T00:00:41Z",
},
],
},
],
},
{
total_count: 4,
jobs: [
{
name: "Queued lane",
status: "queued",
created_at: "2026-10-13T00:00:00Z",
started_at: "2026-10-13T00:00:00Z",
completed_at: null,
steps: [],
},
{
name: "Running lane",
status: "in_progress",
created_at: "2026-10-13T00:00:00Z",
started_at: "2026-10-13T00:00:15Z",
completed_at: null,
steps: [
{
name: "Download artifacts",
started_at: "2026-10-13T00:00:20Z",
completed_at: "2026-10-13T00:00:22Z",
},
],
},
{
name: "Skipped lane",
status: "completed",
conclusion: "skipped",
created_at: "2026-10-13T00:00:00Z",
started_at: "2026-10-13T00:00:00Z",
completed_at: "2026-10-13T00:00:00Z",
steps: [],
},
],
},
]);
const result = await runReporter(fixture);
const report = JSON.parse(
await readFile(join(fixture.outputDirectory, "report.json"), "utf8"),
);
expect(report.run.snapshot).toContain("partial");
expect(report.counts).toMatchObject({
jobs: 4,
completed: 2,
skipped: 1,
queueUnknown: 2,
executionUnknown: 3,
completedExecutionUnknown: 0,
});
expect(report.aggregates.queueSeconds).toMatchObject({
count: 2,
p50: 10,
p95: 15,
});
expect(report.aggregates.completedExecutionSeconds).toMatchObject({
count: 1,
p50: 60,
p95: 60,
});
expect(report.aggregates.summedCompletedJobElapsedSeconds).toMatchObject({
value: 60,
knownCount: 1,
unknownCount: 0,
label: expect.stringContaining("not CPU time"),
});
expect(report.jobs[1].queue).toEqual({
seconds: null,
reason: "job start is not established by its status",
});
expect(report.jobs[3].queue).toEqual({
seconds: null,
reason: "job was skipped",
});
expect(report.jobs[3].execution).toEqual({
seconds: null,
reason: "job was skipped",
});
expect(report.stepCosts.npmCi[0].duration.seconds).toBe(30);
expect(report.stepCosts.artifactUpload[0].duration.seconds).toBe(1);
expect(report.stepCosts.artifactDownload[0].duration.seconds).toBe(2);
expect(report.stepCostInterpretation).toContain(
"not total artifact transfer costs",
);
expect(report.cpuSeconds).toBeUndefined();
expect(result.stdout).toContain("4 job timing records");
expect(
await readFile(join(fixture.outputDirectory, "jobs.json"), "utf8"),
).toBe(fixture.raw);
const summary = await readFile(fixture.summary, "utf8");
expect(summary).toContain("point-in-time snapshot");
expect(summary).toContain("unknown");
expect(summary).toContain("not CPU time");
expect(summary).toContain("60 s");
expect(summary).toContain("not total artifact transfer costs");
const args = JSON.parse(await readFile(fixture.args, "utf8"));
expect(args).toEqual([
"api",
"--paginate",
"--slurp",
"repos/example/rea/actions/runs/42/attempts/2/jobs?per_page=100",
]);
});
it("marks invalid and out-of-order timestamps unknown instead of emitting negative durations", async () => {
const fixture = await createFixture([
{
jobs: [
{
name: "Invalid dates",
status: "completed",
created_at: "not-a-date",
started_at: "2026-10-13T00:00:10Z",
completed_at: "2026-10-13T00:00:05Z",
steps: [],
},
],
},
]);
await runReporter(fixture);
const report = JSON.parse(
await readFile(join(fixture.outputDirectory, "report.json"), "utf8"),
);
expect(report.jobs[0].queue).toEqual({
seconds: null,
reason: "invalid timestamp",
});
expect(report.jobs[0].execution).toEqual({
seconds: null,
reason: "timestamps out of order",
});
expect(report.counts.completedExecutionUnknown).toBe(1);
expect(report.aggregates.summedCompletedJobElapsedSeconds).toMatchObject({
value: null,
unknownCount: 1,
});
});
it("fails on malformed pages or a failed GitHub API request", async () => {
const malformed = await createFixture([{ jobs: "not-an-array" }]);
await expect(runReporter(malformed)).rejects.toThrow();
await expect(
readFile(join(malformed.outputDirectory, "jobs.json"), "utf8"),
).resolves.toBe(malformed.raw);
const failedApi = await createFixture([], { exitCode: 1 });
await expect(runReporter(failedApi)).rejects.toThrow();
});
async function createFixture(
pages: unknown[],
{ exitCode = 0 }: { exitCode?: number } = {},
) {
const directory = await createTestTempDirectory("rea-ci-job-timings-");
await workflowGit(directory, ["init", "--initial-branch=main"]);
const fakeBin = join(directory, "bin");
await mkdir(fakeBin);
const raw = JSON.stringify(pages);
const rawPath = join(directory, "gh-response.json");
const args = join(directory, "gh-args.json");
await writeFile(rawPath, raw);
const fakeGh = join(fakeBin, "gh");
await writeFile(
fakeGh,
`#!/usr/bin/env node\nconst fs = require("node:fs");\nfs.writeFileSync(process.env.CI_GH_ARGS, JSON.stringify(process.argv.slice(2)));\nprocess.stdout.write(fs.readFileSync(process.env.CI_GH_RESPONSE, "utf8"));\nprocess.exitCode = ${String(exitCode)};\n`,
);
await chmod(fakeGh, 0o755);
const outputDirectory = join(directory, ".cache", "ci-job-timings");
const summary = join(directory, "summary.md");
await writeFile(summary, "");
return {
directory,
fakeBin,
raw,
rawPath,
args,
outputDirectory,
summary,
};
}
function runReporter(fixture: Awaited<ReturnType<typeof createFixture>>) {
return executeWorkflowFixture(process.execPath, [reporter], {
cwd: fixture.directory,
env: {
...process.env,
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
CI_REPOSITORY: "example/rea",
CI_JOB_TIMINGS_DIRECTORY: fixture.outputDirectory,
CI_GH_ARGS: fixture.args,
CI_GH_RESPONSE: fixture.rawPath,
CI_TIMING_RUN_ID: "42",
CI_TIMING_RUN_ATTEMPT: "2",
GITHUB_STEP_SUMMARY: fixture.summary,
},
});
}
@@ -0,0 +1,288 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, join } from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
const batchScript = fileURLToPath(
new URL("../../../scripts/ci/job-timings-batch.mjs", import.meta.url),
);
it("collects the latest attempt for recent completed CI and Release runs into a bounded manifest", async () => {
const recent = new Date().toISOString();
const fixture = await createFixture([
{
workflow_runs: [
{
id: 42,
run_attempt: 3,
status: "completed",
conclusion: "failure",
created_at: new Date(Date.parse(recent) + 1_000).toISOString(),
},
{
id: 41,
run_attempt: 1,
status: "completed",
conclusion: "cancelled",
created_at: recent,
},
],
},
{
workflow_runs: [
{
id: 43,
run_attempt: 2,
status: "completed",
conclusion: "success",
created_at: new Date(Date.parse(recent) + 2_000).toISOString(),
},
],
},
[
{
jobs: [
{
name: "Release lane",
status: "completed",
conclusion: "success",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 50_000).toISOString(),
steps: [],
},
],
},
],
[
{
jobs: [
{
name: "Failed lane",
status: "completed",
conclusion: "failure",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 55_000).toISOString(),
steps: [],
},
],
},
],
[
{
jobs: [
{
name: "Cancelled lane",
status: "completed",
conclusion: "cancelled",
created_at: recent,
started_at: recent,
completed_at: new Date(Date.parse(recent) + 30_000).toISOString(),
steps: [],
},
],
},
],
]);
const result = await runBatch(fixture);
const manifest = JSON.parse(
await readFile(
join(fixture.outputDirectory, "batch-manifest.json"),
"utf8",
),
);
const calls = JSON.parse(await readFile(fixture.calls, "utf8"));
expect(result.stdout).toContain("3 collected runs");
expect(
await readFile(join(fixture.directory, "summary.md"), "utf8"),
).toContain("collected 3 of 3");
expect(manifest.selection).toMatchObject({
lookbackHours: 24,
maxRunsPerWorkflow: 100,
exhaustive: false,
});
expect(manifest.coverage).toMatchObject({
workflowsQueried: ["CI", "Release"],
selectedRuns: 3,
collectedRuns: 3,
});
expect(manifest.truncation).toEqual([]);
expect(manifest.errors).toEqual([]);
expect(manifest.runs).toMatchObject([
{ id: 43, runAttempt: 2, conclusion: "success" },
{ id: 42, runAttempt: 3, conclusion: "failure" },
{ id: 41, runAttempt: 1, conclusion: "cancelled" },
]);
expect(manifest.runs).toEqual(
expect.arrayContaining([
expect.objectContaining({
status: "collected",
rawPath: expect.any(String),
reportPath: expect.any(String),
}),
]),
);
expect(calls).toContainEqual([
"api",
"--paginate",
"--slurp",
"repos/example/rea/actions/runs/42/attempts/3/jobs?per_page=100",
]);
expect(calls).toContainEqual([
"api",
"--paginate",
"--slurp",
"repos/example/rea/actions/runs/41/attempts/1/jobs?per_page=100",
]);
const failedReport = JSON.parse(
await readFile(
join(fixture.outputDirectory, "runs/42-3/report.json"),
"utf8",
),
);
expect(failedReport.jobs[0].conclusion).toBe("failure");
expect(failedReport.jobs[0].execution.seconds).toBe(55);
});
it("records listing and per-run errors without claiming complete coverage", async () => {
const recent = new Date().toISOString();
const fixture = await createFixture([
{
workflow_runs: [
{
id: 7,
run_attempt: 2,
status: "completed",
conclusion: "success",
created_at: recent,
},
],
},
{ error: true },
{ jobs: "malformed" },
]);
await runBatch(fixture);
const manifest = JSON.parse(
await readFile(
join(fixture.outputDirectory, "batch-manifest.json"),
"utf8",
),
);
expect(manifest.selection.exhaustive).toBe(false);
expect(manifest.errors).toMatchObject([
{ scope: "workflow:Release" },
{ scope: "run" },
]);
expect(manifest.runs[0]).toMatchObject({
id: 7,
runAttempt: 2,
status: "error",
});
});
it("retains a partial manifest when its collection budget is exhausted", async () => {
const fixture = await createFixture([
{
workflow_runs: [8, 9].map((id) => ({
id,
run_attempt: 1,
status: "completed",
created_at: new Date().toISOString(),
})),
},
{ workflow_runs: [] },
]);
await executeWorkflowFixture(
process.execPath,
[
"--input-type=module",
"-e",
`
const { collectJobTimingsBatch } = await import(${JSON.stringify(pathToFileURL(batchScript).href)});
let clock = Date.now();
await collectJobTimingsBatch({
repository: "example/rea",
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY,
now: () => clock,
collect: async () => {
clock += 240001;
throw new Error("fixture request exhausted the remaining budget");
},
});
`,
],
{ cwd: fixture.directory, env: fixtureEnvironment(fixture) },
);
const manifest = JSON.parse(
await readFile(
join(fixture.outputDirectory, "batch-manifest.json"),
"utf8",
),
);
expect(manifest.coverage).toMatchObject({
selectedRuns: 2,
collectedRuns: 0,
});
expect(manifest.runs).toHaveLength(1);
expect(manifest.runs[0]).toMatchObject({ status: "error" });
expect(manifest.truncation.join(" ")).toContain("Time budget");
});
async function createFixture(responses: unknown[]) {
const directory = await createTestTempDirectory("rea-ci-job-timings-batch-");
const fakeBin = join(directory, "bin");
await mkdir(fakeBin);
const responsePath = join(directory, "responses.json");
const calls = join(directory, "calls.json");
await writeFile(responsePath, JSON.stringify(responses));
await writeFile(calls, "[]");
const fakeGh = join(fakeBin, "gh");
await writeFile(
fakeGh,
`#!/usr/bin/env node
const fs = require("node:fs");
const calls = JSON.parse(fs.readFileSync(process.env.CI_GH_CALLS, "utf8"));
calls.push(process.argv.slice(2));
fs.writeFileSync(process.env.CI_GH_CALLS, JSON.stringify(calls));
const responses = JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSES, "utf8"));
const response = responses[calls.length - 1];
if (response && response.error) { process.stderr.write("fixture API error"); process.exitCode = 1; }
else process.stdout.write(JSON.stringify(response));
`,
);
await chmod(fakeGh, 0o755);
return {
directory,
fakeBin,
responses: responsePath,
calls,
outputDirectory: join(directory, ".cache", "ci-job-timings"),
};
}
function runBatch(fixture: Awaited<ReturnType<typeof createFixture>>) {
return executeWorkflowFixture(process.execPath, [batchScript], {
cwd: fixture.directory,
env: fixtureEnvironment(fixture),
});
}
function fixtureEnvironment(
fixture: Awaited<ReturnType<typeof createFixture>>,
) {
return {
...process.env,
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
CI_REPOSITORY: "example/rea",
CI_JOB_TIMINGS_DIRECTORY: fixture.outputDirectory,
CI_GH_CALLS: fixture.calls,
CI_GH_RESPONSES: fixture.responses,
GITHUB_STEP_SUMMARY: join(fixture.directory, "summary.md"),
};
}
@@ -0,0 +1,98 @@
import { readFile } from "node:fs/promises";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
async function workflow(name: string) {
return z
.object({
on: z.record(z.string(), z.unknown()),
permissions: z.record(z.string(), z.string()),
concurrency: z.object({ group: z.string() }).optional(),
jobs: z.record(
z.string(),
z.object({
name: z.string().optional(),
if: z.string().optional(),
uses: z.string().optional(),
with: z.record(z.string(), z.string()).optional(),
steps: z
.array(
z.object({
run: z.string().optional(),
uses: z.string().optional(),
env: z.record(z.string(), z.string()).optional(),
}),
)
.optional(),
}),
),
})
.parse(
parse(
await readFile(
new URL(`../../../.github/workflows/${name}`, import.meta.url),
"utf8",
),
),
);
}
it("refreshes title validation on edits without checking out PR code or cancelling CI", async () => {
const title = await workflow("pr-title.yml");
expect(title.on.pull_request).toMatchObject({
types: ["opened", "synchronize", "reopened", "edited"],
});
expect(title.permissions).toEqual({});
expect(title.concurrency?.group).not.toBe(
(await workflow("ci.yml")).concurrency?.group,
);
const jobs = Object.values(title.jobs);
expect(jobs).toHaveLength(1);
expect(jobs[0]?.name).toBe("Conventional PR title");
expect(jobs[0]?.if).toBeUndefined(); // A skipped check must not green-light an invalid title on body edits.
const steps = jobs[0]?.steps ?? [];
expect(steps).toHaveLength(1);
expect(steps[0]?.uses).toBeUndefined();
expect(steps[0]?.env).toEqual({
PR_TITLE: "${{ github.event.pull_request.title }}",
});
const command = z.string().parse(steps[0]?.run);
for (const title of [
"fix(ci): repair workflow",
"feat!: changed contract",
"docs: literal $(exit 91) `exit 92` text",
]) {
await expect(
executeWorkflowFixture("bash", ["-c", command], {
env: { ...process.env, PR_TITLE: title },
}),
).resolves.toMatchObject({ stderr: "" });
}
for (const title of ["", "Fix CI", "feat: "]) {
await expect(
executeWorkflowFixture("bash", ["-c", command], {
env: { ...process.env, PR_TITLE: title },
}),
).rejects.toMatchObject({ code: 1 });
}
expect((await workflow("ci.yml")).jobs["pr-title"]).toBeUndefined();
});
it("qualifies only ci:full label additions at the caller's merge revision", async () => {
const full = await workflow("ci-full-pr.yml");
expect(full.on.pull_request).toMatchObject({ types: ["labeled"] });
expect(full.permissions).toEqual({ contents: "read", actions: "read" });
expect(Object.values(full.jobs)).toEqual([
{
name: "Qualify labeled PR revision",
if: "github.event.label.name == 'ci:full'",
uses: "./.github/workflows/ci.yml",
with: { source_sha: "${{ github.sha }}" },
},
]);
// Metadata events must not create skipped replacements for the main CI checks.
expect(full.jobs.required).toBeUndefined();
expect(full.concurrency).toBeUndefined();
});
@@ -0,0 +1,375 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
executeWorkflowFixture,
workflowGit,
} from "../../support/workflowGit.js";
const qualificationScript = fileURLToPath(
new URL("../../../scripts/ci/qualification.mjs", import.meta.url),
);
const requiredScript = fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
);
const now = Date.now();
const validSha = "a".repeat(40);
const repository = "example/rea";
function artifact(overrides: Record<string, unknown> = {}) {
return {
id: 42,
name: "ci-full-qualification-v1",
expired: false,
created_at: new Date(now - 60_000).toISOString(),
workflow_run: {
id: 7,
repository_id: 1,
head_repository_id: 1,
head_branch: "main",
head_sha: validSha,
},
...overrides,
};
}
function run(overrides: Record<string, unknown> = {}) {
return {
id: 7,
status: "completed",
conclusion: "success",
path: ".github/workflows/ci.yml",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_sha: validSha,
head_branch: "main",
head_repository: { full_name: repository },
...overrides,
};
}
async function admissionFixture(
candidate: Record<string, unknown> = artifact(),
candidateRun: Record<string, unknown> = run(),
options: { apiError?: boolean; malformed?: boolean; force?: boolean } = {},
) {
const directory = await createTestTempDirectory("rea-ci-qualification-");
const bin = join(directory, "bin");
await mkdir(bin);
const fixturePath = join(directory, "api.json");
await writeFile(
fixturePath,
options.malformed
? "not-json"
: JSON.stringify({ artifacts: [candidate], run: candidateRun }),
);
const gh = join(bin, "gh");
await writeFile(
gh,
[
"#!/bin/sh",
'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi',
'case "$*" in',
' *actions/artifacts*) cat "$GH_FIXTURE_RESPONSE" ;;',
" *actions/runs/7*) node -e 'const x=require(process.env.GH_FIXTURE_RESPONSE); process.stdout.write(JSON.stringify(x.run))' ;;",
" *) exit 2 ;;",
"esac",
"",
].join("\n"),
);
await chmod(gh, 0o755);
const output = join(directory, "github-output");
await writeFile(output, "");
const env = {
...process.env,
PATH: `${bin}:${process.env.PATH ?? ""}`,
GH_FIXTURE_RESPONSE: fixturePath,
GH_API_ERROR: String(options.apiError ?? false),
CI_REPOSITORY: repository,
CI_REPOSITORY_ID: "1",
CI_DEFAULT_BRANCH: "main",
HEAD_SHA: validSha,
GITHUB_RUN_ID: "100",
GITHUB_OUTPUT: output,
...(options.force ? { CI_FORCE_FULL: "true" } : {}),
};
return { directory, output, env };
}
async function decide(
candidate?: Record<string, unknown>,
candidateRun?: Record<string, unknown>,
options?: { apiError?: boolean; malformed?: boolean; force?: boolean },
) {
const fixture = await admissionFixture(candidate, candidateRun, options);
const result = await executeWorkflowFixture(
process.execPath,
[qualificationScript],
{
cwd: fixture.directory,
env: fixture.env,
},
);
return {
...fixture,
decision: z
.record(z.string(), z.unknown())
.parse(JSON.parse(result.stdout)),
outputText: await readFile(fixture.output, "utf8"),
};
}
it("reuses only a recent trusted full qualification for the exact default-branch SHA", async () => {
const { decision, outputText } = await decide();
expect(decision).toMatchObject({ run_full: false, run_id: 7 });
expect(Object.keys(decision).sort()).toEqual([
"reason",
"run_full",
"run_id",
]);
expect(outputText).toContain("run_full=false\n");
expect(outputText).toContain("run_id=7\n");
});
const rejectedCandidates: Array<
[string, () => Record<string, unknown>, () => Record<string, unknown>]
> = [
[
"old",
() =>
artifact({
created_at: new Date(now - 24 * 60 * 60 * 1000 - 1).toISOString(),
}),
run,
],
[
"future",
() => artifact({ created_at: new Date(now + 60_000).toISOString() }),
run,
],
["expired", () => artifact({ expired: true }), run],
[
"foreign repository",
() =>
artifact({
workflow_run: { ...artifact().workflow_run, repository_id: 2 },
}),
run,
],
[
"foreign branch",
() =>
artifact({
workflow_run: { ...artifact().workflow_run, head_branch: "other" },
}),
run,
],
[
"foreign SHA",
() =>
artifact({
workflow_run: { ...artifact().workflow_run, head_sha: "b".repeat(40) },
}),
run,
],
[
"untrusted workflow",
artifact,
() => run({ path: ".github/workflows/release.yml" }),
],
["failed run", artifact, () => run({ conclusion: "failure" })],
[
"old producer rerun with a fresh artifact",
artifact,
() =>
run({ created_at: new Date(now - 25 * 60 * 60 * 1000).toISOString() }),
],
["wrong workflow event", artifact, () => run({ event: "push" })],
];
it.each(rejectedCandidates)(
"runs full qualification for a %s candidate",
async (_name, makeArtifact, makeRun) => {
const { decision } = await decide(makeArtifact(), makeRun());
expect(decision.run_full).toBe(true);
},
);
it.each([
["API error", { apiError: true }],
["malformed query response", { malformed: true }],
] as const)("fails closed on %s", async (_name, options) => {
const { decision } = await decide(undefined, undefined, options);
expect(decision.run_full).toBe(true);
});
it("lets the daily force-full setting bypass the GitHub API", async () => {
const { decision } = await decide(undefined, undefined, {
force: true,
apiError: true,
});
expect(decision).toMatchObject({ run_full: true });
expect(decision.reason).toContain("Forced");
});
async function workflowContract() {
const workflowPath = new URL(
"../../../.github/workflows/ci.yml",
import.meta.url,
);
const parsed = parse(await readFile(workflowPath, "utf8"));
return z
.object({
jobs: z.record(
z.string(),
z.object({ needs: z.unknown().optional() }).passthrough(),
),
})
.parse(parsed).jobs;
}
async function requiredRun(
changes: Record<string, string>,
overrides: { failed?: string; runAttempt?: string; event?: string } = {},
) {
const directory = await createTestTempDirectory("rea-ci-receipt-");
await workflowGit(directory, ["init", "--initial-branch=main"]);
await writeFile(join(directory, "fixture.txt"), "qualified source\n");
await workflowGit(directory, ["add", "."]);
await workflowGit(directory, ["commit", "-m", "fixture source"]);
const head = await workflowGit(directory, ["rev-parse", "HEAD"]);
const jobs = await workflowContract();
const results: Record<
string,
{ result: string; outputs?: Record<string, string> }
> = {};
for (const name of z.array(z.string()).parse(jobs.required?.needs)) {
results[name] = {
result:
name === overrides.failed
? "failure"
: changes.tier === "reused"
? "skipped"
: "success",
};
}
results.changes = { result: "success", outputs: changes };
const receiptDirectory = join(directory, ".cache", "receipt");
const output = join(directory, "github-output");
await writeFile(output, "");
const env = {
...process.env,
CI_JOBS: JSON.stringify(results),
CI_QUALIFICATION_DIRECTORY: receiptDirectory,
GITHUB_EVENT_NAME: overrides.event ?? "schedule",
GITHUB_SHA: head,
GITHUB_RUN_ID: "100",
GITHUB_RUN_ATTEMPT: overrides.runAttempt ?? "1",
GITHUB_REPOSITORY: repository,
GITHUB_OUTPUT: output,
};
const result = executeWorkflowFixture(process.execPath, [requiredScript], {
cwd: directory,
env,
});
return { directory, head, output, result };
}
it("writes a full receipt only after every canonical flag and workflow need pass", async () => {
const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url);
const { CI_FLAGS } = await import(scopesPath.href);
const changes = Object.fromEntries(
CI_FLAGS.map((flag: string) => [flag, "true"]),
);
changes.tier = "full";
const { directory, head, output, result } = await requiredRun(changes);
await expect(result).resolves.toMatchObject({ stderr: "" });
const receiptPath = join(directory, ".cache", "receipt", "receipt.json");
const receipt = z
.object({
sourceSha: z.string(),
runId: z.number(),
runAttempt: z.number(),
})
.parse(JSON.parse(await readFile(receiptPath, "utf8")));
expect(receipt).toMatchObject({ sourceSha: head, runId: 100, runAttempt: 1 });
expect(await readFile(output, "utf8")).toContain(
`receipt_path=${receiptPath}\n`,
);
expect(await readFile(output, "utf8")).toContain(
"receipt_name=ci-full-qualification-v1\n",
);
});
it.each([
["failed selected lane", { failed: "sqlite-runtime" }],
["partial full scope", {}],
] as const)(
"does not issue a receipt after %s",
async (caseName, overrides) => {
const scopesPath = new URL(
"../../../scripts/ci/scopes.mjs",
import.meta.url,
);
const { CI_FLAGS } = await import(scopesPath.href);
const changes = Object.fromEntries(
CI_FLAGS.map((flag: string) => [flag, "true"]),
);
changes.tier = "full";
if (caseName === "partial full scope") changes.sqlite = "false";
const outcome = await requiredRun(changes, overrides);
await expect(outcome.result).rejects.toMatchObject({ code: 1 });
await expect(
readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")),
).rejects.toMatchObject({ code: "ENOENT" });
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
},
);
it("never writes a fresh receipt for a reused scheduled qualification", async () => {
const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url);
const { CI_FLAGS } = await import(scopesPath.href);
const changes = Object.fromEntries(
CI_FLAGS.map((flag: string) => [flag, "false"]),
);
changes.tier = "reused";
changes.qualification_run_id = "7";
const outcome = await requiredRun(changes);
const result = await outcome.result;
await expect(
readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")),
).rejects.toMatchObject({ code: "ENOENT" });
expect(result.stdout).toContain("Qualification reused from run 7");
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
});
it.each([
["missing receipt run", { qualification_run_id: "" }, "schedule"],
["selected validation lane", { sqlite: "true" }, "schedule"],
["manual dispatch", {}, "workflow_dispatch"],
] as const)(
"rejects qualification reuse with %s",
async (_name, override, event) => {
const { CI_FLAGS } = await import(
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
);
const changes: Record<string, string> = {
...Object.fromEntries(CI_FLAGS.map((flag: string) => [flag, "false"])),
tier: "reused",
qualification_run_id: "7",
...override,
};
const outcome = await requiredRun(changes, { event });
await expect(outcome.result).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("scheduled admission receipt"),
});
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
},
);
+424 -22
View File
@@ -26,7 +26,19 @@ const workflow = async (filename = "ci.yml") =>
),
);
const scopeFlags = async () =>
z
.object({ CI_FLAGS: z.array(z.string()) })
.parse(
await import(
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
),
).CI_FLAGS;
it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => {
expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe(
"needs.changes.outputs.sqlite == 'true'",
);
const steps = z.array(
z.object({
uses: z.string().optional(),
@@ -52,6 +64,77 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and
expect(downloads[0]?.with).toEqual({ name, path: "." });
});
it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => {
const { jobs } = await workflow();
const CI_FLAGS = await scopeFlags();
const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs);
expect(jobs.changes?.permissions).toEqual({
contents: "read",
actions: "read",
});
expect(jobs.build?.permissions).toEqual({
contents: "read",
actions: "read",
});
for (const flag of CI_FLAGS)
expect(outputs[flag]).toBe(`\${{ steps.plan.outputs.${flag} }}`);
const steps = z
.array(
z.object({
run: z.string().optional(),
if: z.string().optional(),
}),
)
.parse(jobs.required?.steps);
const telemetry = steps.find((step) =>
step.run?.includes("scripts/ci/job-timings.mjs"),
);
expect(telemetry).toBeUndefined();
const timings = z
.object({
on: z.object({
schedule: z.array(z.object({ cron: z.string() })),
workflow_dispatch: z.unknown(),
workflow_run: z.unknown().optional(),
}),
permissions: z.record(z.string(), z.string()),
jobs: z.object({
report: z.object({
steps: z.array(
z.object({
uses: z.string().optional(),
run: z.string().optional(),
with: z.record(z.string(), z.unknown()).optional(),
env: z.record(z.string(), z.unknown()).optional(),
}),
),
}),
}),
})
.parse(
parse(
await readFile(
new URL("../../../.github/workflows/ci-timings.yml", import.meta.url),
"utf8",
),
),
);
expect(timings.on.workflow_run).toBeUndefined();
expect(timings.on.schedule).toHaveLength(1);
expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
const checkout = timings.jobs.report.steps.find((step) =>
step.uses?.startsWith("actions/checkout@"),
);
expect(checkout?.with).toEqual({
ref: "${{ github.event.repository.default_branch }}",
"persist-credentials": false,
});
const report = timings.jobs.report.steps.find((step) =>
step.run?.includes("scripts/ci/job-timings-batch.mjs"),
);
expect(report?.env?.CI_REPOSITORY).toBe("${{ github.repository }}");
});
const requiredNeeds = async () => {
const { jobs } = await workflow();
return { jobs, needs: z.array(z.string()).parse(jobs.required?.needs) };
@@ -64,6 +147,240 @@ it("makes the required gate wait for every CI lane, including future additions",
);
});
it("holds native and provider fan-out behind static validation while portable feedback stays independent", async () => {
const { jobs } = await workflow();
const earlyJobs = new Set([
"changes",
"static",
"build",
"test-shard",
"test",
"docs",
"required",
]);
for (const [name, job] of Object.entries(jobs)) {
if (earlyJobs.has(name)) continue;
expect(job.needs, `${name} must wait for static validation`).toContain(
"static",
);
expect(job.if).not.toMatch(/always\(|cancelled\(/u);
}
expect(jobs["test-shard"]?.needs).not.toContain("static");
});
it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => {
const { jobs } = await workflow();
const consumers = [
"browser",
"binary_layout",
"evm",
"javascript_recovery",
"recorded_crash",
"web_network",
"web_runtime",
"web_source_map",
];
const buildTerms = z
.string()
.parse(jobs.build?.if)
.split("||")
.map((term) => term.trim());
// Provider workflow edits select their lane without code=true. The build
// producer must still run, without selecting the portable test shards.
expect(new Set(buildTerms)).toEqual(
new Set(
["code", ...consumers].map(
(scope) => `needs.changes.outputs.${scope} == 'true'`,
),
),
);
expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'");
for (const name of consumers) {
const caller = jobs[name];
expect(caller?.needs).toContain("build");
expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" });
const path = z.string().parse(caller?.uses);
const called = z
.object({
on: z.object({
workflow_call: z.object({
inputs: z.record(z.string(), z.unknown()),
}),
workflow_dispatch: z.unknown(),
}),
jobs: z.record(
z.string(),
z.object({
steps: z.array(
z.object({
uses: z.string().optional(),
with: z.record(z.string(), z.unknown()).optional(),
}),
),
}),
),
})
.parse(
parse(
await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"),
),
);
expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({
type: "string",
default: "",
});
expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true);
for (const job of Object.values(called.jobs))
expect(
job.steps.find(
(step) => step.uses === "./.github/actions/portable-build-cache",
)?.with,
).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" });
}
const action = z
.object({
runs: z.object({
steps: z.array(
z.object({
id: z.string().optional(),
uses: z.string().optional(),
if: z.string().optional(),
with: z.record(z.string(), z.unknown()).optional(),
"continue-on-error": z.boolean().optional(),
}),
),
}),
})
.parse(
parse(
await readFile(
new URL(
"../../../.github/actions/portable-build-cache/action.yml",
import.meta.url,
),
"utf8",
),
),
);
const download = action.runs.steps.find(
(step) => step.uses === "actions/download-artifact@v4",
);
expect(download?.with).toEqual({
name: "${{ inputs.build_cache_artifact }}",
path: ".turbo/cache/",
});
expect(download?.if).toBe("inputs.build_cache_artifact != ''");
expect(download?.["continue-on-error"]).not.toBe(true);
for (const step of action.runs.steps.filter(
(step) => step.id === "plan" || step.uses?.startsWith("actions/cache"),
))
expect(step.if).toContain("inputs.build_cache_artifact == ''");
});
it("runs workflow validation inside required static checks when selected", async () => {
const { jobs } = await workflow();
const steps = z
.array(z.object({ run: z.string().optional(), if: z.string().optional() }))
.parse(jobs.static?.steps);
const validation = steps.filter((step) => step.run?.includes("actionlint"));
expect(validation).toHaveLength(1);
expect(validation[0]?.if).toBe("needs.changes.outputs.workflows == 'true'");
expect(jobs.workflows).toBeUndefined();
});
it("leaves cancellation ownership with CI so child lanes cannot cancel another tier", async () => {
const { jobs } = await workflow();
const children = Object.values(jobs).flatMap((job) =>
typeof job.uses === "string" && job.uses.startsWith("./.github/workflows/")
? [job.uses]
: [],
);
expect(children.length).toBeGreaterThan(0);
for (const path of children) {
const child = z
.object({ concurrency: z.unknown().optional() })
.parse(
parse(
await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"),
),
);
expect(
child.concurrency,
`${path} must use caller scheduling`,
).toBeUndefined();
}
});
it("keeps fast feedback separate from complete qualification and release proof", async () => {
const ci = z
.object({
on: z.object({
push: z.unknown(),
pull_request: z.unknown(),
schedule: z.array(z.object({ cron: z.string() })),
workflow_call: z.object({
inputs: z.object({
source_sha: z.object({
required: z.literal(true),
type: z.literal("string"),
}),
}),
}),
}),
concurrency: z.object({
group: z.string(),
"cancel-in-progress": z.string(),
}),
jobs: z.record(
z.string(),
z.object({
if: z.string().optional(),
steps: z
.array(
z.object({
run: z.string().optional(),
if: z.string().optional(),
with: z.record(z.string(), z.unknown()).optional(),
}),
)
.optional(),
}),
),
})
.parse(
parse(
await readFile(
new URL("../../../.github/workflows/ci.yml", import.meta.url),
"utf8",
),
),
);
expect(ci.on.schedule).toEqual([
{ cron: "23 0-2,4-23 * * *" },
{ cron: "23 3 * * *" },
]);
expect(ci.concurrency.group).toContain("'qualification' || 'feedback'");
expect(ci.concurrency.group).toContain("inputs.source_sha ||");
expect(ci.concurrency.group).toContain("github.ref");
expect(ci.concurrency["cancel-in-progress"]).toBe(
"${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}",
);
expect(ci.jobs.required?.if).toBe("${{ always() }}");
const shards = ci.jobs["test-shard"]?.steps ?? [];
const fast = shards.find((step) => step.run?.includes("npm run test:run"));
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
expect(fast?.run).toContain("--reporter=blob");
expect(fast?.run).not.toContain("--coverage");
expect(full?.if).toBe("needs.changes.outputs.tier == 'full'");
const proof = ci.jobs["package-e2e"]?.steps?.find(
(step) => step.with?.name === "verified-release-package",
);
expect(proof?.if).toContain("needs.changes.outputs.tier == 'full'");
expect(proof?.if).toContain("matrix.platform == 'linux'");
expect(proof?.if).toContain("matrix.architecture == 'x64'");
});
it.each([
["true", "success", true],
["true", "failure", false],
@@ -71,34 +388,119 @@ it.each([
["true", "skipped", false],
["true", "missing", false],
["false", "skipped", true],
["false", "failure", false],
] as const)(
"enforces the SQLite required gate for code=%s and result=%s",
async (code, result, passes) => {
"enforces mapped required scopes for selected=%s and result=%s",
async (selected, result, passes) => {
const workflow = await requiredNeeds();
// GitHub exposes only direct dependencies in the required job's needs context.
const needs: Record<
string,
{ result: string; outputs?: Record<string, string> }
> = Object.fromEntries(
workflow.needs.map((name) => [name, { result: "success" }]),
);
needs.changes = { result: "success", outputs: { code } };
if (result === "missing") delete needs["sqlite-runtime"];
else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result };
const execution = executeWorkflowFixture(
for (const [scope, lane] of [
["sqlite", "sqlite-runtime"],
["workflows", "static"],
] as const) {
// GitHub exposes only direct dependencies in the required job's needs context.
const needs: Record<
string,
{ result: string; outputs?: Record<string, string> }
> = Object.fromEntries(
workflow.needs.map((name) => [name, { result: "success" }]),
);
needs.changes = {
result: "success",
outputs: {
...Object.fromEntries(
(await scopeFlags()).map((flag) => [flag, "false"]),
),
tier: "fast",
code: "true",
[scope]: selected,
},
};
if (result === "missing") delete needs[lane];
else needs[lane] = { result };
const execution = executeWorkflowFixture(
process.execPath,
[
fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
),
],
{ env: { ...process.env, CI_JOBS: JSON.stringify(needs) } },
);
// Static checks remain required even without workflow edits.
const expected = passes && !(lane === "static" && result !== "success");
if (expected)
await expect(execution).resolves.toMatchObject({ stderr: "" });
else
await expect(execution).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining(lane),
});
}
},
);
it("rejects incomplete or malformed classification even when available jobs succeeded", async () => {
const flags = await scopeFlags();
const valid = {
...Object.fromEntries(flags.map((flag) => [flag, "false"])),
tier: "fast",
};
const run = (outputs: unknown) =>
executeWorkflowFixture(
process.execPath,
[
fileURLToPath(
new URL("../../../scripts/ci/required.mjs", import.meta.url),
),
],
{ env: { ...process.env, CI_JOBS: JSON.stringify(needs) } },
{
env: {
...process.env,
CI_JOBS: JSON.stringify({
changes: { result: "success", outputs },
static: { result: "success" },
test: { result: "success" },
}),
CI_QUALIFICATION_DIRECTORY: "",
},
},
);
if (passes) await expect(execution).resolves.toMatchObject({ stderr: "" });
else
await expect(execution).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("sqlite-runtime"),
});
},
);
await expect(run(valid)).resolves.toMatchObject({ stderr: "" });
// Metadata is not a scope, even when its string happens to be "true".
await expect(
run({ ...valid, qualification_run_id: "true" }),
).resolves.toMatchObject({ stderr: "" });
for (const outputs of [
undefined,
null,
{},
{ ...valid, tier: undefined },
{ ...valid, tier: "" },
{ ...valid, tier: "unknown" },
]) {
await expect(run(outputs)).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("valid tier"),
});
}
for (const flag of flags) {
await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining(`scope outputs: ${flag}`),
});
}
for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) {
await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("scope outputs: sqlite"),
});
}
for (const tier of ["full", "reused"]) {
await expect(
run({ ...valid, tier, sqlite: undefined }),
).rejects.toMatchObject({
code: 1,
stderr: expect.stringContaining("scope outputs: sqlite"),
});
}
});
@@ -0,0 +1,188 @@
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { delimiter, dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { executeWorkflowFixture } from "../../support/workflowGit.js";
const script = fileURLToPath(
new URL("../../../scripts/ci/test-history.mjs", import.meta.url),
);
const repository = "example/rea";
const repositoryId = 1;
const now = Date.now();
function artifact(id: number, overrides: Record<string, unknown> = {}) {
return {
id: id + 100,
name: "test-timings",
expired: false,
created_at: new Date(now - 60_000).toISOString(),
workflow_run: {
id,
repository_id: repositoryId,
head_repository_id: repositoryId,
head_branch: "main",
},
...overrides,
};
}
function run(id: number, overrides: Record<string, unknown> = {}) {
return {
id,
status: "completed",
conclusion: "success",
path: ".github/workflows/ci.yml",
event: "schedule",
created_at: new Date(now - 120_000).toISOString(),
head_branch: "main",
head_repository: { full_name: repository },
...overrides,
};
}
async function invoke(
artifacts: unknown[],
runs: Record<number, unknown>,
options: {
apiError?: boolean;
repository?: string;
repositoryId?: string;
} = {},
) {
const directory = await createTestTempDirectory("rea-ci-test-history-");
const fakeBin = join(directory, "bin");
await mkdir(fakeBin);
const fixture = join(directory, "api.json");
const calls = join(directory, "api-calls");
const output = join(directory, "github-output");
await writeFile(fixture, JSON.stringify({ artifacts, runs }));
await writeFile(output, "");
await writeFile(calls, "");
const fakeGh = join(fakeBin, "gh");
await writeFile(
fakeGh,
[
"#!/bin/sh",
'printf \'%s\\n\' "$*" >> "$GH_API_CALLS"',
'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi',
'if [ "$4" = "repos/example/rea/actions/artifacts" ]; then',
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify({artifacts:x.artifacts}));\'',
"else",
' run_id="${4##*/}"',
' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify(x.runs[Number(process.argv[1])] ?? {}));\' "$run_id"',
"fi",
"",
].join("\n"),
);
await chmod(fakeGh, 0o755);
const node24 = dirname(process.execPath);
const result = await executeWorkflowFixture(process.execPath, [script], {
cwd: directory,
env: {
...process.env,
PATH: [fakeBin, node24, process.env.PATH ?? ""].join(delimiter),
GH_FIXTURE_RESPONSE: fixture,
GH_API_CALLS: calls,
GH_API_ERROR: String(options.apiError ?? false),
GH_TOKEN: "fixture-token",
CI_REPOSITORY: options.repository ?? repository,
CI_REPOSITORY_ID: options.repositoryId ?? String(repositoryId),
CI_DEFAULT_BRANCH: "main",
GITHUB_OUTPUT: output,
},
});
return {
result,
output: await readFile(output, "utf8"),
calls: await readFile(calls, "utf8"),
};
}
it("selects the latest trusted full run that actually published timing history", async () => {
const { result, output, calls } = await invoke(
[
artifact(15),
artifact(14),
artifact(13, {
workflow_run: { ...artifact(13).workflow_run, head_repository_id: 2 },
}),
artifact(12, {
workflow_run: { ...artifact(12).workflow_run, head_branch: "feature" },
}),
artifact(11),
],
{
15: run(15, { conclusion: "failure" }),
14: run(14, { path: ".github/workflows/release.yml" }),
11: run(11, { event: "workflow_dispatch" }),
},
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "11" });
expect(output).toBe("run-id=11\n");
expect(calls).toContain("name=test-timings");
expect(calls).toContain("per_page=100");
expect(calls.match(/actions\/runs\//gu)).toHaveLength(3);
expect(calls).not.toContain("actions/workflows");
});
it("rejects expired, foreign, old, and non-full run candidates", async () => {
const { result, output } = await invoke(
[
artifact(15, { expired: true }),
artifact(14, {
workflow_run: { ...artifact(14).workflow_run, head_repository_id: 2 },
}),
artifact(13, {
created_at: new Date(now - 31 * 24 * 60 * 60 * 1000).toISOString(),
}),
artifact(12),
],
{
12: run(12, { event: "pull_request" }),
},
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(output).toBe("run-id=\n");
});
it("writes an empty run id when GitHub history is unavailable", async () => {
const { result, output } = await invoke([], {}, { apiError: true });
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(output).toBe("run-id=\n");
});
it("bounds run lookups to five artifacts even when inventory has more", async () => {
const ids = [25, 24, 23, 22, 21, 20, 19];
const { result, calls } = await invoke(
ids.map((id) => artifact(id)),
Object.fromEntries(
ids.map((id, index) => [
id,
run(id, index < 5 ? { conclusion: "failure" } : {}),
]),
),
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(calls.match(/actions\/runs\//gu)).toHaveLength(5);
});
it("returns empty history without querying for an invalid repository identity", async () => {
const { result, output, calls } = await invoke(
[],
{},
{
repository: "owner/repo?per_page=100",
repositoryId: "not-an-id",
},
);
expect(JSON.parse(result.stdout)).toEqual({ run_id: "" });
expect(output).toBe("run-id=\n");
expect(calls).toBe("");
});
+297 -12
View File
@@ -1,5 +1,5 @@
import { execFile } from "node:child_process";
import { readFile, writeFile } from "node:fs/promises";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
@@ -8,8 +8,11 @@ import { expect, it } from "vitest";
import { parse } from "yaml";
import { z } from "zod";
import { planTestSelection } from "../../../scripts/ci/test-selection.mjs";
import { parseTestDurations } from "../../ciTestSequencer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { workflowGit } from "../../support/workflowGit.js";
const execute = promisify(execFile);
const script = new URL("../../../scripts/ci/test-report.mjs", import.meta.url);
@@ -53,6 +56,23 @@ it("collects failed-shard diagnostics before enforcing the shard result", async
expect(upload).toBeGreaterThan(merge);
expect(gate).toBeGreaterThan(upload);
expect(steps[merge]?.if).not.toContain("needs.test-shard.result");
const merges = steps.filter((step) => step.run?.includes("--reporter=json"));
expect(merges).toHaveLength(2);
for (const step of merges)
expect(step.run).toContain(
"--reporter=./scripts/ci/test-timing-reporter.mjs",
);
expect(steps[upload]?.with?.path).toContain("test-timings.json");
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
expect(steps[upload]?.with?.path).toContain(
".cache/ci-test-selection/report.json",
);
const summary = steps.find((step) =>
step.run?.includes("scripts/ci/test-report.mjs"),
);
expect(summary?.env?.CI_TEST_SELECTION).toBe(
"${{ needs.changes.outputs.test_selection }}",
);
const command = steps[gate]?.run;
if (command === undefined) throw new Error("Missing shard result gate");
for (const result of ["failure", "cancelled", "skipped", ""]) {
@@ -77,11 +97,67 @@ it("keeps diagnosis available after merge failure and publishes only complete su
const history = steps.find((step) => step.with?.name === "test-timings");
expect(history?.if).toContain("success()");
expect(history?.if).toContain("needs.test-shard.result == 'success'");
expect(history?.if).toContain("needs.changes.outputs.tier == 'full'");
expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json");
const download = (await workflow()).build!.steps.find(
const buildSteps = (await workflow()).build!.steps;
const lookup = buildSteps.find((step) =>
step.run?.includes("test-history.mjs"),
);
expect(lookup?.env).toMatchObject({
CI_REPOSITORY: "${{ github.repository }}",
CI_REPOSITORY_ID: "${{ github.repository_id }}",
CI_DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}",
});
const download = buildSteps.find(
(step) => step.name === "Download previous timing report",
);
expect(download?.with?.name).toBe(history?.with?.name);
expect(download?.with?.["run-id"]).toBe(
"${{ steps.history.outputs.run-id }}",
);
});
it("rejects successful shard reports with missing or duplicate portable files", async () => {
const root = await createTestTempDirectory("rea-ci-report-inventory-");
await workflowGit(root, ["init"]);
await mkdir(join(root, "tests"));
for (const name of ["first", "second"])
await writeFile(join(root, `tests/${name}.test.ts`), "// fixture\n");
await workflowGit(root, ["add", "tests"]);
const run = async (names: readonly string[]) => {
await writeFile(
join(root, "test-timings.json"),
JSON.stringify({
version: 1,
success: true,
files: names.map((name) => ({ name, durationMs: 1 })),
}),
);
return execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
SHARD_RESULT: "success",
MERGE_RESULT: "success",
CI_VERIFY_TEST_INVENTORY: "true",
},
});
};
await expect(
run(["tests/first.test.ts", "tests/second.test.ts"]),
).resolves.toMatchObject({ stderr: "" });
for (const names of [
["tests/first.test.ts"],
["tests/first.test.ts", "tests/second.test.ts", "tests/unplanned.test.ts"],
["tests/first.test.ts", "tests/first.test.ts", "tests/second.test.ts"],
]) {
await expect(run(names)).rejects.toMatchObject({
stderr: expect.stringContaining("Incomplete portable test inventory"),
});
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
}
});
it("runs the existing onboarding and Python producer guards from their selected CI owners", async () => {
@@ -116,20 +192,25 @@ it("exports portable timing history and removes it on a failed, partial, or inva
const summary = join(root, "summary.md");
const history = join(root, ".cache/ci-test-history/test-results.json");
const report = {
version: 1,
success: true,
numFailedTests: 0,
testResults: [
files: [
{
name: join(root, "tests/slow.test.ts"),
startTime: 10,
endTime: 110,
assertionResults: [{ large: "not timing data" }],
durationMs: 100,
phases: {
execution: 20,
preparation: 20,
environment: 10,
imports: 40,
setup: 10,
},
},
{ name: join(root, "tests/fast.test.ts"), startTime: 20, endTime: 30 },
{ name: join(root, "tests/fast.test.ts"), durationMs: 10 },
],
};
const run = async (value: unknown, shards = "success", merge = "success") => {
await writeFile(join(root, "test-results.json"), JSON.stringify(value));
await writeFile(join(root, "test-timings.json"), JSON.stringify(value));
await writeFile(summary, "");
return execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
@@ -143,7 +224,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva
};
await run(report);
const contents = await readFile(history, "utf8");
expect(contents).not.toContain("assertionResults");
expect(contents).not.toContain("phases");
expect(contents).not.toContain("startTime");
expect(contents).not.toContain(root);
expect(
parseTestDurations(contents, resolve(root, "different-checkout")),
@@ -154,7 +236,7 @@ it("exports portable timing history and removes it on a failed, partial, or inva
]),
);
for (const [value, shards, merge] of [
[{ ...report, success: false, numFailedTests: 1 }, "failure", "failure"],
[{ ...report, success: false }, "failure", "failure"],
[report, "cancelled", "success"],
[report, "success", "failure"],
] as const) {
@@ -167,8 +249,211 @@ it("exports portable timing history and removes it on a failed, partial, or inva
await expect(
run({
...report,
testResults: [{ name: "../outside.test.ts", startTime: 1, endTime: 3 }],
files: [{ name: "../outside.test.ts", durationMs: 2 }],
}),
).rejects.toMatchObject({ code: 1 });
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
await run(report);
await expect(
run({
...report,
files: [{ name: "tests/invalid.test.ts", durationMs: -1 }],
}),
).rejects.toMatchObject({ code: 1 });
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
});
it("recovers complete module costs from real blob reports without executing tests again", async () => {
const root = await createTestTempDirectory("rea-ci-module-timing-");
const config = join(root, "vitest.config.mjs");
await writeFile(
config,
'export default { test: { globals: true, include: ["sample.test.js"], maxWorkers: 1 } };',
);
await writeFile(
join(root, "sample.test.js"),
[
'import { writeFileSync } from "node:fs";',
"beforeAll(() => new Promise(resolve => setTimeout(resolve, 80)));",
"afterAll(() => new Promise(resolve => setTimeout(resolve, 80)));",
'it("has cheap assertions and expensive outer hooks", () => { writeFileSync("executed", "yes"); expect(1).toBe(1); });',
].join("\n"),
);
const vitest = fileURLToPath(
new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url),
);
const reporter = fileURLToPath(
new URL("../../../scripts/ci/test-timing-reporter.mjs", import.meta.url),
);
const run = (args: string[]) =>
execute(process.execPath, [vitest, ...args, "--config", config], {
cwd: root,
env: process.env,
timeout: 20_000,
});
await run(["run", "--reporter=blob"]);
expect(await readFile(join(root, "executed"), "utf8")).toBe("yes");
await rm(join(root, "executed"));
await run([
"--merge-reports",
`--reporter=${reporter}`,
"--reporter=json",
"--outputFile.json=body.json",
]);
await expect(readFile(join(root, "executed"))).rejects.toMatchObject({
code: "ENOENT",
});
const report = z
.object({
version: z.literal(1),
success: z.literal(true),
files: z.array(
z.object({
name: z.string(),
durationMs: z.number(),
phases: z.record(z.string(), z.number()),
}),
),
})
.parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")));
expect(
JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")),
).toMatchObject({
reason: "passed",
unhandledErrors: 0,
files: [{ state: "passed" }],
});
expect(report.files).toHaveLength(1);
const file = report.files[0]!;
const json = z
.object({
testResults: z.array(
z.object({ startTime: z.number(), endTime: z.number() }),
),
})
.parse(JSON.parse(await readFile(join(root, "body.json"), "utf8")));
const body = json.testResults[0]!;
expect(
file.phases.execution! - (body.endTime - body.startTime),
).toBeGreaterThan(100);
expect(file.phases.imports).toBeGreaterThan(0);
expect(file.phases.preparation).toBeGreaterThan(0);
expect(file.durationMs).toBeCloseTo(
Object.values(file.phases).reduce((sum, value) => sum + value, 0),
);
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
});
expect(
parseTestDurations(
await readFile(
join(root, ".cache/ci-test-history/test-results.json"),
"utf8",
),
root,
)?.get("sample.test.js"),
).toBe(file.durationMs);
await writeFile(
join(root, "sample.test.js"),
'it("fails", () => { throw new Error("fixture failure"); });',
);
await expect(run(["run", "--reporter=blob"])).rejects.toMatchObject({
code: 1,
});
await expect(
run(["--merge-reports", `--reporter=${reporter}`]),
).rejects.toMatchObject({ code: 1 });
const failed = JSON.parse(
await readFile(join(root, "test-timings.json"), "utf8"),
) as unknown;
expect(failed).toMatchObject({
version: 1,
success: false,
reason: "failed",
unhandledErrors: 0,
files: [{ state: "failed" }],
});
// Even a caller accidentally marking the merge successful cannot publish this report.
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
});
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
});
it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => {
const root = await createTestTempDirectory("rea-ci-selection-report-");
await workflowGit(root, ["init"]);
const paths = [
"src/sqlite/provider.test.ts",
"src/evm/provider.test.ts",
"tests/boundary/mcp/workflow.test.ts",
];
for (const path of paths) {
await mkdir(join(root, path, ".."), { recursive: true });
await writeFile(join(root, path), "// fixture\n");
}
await workflowGit(root, ["add", "src", "tests"]);
const timing = {
version: 1,
success: false,
reason: "failed",
unhandledErrors: 0,
files: paths.map((name, index) => ({
name,
durationMs: 10,
state: index === 1 ? "failed" : "passed",
})),
};
const run = async (report: unknown) => {
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
await execute(process.execPath, [fileURLToPath(script)], {
cwd: root,
env: {
...process.env,
SHARD_RESULT: "failure",
MERGE_RESULT: "failure",
CI_VERIFY_TEST_INVENTORY: "true",
CI_TEST_SELECTION: JSON.stringify(
planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"),
),
GITHUB_SHA: "merge-sha",
CI_PR_HEAD_SHA: "pr-head-sha",
},
});
return JSON.parse(
await readFile(
join(root, ".cache/ci-test-selection/report.json"),
"utf8",
),
) as unknown;
};
expect(await run(timing)).toMatchObject({
observational: true,
complete: true,
selected: [paths[0], paths[2]],
omitted: [paths[1]],
missedFailures: [paths[1]],
executionSha: "merge-sha",
prHeadSha: "pr-head-sha",
});
for (const report of [
{ ...timing, files: timing.files.slice(0, 1) },
{ ...timing, unhandledErrors: 1 },
{ ...timing, reason: "interrupted" },
])
expect(await run(report)).toMatchObject({ complete: false });
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
code: 1,
});
await expect(
readFile(join(root, ".cache/ci-test-selection/report.json")),
).rejects.toMatchObject({ code: "ENOENT" });
await expect(
readFile(join(root, ".cache/ci-test-history/test-results.json")),
).rejects.toMatchObject({ code: "ENOENT" });
});
@@ -17,24 +17,24 @@ describe("CI test history", () => {
const root = resolve("/workspace/rea");
const durations = parseTestDurations(
JSON.stringify({
version: 1,
success: true,
numFailedTests: 0,
testResults: [
files: [
{
name: join(root, "tests/slow.test.ts"),
startTime: 10,
endTime: 110,
durationMs: 100,
},
{
name: join(root, "tests/slow.test.ts"),
startTime: 20,
endTime: 80,
durationMs: 60,
},
{
name: resolve(root, "../outside.test.ts"),
startTime: 10,
endTime: 1000,
durationMs: 990,
},
{ name: "tests/negative.test.ts", durationMs: -1 },
{ name: "tests/zero.test.ts", durationMs: 0 },
{ name: "tests/invalid.test.ts", durationMs: "100" },
null,
],
}),
@@ -43,7 +43,20 @@ describe("CI test history", () => {
expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]]));
expect(
parseTestDurations('{"success": false, "numFailedTests": 1}', root),
parseTestDurations('{"version": 1, "success": false, "files": []}', root),
).toBeUndefined();
// Legacy test-body spans must not be mixed with complete module costs.
expect(
parseTestDurations(
JSON.stringify({
success: true,
numFailedTests: 0,
testResults: [
{ name: "tests/legacy.test.ts", startTime: 0, endTime: 100 },
],
}),
root,
),
).toBeUndefined();
expect(parseTestDurations("not json", root)).toBeUndefined();
expect(parseTestDurations("null", root)).toBeUndefined();
@@ -0,0 +1,137 @@
import { createHash } from "node:crypto";
import { mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { expect, it } from "vitest";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import {
executeWorkflowFixture,
workflowGit,
} from "../../support/workflowGit.js";
const validator = fileURLToPath(
new URL("../../../scripts/ci/verified-package.mjs", import.meta.url),
);
type Fixture = Awaited<ReturnType<typeof createFixture>>;
type FixtureOptions = {
override?: Record<string, string | number>;
manifestContents?: string;
writeManifest?: boolean;
};
it("validates a tarball against the exact checkout and current workflow attempt", async () => {
const fixture = await createFixture();
const result = await validate(fixture);
expect(result.stdout.trim()).toBe(
join(fixture.artifactDirectory, "package.tgz"),
);
expect(await readFile(fixture.output, "utf8")).toBe(
`tarball=${join(fixture.artifactDirectory, "package.tgz")}\n`,
);
});
it("rejects a workflow SHA that differs from the checked out HEAD", async () => {
const fixture = await createFixture();
await expect(validate(fixture, "c".repeat(40))).rejects.toThrow();
});
it.each([
["wrong checkout SHA", { sourceSha: "c".repeat(40) }],
["wrong workflow run", { runId: 124 }],
["future workflow attempt", { runAttempt: 3 }],
["wrong verifier platform", { platform: "darwin" }],
["wrong verifier architecture", { architecture: "arm64" }],
["wrong package digest", { sha256: "c".repeat(64) }],
])("rejects %s", async (_label, override) => {
const fixture = await createFixture({ override });
await expect(validate(fixture)).rejects.toThrow();
});
it("accepts an earlier successful attempt from the same workflow run", async () => {
const fixture = await createFixture({ override: { runAttempt: 1 } });
await expect(validate(fixture)).resolves.toMatchObject({
stdout: expect.any(String),
});
});
it.each([
["malformed manifest", "{"],
["manifest with missing fields", JSON.stringify({ schemaVersion: 1 })],
])("rejects %s", async (_label, contents) => {
const fixture = await createFixture({ manifestContents: contents });
await expect(validate(fixture)).rejects.toThrow();
});
it("rejects an absent manifest", async () => {
const fixture = await createFixture({ writeManifest: false });
await expect(validate(fixture)).rejects.toThrow();
});
async function createFixture({
override = {},
manifestContents,
writeManifest = true,
}: FixtureOptions = {}) {
const directory = await createTestTempDirectory("rea-verified-package-");
await workflowGit(directory, ["init", "--initial-branch=main"]);
await writeFile(
join(directory, "package.json"),
'{"name":"fixture","version":"1.2.3"}\n',
);
await workflowGit(directory, ["add", "package.json"]);
await workflowGit(directory, [
"commit",
"-m",
"test: verified package fixture",
]);
const sourceSha = await workflowGit(directory, ["rev-parse", "HEAD"]);
const artifactDirectory = join(directory, "artifact");
await mkdir(artifactDirectory);
const archive = "verified tarball fixture";
await writeFile(join(artifactDirectory, "package.tgz"), archive);
const manifest = {
schemaVersion: 1,
sourceSha,
runId: 123,
runAttempt: 2,
platform: "linux",
architecture: "x64",
packageVersion: "1.2.3",
nodeVersion: process.version,
sha256: createHash("sha256").update(archive).digest("hex"),
...override,
};
if (writeManifest)
await writeFile(
join(artifactDirectory, "manifest.json"),
manifestContents ?? JSON.stringify(manifest),
);
const output = join(directory, "github-output");
await writeFile(output, "");
return { directory, sourceSha, artifactDirectory, output };
}
function validate(fixture: Fixture, sourceSha = fixture.sourceSha) {
return executeWorkflowFixture(
process.execPath,
[validator, "validate", fixture.artifactDirectory],
{
cwd: fixture.directory,
env: {
...process.env,
GITHUB_SHA: sourceSha,
GITHUB_RUN_ID: "123",
GITHUB_RUN_ATTEMPT: "2",
GITHUB_OUTPUT: fixture.output,
},
},
);
}
@@ -649,9 +649,8 @@ it.skipIf(process.platform === "win32").each([
const sourceSha = await candidate(f, valid ? "6.0.0" : "5.1.0");
const workflow = z
.object({
jobs: z.record(
z.string(),
z.object({
jobs: z.object({
[job]: z.object({
steps: z.array(
z.object({
name: z.string().optional(),
@@ -659,7 +658,7 @@ it.skipIf(process.platform === "win32").each([
}),
),
}),
),
}),
})
.parse(
parse(
@@ -34,6 +34,12 @@ async function readReleaseWorkflow() {
"release-proposal": jobSchema,
"release-please": jobSchema,
"merged-release": jobSchema,
qualification: z.object({
if: z.string(),
needs: z.array(z.string()),
uses: z.string(),
with: z.record(z.string(), z.string()),
}),
publish: jobSchema,
"publish-mcp": jobSchema,
}),
@@ -271,10 +277,19 @@ it("binds npm and MCP publication to the same immutable release SHA", async () =
release_created: "${{ steps.release.outputs.release_created }}",
sha: "${{ steps.release.outputs.sha }}",
});
expect(workflow.jobs.publish.if).toContain("inputs.phase == 'publish'");
expect(workflow.jobs.publish.if).toContain(
expect(workflow.jobs.qualification.if).toContain("inputs.phase == 'publish'");
expect(workflow.jobs.qualification.if).toContain(
"needs.release-please.outputs.release_created",
);
expect(workflow.jobs.qualification.uses).toBe("./.github/workflows/ci.yml");
expect(workflow.jobs.qualification.with).toEqual({
source_sha:
"${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }}",
});
expect(workflow.jobs.publish.needs).toContain("qualification");
expect(workflow.jobs.publish.if).toContain(
"needs.qualification.result == 'success'",
);
expect(workflow.jobs["publish-mcp"].if).toContain(
"needs.publish.result == 'success'",
);
@@ -309,7 +324,7 @@ it("binds npm and MCP publication to the same immutable release SHA", async () =
)?.run;
expect(publishCommand).toContain("scripts/release-npm-tag.mjs");
expect(publishCommand).toContain(
'npm publish --access public --tag "${tag}"',
'npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}"',
);
for (const [version, tag] of [
["6.1.0", "latest"],
@@ -644,7 +659,7 @@ it("routes website verification through the central PR policy", async () => {
// Publishing is irreversible. Keep the release authority invariant as a static
// boundary check; it is not proof that package installation or publishing works.
it("requires package verification before publishing and retains the registry canary", async () => {
it("requires qualified package validation before publishing and retains the registry canary", async () => {
const workflow = z
.object({
jobs: z.object({
@@ -663,7 +678,7 @@ it("requires package verification before publishing and retains the registry can
);
const commands = workflow.jobs.publish.steps.map(({ run }) => run ?? "");
const verify = commands.findIndex((command) =>
command.includes("npm run verify:package"),
command.includes("scripts/ci/verified-package.mjs validate"),
);
const publish = commands.findIndex((command) =>
command.includes("npm publish"),
@@ -489,4 +489,28 @@ describe("safe artifact output Windows stream paths", () => {
await expect(access(output)).rejects.toMatchObject({ code: "ENOENT" });
expect(await readdir(parent)).toEqual([]);
});
it.each(["Folder/Icon\r", "Fold\ter/file.txt"])(
"refuses Windows control characters before creating %j",
async (path) => {
const parent = await createTestTempDirectory("rea-safe-output-control-");
const output = join(parent, "published");
const tree = await SafeOutputTree.create(output, "win32");
const bytes = Buffer.from("regular file content");
await expect(
tree.write(path, Readable.from(bytes), {
sha256: createHash("sha256").update(bytes).digest("hex"),
bytes: bytes.byteLength,
}),
).rejects.toMatchObject({
reason: "path",
message: `Windows destination cannot materialize artifact path: ${JSON.stringify(path)}; Windows names cannot contain U+0001-U+001F.`,
});
expect(await readdir(output)).toEqual([]);
expect(await tree.rollback()).toEqual({
status: "complete",
residualPaths: [],
});
},
);
});
@@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { cp, readFile, writeFile } from "node:fs/promises";
import { basename, join } from "node:path";
import { promisify } from "node:util";
import { expect, it } from "vitest";
@@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")(
await workflowGit(root, ["init", "--initial-branch=main"]);
await workflowGit(root, ["add", "."]);
await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]);
const run = () =>
const run = (cwd = root) =>
execute(
process.execPath,
[
@@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")(
"test:run",
"--output-logs=errors-only",
],
{ cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } },
{ cwd, env: { ...process.env, TURBO_DAEMON: "false" } },
);
await run();
await writeFile(join(root, "README_zh.md"), "wrong command");
@@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")(
"prepared\n",
);
}
// Provider jobs receive the producer's cache in a fresh checkout, not its
// outputs. Turbo must restore identical inputs and rebuild changed ones.
const consumer = await createTestTempDirectory("rea-cache-consumer-");
await cp(root, consumer, {
recursive: true,
filter: (path) =>
basename(path) !== ".turbo" &&
!/^prepared-\d+\.txt$/u.test(basename(path)),
});
await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), {
recursive: true,
});
const restored = await run(consumer);
expect(restored.stdout).toContain(`${dependencies.length} cached`);
for (const index of dependencies.keys())
expect(
await readFile(join(consumer, `prepared-${index}.txt`), "utf8"),
).toBe("prepared\n");
await writeFile(
join(consumer, "artifact-input.txt"),
"changed producer input",
);
await run(consumer);
for (const index of dependencies.keys())
expect(
await readFile(join(consumer, `prepared-${index}.txt`), "utf8"),
).toBe("prepared\nprepared\n");
},
);
@@ -144,14 +144,26 @@ describe("enhanced MCP tools", () => {
);
expect(result).toEqual({
document: "fixture",
image_base: "0x1000",
segments: [
{ name: "__TEXT", start: "0x1000", end: "0x1800", length: 2048 },
{ name: "__DATA", start: "0x1800", end: "0x2000", length: 2048 },
{
name: "__TEXT",
start: "0x1000",
end: "0x1800",
length: 2048,
image_base: "0x1000",
},
{
name: "__DATA",
start: "0x1800",
end: "0x2000",
length: 2048,
image_base: "0x1000",
},
],
segment_count: 2,
procedure_count: 2,
string_count: 700,
image_base: "0x1000",
});
});
@@ -0,0 +1,121 @@
import { afterEach, expect, it } from "vitest";
import { Ajv2020 } from "ajv/dist/2020.js";
import type { AnySchemaObject } from "ajv";
import {
closeEnhancedToolResources,
connect,
jsonResult,
} from "./enhancedToolsHarness.js";
import { observed as ok } from "../../fixtures/analysisExecution.js";
import type { SegmentSummary } from "../../../src/domain/hopperValues.js";
afterEach(closeEnhancedToolResources);
const segment = {
name: ".text",
start: "0x101000",
end: "0x101100",
readable: true,
writable: false,
executable: true,
image_base: "0x100000",
address_space: "ram",
};
const overviewClient = (segments: SegmentSummary[]) =>
connect({
execute: (name) => {
switch (name) {
case "current_document":
return Promise.resolve(ok("fixture"));
case "list_documents":
return Promise.resolve(ok(["fixture"]));
case "list_segments":
return Promise.resolve(ok(segments));
case "list_procedures":
return Promise.resolve(ok([{ address: "0x101013", value: "entry" }]));
case "list_strings":
return Promise.resolve(ok([]));
default:
throw new Error(`Unexpected operation: ${name}`);
}
},
});
it("preserves observed coordinates and main's optional-string output contract after SDK conversion", async () => {
const client = await overviewClient([segment]);
const schema = (await client.listTools()).tools.find(
({ name }) => name === "binary_overview",
)?.outputSchema;
if (schema === undefined) throw new Error("Missing overview output schema");
const validate = new Ajv2020({ strict: false }).compile(
schema as AnySchemaObject,
);
const response = await client.callTool({
name: "binary_overview",
arguments: {},
});
expect(validate(response.structuredContent)).toBe(true);
expect(jsonResult(response)).toMatchObject({
image_base: "0x100000",
segments: [
{ start: "0x101000", image_base: "0x100000", address_space: "ram" },
],
});
const absent = JSON.parse(JSON.stringify(response.structuredContent));
delete absent.normalized_result.image_base;
expect(validate(absent)).toBe(true);
absent.normalized_result.image_base = null;
expect(validate(absent)).toBe(false);
});
it("omits unreported bases without inventing zero and preserves explicit zero", async () => {
const { image_base: _base, ...unreported } = segment;
const client = await overviewClient([unreported]);
expect(
jsonResult(
await client.callTool({ name: "binary_overview", arguments: {} }),
),
).not.toHaveProperty("image_base");
const zero = await overviewClient([{ ...segment, image_base: "0x0" }]);
expect(
jsonResult(await zero.callTool({ name: "binary_overview", arguments: {} })),
).toMatchObject({ image_base: "0x0" });
});
it("keeps a partially reported base unknown while retaining the reported segment observation", async () => {
const { image_base: _base, ...unreported } = segment;
const client = await overviewClient([
segment,
{ ...unreported, name: ".data" },
]);
const result = jsonResult(
await client.callTool({ name: "binary_overview", arguments: {} }),
);
expect(result).not.toHaveProperty("image_base");
expect(result).toMatchObject({
segments: [{ image_base: "0x100000" }, { name: ".data" }],
});
});
it("retains all conflicting observations after repeated early bases without selecting one", async () => {
const { address_space: _space, ...unreported } = segment;
const client = await overviewClient([
...Array.from({ length: 40 }, () => unreported),
{
...segment,
name: ".data",
start: "other%20space:0x201000",
end: "other%20space:0x201100",
image_base: "0x200000",
address_space: "other space",
},
]);
const result = jsonResult(
await client.callTool({ name: "binary_overview", arguments: {} }),
);
expect(result).not.toHaveProperty("image_base");
expect(result).toHaveProperty("segments.length", 41);
expect(result).toHaveProperty("segments.40.image_base", "0x200000");
expect(result).toHaveProperty("segments.40.address_space", "other space");
expect(result).not.toHaveProperty("segments.0.address_space");
});
@@ -116,6 +116,66 @@ describe("managed member comparison", () => {
});
});
describe("managed member comparison floating-point constants", () => {
it.each([
{
opcode: "ldc.r4",
positive: "22000000002a",
negative: "22000000802a",
type: 0x0c,
},
{
opcode: "ldc.r8",
positive: "2300000000000000002a",
negative: "2300000000000000802a",
type: 0x0d,
},
])(
"detects a change from positive to negative zero in $opcode",
({ opcode, positive, negative, type }) => {
const observe = (hex: string, path: string) => {
const il = Buffer.from(hex, "hex");
return inspect(
buildManagedPeFixture({
methodSignature: Buffer.from([0x00, 0x00, type]),
ilBody: Buffer.concat([Buffer.from([(il.length << 2) | 2]), il]),
}),
path,
);
};
const left = observe(positive, "/tmp/positive-zero.dll");
const right = observe(negative, "/tmp/negative-zero.dll");
const comparison = compareManagedMembers(left, right);
expect(comparison.methods[0]).toMatchObject({
status: "changed",
match: { status: "matched", basis: "exact-signature" },
dimensions: ["cil"],
});
for (const [observation, operand] of [
[left, "0"],
[right, "-0"],
] as const) {
expect(observation.result.methods[0]?.body).toMatchObject({
status: "present",
anchors: [
{ il_offset: 0, opcode, operand_kind: "constant", operand },
],
});
}
expect(left.result.methods[0]?.body.normalized_il_sha256).not.toBe(
right.result.methods[0]?.body.normalized_il_sha256,
);
const repeated = observe(negative, "/tmp/negative-zero-copy.dll");
expect(compareManagedMembers(right, repeated).methods[0]).toMatchObject({
status: "unchanged",
match: { status: "matched", basis: "exact-il-signature" },
dimensions: [],
});
},
);
});
describe("managed member comparison uncertainty", () => {
it("does not infer a structural method match from undecoded signature fields", () => {
const left = inspect(
@@ -61,7 +61,13 @@ afterEach(async () => {
});
describe("native macOS provider discovery", () => {
it.each(["ordinary.fixture", " spaced.fixture ", " leading", "trailing "])(
it.each([
"ordinary.fixture",
" spaced.fixture ",
" leading",
"trailing ",
"literal\r\nidentifier\r",
])(
"preserves the exact signing identifier %j reported by codesign",
(identifier) => {
const parsed = parseCodeSignature(
@@ -74,31 +80,68 @@ describe("native macOS provider discovery", () => {
},
);
it("separates echoed path and identifier text from signature fields", () => {
const path = "/apps/x\nAuthority=Forged\nIdentifier=forged";
const identifier = "x\nFormat=forged\nAuthority=Forged-5555";
const parsed = parseCodeSignature(
[
`Executable=${path}`,
`Identifier=${identifier}`,
"Format=Mach-O thin (arm64)",
"CodeDirectory v=20400 size=342 flags=0x2(adhoc) hashes=2+2",
"CDHash=6aee90e6",
"Signature=adhoc",
"TeamIdentifier=not set",
"",
].join("\r\n"),
false,
["/elsewhere", path],
);
expect(parsed).toMatchObject({
identifier,
format: "Mach-O thin (arm64)",
authorities: [],
cdhashes: ["6aee90e6"],
team_identifier: null,
});
});
it.each(["\n", "\r\n"])(
"separates echoed path and identifier text from signature fields (%j)",
(newline) => {
const path = "/apps/x\r\nAuthority=Forged\nIdentifier=forged";
const identifier = "x\r\nFormat=forged\nAuthority=Forged-5555\r";
const parsed = parseCodeSignature(
[
`Executable=${path}`,
`Identifier=${identifier}`,
"Format=Mach-O thin (arm64)",
"CodeDirectory v=20400 size=342 flags=0x2(adhoc) hashes=2+2",
"CDHash=6aee90e6",
"Signature=adhoc",
"TeamIdentifier=not set",
"",
].join(newline),
false,
["/elsewhere", path],
);
expect(parsed).toMatchObject({
identifier,
format: "Mach-O thin (arm64)",
authorities: [],
cdhashes: ["6aee90e6"],
team_identifier: null,
});
},
);
it.each([
{ executable: "", paths: [], newline: "\r\n", identifier: "fixture" },
{
executable: "Executable=/apps/fixture\r\n",
paths: [],
newline: "\r\n",
identifier: "fixture\nFormat=payload\r",
},
{
executable: "Executable=/apps/fixture\n",
paths: ["/elsewhere"],
newline: "\n",
identifier: "fixture\r\nFormat=payload\r",
},
{
executable: "Executable=/apps/fixture\r\n",
paths: ["/elsewhere"],
newline: "\r\n",
identifier: "fixture\r",
},
])(
"frames the identifier without a matching executable path: %j",
({ executable, paths, newline, identifier }) => {
const parsed = parseCodeSignature(
`${executable}Identifier=${identifier}${newline}Format=Mach-O thin (arm64)${newline}TeamIdentifier=not set${newline}`,
false,
paths,
);
expect(parsed.identifier).toBe(identifier);
expect(parsed.format).toBe("Mach-O thin (arm64)");
expect(parsed.team_identifier).toBeNull();
},
);
it.each(["timeout", "output-limit"] as const)(
"preserves %s resolution failures with their captured output and cleanup status",
@@ -1,7 +1,11 @@
import { expect, it } from "vitest";
import { CPU } from "../../../../src/artifacts/apple/MachoImage.fixture.js";
import { inspectMachoSchema } from "../../../../src/domain/native/nativeInspection.js";
import { err } from "../../../../src/domain/result.js";
import { NativeCommandFailure } from "../../../../src/native/CommandRunner.js";
import {
NativeCommandFailure,
type NativeCommandRunner,
} from "../../../../src/native/CommandRunner.js";
import { NativeMacOSProvider } from "../../../../src/native/NativeMacOSProvider.js";
import {
NativeFixtureRunner,
@@ -9,6 +13,35 @@ import {
nativeMachoTarget,
} from "../../../fixtures/nativeCommands.js";
/** Replay fixtures, failing any slice command that does not name the expected slice. */
const sliceSelectingRunner = (
fixtures: NativeFixtureRunner,
selected: string,
dyldArchitecture: string,
calls: {
readonly tool: string;
readonly arguments_: readonly string[];
}[] = [],
): NativeCommandRunner => ({
run(tool, arguments_) {
calls.push({ tool, arguments_ });
const expected = tool === "dyld_info" ? dyldArchitecture : selected;
if (
!["file", "lipo"].includes(tool) &&
!arguments_.some(
(argument, index) =>
argument === expected ||
argument === `--arch=${expected}` ||
(argument === "-arch" && arguments_[index + 1] === expected),
)
)
return Promise.resolve(
err(new NativeCommandFailure(tool, "nonzero-exit", 1)),
);
return fixtures.run(tool, arguments_);
},
});
it("does not attribute the first universal slice's UUID and segments to the selected target", async () => {
const singleSlice = await nativeFixture("otool-load.txt");
const selected = new NativeFixtureRunner();
@@ -58,18 +91,39 @@ it.each([
selected: "arm64e",
dyldArchitecture: "arm64e.v1",
},
])(
{
architectures: ["arm64v8", "arm64e.v1"],
selected: "arm64e",
dyldArchitecture: "arm64e.v1",
},
{
architectures: ["arm64e.v1", "arm64v8"],
selected: "arm64e",
dyldArchitecture: "arm64e.v1",
},
] as const)(
"selects the $dyldArchitecture slice of a universal arm64e target",
async ({ architectures, selected, dyldArchitecture }) => {
const calls: {
readonly tool: string;
readonly arguments_: readonly string[];
}[] = [];
const cpus = {
x86_64: CPU.x86_64,
arm64e: CPU.arm64e,
arm64v8: { type: CPU.arm64.type, subtype: 1 },
"arm64e.x1": { type: CPU.arm64.type, subtype: 0x8000000c },
"arm64e.v1": { type: CPU.arm64.type, subtype: 0x81000002 },
};
const lipo = architectures
.map(
(architecture, index) =>
`architecture ${architecture}\n cputype 16777228\n cpusubtype 2\n offset ${index * 16384}\n size 8192\n align 2^14 (16384)`,
)
.map((architecture, index) => {
const cpu = cpus[architecture];
const subtype =
architecture === "arm64e.v1"
? "CPU_SUBTYPE_ARM64E\n capabilities PTR_AUTH_VERSION USERSPACE 1"
: String(cpu.subtype);
return `architecture ${architecture}\n cputype ${cpu.type}\n cpusubtype ${subtype}\n offset ${index * 16384}\n size 8192\n align 2^14 (16384)`;
})
.join("\n");
const fixtures = new NativeFixtureRunner({
lipo,
@@ -77,25 +131,7 @@ it.each([
});
const client = new NativeMacOSProvider(
{},
{
run(tool, arguments_) {
calls.push({ tool, arguments_ });
const expected = tool === "dyld_info" ? dyldArchitecture : selected;
if (
!["file", "lipo"].includes(tool) &&
!arguments_.some(
(argument, index) =>
argument === expected ||
argument === `--arch=${expected}` ||
(argument === "-arch" && arguments_[index + 1] === expected),
)
)
return Promise.resolve(
err(new NativeCommandFailure(tool, "nonzero-exit", 1)),
);
return fixtures.run(tool, arguments_);
},
},
sliceSelectingRunner(fixtures, selected, dyldArchitecture, calls),
"darwin",
).createClient(nativeMachoTarget("/owned/universal"));
@@ -104,6 +140,11 @@ it.each([
if (!result.ok) throw result.error;
expect(result.value.result).toMatchObject({
uuid: "01234567-89AB-CDEF-0123-456789ABCDEF",
selected_architecture: {
name: dyldArchitecture,
container_file_offset:
architectures.findIndex((name) => name === dyldArchitecture) * 16384,
},
architectures: {
items: architectures.map((name) => ({ name })),
},
@@ -116,6 +157,36 @@ it.each([
},
);
it.each([
{ capture: "calculator.txt", selected: "arm64e" },
{ capture: "shortcut-droplet.txt", selected: "arm64" },
])(
"selects one slice when every universal slice has the target's CPU type ($capture)",
async ({ capture, selected }) => {
const base = nativeMachoTarget("/owned/universal");
if (base.kind !== "executable")
throw new Error("Expected executable fixture");
// Admission names arm64, arm64e and arm64e.x1 slices by one CPU family.
const target = { ...base, availableArchitectures: ["arm64"] as const };
const fixtures = new NativeFixtureRunner({
lipo: await nativeFixture(`lipo-same-cpu/${capture}`),
});
const result = await new NativeMacOSProvider(
{},
sliceSelectingRunner(fixtures, selected, selected),
"darwin",
)
.createClient(target)
.execute("inspect_macho", {});
if (!result.ok) throw result.error;
expect(result.value.result).toMatchObject({
selected_architecture: { name: selected, container_file_offset: 16384 },
architectures: { total: 2 },
});
},
);
it("projects universal segment evidence into the selected slice's container offsets", async () => {
const client = new NativeMacOSProvider(
{},
+6 -10
View File
@@ -5,14 +5,13 @@ import { BaseSequencer, type TestSpecification } from "vitest/node";
import { z } from "zod";
const reportSchema = z.object({
version: z.literal(1),
success: z.literal(true),
numFailedTests: z.literal(0),
testResults: z.array(z.unknown()),
files: z.array(z.unknown()),
});
const fileSchema = z.object({
name: z.string(),
startTime: z.number(),
endTime: z.number(),
durationMs: z.number().finite().positive(),
});
export function parseTestDurations(
@@ -30,15 +29,12 @@ export function parseTestDurations(
if (!report.success) return undefined;
const durations = new Map<string, number>();
for (const value of report.data.testResults) {
for (const value of report.data.files) {
const result = fileSchema.safeParse(value);
if (!result.success) continue;
const { name, startTime, endTime } = result.data;
const duration = endTime - startTime;
const { name, durationMs } = result.data;
const path = relative(root, resolve(root, name));
if (
!Number.isFinite(duration) ||
duration <= 0 ||
path === "" ||
path === ".." ||
path.startsWith(".." + sep) ||
@@ -46,7 +42,7 @@ export function parseTestDurations(
)
continue;
const key = path.split(sep).join("/");
durations.set(key, Math.max(durations.get(key) ?? 0, duration));
durations.set(key, Math.max(durations.get(key) ?? 0, durationMs));
}
return durations.size ? durations : undefined;
}
@@ -0,0 +1,7 @@
__attribute__((noinline)) int rea_pie_add(int value) {
return value + 7;
}
int rea_pie_entry(void) {
return rea_pie_add(35);
}
@@ -0,0 +1,118 @@
//@category REA Verification
import java.io.ByteArrayInputStream;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.Locale;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;
import ghidra.app.script.GhidraScriptUtil;
import ghidra.app.util.headless.HeadlessScript;
import ghidra.program.model.address.AddressSet;
import ghidra.program.model.listing.Function;
import ghidra.program.model.symbol.SourceType;
import ghidra.app.decompiler.DecompInterface;
import ghidra.app.decompiler.DecompileResults;
import ghidra.app.decompiler.DecompileProcess.DisposeState;
import ghidra.util.task.TaskMonitor;
/** Real source-identity cases unavailable through ordinary single-file CLI imports. */
public final class ReaFileOffsetSourceProbe extends HeadlessScript {
@Override
public void run() throws Exception {
var file = GhidraScriptUtil.findScriptByName("ReaGhidraBridge.java");
var bridge = GhidraScriptUtil.getProvider(file).getScriptInstance(file, new PrintWriter(System.out, true));
bridge.set(state, monitor, new PrintWriter(System.out, true));
var space = bridge.getClass().getDeclaredField("sessionDefaultAddressSpace");
space.setAccessible(true);
space.set(null, currentProgram.getAddressFactory().getDefaultAddressSpace());
var identity = bridge.getClass().getDeclaredField("admittedTargetSha256");
identity.setAccessible(true);
identity.set(bridge, currentProgram.getExecutableSHA256().toLowerCase(Locale.ROOT));
var resolve = bridge.getClass().getDeclaredMethod("resolveDecompileProcedure", String.class, JsonObject.class);
resolve.setAccessible(true);
var reverse = bridge.getClass().getDeclaredMethod("addressToFileOffset", JsonObject.class);
reverse.setAccessible(true);
var memory = currentProgram.getMemory();
var original = memory.getAllFileBytes().get(0);
byte[] bytes = new byte[Math.toIntExact(original.getSize())];
require(original.getOriginalBytes(0, bytes) == bytes.length, "complete fixture source");
byte[] auxiliary = new byte[0x1000];
auxiliary[0x910] = (byte)0xc3;
var foreign = memory.createFileBytes("auxiliary.bin", 0, auxiliary.length,
new ByteArrayInputStream(auxiliary), monitor);
var address = toAddr(0x140100000L);
memory.createInitializedBlock("auxiliary", address, foreign, 0, auxiliary.length, false);
var entry = address.add(0x910);
currentProgram.getFunctionManager().createFunction("foreign_only", entry,
new AddressSet(entry), SourceType.USER_DEFINED);
rejected(resolve, bridge, "no loaded source mapping", "file-offset:0x910", new JsonObject());
rejected(reverse, bridge, "0 authoritative target-file mappings", reverseInput("0x140100910"));
// A source object's declared original-file start does not make its local
// offsets a complete, digest-matched target representation.
var partial = memory.createFileBytes("partial.bin", 0x200, 0x200,
new ByteArrayInputStream(bytes, 0x200, 0x200), monitor);
memory.createInitializedBlock("partial", toAddr(0x140300000L), partial, 0, 0x200, false);
rejected(reverse, bridge, "0 authoritative target-file mappings", reverseInput("0x140300010"));
var selected = (Function)resolve.invoke(bridge, "file-offset:0x210", new JsonObject());
require(selected.getEntryPoint().equals(toAddr(0x140001010L)), "original target selected despite other sources");
var mapped = (JsonObject)reverse.invoke(bridge, reverseInput("0x140001010"));
require(mapped.get("file_offset").getAsLong() == 0x210, "original target round trip");
// Inject backend failures after real source/function resolution. This
// verifies context propagation, not a naturally failing decompiler.
var pseudocode = bridge.getClass().getDeclaredMethod("procedurePseudocode", JsonObject.class);
pseudocode.setAccessible(true);
var decompiler = bridge.getClass().getDeclaredField("decompiler");
decompiler.setAccessible(true);
JsonObject request = new JsonObject();
request.add("document", JsonNull.INSTANCE);
request.addProperty("procedure", "file-offset:0x210");
for (DisposeState disposition : new DisposeState[] { DisposeState.NOT_DISPOSED, DisposeState.DISPOSED_ON_CANCEL }) {
decompiler.set(bridge, new DecompInterface() {
@Override
public DecompileResults decompileFunction(Function function, int timeout, TaskMonitor taskMonitor) {
return new DecompileResults(function, currentProgram.getLanguage(),
currentProgram.getCompilerSpec(), null, "Injected backend failure", null, disposition);
}
});
Throwable failure = rejected(pseudocode, bridge,
"Selector file-offset:0x210 mapped to analysis address 0x140001010 and procedure 0x140001010", request);
var code = failure.getClass().getDeclaredField("code");
code.setAccessible(true);
require(code.get(failure).equals(disposition == DisposeState.DISPOSED_ON_CANCEL
? "decompile_cancelled" : "decompile_failed"), "backend failure code retained");
}
var duplicate = memory.createFileBytes("duplicate.bin", 0, bytes.length,
new ByteArrayInputStream(bytes), monitor);
memory.createInitializedBlock("duplicate", toAddr(0x140201000L), duplicate, 0x200, 0x200, false);
rejected(resolve, bridge, "multiple loaded mappings", "file-offset:0x210", new JsonObject());
println("REA_FILE_OFFSET_SOURCE_OK foreign/partial rejected; original round trip; duplicate ambiguity; injected backend failure context retained");
}
private JsonObject reverseInput(String address) {
JsonObject params = new JsonObject();
params.add("document", JsonNull.INSTANCE);
params.addProperty("address", address);
return params;
}
private Throwable rejected(Method method, Object bridge, String reason, Object... arguments) throws Exception {
try {
method.invoke(bridge, arguments);
} catch (InvocationTargetException rejected) {
require(rejected.getCause().getMessage().contains(reason), "expected mapping failure: " + reason);
return rejected.getCause();
}
throw new AssertionError("Expected mapping failure: " + reason);
}
private void require(boolean condition, String message) {
if (!condition) throw new AssertionError(message);
}
}
+240
View File
@@ -0,0 +1,240 @@
import { expect, it } from "vitest";
import {
compareTestSelection,
planTestSelection,
} from "../../scripts/ci/test-selection.mjs";
const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts";
const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts";
const adbSource = "src/adb/AdbProvider.test.ts";
const adbBoundary =
"tests/boundary/providers/adb/adbBugreportPublication.test.ts";
const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts";
const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts";
const processBoundary =
"tests/boundary/process/processCaptureLifecycle.test.ts";
const acceptance = "tests/acceptance/tools/localToolContracts.test.ts";
const composition = "tests/composition/directAnalysisCleanup.test.ts";
const evaluation = "tests/evaluation/testDiscovery.test.ts";
const passed = (name: string) => ({ name, durationMs: 10, state: "passed" });
it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary],
"pull_request",
"fast",
);
const inventory = [
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
];
const report = compareTestSelection(plan, inventory, inventory.map(passed));
expect(plan).toMatchObject({
mode: "candidate",
owners: ["adb", "sqlite"],
});
expect(report.selected).toEqual(
expect.arrayContaining([
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
]),
);
expect(report.omitted).toEqual([]);
});
it("omits only unrelated owned tests while retaining dynamic boundary suites", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const inventory = [
sqliteSource,
sqliteBoundary,
adbSource,
adbBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
];
const report = compareTestSelection(plan, inventory, inventory.map(passed));
expect(report.selected).toEqual(
expect.arrayContaining([
sqliteSource,
sqliteBoundary,
cliBoundary,
mcpBoundary,
processBoundary,
acceptance,
composition,
evaluation,
]),
);
expect(report.omitted).toEqual([adbSource, adbBoundary]);
expect(report.complete).toBe(true);
});
it.each([
["unowned shared source", "src/application/Workflow.ts"],
["fixture input", "tests/conformance/c/fixture.c"],
["configuration", "vitest.config.ts"],
["documentation", "README.md"],
["CLI implementation", "src/cli/apktoolCommands.ts"],
["MCP implementation", "src/server/registerSqliteTools.ts"],
["process ownership", "src/process/ProcessOwnership.ts"],
["acceptance test", acceptance],
["composition test", composition],
])("uses full fallback for %s changes", (_description, path) => {
expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({
mode: "full",
owners: [],
changedTests: [],
});
});
it("uses full fallback outside ordinary fast pull requests or without diff paths", () => {
expect(
planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast")
.mode,
).toBe("full");
expect(
planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"full",
).mode,
).toBe("full");
expect(planTestSelection([], "pull_request", "fast").mode).toBe("full");
});
it("keeps new changed tests and falls back when a changed test was removed", () => {
const addedTest = "src/sqlite/NewDatabaseRule.test.ts";
const candidate = planTestSelection([addedTest], "pull_request", "fast");
expect(candidate.changedTests).toEqual([addedTest]);
const inventory = [addedTest, adbSource];
expect(
compareTestSelection(candidate, inventory, inventory.map(passed)).selected,
).toContain(addedTest);
const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts";
const report = compareTestSelection(
planTestSelection([removedTest], "pull_request", "fast"),
[sqliteSource, adbSource],
[passed(sqliteSource), passed(adbSource)],
);
expect(report.plan.mode).toBe("full");
expect(report.selected).toEqual([adbSource, sqliteSource]);
expect(report.omitted).toEqual([]);
});
it("marks incomplete inventories and nonterminal outcomes as incomplete", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const inventory = [sqliteSource, adbSource];
const completeResults = inventory.map(passed);
expect(
compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete,
).toBe(false);
expect(
compareTestSelection(plan, inventory, [
...completeResults,
passed("tests/evaluation/unexpected.test.ts"),
]),
).toMatchObject({
complete: false,
unexpected: ["tests/evaluation/unexpected.test.ts"],
});
expect(
compareTestSelection(plan, inventory, [
...completeResults,
passed(adbSource),
]),
).toMatchObject({ complete: false, duplicateFiles: 1 });
for (const state of ["pending", "skipped"])
expect(
compareTestSelection(plan, inventory, [
passed(sqliteSource),
{ name: adbSource, durationMs: 0, state },
]),
).toMatchObject({ complete: false, unobserved: [adbSource] });
});
it("reports a failure from an omitted test, including collection or module failures", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
const report = compareTestSelection(
plan,
[sqliteSource, adbSource],
[passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }],
);
expect(report.omitted).toContain(adbSource);
expect(report.missedFailures).toEqual([adbSource]);
});
it("rejects plans that name unowned owners or malformed changed tests", () => {
const plan = planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
);
expect(() =>
compareTestSelection(
{ ...plan, owners: ["unknown"] },
[sqliteSource],
[passed(sqliteSource)],
),
).toThrow("Invalid observational test selection plan");
expect(() =>
compareTestSelection(
{ ...plan, changedTests: ["src/sqlite/not-a-test.ts"] },
[sqliteSource],
[passed(sqliteSource)],
),
).toThrow("Invalid observational test selection plan");
});
it("uses full fallback when a changed owner has no executable tests", () => {
const report = compareTestSelection(
planTestSelection(
["src/sqlite/SqliteDatabaseProvider.ts"],
"pull_request",
"fast",
),
[adbSource],
[passed(adbSource)],
);
expect(report.plan.mode).toBe("full");
expect(report.omitted).toEqual([]);
});
@@ -0,0 +1,90 @@
import { join } from "node:path";
import { expect, it, vi } from "vitest";
import { buildNativeAotFixture } from "../../../scripts/lib/nativeaot-build.mjs";
const options = {
dotnet: "dotnet",
project: "fixture.csproj",
runtime: "win-x64",
output: "fixture-output",
};
it("restores the exact publish target before compiling without network restore", async () => {
const run = vi.fn<(command: string, args: string[]) => void>();
await buildNativeAotFixture(run, options);
expect(run).toHaveBeenCalledTimes(2);
const restore = run.mock.calls[0]?.[1] ?? [];
const publish = run.mock.calls[1]?.[1] ?? [];
expect(restore[0]).toBe("restore");
expect(publish[0]).toBe("publish");
expect(publish).toContain("--no-restore");
for (const argument of [
options.project,
options.runtime,
"-p:Configuration=Release",
`-p:BaseIntermediateOutputPath=${join(options.output, "obj")}/`,
]) {
expect(restore).toContain(argument);
expect(publish).toContain(argument);
}
});
it("retries without logging credential-bearing restore diagnostics", async () => {
const logged = vi.spyOn(console, "error").mockImplementation(() => {});
const timeout = Object.assign(
new Error(
"dotnet restore ETIMEDOUT https://fixture-user:fixture-password@feed.invalid/v3/index.json",
),
{
code: "ETIMEDOUT",
},
);
const run = vi
.fn<(command: string, args: string[]) => void>()
.mockImplementationOnce(() => {
throw timeout;
});
try {
await buildNativeAotFixture(run, options);
expect(
logged.mock.calls.flat().every((value) => typeof value === "string"),
).toBe(true);
expect(logged.mock.calls.flat().join("\n")).not.toContain(
"fixture-password",
);
} finally {
logged.mockRestore();
}
expect(run.mock.calls.map((call) => call[1][0])).toEqual([
"restore",
"restore",
"publish",
]);
});
it("stops after two restore failures and preserves the final diagnostic", async () => {
const failure = new Error("NuGet source unavailable");
const run = vi.fn<(command: string, args: string[]) => void>(() => {
throw failure;
});
await expect(buildNativeAotFixture(run, options)).rejects.toMatchObject({
message: "NativeAOT dependency restore failed after two attempts",
cause: failure,
});
expect(run).toHaveBeenCalledTimes(2);
expect(run.mock.calls.every((call) => call[1][0] === "restore")).toBe(true);
});
it("does not retry compilation failures", async () => {
const failure = new Error("Native compiler rejected the fixture");
const run = vi
.fn<(command: string, args: string[]) => void>()
.mockImplementationOnce(() => {})
.mockImplementationOnce(() => {
throw failure;
});
await expect(buildNativeAotFixture(run, options)).rejects.toBe(failure);
expect(run).toHaveBeenCalledTimes(2);
});
@@ -0,0 +1,150 @@
import { execFile } from "node:child_process";
import { readdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { expect, it } from "vitest";
import {
nativeAotCommandError,
supportsNativeAotFixtureHost,
} from "../../../scripts/lib/nativeaot-build.mjs";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const generator = fileURLToPath(
new URL("../../../scripts/generate-nativeaot-fixtures.mjs", import.meta.url),
);
const supportedHost = supportsNativeAotFixtureHost(
process.platform,
process.arch,
);
it.skipIf(!supportedHost).each(["restore", "publish"])(
"redacts authenticated feed URLs from the uncaught %s diagnostic",
async (operation) => {
const directory = await createTestTempDirectory(
"rea-nativeaot-diagnostic-",
);
const hook = join(directory, "child-output.cjs");
await writeFile(
hook,
`
const child = require("node:child_process");
child.spawnSync = (_command, args) => {
if (args[0] === "--version") return { status: 0, stdout: "8.0.416\\n", stderr: "" };
if (args[0] === ${JSON.stringify(operation)}) return {
status: 1,
error: new Error("failed HTTPS://error-user:error-secret@feed.invalid/v3/index.json"),
stdout: "NU1301 https://stdout-user:stdout-secret@feed.invalid/v3/index.json\\n",
stderr: "source https://stderr-user:stderr-secret%40suffix@feed.invalid/v3/index.json\\nkeep https://feed.invalid/path/user@name?label=token-name\\n"
};
return { status: 0, stdout: "", stderr: "" };
};
require("node:module").syncBuiltinESMExports();
`,
"utf8",
);
const failure = await promisify(execFile)(
process.execPath,
["--require", hook, generator],
{
env: {
...process.env,
REA_NATIVEAOT_FIXTURE_ROOT: join(directory, "output"),
},
timeout: 15_000,
},
).then(
() => {
throw new Error("Expected fixture generation to fail");
},
(error: { stderr: string; code: number }) => error,
);
expect(failure.code).toBe(1);
for (const secret of [
"error-user",
"error-secret",
"stdout-user",
"stdout-secret",
"stderr-user",
"stderr-secret",
])
expect(failure.stderr).not.toContain(secret);
expect(failure.stderr).toContain("NU1301");
expect(failure.stderr).toContain("feed.invalid/v3/index.json");
expect(failure.stderr).toContain(
"https://feed.invalid/path/user@name?label=token-name",
);
if (operation === "restore") {
expect(failure.stderr).toContain("attempt 2/2");
expect(failure.stderr).toContain("failed after two attempts");
expect(failure.stderr).toContain("[cause]");
}
},
);
it.skipIf(supportedHost)(
"refuses unsupported fixture hosts before tooling or output creation",
async () => {
const directory = await createTestTempDirectory("rea-nativeaot-host-");
const hook = join(directory, "unexpected-child.cjs");
await writeFile(
hook,
`const child = require("node:child_process");
child.spawnSync = () => { throw new Error("Unexpected fixture tooling invocation"); };
require("node:module").syncBuiltinESMExports();
`,
"utf8",
);
const failure = await promisify(execFile)(
process.execPath,
["--require", hook, generator],
{
env: {
...process.env,
REA_NATIVEAOT_FIXTURE_ROOT: join(directory, "output"),
},
timeout: 15_000,
},
).then(
() => {
throw new Error("Expected unsupported fixture host refusal");
},
(error: { stderr: string; code: number }) => error,
);
expect(failure.code).toBe(1);
expect(failure.stderr).toContain("requires Linux or Windows x64");
expect(failure.stderr).not.toContain(
"Unexpected fixture tooling invocation",
);
expect(await readdir(directory)).toEqual(["unexpected-child.cjs"]);
},
);
it("preserves diagnostics without transport authentication exactly", () => {
const error = nativeAotCommandError(
"C:/token projects/dotnet.exe",
"restore",
{
status: 1,
stdout: "NU1301 https://feed.invalid/user@name?label=password\n",
stderr: "project C:/token projects/fixture.csproj unavailable",
},
);
expect(error.message).toBe(
"NativeAOT fixture lane requires/action failed C:/token projects/dotnet.exe restore: exit 1\nNU1301 https://feed.invalid/user@name?label=password\nproject C:/token projects/fixture.csproj unavailable",
);
expect(error.cause).toBeUndefined();
});
it("treats HTTP backslashes as path separators rather than authentication", () => {
const output = String.raw`https://feed.invalid\path\user@name`;
expect(new URL(output).username).toBe("");
const error = nativeAotCommandError("dotnet", "restore", {
status: 1,
stdout: output,
});
expect(error.message).toContain(output);
});
+19
View File
@@ -0,0 +1,19 @@
# Universal Mach-O captures whose slices share one CPU type
Captured on macOS 27.0.1 (26A434), arm64, from two system executables whose
FAT slices all use `CPU_TYPE_ARM64`:
```sh
lipo -detailed_info /System/Applications/Calculator.app/Contents/MacOS/Calculator > calculator.txt
lipo -detailed_info /System/Library/CoreServices/ShortcutDroplet.app/Contents/MacOS/ShortcutDroplet > shortcut-droplet.txt
```
Calculator holds `arm64e` (subtype `0x80000002`) and `arm64e.x1` (subtype
`0x8000000c`); ShortcutDroplet holds `arm64` (subtype `0`) and `arm64e.x1`.
Target admission names both files by the single CPU family `arm64`, while lipo
lists two slices.
The executable path on the `Fat header in:` line was replaced with `fixture`,
and trailing whitespace (lipo prints `capabilities ` for a slice without
capabilities) and the final blank line were removed. Tool fields are otherwise
unchanged.
@@ -0,0 +1,17 @@
Fat header in: fixture
fat_magic 0xcafebabe
nfat_arch 2
architecture arm64e
cputype CPU_TYPE_ARM64
cpusubtype CPU_SUBTYPE_ARM64E
capabilities PTR_AUTH_VERSION USERSPACE 0
offset 16384
size 1727552
align 2^14 (16384)
architecture arm64e.x1
cputype CPU_TYPE_ARM64
cpusubtype unknown arm64 cpusubtype
capabilities PTR_AUTH_VERSION USERSPACE 0
offset 1753088
size 1711088
align 2^14 (16384)
@@ -0,0 +1,17 @@
Fat header in: fixture
fat_magic 0xcafebabe
nfat_arch 2
architecture arm64
cputype CPU_TYPE_ARM64
cpusubtype CPU_SUBTYPE_ARM64_ALL
capabilities
offset 16384
size 52000
align 2^14 (16384)
architecture arm64e.x1
cputype CPU_TYPE_ARM64
cpusubtype unknown arm64 cpusubtype
capabilities PTR_AUTH_VERSION USERSPACE 0
offset 81920
size 52000
align 2^14 (16384)