perf(tests): reuse stateless workflows and immutable fixtures
This commit is contained in:
+11
-3
@@ -23,6 +23,11 @@ cases. Share setup without hiding inputs or expected evidence. For successful
|
||||
`.ok` assertion adds no coverage. Protocol fixtures must reject unmodeled
|
||||
commands and preserve actual producer reply shapes.
|
||||
|
||||
Stateless source-analysis matrices may share a file-owned stdio MCP connection
|
||||
and retain representative compiled CLI parity checks. Every distinct source
|
||||
scenario still runs through the public workflow; session-lifecycle scenarios
|
||||
need their own sessions and cleanup assertions.
|
||||
|
||||
Report unavailable authority as a named skip. Await owned cleanup before a
|
||||
verifier reports success. Native process inspectors are released at Vitest
|
||||
worker teardown; fork termination does not run Node exit hooks, and per-file
|
||||
@@ -930,9 +935,12 @@ provider evidence.
|
||||
Vitest projects use up to two workers, bounded by host parallelism.
|
||||
`process-boundary` runs later with serial files because process-tree sampling
|
||||
shares host resources. Acceptance and process-global cases use isolated forks
|
||||
without serial scheduling. Pure domain/service and MCP boundary projects share
|
||||
immutable module graphs; each MCP session still owns its resources. Other
|
||||
projects retain per-file isolation. See `vitest.config.ts` for current settings.
|
||||
without serial scheduling. Pure domain/service, composition, conformance,
|
||||
evaluation, and MCP boundary projects share module graphs; their tests avoid
|
||||
process-global state and keep resource cleanup test-scoped. Each MCP session
|
||||
still owns its resources. Adapter, non-MCP boundary, acceptance, and
|
||||
process-global projects retain per-file isolation. See `vitest.config.ts` for
|
||||
current settings.
|
||||
|
||||
Build/documentation writers hold checkout-local output locks. Tests do not
|
||||
hold a broad command lock; `check:pr` finishes tests before document validation.
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { copyFile, mkdir, rm, writeFile } from "node:fs/promises";
|
||||
import { copyFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import { createRequire } from "node:module";
|
||||
import { join, relative, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { afterAll, describe, expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { writeElectronApiAliasApplication } from "../../fixtures/electronApiAliasApplication.js";
|
||||
@@ -28,7 +28,33 @@ import { ELECTRON_IDENTITY_LIMITATION } from "../../../src/domain/javascript/jav
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const requireFixture = createRequire(import.meta.url);
|
||||
const temporary: string[] = [];
|
||||
// Analysis requests are stateless; retain one compiled server and its SDK
|
||||
// validators while every scenario still owns a separate application directory.
|
||||
const mcpTransport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [resolve("scripts/rea.mjs"), "mcp"],
|
||||
cwd: process.cwd(),
|
||||
env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
|
||||
stderr: "pipe",
|
||||
});
|
||||
mcpTransport.stderr?.on("data", () => undefined);
|
||||
const mcpClient = new Client({
|
||||
name: "application-workflow-parity",
|
||||
version: "1",
|
||||
});
|
||||
let mcpConnection: Promise<void> | undefined;
|
||||
const connectedMcpClient = async (): Promise<Client> => {
|
||||
await (mcpConnection ??= mcpClient.connect(mcpTransport));
|
||||
return mcpClient;
|
||||
};
|
||||
|
||||
afterAll(async () => {
|
||||
try {
|
||||
await mcpClient.close();
|
||||
} finally {
|
||||
await mcpTransport.close();
|
||||
}
|
||||
});
|
||||
const WEBPACK_FACTORY_BODY = `
|
||||
const key = "dynamic";
|
||||
const __webpack_require__ = {
|
||||
@@ -57,14 +83,6 @@ const WEBPACK_FACTORY_BODY = `
|
||||
exports = function ignored() { return 8; };
|
||||
`;
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(
|
||||
temporary
|
||||
.splice(0)
|
||||
.map(async (path) => rm(path, { recursive: true, force: true })),
|
||||
);
|
||||
});
|
||||
|
||||
const writeSelfImportFixture = async (root: string): Promise<void> => {
|
||||
await Promise.all([
|
||||
writeFile(
|
||||
@@ -162,51 +180,42 @@ const expectSelfImportEvidence = (
|
||||
const analyzeThroughStdioMcp = async (
|
||||
inputPath: string,
|
||||
): Promise<JavaScriptApplicationAnalysisResult> => {
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [resolve("scripts/rea.mjs"), "mcp"],
|
||||
cwd: process.cwd(),
|
||||
env: {
|
||||
PATH: process.env.PATH ?? "",
|
||||
REA_LOG_LEVEL: "silent",
|
||||
},
|
||||
stderr: "pipe",
|
||||
const client = await connectedMcpClient();
|
||||
const response = await client.callTool({
|
||||
name: "analyze_javascript_application",
|
||||
arguments: { input_path: inputPath },
|
||||
});
|
||||
const client = new Client({
|
||||
name: "javascript-self-import-parity",
|
||||
version: "1",
|
||||
expect(response.isError).not.toBe(true);
|
||||
expect(response.content).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ type: "text" })]),
|
||||
);
|
||||
const result = z
|
||||
.object({ normalized_result: javascriptApplicationAnalysisResultSchema })
|
||||
.parse(response.structuredContent).normalized_result;
|
||||
expect(result).toMatchObject({
|
||||
input_path: inputPath,
|
||||
format: "directory",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const response = await client.callTool({
|
||||
name: "analyze_javascript_application",
|
||||
arguments: { input_path: inputPath },
|
||||
});
|
||||
expect(response.isError).not.toBe(true);
|
||||
expect(response.content).toEqual(
|
||||
expect.arrayContaining([expect.objectContaining({ type: "text" })]),
|
||||
);
|
||||
const result = z
|
||||
.object({ normalized_result: javascriptApplicationAnalysisResultSchema })
|
||||
.parse(response.structuredContent).normalized_result;
|
||||
expect(result).toMatchObject({
|
||||
input_path: inputPath,
|
||||
format: "directory",
|
||||
});
|
||||
return result;
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
describe("Electron alias identity CLI and MCP", () => {
|
||||
const expectCliAnalysisParity = async (
|
||||
root: string,
|
||||
mcp: JavaScriptApplicationAnalysisResult,
|
||||
): Promise<void> => {
|
||||
const cli = z
|
||||
.object({ normalized_result: javascriptApplicationAnalysisResultSchema })
|
||||
.parse(
|
||||
await runCli(["analyze-javascript-application", root, "--json"]),
|
||||
).normalized_result;
|
||||
expect(cli).toEqual(mcp);
|
||||
};
|
||||
|
||||
describe("Electron alias identity through stdio MCP", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "CommonJS binding chains",
|
||||
cliParity: true,
|
||||
esm: false,
|
||||
main: 'const { BrowserWindow, ipcMain } = require("electron"); const First = BrowserWindow; const Win = First; const ipc = ipcMain;',
|
||||
preload:
|
||||
@@ -235,16 +244,16 @@ describe("Electron alias identity CLI and MCP", () => {
|
||||
},
|
||||
{
|
||||
name: "ESM binding aliases",
|
||||
cliParity: true,
|
||||
esm: true,
|
||||
main: 'import { BrowserWindow, ipcMain } from "electron"; const Win = BrowserWindow; const ipc = ipcMain;',
|
||||
preload:
|
||||
'import { contextBridge, ipcRenderer } from "electron"; const bridge = contextBridge; const ipc = ipcRenderer;',
|
||||
},
|
||||
])(
|
||||
"recovers Electron boundaries through $name in CLI and MCP (#1656)",
|
||||
async ({ main, preload, esm }) => {
|
||||
"recovers Electron boundaries through $name (#1656)",
|
||||
async ({ cliParity, main, preload, esm }) => {
|
||||
const root = await createTestTempDirectory("rea-electron-alias-cli-");
|
||||
temporary.push(root);
|
||||
await Promise.all([
|
||||
writeFile(
|
||||
join(root, "package.json"),
|
||||
@@ -264,15 +273,11 @@ describe("Electron alias identity CLI and MCP", () => {
|
||||
`${preload}\nbridge.exposeInMainWorld("api", { ping: () => ipc.invoke("ch:ping") });\nfunction shadow(bridge, ipc) { bridge.exposeInMainWorld("decoy", {}); ipc.invoke("ch:decoy"); }\n`,
|
||||
),
|
||||
]);
|
||||
const cli = z
|
||||
.object({
|
||||
normalized_result: javascriptApplicationAnalysisResultSchema,
|
||||
})
|
||||
.parse(
|
||||
await runCli(["analyze-javascript-application", root, "--json"]),
|
||||
).normalized_result;
|
||||
const mcp = await analyzeThroughStdioMcp(root);
|
||||
expect(cli.summary).toMatchObject({
|
||||
const result = await analyzeThroughStdioMcp(root);
|
||||
// Keep CLI parity for both module formats; the other rows vary only AST
|
||||
// semantics already exercised by the same public analysis workflow.
|
||||
if (cliParity) await expectCliAnalysisParity(root, result);
|
||||
expect(result.summary).toMatchObject({
|
||||
browser_windows: 1,
|
||||
explicit_web_preferences: 1,
|
||||
preload_entrypoints: 1,
|
||||
@@ -286,67 +291,65 @@ describe("Electron alias identity CLI and MCP", () => {
|
||||
paired_renderer_transmissions: 1,
|
||||
},
|
||||
});
|
||||
expect(mcp.summary).toEqual(cli.summary);
|
||||
for (const result of [cli, mcp]) {
|
||||
expect(result.limitations).toContain(ELECTRON_IDENTITY_LIMITATION);
|
||||
expect(result.graph.nodes).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
kind: "ipc-channel",
|
||||
observations: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
channel: "ch:ping",
|
||||
operation: "handle",
|
||||
}),
|
||||
evidence: expect.objectContaining({
|
||||
authority: "ast-static-analysis",
|
||||
location: expect.objectContaining({
|
||||
available: true,
|
||||
value: expect.objectContaining({
|
||||
kind: "source-range",
|
||||
source: "main.js",
|
||||
}),
|
||||
expect(result.limitations).toContain(ELECTRON_IDENTITY_LIMITATION);
|
||||
expect(result.graph.nodes).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
kind: "ipc-channel",
|
||||
observations: expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
channel: "ch:ping",
|
||||
operation: "handle",
|
||||
}),
|
||||
evidence: expect.objectContaining({
|
||||
authority: "ast-static-analysis",
|
||||
location: expect.objectContaining({
|
||||
available: true,
|
||||
value: expect.objectContaining({
|
||||
kind: "source-range",
|
||||
source: "main.js",
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
channel: "ch:ping",
|
||||
operation: "invoke",
|
||||
}),
|
||||
evidence: expect.objectContaining({
|
||||
authority: "ast-static-analysis",
|
||||
location: expect.objectContaining({
|
||||
available: true,
|
||||
value: expect.objectContaining({
|
||||
kind: "source-range",
|
||||
source: "preload.js",
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
properties: expect.objectContaining({
|
||||
channel: "ch:ping",
|
||||
operation: "invoke",
|
||||
}),
|
||||
evidence: expect.objectContaining({
|
||||
authority: "ast-static-analysis",
|
||||
location: expect.objectContaining({
|
||||
available: true,
|
||||
value: expect.objectContaining({
|
||||
kind: "source-range",
|
||||
source: "preload.js",
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(result.graph.edges).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ relation: "handles" }),
|
||||
expect.objectContaining({ relation: "invokes" }),
|
||||
expect.objectContaining({ relation: "loads" }),
|
||||
expect.objectContaining({ relation: "exposes" }),
|
||||
]),
|
||||
);
|
||||
}
|
||||
}),
|
||||
]),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(result.graph.edges).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ relation: "handles" }),
|
||||
expect.objectContaining({ relation: "invokes" }),
|
||||
expect.objectContaining({ relation: "loads" }),
|
||||
expect.objectContaining({ relation: "exposes" }),
|
||||
]),
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("Electron unavailable identity CLI and MCP", () => {
|
||||
describe("Electron unavailable identity through stdio MCP", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "reassigned aliases",
|
||||
cliParity: true,
|
||||
main: 'const { BrowserWindow, ipcMain } = require("electron"); let Win = BrowserWindow; let ipc = ipcMain; Win = class {}; ipc = { handle() {} };',
|
||||
preload:
|
||||
'const { contextBridge, ipcRenderer } = require("electron"); let bridge = contextBridge; let ipc = ipcRenderer; bridge = { exposeInMainWorld() {} }; ipc = { invoke() {} };',
|
||||
@@ -406,10 +409,9 @@ describe("Electron unavailable identity CLI and MCP", () => {
|
||||
'const electron = require("electron"); const bridge = electron["fake.contextBridge"]; const ipc = electron["fake.ipcRenderer"];',
|
||||
},
|
||||
])(
|
||||
"keeps Electron identity unresolved for $name in CLI and MCP (#1656)",
|
||||
async ({ main, preload }) => {
|
||||
"keeps Electron identity unresolved for $name (#1656)",
|
||||
async ({ cliParity, main, preload }) => {
|
||||
const root = await createTestTempDirectory("rea-electron-alias-unknown-");
|
||||
temporary.push(root);
|
||||
await Promise.all([
|
||||
writeFile(
|
||||
join(root, "package.json"),
|
||||
@@ -428,25 +430,17 @@ describe("Electron unavailable identity CLI and MCP", () => {
|
||||
`${preload}\nbridge.exposeInMainWorld("api", { ping: () => ipc.invoke("ch:ping") });\n`,
|
||||
),
|
||||
]);
|
||||
const cli = z
|
||||
.object({
|
||||
normalized_result: javascriptApplicationAnalysisResultSchema,
|
||||
})
|
||||
.parse(
|
||||
await runCli(["analyze-javascript-application", root, "--json"]),
|
||||
).normalized_result;
|
||||
const mcp = await analyzeThroughStdioMcp(root);
|
||||
for (const result of [cli, mcp]) {
|
||||
expect(result.summary).toMatchObject({
|
||||
browser_windows: 0,
|
||||
context_bridge_apis: 0,
|
||||
ipc: { operations: 0 },
|
||||
});
|
||||
expect(
|
||||
result.graph.nodes.some(({ kind }) => kind === "ipc-channel"),
|
||||
).toBe(false);
|
||||
expect(result.limitations).toContain(ELECTRON_IDENTITY_LIMITATION);
|
||||
}
|
||||
const result = await analyzeThroughStdioMcp(root);
|
||||
if (cliParity) await expectCliAnalysisParity(root, result);
|
||||
expect(result.summary).toMatchObject({
|
||||
browser_windows: 0,
|
||||
context_bridge_apis: 0,
|
||||
ipc: { operations: 0 },
|
||||
});
|
||||
expect(
|
||||
result.graph.nodes.some(({ kind }) => kind === "ipc-channel"),
|
||||
).toBe(false);
|
||||
expect(result.limitations).toContain(ELECTRON_IDENTITY_LIMITATION);
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -454,7 +448,6 @@ describe("Electron unavailable identity CLI and MCP", () => {
|
||||
describe("JavaScript application path CLI", () => {
|
||||
it("accepts a relative local application path and preserves canonical Evidence identity", async () => {
|
||||
const root = await createTestTempDirectory("rea-relative-application-cli-");
|
||||
temporary.push(root);
|
||||
await writeFile(join(root, "app.js"), "export const value = 1;\n");
|
||||
const absolute = await analyzeJavaScriptApplication({ input_path: root });
|
||||
if (!absolute.ok) throw absolute.error;
|
||||
@@ -482,7 +475,6 @@ describe("JavaScript application path CLI", () => {
|
||||
const root = await createTestTempDirectory(
|
||||
"rea-self-import-application-cli-",
|
||||
);
|
||||
temporary.push(root);
|
||||
await writeSelfImportFixture(root);
|
||||
|
||||
const analyzed = await runCli([
|
||||
@@ -504,7 +496,6 @@ describe("JavaScript application path CLI", () => {
|
||||
|
||||
it("preserves canonical Electron API aliases through CLI and stdio MCP", async () => {
|
||||
const root = await createTestTempDirectory("rea-electron-api-alias-");
|
||||
temporary.push(root);
|
||||
await writeElectronApiAliasApplication(root);
|
||||
|
||||
const cliResult = z
|
||||
@@ -588,7 +579,6 @@ describe("application workflow CLI parity", () => {
|
||||
});
|
||||
|
||||
const root = await createTestTempDirectory("rea-application-cli-");
|
||||
temporary.push(root);
|
||||
const comparisonPath = join(root, "comparison.json");
|
||||
await writeFile(
|
||||
comparisonPath,
|
||||
@@ -654,7 +644,6 @@ describe("application workflow CLI parity", () => {
|
||||
|
||||
it("traces the same authenticated semantic graph through the CLI", async () => {
|
||||
const root = await createTestTempDirectory("rea-semantic-cli-");
|
||||
temporary.push(root);
|
||||
await writeFile(
|
||||
join(root, "app.js"),
|
||||
"function add(value) { return value + 1; } add(2);",
|
||||
@@ -777,7 +766,6 @@ describe("empty property key application CLI", () => {
|
||||
describe("Webpack factory CommonJS export keys", () => {
|
||||
it("keeps exact static keys and omits computed names through CLI and MCP", async () => {
|
||||
const root = await createTestTempDirectory("rea-webpack-export-keys-");
|
||||
temporary.push(root);
|
||||
const bundle = `globalThis.webpackChunkStaticExports.push([["main"], {
|
||||
1: function(module, exports, __webpack_require__) {${WEBPACK_FACTORY_BODY}},
|
||||
2: function(module) { module.exports = function realDefault() {}; },
|
||||
@@ -798,7 +786,6 @@ describe("Webpack factory CommonJS export keys", () => {
|
||||
const oracleRoot = await createTestTempDirectory(
|
||||
"rea-webpack-export-oracle-",
|
||||
);
|
||||
temporary.push(oracleRoot);
|
||||
const oraclePath = join(oracleRoot, "oracle.cjs");
|
||||
await writeFile(oraclePath, WEBPACK_FACTORY_BODY);
|
||||
const actual = requireFixture(oraclePath) as Record<string, unknown>;
|
||||
@@ -864,32 +851,16 @@ describe("Webpack factory CommonJS export keys", () => {
|
||||
"--json",
|
||||
]),
|
||||
);
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [resolve("scripts/rea.mjs"), "mcp"],
|
||||
cwd: process.cwd(),
|
||||
env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
|
||||
stderr: "pipe",
|
||||
const client = await connectedMcpClient();
|
||||
const response = await client.callTool({
|
||||
name: "analyze_javascript_application",
|
||||
arguments: { input_path: root, format: "directory" },
|
||||
});
|
||||
transport.stderr?.on("data", () => undefined);
|
||||
const client = new Client({ name: "webpack-export-keys", version: "1" });
|
||||
let mcpEvidence: z.infer<typeof applicationEvidenceSchema>;
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const response = await client.callTool({
|
||||
name: "analyze_javascript_application",
|
||||
arguments: { input_path: root, format: "directory" },
|
||||
});
|
||||
expect(response.isError).not.toBe(true);
|
||||
mcpEvidence = applicationEvidenceSchema.parse(response.structuredContent);
|
||||
expect(await client.ping()).toEqual({});
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
expect(response.isError).not.toBe(true);
|
||||
const mcpEvidence = applicationEvidenceSchema.parse(
|
||||
response.structuredContent,
|
||||
);
|
||||
expect(await client.ping()).toEqual({});
|
||||
expect(mcpEvidence.evidence_id).toBe(cliEvidence.evidence_id);
|
||||
|
||||
const expectBundleExports = (
|
||||
@@ -947,7 +918,6 @@ describe("application workflow CLI input", () => {
|
||||
describe("application workflow CLI copy boundaries", () => {
|
||||
it("compares source values excluded from escaped rest and spread copies", async () => {
|
||||
const root = await createTestTempDirectory("rea-copy-boundary-cli-");
|
||||
temporary.push(root);
|
||||
const sources = [
|
||||
`export default function make() {
|
||||
function mutate(value) { value.changed = true; }
|
||||
@@ -1020,7 +990,6 @@ describe("application workflow CLI copy boundaries", () => {
|
||||
describe("application workflow CLI and MCP capture lifetimes", () => {
|
||||
it("compares captured primitives and source values after aliases are replaced", async () => {
|
||||
const root = await createTestTempDirectory("rea-capture-lifetime-cli-");
|
||||
temporary.push(root);
|
||||
const fields = [
|
||||
"rebound",
|
||||
"rest",
|
||||
@@ -1122,7 +1091,6 @@ describe("application workflow CLI and MCP capture lifetimes", () => {
|
||||
describe("application workflow CLI export Evidence", () => {
|
||||
it("preserves uncertainty after a helper can mutate an awaited return object", async () => {
|
||||
const root = await createTestTempDirectory("rea-awaited-export-shape-cli-");
|
||||
temporary.push(root);
|
||||
const sources = [
|
||||
`export default async function make() {
|
||||
const result = { kind: "record" };
|
||||
@@ -1159,7 +1127,6 @@ describe("application workflow CLI export Evidence", () => {
|
||||
|
||||
it("compares exact export shapes from file-backed Evidence", async () => {
|
||||
const root = await createTestTempDirectory("rea-export-shape-cli-");
|
||||
temporary.push(root);
|
||||
const leftRoot = join(root, "left");
|
||||
const rightRoot = join(root, "right");
|
||||
await Promise.all([mkdir(leftRoot), mkdir(rightRoot)]);
|
||||
@@ -1223,7 +1190,6 @@ describe("application workflow CLI export property presence", () => {
|
||||
const root = await createTestTempDirectory(
|
||||
"rea-export-shape-presence-cli-",
|
||||
);
|
||||
temporary.push(root);
|
||||
const leftRoot = join(root, "left");
|
||||
const rightRoot = join(root, "right");
|
||||
await Promise.all([mkdir(leftRoot), mkdir(rightRoot)]);
|
||||
@@ -1349,32 +1315,13 @@ describe("application workflow CLI validation", () => {
|
||||
const compareThroughStdioMcp = async (
|
||||
input: Record<string, unknown>,
|
||||
): Promise<unknown> => {
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [resolve("scripts/rea.mjs"), "mcp"],
|
||||
cwd: process.cwd(),
|
||||
env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
|
||||
stderr: "pipe",
|
||||
const client = await connectedMcpClient();
|
||||
const response = await client.callTool({
|
||||
name: "compare_javascript_export_shapes",
|
||||
arguments: input,
|
||||
});
|
||||
const client = new Client({
|
||||
name: "javascript-capture-parity",
|
||||
version: "1",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const response = await client.callTool({
|
||||
name: "compare_javascript_export_shapes",
|
||||
arguments: input,
|
||||
});
|
||||
expect(response.isError).not.toBe(true);
|
||||
return response.structuredContent;
|
||||
} finally {
|
||||
try {
|
||||
await client.close();
|
||||
} finally {
|
||||
await transport.close();
|
||||
}
|
||||
}
|
||||
expect(response.isError).not.toBe(true);
|
||||
return response.structuredContent;
|
||||
};
|
||||
|
||||
const analyzeCliSources = (
|
||||
|
||||
@@ -18,7 +18,12 @@ const requireFixture = createRequire(import.meta.url);
|
||||
const comparisonEnvelope = z.object({
|
||||
normalized_result: javaScriptExportShapeComparisonResultSchema,
|
||||
});
|
||||
let mcpComparisonConnection:
|
||||
const applicationEnvelope = z.object({
|
||||
normalized_result: javascriptApplicationAnalysisResultSchema,
|
||||
});
|
||||
// Source semantics use the real stdio workflow throughout the matrix. The
|
||||
// missing-export and count-change cases also check compiled CLI adapter parity.
|
||||
let mcpConnection:
|
||||
| {
|
||||
readonly client: Client;
|
||||
readonly transport: StdioClientTransport;
|
||||
@@ -27,20 +32,20 @@ let mcpComparisonConnection:
|
||||
| undefined;
|
||||
|
||||
afterAll(async () => {
|
||||
if (mcpComparisonConnection === undefined) return;
|
||||
if (mcpConnection === undefined) return;
|
||||
try {
|
||||
await mcpComparisonConnection.ready;
|
||||
await mcpConnection.ready;
|
||||
} catch {
|
||||
// Keep the failed connection owned until both resources are closed below.
|
||||
}
|
||||
try {
|
||||
await mcpComparisonConnection.client.close();
|
||||
await mcpConnection.client.close();
|
||||
} finally {
|
||||
await mcpComparisonConnection.transport.close();
|
||||
await mcpConnection.transport.close();
|
||||
}
|
||||
});
|
||||
|
||||
describe("exported function binding resolution through the CLI", () => {
|
||||
describe("exported function binding resolution through stdio MCP and CLI", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "a function declaration exported by a specifier",
|
||||
@@ -100,7 +105,7 @@ describe("exported function binding resolution through the CLI", () => {
|
||||
async ({ source, exportName }) => {
|
||||
const fixture = await analyzeVersions(source, exportName);
|
||||
await expectRuntimeExports(fixture, (count) => ({ count }));
|
||||
const result = await compareThroughCli(fixture);
|
||||
const result = await compareThroughMcp(fixture.input);
|
||||
expectUncertainComparison(result);
|
||||
for (const evidence of [fixture.input.left, fixture.input.right]) {
|
||||
const { normalized_result: analysis } = z
|
||||
@@ -121,7 +126,7 @@ describe("exported function binding resolution through the CLI", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("does not associate a private function name with a numeric export in CLI or MCP", async () => {
|
||||
it("does not associate a private function name with a numeric export through stdio MCP", async () => {
|
||||
const fixture = await analyzeVersions(
|
||||
(count) =>
|
||||
`const hidden = function current() { return { count: 0 }; };
|
||||
@@ -130,9 +135,8 @@ describe("exported function binding resolution through the CLI", () => {
|
||||
"default",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => count);
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectUncertainComparison(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectUncertainComparison(comparison);
|
||||
});
|
||||
|
||||
it("does not treat rebinding the CommonJS exports alias as a default export", async () => {
|
||||
@@ -148,13 +152,14 @@ describe("exported function binding resolution through the CLI", () => {
|
||||
expect(requireFixture(fixture.leftPath)).toEqual({});
|
||||
expect(requireFixture(fixture.rightPath)).toEqual({});
|
||||
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const mcp = await compareThroughMcp(fixture.input);
|
||||
await expectCliAnalysisParity(fixture);
|
||||
expect(mcp).toMatchObject({
|
||||
left: { status: "missing" },
|
||||
right: { status: "missing" },
|
||||
summary: { added: 0, removed: 0, changed: 0 },
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
expect(await compareThroughCli(fixture)).toEqual(mcp);
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -179,14 +184,17 @@ describe("exported function binding resolution through the CLI", () => {
|
||||
});
|
||||
for (const evidence of [fixture.input.left, fixture.input.right])
|
||||
expectExportedCallableBinding(evidence, "parse");
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectCountChange(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectCountChange(comparison);
|
||||
if (target === "exports") {
|
||||
await expectCliAnalysisParity(fixture);
|
||||
expect(await compareThroughCli(fixture)).toEqual(comparison);
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("CommonJS assignment-chain receiver identity through the CLI", () => {
|
||||
describe("CommonJS assignment-chain receiver identity through stdio MCP", () => {
|
||||
it("does not export a named write whose captured module.exports receiver is replaced", async () => {
|
||||
const fixture = await analyzeVersions(
|
||||
(count) =>
|
||||
@@ -215,13 +223,12 @@ describe("CommonJS assignment-chain receiver identity through the CLI", () => {
|
||||
),
|
||||
).toEqual([]);
|
||||
}
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expect(comparison).toMatchObject({
|
||||
left: { status: "missing" },
|
||||
right: { status: "missing" },
|
||||
summary: { added: 0, removed: 0, changed: 0 },
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
});
|
||||
|
||||
it("keeps a named property export when its receiver remains stable through the RHS chain", async () => {
|
||||
@@ -236,9 +243,8 @@ describe("CommonJS assignment-chain receiver identity through the CLI", () => {
|
||||
kind: "result",
|
||||
count: path === fixture.leftPath ? 1 : 2,
|
||||
});
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectCountChange(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectCountChange(comparison);
|
||||
});
|
||||
|
||||
it("keeps receiver identity unresolved when module.exports is reassigned to the same function", async () => {
|
||||
@@ -257,12 +263,11 @@ describe("CommonJS assignment-chain receiver identity through the CLI", () => {
|
||||
}
|
||||
for (const evidence of [fixture.input.left, fixture.input.right])
|
||||
expectUnresolvedExportBinding(evidence, "parse");
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expect(comparison).toMatchObject({
|
||||
left: { status: "unavailable" },
|
||||
right: { status: "unavailable" },
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
});
|
||||
|
||||
it("keeps receiver replacement by an existing function unresolved", async () => {
|
||||
@@ -282,12 +287,11 @@ describe("CommonJS assignment-chain receiver identity through the CLI", () => {
|
||||
}
|
||||
for (const evidence of [fixture.input.left, fixture.input.right])
|
||||
expectUnresolvedExportBinding(evidence, "parse");
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expect(comparison).toMatchObject({
|
||||
left: { status: "unavailable" },
|
||||
right: { status: "unavailable" },
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
});
|
||||
|
||||
it("resolves a nested lexical parse binding in an ordinary module.exports chain", async () => {
|
||||
@@ -301,12 +305,11 @@ describe("CommonJS assignment-chain receiver identity through the CLI", () => {
|
||||
expect(requireFixture(path)()).toEqual({ count: 7 });
|
||||
for (const evidence of [fixture.input.left, fixture.input.right])
|
||||
expectExportedCallableBinding(evidence, "parse");
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expect(comparison).toMatchObject({
|
||||
summary: { added: 0, removed: 0, changed: 0, unknown: 0 },
|
||||
changes: [],
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -323,13 +326,12 @@ describe("CommonJS export identity and scope regressions", () => {
|
||||
expect(typeof value).toBe("function");
|
||||
expect(Reflect.ownKeys(value)).toEqual(["length", "name"]);
|
||||
}
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expect(comparison).toMatchObject({
|
||||
left: { status: "missing" },
|
||||
right: { status: "missing" },
|
||||
summary: { added: 0, removed: 0, changed: 0 },
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
});
|
||||
|
||||
it("keeps a dead var shadow of the alias-chain RHS unavailable", async () => {
|
||||
@@ -341,9 +343,8 @@ describe("CommonJS export identity and scope regressions", () => {
|
||||
);
|
||||
expect(requireFixture(fixture.leftPath)).toBeUndefined();
|
||||
expect(requireFixture(fixture.rightPath)).toBeUndefined();
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectUncertainComparison(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectUncertainComparison(comparison);
|
||||
});
|
||||
|
||||
it.each(["+=", "||=", "??="] as const)(
|
||||
@@ -364,9 +365,8 @@ describe("CommonJS export identity and scope regressions", () => {
|
||||
expect(left).toEqual({});
|
||||
expect(right).toEqual({});
|
||||
}
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectUncertainComparison(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectUncertainComparison(comparison);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -406,18 +406,17 @@ describe("CommonJS export identity and scope regressions", () => {
|
||||
}),
|
||||
]);
|
||||
}
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expect(cli).toMatchObject({
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expect(comparison).toMatchObject({
|
||||
left: { status: "selected" },
|
||||
right: { status: "selected" },
|
||||
summary: { added: 0, removed: 0, changed: 0 },
|
||||
changes: [],
|
||||
});
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
});
|
||||
});
|
||||
|
||||
describe("stable exported function bindings through the CLI", () => {
|
||||
describe("stable exported function bindings through stdio MCP", () => {
|
||||
it.each([
|
||||
{
|
||||
name: "an anonymous default",
|
||||
@@ -460,7 +459,7 @@ describe("stable exported function bindings through the CLI", () => {
|
||||
},
|
||||
])("retains exact comparison for $name", async ({ source }) => {
|
||||
const fixture = await analyzeVersions(source, "default");
|
||||
expectCountChange(await compareThroughCli(fixture));
|
||||
expectCountChange(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -481,7 +480,7 @@ describe("export bindings with loop and assignment-only writes", () => {
|
||||
return `${statement} export { current as default };`;
|
||||
}, "default");
|
||||
await expectRuntimeExports(fixture, () => undefined);
|
||||
expectUncertainComparison(await compareThroughCli(fixture));
|
||||
expectUncertainComparison(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it("retains a var initializer in an unconditional nested block", async () => {
|
||||
@@ -492,7 +491,7 @@ describe("export bindings with loop and assignment-only writes", () => {
|
||||
"default",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => ({ kind: "result", count }));
|
||||
expectCountChange(await compareThroughCli(fixture));
|
||||
expectCountChange(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -516,7 +515,7 @@ describe("export bindings with loop and assignment-only writes", () => {
|
||||
"app.cjs",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => ({ count }));
|
||||
expectUncertainComparison(await compareThroughCli(fixture));
|
||||
expectUncertainComparison(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it("does not associate a for-in key with the overwritten function", async () => {
|
||||
@@ -529,7 +528,7 @@ describe("export bindings with loop and assignment-only writes", () => {
|
||||
"app.cjs",
|
||||
);
|
||||
await expectRuntimeExports(fixture, String);
|
||||
expectUncertainComparison(await compareThroughCli(fixture));
|
||||
expectUncertainComparison(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it("preserves an outer export shadowed by a lexical loop declaration", async () => {
|
||||
@@ -540,7 +539,7 @@ describe("export bindings with loop and assignment-only writes", () => {
|
||||
"current",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => ({ kind: "result", count }));
|
||||
expectCountChange(await compareThroughCli(fixture));
|
||||
expectCountChange(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -573,9 +572,8 @@ describe("export bindings with loop and assignment-only writes", () => {
|
||||
])("does not infer $name from its only assignment", async ({ source }) => {
|
||||
const fixture = await analyzeVersions(source, "default", "app.cjs");
|
||||
await expectRuntimeExports(fixture, (count) => count);
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectUncertainComparison(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectUncertainComparison(comparison);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -597,7 +595,7 @@ describe("export binding lexical scope and declaration order", () => {
|
||||
])("keeps $name uncertain", async ({ source }) => {
|
||||
const fixture = await analyzeVersions(source, "current");
|
||||
await expectRuntimeExports(fixture, (count) => ({ count }));
|
||||
expectUncertainComparison(await compareThroughCli(fixture));
|
||||
expectUncertainComparison(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it("does not let a later local declaration contaminate the exported outer binding", async () => {
|
||||
@@ -612,7 +610,7 @@ describe("export binding lexical scope and declaration order", () => {
|
||||
"current",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => ({ kind: "result", count }));
|
||||
expectCountChange(await compareThroughCli(fixture));
|
||||
expectCountChange(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it("does not associate a nested CommonJS numeric export with an outer function", async () => {
|
||||
@@ -625,9 +623,8 @@ describe("export binding lexical scope and declaration order", () => {
|
||||
"app.cjs",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => count);
|
||||
const cli = await compareThroughCli(fixture);
|
||||
expectUncertainComparison(cli);
|
||||
expect(await compareThroughMcp(fixture.input)).toEqual(cli);
|
||||
const comparison = await compareThroughMcp(fixture.input);
|
||||
expectUncertainComparison(comparison);
|
||||
});
|
||||
|
||||
it.each([
|
||||
@@ -651,7 +648,7 @@ describe("export binding lexical scope and declaration order", () => {
|
||||
])("compares $name through its real CommonJS binding", async ({ source }) => {
|
||||
const fixture = await analyzeVersions(source, "default", "app.cjs");
|
||||
await expectRuntimeExports(fixture, (count) => ({ kind: "result", count }));
|
||||
expectCountChange(await compareThroughCli(fixture));
|
||||
expectCountChange(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -756,7 +753,7 @@ describe("export writes in function evaluation contexts", () => {
|
||||
])("keeps an outer export write in $name uncertain", async ({ source }) => {
|
||||
const fixture = await analyzeVersions(source, "current");
|
||||
await expectRuntimeExports(fixture, (count) => ({ count }));
|
||||
expectUncertainComparison(await compareThroughCli(fixture));
|
||||
expectUncertainComparison(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
|
||||
it("resolves a CommonJS export in a parameter default outside the function body", async () => {
|
||||
@@ -769,7 +766,7 @@ describe("export writes in function evaluation contexts", () => {
|
||||
"app.cjs",
|
||||
);
|
||||
await expectRuntimeExports(fixture, (count) => ({ kind: "result", count }));
|
||||
expectCountChange(await compareThroughCli(fixture));
|
||||
expectCountChange(await compareThroughMcp(fixture.input));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -897,8 +894,8 @@ const analyzeVersions = async (
|
||||
writeFile(rightPath, source(2)),
|
||||
]);
|
||||
const [left, right] = await Promise.all([
|
||||
runCli(["analyze-javascript-application", leftRoot, "--json"]),
|
||||
runCli(["analyze-javascript-application", rightRoot, "--json"]),
|
||||
analyzeThroughMcp(leftRoot),
|
||||
analyzeThroughMcp(rightRoot),
|
||||
]);
|
||||
return {
|
||||
root,
|
||||
@@ -915,6 +912,31 @@ const analyzeVersions = async (
|
||||
};
|
||||
};
|
||||
|
||||
const analyzeThroughMcp = async (inputPath: string) => {
|
||||
const client = await getMcpClient();
|
||||
const response = await client.callTool({
|
||||
name: "analyze_javascript_application",
|
||||
arguments: { input_path: inputPath, format: "auto" },
|
||||
});
|
||||
expect(response.isError).not.toBe(true);
|
||||
return response.structuredContent;
|
||||
};
|
||||
|
||||
const expectCliAnalysisParity = async (
|
||||
fixture: Awaited<ReturnType<typeof analyzeVersions>>,
|
||||
): Promise<void> => {
|
||||
for (const [root, evidence] of [
|
||||
[join(fixture.root, "left"), fixture.input.left],
|
||||
[join(fixture.root, "right"), fixture.input.right],
|
||||
] as const) {
|
||||
const cli = applicationEnvelope.parse(
|
||||
await runCli(["analyze-javascript-application", root, "--json"]),
|
||||
);
|
||||
const mcp = applicationEnvelope.parse(evidence);
|
||||
expect(cli.normalized_result).toEqual(mcp.normalized_result);
|
||||
}
|
||||
};
|
||||
|
||||
const compareThroughCli = async (
|
||||
fixture: Awaited<ReturnType<typeof analyzeVersions>>,
|
||||
) => {
|
||||
@@ -926,7 +948,7 @@ const compareThroughCli = async (
|
||||
};
|
||||
|
||||
const compareThroughMcp = async (input: Record<string, unknown>) => {
|
||||
const client = await getMcpComparisonClient();
|
||||
const client = await getMcpClient();
|
||||
const response = await client.callTool({
|
||||
name: "compare_javascript_export_shapes",
|
||||
arguments: input,
|
||||
@@ -936,8 +958,8 @@ const compareThroughMcp = async (input: Record<string, unknown>) => {
|
||||
return comparisonEnvelope.parse(response.structuredContent).normalized_result;
|
||||
};
|
||||
|
||||
const getMcpComparisonClient = async (): Promise<Client> => {
|
||||
if (mcpComparisonConnection === undefined) {
|
||||
const getMcpClient = async (): Promise<Client> => {
|
||||
if (mcpConnection === undefined) {
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [resolve("scripts/rea.mjs"), "mcp"],
|
||||
@@ -945,18 +967,19 @@ const getMcpComparisonClient = async (): Promise<Client> => {
|
||||
env: { PATH: process.env.PATH ?? "", REA_LOG_LEVEL: "silent" },
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", () => undefined);
|
||||
const client = new Client({
|
||||
name: "exported-function-bindings",
|
||||
version: "1",
|
||||
});
|
||||
mcpComparisonConnection = {
|
||||
mcpConnection = {
|
||||
client,
|
||||
transport,
|
||||
ready: client.connect(transport),
|
||||
};
|
||||
}
|
||||
await mcpComparisonConnection.ready;
|
||||
return mcpComparisonConnection.client;
|
||||
await mcpConnection.ready;
|
||||
return mcpConnection.client;
|
||||
};
|
||||
|
||||
const runtimeExport = async (
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { expect } from "vitest";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
const cases = [
|
||||
{
|
||||
client: "gemini_cli",
|
||||
env: { GEMINI_CLI_HOME: "profile" },
|
||||
config: "profile/.gemini/settings.json",
|
||||
skill: "profile/.agents/skills",
|
||||
},
|
||||
{
|
||||
client: "opencode",
|
||||
env: { OPENCODE_CONFIG_DIR: "profile" },
|
||||
config: "profile/opencode.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "opencode",
|
||||
env: { OPENCODE_CONFIG_DIR: "" },
|
||||
config: ".config/opencode/opencode.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "codex",
|
||||
env: { CODEX_HOME: "" },
|
||||
config: ".codex/config.toml",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "grok_build",
|
||||
env: { GROK_HOME: "" },
|
||||
config: ".grok/config.toml",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "omp",
|
||||
env: { PI_CODING_AGENT_DIR: "profile" },
|
||||
config: "profile/mcp.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "devin",
|
||||
env: { XDG_CONFIG_HOME: "profile" },
|
||||
config: "profile/devin/mcp_config.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "antigravity",
|
||||
env: {},
|
||||
config: ".gemini/config/mcp_config.json",
|
||||
skill: ".gemini/config/skills",
|
||||
},
|
||||
];
|
||||
for (const testCase of cases.filter(
|
||||
({ client }) => process.platform !== "win32" || client !== "devin",
|
||||
)) {
|
||||
cliTest(
|
||||
`${testCase.client} lifecycle follows the selected roots (${testCase.config})`,
|
||||
async ({ cli }) => {
|
||||
const home = await createTestTempDirectory("rea-client-profile-");
|
||||
const environment: NodeJS.ProcessEnv = { HOME: home, USERPROFILE: home };
|
||||
for (const [key, value] of Object.entries(testCase.env))
|
||||
environment[key] = value === "profile" ? join(home, value) : value;
|
||||
// OMP's relative override is relative to the caller, not silently ignored.
|
||||
if (testCase.client === "omp")
|
||||
environment.PI_CODING_AGENT_DIR = "profile";
|
||||
const config = join(home, testCase.config);
|
||||
const skillDirectory = join(
|
||||
home,
|
||||
testCase.skill,
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
const run = (arguments_: readonly string[]) =>
|
||||
cli.run({
|
||||
arguments: arguments_,
|
||||
cwd: home,
|
||||
environment,
|
||||
timeoutMs: 20_000,
|
||||
});
|
||||
const planned = await run([
|
||||
"setup",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--dry-run",
|
||||
"--json",
|
||||
]);
|
||||
expect(planned.exitCode).toBe(0);
|
||||
expect(planned.json).toMatchObject({
|
||||
status: "planned",
|
||||
plannedActions: expect.arrayContaining([
|
||||
expect.objectContaining({ target: config }),
|
||||
expect.objectContaining({ target: skillDirectory }),
|
||||
]),
|
||||
});
|
||||
await expect(access(config)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const setup = await run([
|
||||
"setup",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--yes",
|
||||
"--json",
|
||||
]);
|
||||
expect(setup.exitCode).toBe(0);
|
||||
expect(setup.json).toMatchObject({ status: "ready" });
|
||||
await expect(access(config)).resolves.toBeUndefined();
|
||||
expect(
|
||||
await readFile(join(skillDirectory, "SKILL.md"), "utf8"),
|
||||
).toContain("reverse-engineer-anything");
|
||||
await expect(access(join(home, "config.toml"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
const doctor = await run([
|
||||
"doctor",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--skill",
|
||||
"--json",
|
||||
]);
|
||||
expect(doctor.exitCode).toBe(0);
|
||||
expect(doctor.json).toMatchObject({
|
||||
healthy: true,
|
||||
identity: { skill: { state: "aligned" } },
|
||||
});
|
||||
if (testCase.client === "grok_build") {
|
||||
const original = await readFile(config, "utf8");
|
||||
await writeFile(
|
||||
config,
|
||||
original.replace(
|
||||
"[mcp_servers.rea]",
|
||||
"[mcp_servers.rea]\ndisabled = true",
|
||||
),
|
||||
);
|
||||
expect(
|
||||
(await run(["doctor", "--client", "grok_build", "--json"])).exitCode,
|
||||
).toBe(0);
|
||||
await writeFile(config, original);
|
||||
}
|
||||
const repeat = await run([
|
||||
"setup",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--yes",
|
||||
"--json",
|
||||
]);
|
||||
expect(repeat.json).toMatchObject({
|
||||
status: "ready",
|
||||
appliedActions: [],
|
||||
});
|
||||
const uninstall = await run(["uninstall", "--json"]);
|
||||
expect(uninstall.exitCode).toBe(0);
|
||||
expect(uninstall.json).toMatchObject({ status: "complete" });
|
||||
await expect(
|
||||
access(join(skillDirectory, "SKILL.md")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -6,161 +6,6 @@ import { PRODUCT_IDENTITY } from "../../../src/identity.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { cliTest } from "../../support/cli/cliFixture.js";
|
||||
|
||||
const cases = [
|
||||
{
|
||||
client: "gemini_cli",
|
||||
env: { GEMINI_CLI_HOME: "profile" },
|
||||
config: "profile/.gemini/settings.json",
|
||||
skill: "profile/.agents/skills",
|
||||
},
|
||||
{
|
||||
client: "opencode",
|
||||
env: { OPENCODE_CONFIG_DIR: "profile" },
|
||||
config: "profile/opencode.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "opencode",
|
||||
env: { OPENCODE_CONFIG_DIR: "" },
|
||||
config: ".config/opencode/opencode.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "codex",
|
||||
env: { CODEX_HOME: "" },
|
||||
config: ".codex/config.toml",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "grok_build",
|
||||
env: { GROK_HOME: "" },
|
||||
config: ".grok/config.toml",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "omp",
|
||||
env: { PI_CODING_AGENT_DIR: "profile" },
|
||||
config: "profile/mcp.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "devin",
|
||||
env: { XDG_CONFIG_HOME: "profile" },
|
||||
config: "profile/devin/mcp_config.json",
|
||||
skill: ".agents/skills",
|
||||
},
|
||||
{
|
||||
client: "antigravity",
|
||||
env: {},
|
||||
config: ".gemini/config/mcp_config.json",
|
||||
skill: ".gemini/config/skills",
|
||||
},
|
||||
];
|
||||
for (const testCase of cases.filter(
|
||||
({ client }) => process.platform !== "win32" || client !== "devin",
|
||||
)) {
|
||||
cliTest(
|
||||
`${testCase.client} lifecycle follows the selected roots (${testCase.config})`,
|
||||
async ({ cli }) => {
|
||||
const home = await createTestTempDirectory("rea-client-profile-");
|
||||
const environment: NodeJS.ProcessEnv = { HOME: home, USERPROFILE: home };
|
||||
for (const [key, value] of Object.entries(testCase.env))
|
||||
environment[key] = value === "profile" ? join(home, value) : value;
|
||||
// OMP's relative override is relative to the caller, not silently ignored.
|
||||
if (testCase.client === "omp")
|
||||
environment.PI_CODING_AGENT_DIR = "profile";
|
||||
const config = join(home, testCase.config);
|
||||
const skillDirectory = join(
|
||||
home,
|
||||
testCase.skill,
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
const run = (arguments_: readonly string[]) =>
|
||||
cli.run({
|
||||
arguments: arguments_,
|
||||
cwd: home,
|
||||
environment,
|
||||
timeoutMs: 20_000,
|
||||
});
|
||||
const planned = await run([
|
||||
"setup",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--dry-run",
|
||||
"--json",
|
||||
]);
|
||||
expect(planned.exitCode).toBe(0);
|
||||
expect(planned.json).toMatchObject({
|
||||
status: "planned",
|
||||
plannedActions: expect.arrayContaining([
|
||||
expect.objectContaining({ target: config }),
|
||||
expect.objectContaining({ target: skillDirectory }),
|
||||
]),
|
||||
});
|
||||
await expect(access(config)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const setup = await run([
|
||||
"setup",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--yes",
|
||||
"--json",
|
||||
]);
|
||||
expect(setup.exitCode).toBe(0);
|
||||
expect(setup.json).toMatchObject({ status: "ready" });
|
||||
await expect(access(config)).resolves.toBeUndefined();
|
||||
expect(
|
||||
await readFile(join(skillDirectory, "SKILL.md"), "utf8"),
|
||||
).toContain("reverse-engineer-anything");
|
||||
await expect(access(join(home, "config.toml"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
const doctor = await run([
|
||||
"doctor",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--skill",
|
||||
"--json",
|
||||
]);
|
||||
expect(doctor.exitCode).toBe(0);
|
||||
expect(doctor.json).toMatchObject({
|
||||
healthy: true,
|
||||
identity: { skill: { state: "aligned" } },
|
||||
});
|
||||
if (testCase.client === "grok_build") {
|
||||
const original = await readFile(config, "utf8");
|
||||
await writeFile(
|
||||
config,
|
||||
original.replace(
|
||||
"[mcp_servers.rea]",
|
||||
"[mcp_servers.rea]\ndisabled = true",
|
||||
),
|
||||
);
|
||||
expect(
|
||||
(await run(["doctor", "--client", "grok_build", "--json"])).exitCode,
|
||||
).toBe(0);
|
||||
await writeFile(config, original);
|
||||
}
|
||||
const repeat = await run([
|
||||
"setup",
|
||||
"--client",
|
||||
testCase.client,
|
||||
"--yes",
|
||||
"--json",
|
||||
]);
|
||||
expect(repeat.json).toMatchObject({
|
||||
status: "ready",
|
||||
appliedActions: [],
|
||||
});
|
||||
const uninstall = await run(["uninstall", "--json"]);
|
||||
expect(uninstall.exitCode).toBe(0);
|
||||
expect(uninstall.json).toMatchObject({ status: "complete" });
|
||||
await expect(
|
||||
access(join(skillDirectory, "SKILL.md")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
for (const policy of [{ excluded: ["rea"] }, { allowed: ["other"] }]) {
|
||||
cliTest(
|
||||
`Gemini refuses a registration blocked by ${Object.keys(policy)[0]}`,
|
||||
|
||||
@@ -1,10 +1,15 @@
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { cp, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, it } from "vitest";
|
||||
import { afterAll, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { parse } from "yaml";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
createTestWorkspace,
|
||||
removeTestWorkspace,
|
||||
type TestWorkspace,
|
||||
} from "../../support/workspace/workspaceFixture.js";
|
||||
import {
|
||||
executeWorkflowFixture as execFileAsync,
|
||||
workflowGit as git,
|
||||
@@ -52,27 +57,72 @@ async function metadata(
|
||||
);
|
||||
}
|
||||
|
||||
interface ReleaseSeed {
|
||||
readonly directory: string;
|
||||
readonly releasedSha: string;
|
||||
}
|
||||
|
||||
const seedWorkspaces: TestWorkspace[] = [];
|
||||
const seedPromises = new Map<string, Promise<ReleaseSeed>>();
|
||||
|
||||
const releaseSeed = (versioning?: string): Promise<ReleaseSeed> => {
|
||||
const key = versioning ?? "default";
|
||||
const existing = seedPromises.get(key);
|
||||
if (existing !== undefined) return existing;
|
||||
|
||||
const pending = (async (): Promise<ReleaseSeed> => {
|
||||
const workspace = await createTestWorkspace("rea-release-checkpoint-seed-");
|
||||
seedWorkspaces.push(workspace);
|
||||
const directory = workspace.path(key);
|
||||
await mkdir(directory, { recursive: true });
|
||||
await git(directory, ["init", "--initial-branch=main"]);
|
||||
await metadata(directory, "4.0.0", "Initial release", versioning);
|
||||
await git(directory, ["add", "."]);
|
||||
await git(
|
||||
directory,
|
||||
["commit", "-m", "chore: seed"],
|
||||
"2026-01-01T00:00:00Z",
|
||||
);
|
||||
await git(directory, ["switch", "-c", "release/5.0.0"]);
|
||||
await metadata(directory, "5.0.0", "Previous release", versioning);
|
||||
await git(directory, ["add", "."]);
|
||||
await git(
|
||||
directory,
|
||||
["commit", "-m", "chore: release 5.0.0"],
|
||||
"2026-01-03T00:00:00Z",
|
||||
);
|
||||
const releasedSha = await git(directory, ["rev-parse", "HEAD"]);
|
||||
await git(directory, ["tag", "rea-agents-5.0.0"]);
|
||||
await git(directory, ["switch", "main"]);
|
||||
return { directory, releasedSha };
|
||||
})();
|
||||
seedPromises.set(key, pending);
|
||||
return pending;
|
||||
};
|
||||
|
||||
afterAll(async () => {
|
||||
await Promise.allSettled(seedPromises.values());
|
||||
const cleanup = await Promise.allSettled(
|
||||
seedWorkspaces.map(({ root }) => removeTestWorkspace(root)),
|
||||
);
|
||||
const failures = cleanup.flatMap((result) =>
|
||||
result.status === "rejected" ? [result.reason] : [],
|
||||
);
|
||||
if (failures.length > 0)
|
||||
throw new AggregateError(
|
||||
failures,
|
||||
"Could not remove release checkpoint seeds",
|
||||
);
|
||||
});
|
||||
|
||||
async function fixture(
|
||||
subject = "fix(contracts)!: require absolute paths (#948)",
|
||||
footer = "",
|
||||
options: { mergePullRequest?: boolean; versioning?: string } = {},
|
||||
) {
|
||||
const seed = await releaseSeed(options.versioning);
|
||||
const directory = await createTestTempDirectory("rea-release-ancestry-");
|
||||
await git(directory, ["init", "--initial-branch=main"]);
|
||||
await metadata(directory, "4.0.0", "Initial release", options.versioning);
|
||||
await git(directory, ["add", "."]);
|
||||
await git(directory, ["commit", "-m", "chore: seed"], "2026-01-01T00:00:00Z");
|
||||
await git(directory, ["switch", "-c", "release/5.0.0"]);
|
||||
await metadata(directory, "5.0.0", "Previous release", options.versioning);
|
||||
await git(directory, ["add", "."]);
|
||||
await git(
|
||||
directory,
|
||||
["commit", "-m", "chore: release 5.0.0"],
|
||||
"2026-01-03T00:00:00Z",
|
||||
);
|
||||
const releasedSha = await git(directory, ["rev-parse", "HEAD"]);
|
||||
await git(directory, ["tag", "rea-agents-5.0.0"]);
|
||||
await git(directory, ["switch", "main"]);
|
||||
await cp(seed.directory, directory, { recursive: true, dereference: true });
|
||||
if (options.mergePullRequest) {
|
||||
await git(directory, ["switch", "-c", "change/contracts"]);
|
||||
}
|
||||
@@ -115,7 +165,7 @@ async function fixture(
|
||||
directory,
|
||||
sourceSha,
|
||||
changedSha,
|
||||
releasedSha,
|
||||
releasedSha: seed.releasedSha,
|
||||
versioning: options.versioning,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1,9 +1,20 @@
|
||||
import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
access,
|
||||
chmod,
|
||||
cp,
|
||||
mkdir,
|
||||
readFile,
|
||||
writeFile,
|
||||
} from "node:fs/promises";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, it } from "vitest";
|
||||
import { afterAll, beforeAll, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
createTestWorkspace,
|
||||
removeTestWorkspace,
|
||||
} from "../../support/workspace/workspaceFixture.js";
|
||||
import {
|
||||
executeWorkflowFixture as exec,
|
||||
workflowGit as git,
|
||||
@@ -23,8 +34,20 @@ const page = (nodes: unknown[], cursor: string | null = null) => ({
|
||||
pageInfo: { hasNextPage: cursor !== null, endCursor: cursor },
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const directory = await createTestTempDirectory("rea-release-notes-");
|
||||
interface ReleaseNotesSeed {
|
||||
readonly directory: string;
|
||||
readonly base: string;
|
||||
readonly target: string;
|
||||
}
|
||||
|
||||
let seed: ReleaseNotesSeed | undefined;
|
||||
let seedWorkspaceRoot: string | undefined;
|
||||
|
||||
beforeAll(async () => {
|
||||
const workspace = await createTestWorkspace("rea-release-notes-seed-");
|
||||
seedWorkspaceRoot = workspace.root;
|
||||
const directory = workspace.path("repository");
|
||||
await mkdir(directory, { recursive: true });
|
||||
await git(directory, ["init", "--initial-branch=main"]);
|
||||
await writeFile(join(directory, "product.txt"), "base");
|
||||
await writeFile(
|
||||
@@ -142,6 +165,25 @@ process.stdout.write(JSON.stringify({data:{repository:result}}));
|
||||
`,
|
||||
);
|
||||
await chmod(join(bin, "gh"), 0o755);
|
||||
seed = { directory, base, target };
|
||||
});
|
||||
|
||||
afterAll(async () => {
|
||||
if (seedWorkspaceRoot !== undefined)
|
||||
await removeTestWorkspace(seedWorkspaceRoot);
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const initial = seed;
|
||||
if (initial === undefined)
|
||||
throw new Error("Release notes seed was not initialized");
|
||||
const directory = await createTestTempDirectory("rea-release-notes-");
|
||||
await cp(initial.directory, directory, {
|
||||
recursive: true,
|
||||
dereference: true,
|
||||
});
|
||||
const { base, target } = initial;
|
||||
const bin = join(directory, "bin");
|
||||
const manifest = join(directory, "inventory.json");
|
||||
const invoke = (args: string[]) =>
|
||||
exec(process.execPath, [cli, ...args], {
|
||||
|
||||
@@ -59,6 +59,9 @@ const TEST_PROJECTS = [
|
||||
include: ["tests/composition/**/*.test.ts"],
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Composition tests use provider-neutral sessions and test-scoped cleanup;
|
||||
// sharing their immutable module graph avoids per-file startup work.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "boundary",
|
||||
@@ -106,12 +109,16 @@ const TEST_PROJECTS = [
|
||||
include: ["tests/conformance/**/*.test.ts"],
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Contract cases use recording providers and own no process-global state.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "evaluation",
|
||||
include: ["tests/evaluation/**/*.test.ts"],
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Evaluator tests inspect deterministic inputs without mutating shared state.
|
||||
isolate: false,
|
||||
},
|
||||
];
|
||||
|
||||
|
||||
Reference in New Issue
Block a user