perf(mcp): reduce cold-start work
This commit is contained in:
@@ -0,0 +1,99 @@
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { format } from "prettier";
|
||||
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { MANAGED_WORKFLOW_TOOL_CONTRACTS } from "../dist/contracts/managedWorkflowToolContracts.js";
|
||||
import { ArtifactProvider } from "../dist/artifacts/ArtifactProvider.js";
|
||||
import { ManagedStaticProvider } from "../dist/dotnet/ManagedStaticProvider.js";
|
||||
import { NativeMacOSProvider } from "../dist/native/NativeMacOSProvider.js";
|
||||
import { toolRegistrationOptions } from "../dist/server/toolRegistrationOptions.js";
|
||||
import { ensureGeneratedFile } from "./lib/generated-file.mjs";
|
||||
|
||||
const arguments_ = new Set(process.argv.slice(2));
|
||||
for (const argument of arguments_)
|
||||
if (argument !== "--check")
|
||||
throw new Error(`Unknown MCP tool catalog option: ${argument}`);
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const sessionToolNames = new Set([
|
||||
...TOOL_CONTRACTS.filter(({ kind }) => kind === "session").map(
|
||||
({ name }) => name,
|
||||
),
|
||||
...MANAGED_WORKFLOW_TOOL_CONTRACTS.map(({ name }) => name),
|
||||
]);
|
||||
const catalog = TOOL_CONTRACTS.map((contract) => {
|
||||
const options = toolRegistrationOptions(contract);
|
||||
return {
|
||||
name: contract.name,
|
||||
analysisOperation: [
|
||||
"official-proxy",
|
||||
"enhanced",
|
||||
"native-provider",
|
||||
"artifact-provider",
|
||||
"managed-provider",
|
||||
].includes(contract.kind)
|
||||
? contract.name
|
||||
: null,
|
||||
title: options.title,
|
||||
description: options.description,
|
||||
kind: contract.kind,
|
||||
requiresSession: sessionToolNames.has(contract.name),
|
||||
inputSchema: options.inputSchema["~standard"].jsonSchema.input({
|
||||
target: "draft-2020-12",
|
||||
}),
|
||||
outputSchema: options.outputSchema["~standard"].jsonSchema.output({
|
||||
target: "draft-2020-12",
|
||||
}),
|
||||
annotations: options.annotations,
|
||||
effects: contract.effects,
|
||||
};
|
||||
});
|
||||
const auxiliaryProviders = [
|
||||
new ArtifactProvider(false, false),
|
||||
new NativeMacOSProvider(),
|
||||
new ManagedStaticProvider(),
|
||||
].map((provider) => ({
|
||||
identity: provider.identity(),
|
||||
capabilities: provider.capabilities(),
|
||||
}));
|
||||
const source = await format(
|
||||
`import type { ToolAnnotations } from "@modelcontextprotocol/server";
|
||||
import type {
|
||||
CapabilityDescriptor,
|
||||
ProviderIdentity,
|
||||
} from "./application/AnalysisProvider.js";
|
||||
import type { ToolKind } from "./contracts/toolContractTypes.js";
|
||||
import type { ToolEffects } from "./contracts/toolEffects.js";
|
||||
|
||||
/** Generated from TOOL_CONTRACTS; do not edit. */
|
||||
// prettier-ignore
|
||||
export const GENERATED_MCP_TOOL_CATALOG: readonly {
|
||||
readonly name: string;
|
||||
readonly analysisOperation: CapabilityDescriptor["operation"] | null;
|
||||
readonly title: string;
|
||||
readonly description: string;
|
||||
readonly kind: ToolKind;
|
||||
readonly requiresSession: boolean;
|
||||
readonly inputSchema: Readonly<Record<string, unknown>>;
|
||||
readonly outputSchema: Readonly<Record<string, unknown>>;
|
||||
readonly annotations: ToolAnnotations;
|
||||
readonly effects: ToolEffects;
|
||||
}[] = ${JSON.stringify(catalog)};
|
||||
|
||||
/** Generated lightweight metadata for lazily loaded auxiliary providers. */
|
||||
// prettier-ignore
|
||||
export const GENERATED_AUXILIARY_PROVIDERS: readonly {
|
||||
readonly identity: ProviderIdentity;
|
||||
readonly capabilities: readonly CapabilityDescriptor[];
|
||||
}[] = ${JSON.stringify(auxiliaryProviders)};
|
||||
`,
|
||||
{ parser: "typescript" },
|
||||
);
|
||||
|
||||
await ensureGeneratedFile({
|
||||
path: join(root, "src/generatedMcpToolCatalog.ts"),
|
||||
source,
|
||||
check: arguments_.has("--check"),
|
||||
generateCommand: "npm run mcp-catalog:generate",
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
interface McpStartupPolicy {
|
||||
readonly initializeBudgetMs: number;
|
||||
readonly toolsListBudgetMs: number;
|
||||
readonly firstCatalogBudgetMs: number;
|
||||
readonly doctorDeadlineMs: number;
|
||||
}
|
||||
|
||||
interface McpProcessOptions {
|
||||
readonly command: string;
|
||||
readonly args: readonly string[];
|
||||
readonly environment: Readonly<Record<string, string | undefined>>;
|
||||
readonly policy: McpStartupPolicy;
|
||||
}
|
||||
|
||||
interface McpModuleProfileOptions extends McpProcessOptions {
|
||||
readonly packageName: string;
|
||||
}
|
||||
|
||||
interface RssMeasurement {
|
||||
readonly supported: boolean;
|
||||
readonly method: string | null;
|
||||
readonly peak_bytes: number | null;
|
||||
}
|
||||
|
||||
/** Measure the first usable MCP catalog through a child-process boundary. */
|
||||
export function measureMcpStartup(options: McpProcessOptions): Promise<{
|
||||
readonly process_started_to_initialized_ms: number;
|
||||
readonly initialized_to_tools_list_ms: number;
|
||||
readonly process_started_to_tools_list_ms: number;
|
||||
readonly listed_tool_count: number;
|
||||
readonly rss: RssMeasurement;
|
||||
}>;
|
||||
|
||||
/** Profile ESM file loading in a separate diagnostic MCP child. */
|
||||
export function profileMcpModuleLoading(
|
||||
options: McpModuleProfileOptions,
|
||||
): Promise<{
|
||||
readonly supported: true;
|
||||
readonly method: "node-debug-esm";
|
||||
readonly unique_file_modules: number;
|
||||
readonly unique_package_modules: number;
|
||||
readonly unique_dependency_modules: number;
|
||||
readonly debug_lines: number;
|
||||
readonly truncated: boolean;
|
||||
}>;
|
||||
|
||||
/** Parse Linux procfs VmRSS text into bytes. */
|
||||
export function parseLinuxVmRssBytes(status: string): number | null;
|
||||
|
||||
/** Collect unique file-backed ESM module URLs from Node diagnostics. */
|
||||
export function createEsmModuleProfileCollector(packageName?: string): {
|
||||
append(chunk: string): void;
|
||||
result(): {
|
||||
readonly supported: true;
|
||||
readonly method: "node-debug-esm";
|
||||
readonly unique_file_modules: number;
|
||||
readonly unique_package_modules: number;
|
||||
readonly unique_dependency_modules: number;
|
||||
readonly debug_lines: number;
|
||||
readonly truncated: boolean;
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,217 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { performance } from "node:perf_hooks";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
const RSS_SAMPLE_INTERVAL_MS = 10;
|
||||
const MAXIMUM_PROFILE_BUFFER_BYTES = 2 * 1_024 * 1_024;
|
||||
const MAXIMUM_PROFILE_MODULES = 10_000;
|
||||
|
||||
/**
|
||||
* Measure the first usable MCP catalog through the installed process boundary.
|
||||
*/
|
||||
export async function measureMcpStartup({
|
||||
command,
|
||||
args,
|
||||
environment,
|
||||
policy,
|
||||
}) {
|
||||
const transport = new StdioClientTransport({
|
||||
command,
|
||||
args,
|
||||
env: environment,
|
||||
stderr: "pipe",
|
||||
});
|
||||
let stderr = "";
|
||||
transport.stderr?.on("data", (chunk) => {
|
||||
stderr = boundedAppend(stderr, chunk.toString());
|
||||
});
|
||||
const rss = createLinuxRssSampler(() => transport.pid);
|
||||
const client = new Client({
|
||||
name: "package-startup-probe",
|
||||
version: "1.0.0",
|
||||
});
|
||||
const startedAt = performance.now();
|
||||
rss.start();
|
||||
try {
|
||||
await client.connect(transport, { timeout: policy.initializeBudgetMs });
|
||||
const initializedAt = performance.now();
|
||||
const tools = await client.listTools(undefined, {
|
||||
timeout: policy.toolsListBudgetMs,
|
||||
});
|
||||
const listedAt = performance.now();
|
||||
rss.stop();
|
||||
const measurement = {
|
||||
process_started_to_initialized_ms: elapsed(startedAt, initializedAt),
|
||||
initialized_to_tools_list_ms: elapsed(initializedAt, listedAt),
|
||||
process_started_to_tools_list_ms: elapsed(startedAt, listedAt),
|
||||
listed_tool_count: tools.tools.length,
|
||||
rss: rss.result(),
|
||||
};
|
||||
assertStartupBudgets(measurement, policy);
|
||||
return measurement;
|
||||
} catch (cause) {
|
||||
throw new Error(`packaged MCP startup probe failed: ${stderr}`, { cause });
|
||||
} finally {
|
||||
rss.stop();
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Profile ESM file loading in a separate diagnostic child.
|
||||
*/
|
||||
export async function profileMcpModuleLoading({
|
||||
command,
|
||||
args,
|
||||
environment,
|
||||
policy,
|
||||
packageName,
|
||||
}) {
|
||||
const profile = createEsmModuleProfileCollector(packageName);
|
||||
const transport = new StdioClientTransport({
|
||||
command,
|
||||
args,
|
||||
env: { ...environment, NODE_DEBUG: "esm" },
|
||||
stderr: "pipe",
|
||||
});
|
||||
transport.stderr?.on("data", (chunk) => profile.append(chunk.toString()));
|
||||
const client = new Client({
|
||||
name: "package-module-profile",
|
||||
version: "1.0.0",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport, {
|
||||
timeout: policy.doctorDeadlineMs,
|
||||
});
|
||||
await client.listTools(undefined, {
|
||||
timeout: policy.toolsListBudgetMs,
|
||||
});
|
||||
return profile.result();
|
||||
} finally {
|
||||
await Promise.allSettled([client.close(), transport.close()]);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse Linux procfs VmRSS text into bytes.
|
||||
*/
|
||||
export const parseLinuxVmRssBytes = (status) => {
|
||||
const match = /^VmRSS:\s+(\d+)\s+kB$/mu.exec(status);
|
||||
return match === null ? null : Number(match[1]) * 1_024;
|
||||
};
|
||||
|
||||
/**
|
||||
* Collect unique file-backed ESM module URLs from Node's diagnostic stream.
|
||||
*/
|
||||
export const createEsmModuleProfileCollector = (packageName = "rea-agents") => {
|
||||
const modules = new Set();
|
||||
let buffer = "";
|
||||
let debugLines = 0;
|
||||
let truncated = false;
|
||||
const consume = (line) => {
|
||||
if (line.startsWith("ESM ")) debugLines += 1;
|
||||
for (const match of line.matchAll(/file:\/\/\/[^\s'",}\])]+/gu)) {
|
||||
if (modules.size >= MAXIMUM_PROFILE_MODULES) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
modules.add(match[0]);
|
||||
}
|
||||
};
|
||||
return {
|
||||
append(chunk) {
|
||||
buffer += chunk;
|
||||
if (Buffer.byteLength(buffer) > MAXIMUM_PROFILE_BUFFER_BYTES) {
|
||||
buffer = buffer.slice(-MAXIMUM_PROFILE_BUFFER_BYTES);
|
||||
truncated = true;
|
||||
}
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() ?? "";
|
||||
for (const line of lines) consume(line);
|
||||
},
|
||||
result() {
|
||||
if (buffer.length > 0) consume(buffer);
|
||||
const values = [...modules];
|
||||
const packageSegment = `/node_modules/${packageName}/`;
|
||||
return {
|
||||
supported: true,
|
||||
method: "node-debug-esm",
|
||||
unique_file_modules: values.length,
|
||||
unique_package_modules: values.filter((value) =>
|
||||
value.includes(packageSegment),
|
||||
).length,
|
||||
unique_dependency_modules: values.filter(
|
||||
(value) =>
|
||||
value.includes("/node_modules/") && !value.includes(packageSegment),
|
||||
).length,
|
||||
debug_lines: debugLines,
|
||||
truncated,
|
||||
};
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const createLinuxRssSampler = (pid) => {
|
||||
if (process.platform !== "linux")
|
||||
return {
|
||||
start() {},
|
||||
stop() {},
|
||||
result: () => ({
|
||||
supported: false,
|
||||
method: null,
|
||||
peak_bytes: null,
|
||||
}),
|
||||
};
|
||||
let peakBytes = null;
|
||||
let timer;
|
||||
const sample = () => {
|
||||
const currentPid = pid();
|
||||
if (currentPid === null) return;
|
||||
try {
|
||||
const current = parseLinuxVmRssBytes(
|
||||
readFileSync(`/proc/${String(currentPid)}/status`, "utf8"),
|
||||
);
|
||||
if (current !== null && (peakBytes === null || current > peakBytes))
|
||||
peakBytes = current;
|
||||
} catch {
|
||||
// The short-lived child may exit between resolving its PID and reading.
|
||||
}
|
||||
};
|
||||
return {
|
||||
start() {
|
||||
sample();
|
||||
timer = setInterval(sample, RSS_SAMPLE_INTERVAL_MS);
|
||||
timer.unref();
|
||||
},
|
||||
stop() {
|
||||
if (timer !== undefined) clearInterval(timer);
|
||||
sample();
|
||||
},
|
||||
result: () => ({
|
||||
supported: true,
|
||||
method: "linux-proc-status-vmrss-sampled",
|
||||
peak_bytes: peakBytes,
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
const assertStartupBudgets = (measurement, policy) => {
|
||||
if (measurement.process_started_to_initialized_ms > policy.initializeBudgetMs)
|
||||
throw new Error("packaged MCP exceeded the initialize regression budget");
|
||||
if (measurement.initialized_to_tools_list_ms > policy.toolsListBudgetMs)
|
||||
throw new Error("packaged MCP exceeded the tools/list regression budget");
|
||||
if (
|
||||
measurement.process_started_to_tools_list_ms > policy.firstCatalogBudgetMs
|
||||
)
|
||||
throw new Error(
|
||||
"packaged MCP exceeded the first-catalog regression budget",
|
||||
);
|
||||
};
|
||||
|
||||
const elapsed = (startedAt, completedAt) =>
|
||||
Math.round((completedAt - startedAt) * 100) / 100;
|
||||
|
||||
const boundedAppend = (current, chunk) =>
|
||||
`${current}${chunk}`.slice(-64 * 1_024);
|
||||
@@ -8,6 +8,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { createPackage } from "@electron/asar";
|
||||
|
||||
import { evaluateCodexEvents } from "../dist/evaluation/CodexAgentEval.js";
|
||||
import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const repositoryRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
@@ -346,7 +347,7 @@ async function runCodex(prompt, investigationRoot) {
|
||||
"-c",
|
||||
`mcp_servers.rea.args=${JSON.stringify([join(repositoryRoot, "scripts/rea.mjs"), "mcp"])}`,
|
||||
"-c",
|
||||
"mcp_servers.rea.startup_timeout_sec=30",
|
||||
`mcp_servers.rea.startup_timeout_sec=${String(MCP_STARTUP_POLICY.codexStartupTimeoutSeconds)}`,
|
||||
"-c",
|
||||
mcpEnvironmentOverride,
|
||||
...(optionalModel === undefined ? [] : ["--model", optionalModel]),
|
||||
|
||||
@@ -5,6 +5,7 @@ import { promisify } from "node:util";
|
||||
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js";
|
||||
import * as prompts from "./verify-package-prompts.mjs";
|
||||
import {
|
||||
json,
|
||||
@@ -180,7 +181,7 @@ export async function verifyPackageMcp({
|
||||
(
|
||||
await execute(cli, ["mcp", "doctor", "--json"], {
|
||||
env: environment,
|
||||
timeout: 30_000,
|
||||
timeout: MCP_STARTUP_POLICY.doctorDeadlineMs,
|
||||
})
|
||||
).stdout,
|
||||
);
|
||||
|
||||
@@ -4,6 +4,7 @@ import { join } from "node:path";
|
||||
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { PRODUCT_IDENTITY } from "../dist/identity.js";
|
||||
import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js";
|
||||
import * as prompts from "./verify-package-prompts.mjs";
|
||||
import { verifyPackagedInvestigation } from "./verify-package-investigation.mjs";
|
||||
import { verifyPackageArtifactAndElectron } from "./verify-package-artifact.mjs";
|
||||
@@ -19,6 +20,10 @@ import { verifyPackageSetup } from "./verify-package-setup.mjs";
|
||||
import { verifyManaged } from "./verify-package-managed.mjs";
|
||||
import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import {
|
||||
measureMcpStartup,
|
||||
profileMcpModuleLoading,
|
||||
} from "./lib/mcp-startup-probe.mjs";
|
||||
|
||||
const root = process.cwd();
|
||||
const verifierRun = createVerifierRun();
|
||||
@@ -26,6 +31,8 @@ const workspace = await mkdtemp(join(tmpdir(), "rea-package-"));
|
||||
const evidenceRoot = join(workspace, "evidence");
|
||||
const referenceRoot = join(workspace, "reference-source");
|
||||
let tarball;
|
||||
let mcpStartup;
|
||||
let mcpModuleLoading;
|
||||
|
||||
try {
|
||||
({ tarball } = await verifyPackagePack({ root, workspace }));
|
||||
@@ -42,6 +49,25 @@ try {
|
||||
workspace,
|
||||
environment: environmentData.environment,
|
||||
});
|
||||
mcpStartup = await measureMcpStartup({
|
||||
command: cli,
|
||||
args: ["mcp"],
|
||||
environment: {
|
||||
...environmentData.environment,
|
||||
REA_INVESTIGATION_INPUT_ROOTS_JSON: JSON.stringify([]),
|
||||
},
|
||||
policy: MCP_STARTUP_POLICY,
|
||||
});
|
||||
mcpModuleLoading = await profileMcpModuleLoading({
|
||||
command: cli,
|
||||
args: ["mcp"],
|
||||
environment: {
|
||||
...environmentData.environment,
|
||||
REA_INVESTIGATION_INPUT_ROOTS_JSON: JSON.stringify([]),
|
||||
},
|
||||
policy: MCP_STARTUP_POLICY,
|
||||
packageName: PRODUCT_IDENTITY.packageName,
|
||||
});
|
||||
const { supportedSetupHost } = await verifyPackageDiscovery({
|
||||
cli,
|
||||
environment: environmentData.environment,
|
||||
@@ -102,7 +128,7 @@ try {
|
||||
investigationReplay,
|
||||
});
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), cli: true, analysisCli: true, artifactCli: true, managedCli: true, managedReconstructionCli: true, managedNativeVerificationCli: true, managedRuntimePlanCli: true, managedApplicationGraphCli: true, evidenceCli: true, incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, lifecycleScriptsRequired: false, doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 3 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true, boundedRegexBridge: true })}\n`,
|
||||
`${JSON.stringify({ verifier_run: await completeVerifierRun(verifierRun), cli: true, analysisCli: true, artifactCli: true, managedCli: true, managedReconstructionCli: true, managedNativeVerificationCli: true, managedRuntimePlanCli: true, managedApplicationGraphCli: true, evidenceCli: true, incurMcpCommand: PRODUCT_IDENTITY.mcpCommand, lifecycleScriptsRequired: false, doctor: "platform-appropriate", setup: supportedSetupHost ? "planned-then-idempotent" : "unsupported-host-rejected", setupPlanReadOnly: supportedSetupHost, existingHopperPreserved: supportedSetupHost, clients: supportedSetupHost ? 3 : 0, backupReadback: supportedSetupHost, failureRecovery: supportedSetupHost, configSymlinkLifecycle: supportedSetupHost, skill: supportedSetupHost, skillReferences: supportedSetupHost, mcpTools: TOOL_CONTRACTS.length, mcpPrompts: prompts.names.length, promptCompletion: true, promptCompletionLifecycle: true, evidenceMcp: true, targetFree: true, targetLifecycle: true, boundedRegexBridge: true, mcpStartup, mcpModuleLoading })}\n`,
|
||||
);
|
||||
} finally {
|
||||
if (tarball) await rm(join(root, tarball), { force: true });
|
||||
|
||||
Reference in New Issue
Block a user