refactor(mcp): make analysis states unrepresentable (#468)
* refactor(config): parse capability settings into exact states * refactor(runtime): encode replay and process outcomes exactly * refactor: make analysis states unrepresentable * refactor(mcp): register canonical schemas with SDK * fix: preserve capability and catalog invariants * fix(build): remove stale compiled modules * fix(replay): update real verifier policy dependency * fix(package): verify the complete MCP catalog * fix(test): supply complete replay reload policy * refactor(contracts): centralize output schema lookup * refactor(mcp): delegate protocol boundaries to SDK * fix(browser): approve real source capture * fix(mcp): preserve catalog guidance
This commit is contained in:
@@ -0,0 +1,10 @@
|
||||
import { rm } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
await Promise.all([
|
||||
rm(join(root, "dist"), { recursive: true, force: true }),
|
||||
rm(join(root, ".cache", "tsconfig.build.tsbuildinfo"), { force: true }),
|
||||
]);
|
||||
@@ -1,6 +1,8 @@
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { format } from "prettier";
|
||||
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
|
||||
import { McpServer } from "@modelcontextprotocol/server";
|
||||
|
||||
import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js";
|
||||
import { MANAGED_WORKFLOW_TOOL_CONTRACTS } from "../dist/contracts/managedWorkflowToolContracts.js";
|
||||
@@ -8,7 +10,6 @@ import { ArtifactProvider } from "../dist/artifacts/ArtifactProvider.js";
|
||||
import { ManagedStaticProvider } from "../dist/dotnet/ManagedStaticProvider.js";
|
||||
import { NativeMacOSProvider } from "../dist/native/NativeMacOSProvider.js";
|
||||
import { toolRegistrationOptions } from "../dist/server/toolRegistrationOptions.js";
|
||||
import { applyToolInputJsonSchemaOverride } from "../dist/contracts/toolSchemaJsonOverrides.js";
|
||||
import { ensureGeneratedFile } from "./lib/generated-file.mjs";
|
||||
|
||||
const arguments_ = new Set(process.argv.slice(2));
|
||||
@@ -26,8 +27,11 @@ const sessionToolNames = new Set([
|
||||
),
|
||||
...MANAGED_WORKFLOW_TOOL_CONTRACTS.map(({ name }) => name),
|
||||
]);
|
||||
const advertisedTools = await sdkToolCatalog();
|
||||
const catalog = TOOL_CONTRACTS.map((contract) => {
|
||||
const options = toolRegistrationOptions(contract);
|
||||
const advertised = advertisedTools.get(contract.name);
|
||||
if (advertised === undefined)
|
||||
throw new Error(`MCP SDK omitted registered tool ${contract.name}`);
|
||||
return {
|
||||
name: contract.name,
|
||||
analysisOperation: [
|
||||
@@ -39,23 +43,40 @@ const catalog = TOOL_CONTRACTS.map((contract) => {
|
||||
].includes(contract.kind)
|
||||
? contract.name
|
||||
: null,
|
||||
title: options.title,
|
||||
description: options.description,
|
||||
title: advertised.title,
|
||||
description: advertised.description,
|
||||
kind: contract.kind,
|
||||
requiresSession: sessionToolNames.has(contract.name),
|
||||
inputSchema: applyToolInputJsonSchemaOverride(
|
||||
contract.name,
|
||||
options.inputSchema["~standard"].jsonSchema.input({
|
||||
target: "draft-2020-12",
|
||||
}),
|
||||
),
|
||||
outputSchema: options.outputSchema["~standard"].jsonSchema.output({
|
||||
target: "draft-2020-12",
|
||||
}),
|
||||
annotations: options.annotations,
|
||||
inputSchema: advertised.inputSchema,
|
||||
outputSchema: advertised.outputSchema,
|
||||
annotations: advertised.annotations,
|
||||
effects: contract.effects,
|
||||
};
|
||||
});
|
||||
|
||||
async function sdkToolCatalog() {
|
||||
const server = new McpServer({ name: "rea-catalog-generator", version: "0" });
|
||||
for (const contract of TOOL_CONTRACTS)
|
||||
server.registerTool(
|
||||
contract.name,
|
||||
toolRegistrationOptions(contract),
|
||||
async () => ({
|
||||
content: [{ type: "text", text: "Catalog-only handler" }],
|
||||
isError: true,
|
||||
}),
|
||||
);
|
||||
const client = new Client({ name: "rea-catalog-generator", version: "0" });
|
||||
const [clientTransport, serverTransport] =
|
||||
InMemoryTransport.createLinkedPair();
|
||||
try {
|
||||
await server.connect(serverTransport);
|
||||
await client.connect(clientTransport);
|
||||
const { tools } = await client.listTools();
|
||||
return new Map(tools.map((tool) => [tool.name, tool]));
|
||||
} finally {
|
||||
await Promise.allSettled([client.close(), server.close()]);
|
||||
}
|
||||
}
|
||||
const auxiliaryProviders = [
|
||||
new ArtifactProvider(false, false),
|
||||
new NativeMacOSProvider(undefined, CATALOG_PLATFORM),
|
||||
|
||||
@@ -28,8 +28,8 @@ export const runWithStatus = async (command, args, env) => {
|
||||
/** Parse JSON from a command output string. */
|
||||
export const json = (text) => JSON.parse(text);
|
||||
|
||||
/** Verify that the advertised MCP catalog matches the session's available tools. */
|
||||
export const verifyAvailableToolCatalog = async (client, options) => {
|
||||
/** Verify that the complete advertised MCP catalog matches session metadata. */
|
||||
export const verifyCompleteToolCatalog = async (client, options) => {
|
||||
const listed = await client.listTools(undefined, options);
|
||||
const status = await client.callTool(
|
||||
{ name: "binary_session", arguments: { detail: "full" } },
|
||||
@@ -38,14 +38,11 @@ export const verifyAvailableToolCatalog = async (client, options) => {
|
||||
const availability = status.structuredContent?.result?.tool_availability;
|
||||
if (!Array.isArray(availability))
|
||||
throw new Error("packaged MCP omitted tool availability");
|
||||
const expected = availability
|
||||
.filter(({ available }) => available === true)
|
||||
.map(({ name }) => name)
|
||||
.sort();
|
||||
const expected = availability.map(({ name }) => name).sort();
|
||||
const observed = listed.tools.map(({ name }) => name).sort();
|
||||
if (JSON.stringify(observed) !== JSON.stringify(expected))
|
||||
throw new Error(
|
||||
"packaged MCP tool inventory diverged from session availability",
|
||||
"packaged MCP tool inventory diverged from session metadata",
|
||||
);
|
||||
return observed;
|
||||
};
|
||||
|
||||
@@ -344,11 +344,11 @@ try {
|
||||
assert.equal(reconstructionImport.value.confidence, "inferred");
|
||||
const runtimePlan = await planManagedRuntimeCorrelationEvidence(
|
||||
{
|
||||
policy: {
|
||||
enabled: true,
|
||||
policy: () => ({
|
||||
status: "enabled",
|
||||
roots: [workspace],
|
||||
executablePath: runtimeExecutable,
|
||||
},
|
||||
}),
|
||||
authority: runtimeAuthority.value,
|
||||
},
|
||||
{
|
||||
|
||||
@@ -7,15 +7,12 @@ import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
import { MCP_STARTUP_POLICY } from "../dist/mcpStartupPolicy.js";
|
||||
import * as prompts from "./verify-package-prompts.mjs";
|
||||
import {
|
||||
json,
|
||||
verifyAvailableToolCatalog,
|
||||
} from "./lib/verify-package-core.mjs";
|
||||
import { json, verifyCompleteToolCatalog } from "./lib/verify-package-core.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
|
||||
const verifyMcpToolsAndPrompts = async (client, mcpOptions) => {
|
||||
await verifyAvailableToolCatalog(client, mcpOptions);
|
||||
await verifyCompleteToolCatalog(client, mcpOptions);
|
||||
await prompts.verifyPromptCatalog(client, mcpOptions, prompts.names);
|
||||
await prompts.verifyPromptCompletion(client, mcpOptions, false);
|
||||
};
|
||||
@@ -43,11 +40,21 @@ const verifyMcpReplay = async (client, mcpOptions, investigationReplay) => {
|
||||
};
|
||||
|
||||
const verifyMcpTargetFree = async (client, mcpOptions) => {
|
||||
const names = (await client.listTools(undefined, mcpOptions)).tools.map(
|
||||
({ name }) => name,
|
||||
const status = await client.callTool(
|
||||
{ name: "binary_session", arguments: { detail: "full" } },
|
||||
mcpOptions,
|
||||
);
|
||||
if (names.includes("current_document"))
|
||||
throw new Error("packaged target-free MCP advertised a target-bound tool");
|
||||
const currentDocument =
|
||||
status.structuredContent?.result?.tool_availability?.find(
|
||||
({ name }) => name === "current_document",
|
||||
);
|
||||
if (
|
||||
currentDocument?.available !== false ||
|
||||
currentDocument.reason !== "target_required"
|
||||
)
|
||||
throw new Error(
|
||||
"packaged target-free MCP omitted target-bound availability metadata",
|
||||
);
|
||||
};
|
||||
|
||||
const verifyMcpUnknownProvider = async (client, mcpOptions) => {
|
||||
@@ -96,11 +103,12 @@ const verifyMcpOpenAndBind = async (client, mcpOptions) => {
|
||||
};
|
||||
|
||||
const verifyMcpLinuxToolAvailability = async (client, mcpOptions) => {
|
||||
const names = (await client.listTools(undefined, mcpOptions)).tools.map(
|
||||
({ name }) => name,
|
||||
const current = await client.callTool(
|
||||
{ name: "current_document", arguments: {} },
|
||||
mcpOptions,
|
||||
);
|
||||
if (names.includes("current_document"))
|
||||
throw new Error("packaged Linux MCP advertised unavailable Hopper tools");
|
||||
if (current.isError !== true)
|
||||
throw new Error("packaged Linux MCP executed an unavailable Hopper tool");
|
||||
};
|
||||
|
||||
const verifyMcpNonLinuxCurrentDocument = async (
|
||||
|
||||
@@ -114,6 +114,7 @@ try {
|
||||
observation_ms: 1_000,
|
||||
include_accessibility_text: true,
|
||||
include_script_sources: true,
|
||||
source_capture_approved: true,
|
||||
include_console_text: true,
|
||||
console_text_approved: true,
|
||||
include_json_body_shapes: true,
|
||||
|
||||
@@ -52,7 +52,7 @@ const roots = [
|
||||
),
|
||||
];
|
||||
const policy = {
|
||||
enabled: true,
|
||||
status: "enabled",
|
||||
roots,
|
||||
nodePath: process.execPath,
|
||||
bubblewrapPath:
|
||||
@@ -93,7 +93,7 @@ const authority = new PermissionAuthority(
|
||||
),
|
||||
);
|
||||
const dependencies = {
|
||||
policy,
|
||||
policy: () => policy,
|
||||
host: new SystemJavaScriptReplayHost(),
|
||||
runner: new LinuxJavaScriptReplayRunner(),
|
||||
authority,
|
||||
@@ -131,18 +131,20 @@ const defaultInput = {
|
||||
{ case_id: "plain", arguments: ["plain"] },
|
||||
],
|
||||
generator: { preset: "parser-boundaries", seed: 7, count: 4 },
|
||||
approved: false,
|
||||
};
|
||||
const input =
|
||||
const suppliedPlanInput =
|
||||
suppliedInput === undefined
|
||||
? undefined
|
||||
: (({ approved: _approved, plan_digest: _planDigest, ...planInput }) =>
|
||||
planInput)(suppliedInput);
|
||||
const input =
|
||||
suppliedPlanInput === undefined
|
||||
? defaultInput
|
||||
: {
|
||||
...suppliedInput,
|
||||
...suppliedPlanInput,
|
||||
mode: "plan",
|
||||
left: suppliedLeft,
|
||||
...(suppliedRight === undefined ? {} : { right: suppliedRight }),
|
||||
approved: false,
|
||||
plan_digest: undefined,
|
||||
reproducer_export: undefined,
|
||||
};
|
||||
|
||||
@@ -177,7 +179,21 @@ if (
|
||||
({ status }) => status === "unknown",
|
||||
) === true
|
||||
)
|
||||
throw new Error("Real replay verifier received incomplete Evidence");
|
||||
throw new Error(
|
||||
`Real replay verifier received incomplete Evidence: ${JSON.stringify({
|
||||
authority: evidence?.authority ?? null,
|
||||
isolation: evidence?.environment?.isolation ?? null,
|
||||
termination: evidence?.normalized_result?.termination ?? null,
|
||||
cleanup: evidence?.normalized_result?.cleanup?.state ?? null,
|
||||
outcomes:
|
||||
evidence?.normalized_result?.outcomes?.map(({ outcome }) => outcome) ??
|
||||
[],
|
||||
stderr: evidence?.normalized_result?.stderr ?? null,
|
||||
comparison_statuses:
|
||||
evidence?.normalized_result?.comparison?.map(({ status }) => status) ??
|
||||
[],
|
||||
})}`,
|
||||
);
|
||||
|
||||
if (suppliedInput !== undefined) {
|
||||
process.stdout.write(
|
||||
@@ -222,7 +238,6 @@ const runSingle = async (path, limits = undefined) => {
|
||||
},
|
||||
cases: [{ case_id: "probe", arguments: [] }],
|
||||
...(limits === undefined ? {} : { limits }),
|
||||
approved: false,
|
||||
};
|
||||
const proposed = await runControlledReplay(dependencies, manifest);
|
||||
if (!proposed.ok || proposed.value.plan === null)
|
||||
@@ -288,7 +303,6 @@ const differentialTimeoutInput = {
|
||||
},
|
||||
cases: [{ case_id: "probe", arguments: [] }],
|
||||
limits: { wall_time_ms: 100 },
|
||||
approved: false,
|
||||
};
|
||||
const differentialTimeoutPlan = await runControlledReplay(
|
||||
dependencies,
|
||||
|
||||
@@ -9,7 +9,7 @@ import { promisify } from "node:util";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { verifyAvailableToolCatalog } from "./lib/verify-package-core.mjs";
|
||||
import { verifyCompleteToolCatalog } from "./lib/verify-package-core.mjs";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
@@ -84,7 +84,7 @@ try {
|
||||
let toolCount = 0;
|
||||
try {
|
||||
await client.connect(server);
|
||||
toolCount = (await verifyAvailableToolCatalog(client)).length;
|
||||
toolCount = (await verifyCompleteToolCatalog(client)).length;
|
||||
} finally {
|
||||
await client.close();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user