From 6395e8d3007d5ee453477460e7ee1c51cff76be0 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Mon, 12 Oct 2026 07:40:19 +0800 Subject: [PATCH] perf(mcp)!: compact repeated session discovery facts (#1988) Group repeated session tool availability and provider capability facts, preserve complete named contracts and ordered limitations, and remove duplicate catalog descriptors. Update consumers and migration guidance. Validated by all 31 CI jobs, focused source checks, and actual Claude Code/Linux Hopper/native Windows Ghidra small-target workflows on the current integration tree. Refs #723. BREAKING CHANGE: binary_session no longer includes server_identity.catalog.tools; tool_availability and provider capabilities use grouped facts documented in docs/mcp-contracts.md. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- .../skills/reverse-engineer-anything/SKILL.md | 10 +- .../references/connection-and-recovery.md | 16 +- .../references/native-and-artifacts.md | 17 +- docs/mcp-contracts.md | 47 ++- scripts/lib/mcp-verifier-results.mjs | 58 ++- scripts/lib/native-value-e2e.mjs | 17 +- scripts/lib/verify-package-core.mjs | 13 +- scripts/verify-claude-client.mjs | 85 +++- scripts/verify-package-capabilities.mjs | 12 +- scripts/verify-package-mcp.mjs | 27 +- scripts/verify-packaged-ghidra-windows.mjs | 23 +- scripts/verify-published-package.mjs | 35 +- scripts/verify-real-hopper.mjs | 7 +- scripts/verify/android/failure-recovery.mjs | 23 +- src/application/CapabilityInventory.test.ts | 222 +++++++++- src/application/CapabilityInventory.ts | 72 +++- .../binary/BinarySessionStatus.test.ts | 391 ++++++++++++++++++ src/application/binary/BinarySessionStatus.ts | 146 +++++-- src/catalogIdentity.ts | 11 - src/contracts/toolOutputSchemaPrimitives.ts | 148 ++++++- src/server/registerSessionStatusTool.ts | 14 +- tests/acceptance/applications/runtime.test.ts | 13 +- tests/boundary/mcp/browserMcp.test.ts | 28 +- tests/boundary/mcp/clientCapabilities.test.ts | 19 +- tests/boundary/mcp/ghidraRoutingMcp.test.ts | 65 ++- .../mcp/optionalProviderIsolation.test.ts | 59 +-- tests/boundary/mcp/serverIdentity.test.ts | 81 ++-- tests/boundary/mcp/serverWorkflowMcp.test.ts | 17 +- tests/boundary/mcp/sessionMcp.test.ts | 15 +- .../analysis-sessions/binarySession.test.ts | 47 ++- 30 files changed, 1449 insertions(+), 289 deletions(-) create mode 100644 src/application/binary/BinarySessionStatus.test.ts diff --git a/.agents/skills/reverse-engineer-anything/SKILL.md b/.agents/skills/reverse-engineer-anything/SKILL.md index 45240351d..1960c6d1c 100644 --- a/.agents/skills/reverse-engineer-anything/SKILL.md +++ b/.agents/skills/reverse-engineer-anything/SKILL.md @@ -115,8 +115,14 @@ ask when matches are ambiguous. Never choose an example app on the user's behalf In a target-free session, use `open_binary` to bind any archive/package or native target whose analysis tool operates on the active target. Do not call an unadvertised tool; inspect `binary_session` with `{}` and its -`result.tool_availability` for availability reasons and remediation. The tool -list stays complete; availability depends on the operation, target, and host. +`result.tool_availability` for availability reasons and remediation. Expand each +group's `names` into entries, inheriting the inventory-level client requirements +unless the group overrides them. In its closed effect and annotation vocabularies, +listed flags are true and omitted known flags are false. The grouped name union +is the complete tool inventory; use `tools/list` for canonical tool schemas. +Provider candidate facts describe possible providers, while top-level session +capabilities describe the current route and `analysis_provider_binding` names the +selected provider. Do not treat those scopes as interchangeable. ## Work summary-first diff --git a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md index 5e5f6251d..515a6c9d4 100644 --- a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md +++ b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md @@ -11,14 +11,14 @@ main and an older release expose the same arguments. ## Choose the remedy from the observed failure -| Observation | Meaning and next step | -| --------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. | -| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. | -| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. | -| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. | -| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Resolve only the prerequisite needed for the task; a complete tool catalog does not imply every operation is available. | -| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. | +| Observation | Meaning and next step | +| --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| No REA tools in the active session | Inspect registration with `rea doctor --client codex --json`. A skill installation alone does not register MCP. | +| Missing, malformed or stale registration | Prepare a scoped setup plan; apply only authorized changes with backups. | +| Aligned registration, but no connected tools | Restart/reconnect the client. Doctor checks registration; it cannot establish the current agent's connection. Inspect the client's launch error if reconnection fails. | +| `MCP tool call requires approval, but approval policy is never` | Codex rejected the call before REA executed it. Do not install an engine or repeat setup to solve this client-policy decision. See the approval section below. | +| REA returns `provider_unavailable` or an unavailable tool entry | Read that operation's provider diagnostics or `binary_session.result.tool_availability`. Expand each group's tool names and inherit the shared client requirements unless overridden. Use the operation's availability, reason, and remediation; a complete tool catalog does not imply every operation is available. | +| `rea` is missing in a child shell, or Node is incompatible | Inspect the actual shell PATH and launcher/runtime. A prefix installation may be absent from a login shell's PATH; this does not establish a broken MCP registration. | If the client cannot launch its shell/sandbox helper or read the installed skill, inspect its launch diagnostic. For example, Codex can refuse helper aliases when diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 9f09ad82f..a045e5a62 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -50,10 +50,19 @@ runs and may change files or use the network. Hardened-runtime targets need the `get-task-allow` entitlement. Use `binary_session` with no arguments to check the open target, selected -provider, and alignment. Its default `result.tool_availability` includes the -complete tool inventory with availability reasons and remediation. When choosing -a tool, use its entry in that result; `tools/list` retains the complete catalog -when the target or provider state changes. +provider, and alignment. Its default `result.tool_availability` carries a +complete inventory grouped by shared facts. Expand each group's `names` and +inherit the inventory-level client requirements unless a group overrides them; +availability, reasons, remediation, modes, and flags remain inline. The +`effects` and `annotations` arrays name true flags from fixed vocabularies, so an +omitted known flag is false. `tools/list` remains the canonical source for full +tool schemas when target or provider state changes. + +Provider capability groups carry provider identity and ordered limitations. +Prepend each set's `shared_limitations` to a group's `limitations` to recover the +operation's full ordered list. Candidate capabilities describe possible +providers. Top-level session capabilities describe the current route, and +`analysis_provider_binding` identifies the selected provider. ## DOS MZ with Ghidra diff --git a/docs/mcp-contracts.md b/docs/mcp-contracts.md index b8d8d6fb8..93f890f20 100644 --- a/docs/mcp-contracts.md +++ b/docs/mcp-contracts.md @@ -113,17 +113,48 @@ Schema does not express those cross-field invariants. Call `binary_session` with `{}` and read `result.tool_availability` to choose a callable operation for the current target, provider, host, and negotiated client -capabilities. The default result includes the complete inventory with each -tool's availability, reason, and remediation. Each entry also reports required -and optional negotiated client features plus the currently missing features. -The optional inputs `expected_package_version`, `expected_catalog_digest`, and +capabilities. The default result contains the complete inventory in groups that +share status facts. `tool_availability.client_requirements` is the baseline for +each group; a group's optional `client_requirements` replaces that baseline for +the group's tools. Expand each group's `names` into one tool entry, carrying its +surface, availability, reason, remediation, default-mode status, modes, and +client requirements. Compare the union of group names with `tools/list` when +checking that a client received the full catalog. An operation absent from all +groups is missing from the complete inventory; it is not an implicit +unavailable entry. + +The `effects` and `annotations` arrays list the true members of their fixed +vocabularies. A known member omitted from either array is false. This applies +only to these closed flag sets; it does not make an omitted tool, provider, mode, +or limitation false. Keep availability reasons, remediation, client-feature +requirements, and mode facts inline with the grouped tool names. The optional +inputs `expected_package_version`, `expected_catalog_digest`, and `expected_server_path` compare the live session with the caller's expectations. +`server_identity.catalog` contains catalog counts and schema digests. Grouped +tool availability carries the name set and current per-tool facts; `tools/list` +is the canonical source for complete tool definitions, including input and +output schemas. Clients that previously read `server_identity.catalog.tools` +should expand `tool_availability.groups` and use `tools/list` for schemas. The +same catalog identity summary is returned by CLI `capabilities` and `providers`; +MCP tool definitions remain available through `tools/list`. + +Provider capabilities are grouped by provider identity and shared facts. In each +set, `shared_limitations` is the ordered common prefix; append a group's ordered +`limitations` suffix to recover each operation's exact limitations. Group +`effects` arrays use the same true-only closed-vocabulary rule as tool +availability. Expand every group operation while retaining its provider +identity, availability, reason, rejection code, effects, and ordered +limitations. + `binary_session.analysis_provider_candidates` is authoritative for deep-engine -discovery. Target-free discovery is sorted by provider ID, reports host -availability and `unknown` target support, and does not create an analysis -client. `open_binary.provider_id` accepts a concrete provider ID or `auto`; it -uses the same parser and selection policy as CLI `--provider` and +discovery. Each candidate's capability sets describe that possible provider; +they do not select it. The top-level `capabilities` describe the current route, +while `analysis_provider_binding` identifies the concrete selected route. These +fields are not interchangeable. Target-free discovery is sorted by provider ID, +reports host availability and `unknown` target support, and does not create an +analysis client. `open_binary.provider_id` accepts a concrete provider ID or +`auto`; it uses the same parser and selection policy as CLI `--provider` and `REA_ANALYSIS_PROVIDER`. A successful deep open exposes one immutable provider, concrete version, selection source, and complete analysis profile through `analysis_provider_binding`. Ambiguity and unknown, unavailable, or unsupported diff --git a/scripts/lib/mcp-verifier-results.mjs b/scripts/lib/mcp-verifier-results.mjs index 7ac038a0a..def0effbb 100644 --- a/scripts/lib/mcp-verifier-results.mjs +++ b/scripts/lib/mcp-verifier-results.mjs @@ -1,5 +1,11 @@ import { analysisErrorProjectionSchema } from "../../dist/contracts/errorSchemas.js"; import { toolContract } from "../../dist/contracts/toolContracts.js"; +import { + expandProviderCapabilities, + expandToolAvailability, + providerCapabilitySet, +} from "../../dist/contracts/toolOutputSchemaPrimitives.js"; +import { compareUnicodeCodePoints } from "../../dist/domain/unicodeCodePointOrder.js"; /** Read the textual JSON projection from one MCP tool call. */ export const mcpTextValue = (result) => { @@ -10,6 +16,10 @@ export const mcpTextValue = (result) => { const jsonValue = (result) => JSON.parse(mcpTextValue(result)); +/** Sort canonical contract names using REA's code-point ordering. */ +export const sortVerifierNames = (names) => + [...names].sort(compareUnicodeCodePoints); + /** Read the complete canonical diagnostic from an MCP tool failure. */ export const requireMcpToolError = (result) => { if (result.isError !== true) throw new Error("Expected an MCP tool failure"); @@ -35,6 +45,44 @@ const operationResultKey = (contract) => { return evidence ? "normalized_result" : "result"; }; +/** Expand compact session metadata once at the verifier boundary. */ +export const expandMcpSessionStatus = (status) => ({ + ...status, + capabilities: expandProviderCapabilities(status.capabilities), + analysis_provider_candidates: expandProviderCandidates( + status.analysis_provider_candidates, + ), + tool_availability: expandToolAvailability(status.tool_availability), +}); + +/** Expand provider capability groups while retaining candidate identity/state. */ +export const expandProviderCandidates = (candidates) => + candidates.map((candidate) => ({ + ...candidate, + capabilities: expandProviderCapabilities( + providerCapabilitySet.array().parse(candidate.capabilities), + ), + })); + +/** Expand CLI provider capability groups at the same owning boundary. */ +export const expandProviderOutput = (providers) => ({ + ...providers, + capabilities: expandProviderCapabilities( + providerCapabilitySet.array().parse(providers.capabilities), + ), + analysis_provider_candidates: expandProviderCandidates( + providers.analysis_provider_candidates, + ), +}); + +/** Expand the CLI capabilities command's compact provider descriptor groups. */ +export const expandCapabilitiesOutput = (capabilities) => ({ + ...capabilities, + capabilities: expandProviderCapabilities( + providerCapabilitySet.array().parse(capabilities.capabilities), + ), +}); + /** Validate an Evidence-producing tool output and return its normalized result. */ export const requireMcpEvidenceResult = (result, operation) => { const contract = toolContract(operation); @@ -48,7 +96,10 @@ export const requireMcpLifecycleResult = (result, operation) => { const contract = toolContract(operation); if (operationResultKey(contract) !== "result") throw new Error(`${operation} requires the Evidence result extractor`); - return requireOutput(result, operation, contract).result; + const projected = requireOutput(result, operation, contract).result; + return operation === "binary_session" + ? expandMcpSessionStatus(projected) + : projected; }; /** Project a mixed verifier call using its exact named canonical output schema. */ @@ -58,7 +109,10 @@ export const requireMcpOperationResult = ( contract = toolContract(operation), ) => { const key = operationResultKey(contract); - return requireOutput(result, operation, contract)[key]; + const projected = requireOutput(result, operation, contract)[key]; + return operation === "binary_session" + ? expandMcpSessionStatus(projected) + : projected; }; /** Verify provider identity directly from the tool result. */ diff --git a/scripts/lib/native-value-e2e.mjs b/scripts/lib/native-value-e2e.mjs index e65700b06..513259b30 100644 --- a/scripts/lib/native-value-e2e.mjs +++ b/scripts/lib/native-value-e2e.mjs @@ -11,6 +11,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { parseEvidence } from "../../dist/domain/evidence.js"; import { analysisProfileSchema } from "../../dist/domain/analysisProfile.js"; import { nativeValueTraceSchema } from "../../dist/domain/native/nativeValueTrace.js"; +import { requireMcpLifecycleResult } from "./mcp-verifier-results.mjs"; import { verifyGhidraBoundaries } from "./real-ghidra-boundaries.mjs"; const defaultEntrypoint = fileURLToPath(new URL("../rea.mjs", import.meta.url)); @@ -172,13 +173,16 @@ async function verifyNativeValueAdapters( { timeout: 180000 }, ); assert.notEqual(opened.isError, true, JSON.stringify(opened)); - const inventory = await client.callTool({ + const inventoryReply = await client.callTool({ name: "binary_session", arguments: {}, }); - assert.notEqual(inventory.isError, true, JSON.stringify(inventory)); + const inventory = requireMcpLifecycleResult( + inventoryReply, + "binary_session", + ); assert.ok( - inventory.structuredContent?.result?.tool_availability.some( + inventory.tool_availability.some( (tool) => tool.name === "trace_native_values" && tool.available, ), "Native value tracing is unavailable in real provider discovery", @@ -267,11 +271,14 @@ async function verifyNativeValueAdapters( arguments: {}, }); assert.notEqual(closed.isError, true, JSON.stringify(closed)); - const status = await client.callTool({ + const statusReply = await client.callTool({ name: "binary_session", arguments: {}, }); - assert.equal(status.structuredContent?.result?.open, false); + assert.equal( + requireMcpLifecycleResult(statusReply, "binary_session").open, + false, + ); assert.deepEqual( (await readdir(env.TMPDIR)).filter((name) => name.startsWith("rea-ghidra-"), diff --git a/scripts/lib/verify-package-core.mjs b/scripts/lib/verify-package-core.mjs index 5371edffc..a78b516da 100644 --- a/scripts/lib/verify-package-core.mjs +++ b/scripts/lib/verify-package-core.mjs @@ -2,6 +2,10 @@ import { execFile } from "node:child_process"; import { lstat } from "node:fs/promises"; import { promisify } from "node:util"; import { Ajv2020 } from "ajv/dist/2020.js"; +import { + requireMcpLifecycleResult, + sortVerifierNames, +} from "./mcp-verifier-results.mjs"; /** Promisified child_process execution helper. */ export const exec = promisify(execFile); @@ -46,11 +50,10 @@ export const verifyCompleteToolCatalog = async (client, options) => { { name: "binary_session", arguments: {} }, options, ); - const availability = status.structuredContent?.result?.tool_availability; - if (!Array.isArray(availability)) - throw new Error("packaged MCP omitted tool availability"); - const expected = availability.map(({ name }) => name).sort(); - const observed = listed.tools.map(({ name }) => name).sort(); + const expanded = requireMcpLifecycleResult(status, "binary_session"); + const availability = expanded.tool_availability; + const expected = sortVerifierNames(availability.map(({ name }) => name)); + const observed = sortVerifierNames(listed.tools.map(({ name }) => name)); if (JSON.stringify(observed) !== JSON.stringify(expected)) throw new Error( "packaged MCP tool inventory diverged from session metadata", diff --git a/scripts/verify-claude-client.mjs b/scripts/verify-claude-client.mjs index 30bc66ba7..2c0d79bc0 100644 --- a/scripts/verify-claude-client.mjs +++ b/scripts/verify-claude-client.mjs @@ -5,11 +5,16 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import Ajv2020 from "ajv/dist/2020.js"; +import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; import { TOOL_CONTRACTS, toolContract, } from "../dist/contracts/toolContracts.js"; import { exec } from "./lib/verify-package-core.mjs"; +import { + expandMcpSessionStatus, + sortVerifierNames, +} from "./lib/mcp-verifier-results.mjs"; import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; import { createAnthropicModelFixture } from "./verify/clients/anthropic-model-fixture.mjs"; @@ -58,6 +63,8 @@ await writeFile( 'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n', ); let skillRequested = false; +let sessionRequested = false; +let sessionStatusSeen = false; let skillLoaded = false; let expectedSkillBody; let evidenceSeen = false; @@ -73,7 +80,7 @@ const containsText = (value, text) => { return Object.values(value).some((nested) => containsText(nested, text)); return false; }; -const extractEvidence = (value) => { +const extractInlineResult = (value, matches) => { if (typeof value === "string") { try { // Native Claude appends this resource hint to tool results. Original bodies remain intact in request artifacts. @@ -81,20 +88,16 @@ const extractEvidence = (value) => { /\n\n\n\d+ tokens left<\/total_tokens>\n<\/system-reminder>$/u, "", ); - return extractEvidence(JSON.parse(json)); + return extractInlineResult(JSON.parse(json), matches); } catch { return undefined; } } if (value && typeof value === "object") { - if ( - typeof value.evidence_id === "string" && - value.normalized_result?.graph?.nodes?.length > 0 - ) - return value; + if (matches(value)) return value; for (const nested of Object.values(value)) { - const evidence = extractEvidence(nested); - if (evidence) return evidence; + const result = extractInlineResult(nested, matches); + if (result) return result; } } return undefined; @@ -125,10 +128,12 @@ const fixture = await createAnthropicModelFixture({ 0, -"analyze_javascript_application".length, ); - const expected = TOOL_CONTRACTS.map((t) => prefix + t.name).sort(); + const expected = sortVerifierNames( + TOOL_CONTRACTS.map((t) => prefix + t.name), + ); const reaTools = tools.filter((t) => t.name.startsWith(prefix)); assert.deepEqual( - reaTools.map((t) => t.name).sort(), + sortVerifierNames(reaTools.map((t) => t.name)), expected, "Claude must forward the complete REA catalog after connection", ); @@ -146,11 +151,61 @@ const fixture = await createAnthropicModelFixture({ schemasChecked++; } } + if (mode === "call" && !sessionStatusSeen) { + const observedStatus = extractInlineResult( + body.messages, + (value) => + Boolean(value.result?.server_identity) && + Array.isArray(value.result?.tool_availability?.groups), + ); + if (observedStatus) { + const parsedStatus = + toolContract("binary_session").outputSchema.parse(observedStatus); + const status = { + ...parsedStatus, + result: expandMcpSessionStatus(parsedStatus.result), + }; + assert.deepEqual( + status.result.server_identity.catalog, + CATALOG_IDENTITY, + "binary_session must return the exact compact catalog counts and digests", + ); + assert.equal( + Object.hasOwn(status.result.server_identity.catalog, "tools"), + false, + "binary_session must not inline the duplicated canonical tool catalog", + ); + assert.deepEqual( + sortVerifierNames( + status.result.tool_availability.map((entry) => entry.name), + ), + sortVerifierNames(TOOL_CONTRACTS.map((contract) => contract.name)), + "binary_session must retain complete canonical tool availability", + ); + sessionStatusSeen = true; + } else { + assert( + !sessionRequested, + "Claude must return the inline binary_session result before continuing", + ); + const session = tools.find( + (tool) => tool.name === `${prefix}binary_session`, + ); + assert(session, "Claude must expose the REA binary_session tool"); + sessionRequested = true; + return call(session.name, {}); + } + } skillLoaded ||= expectedSkillBody !== undefined && containsText(body.messages, expectedSkillBody) && containsText(body.messages, skillDirectory); - const observed = extractEvidence(body.messages); + const observed = extractInlineResult( + body.messages, + (value) => + typeof value.evidence_id === "string" && + value.normalized_result?.graph?.nodes?.length > 0, + ); if (observed) { const evidence = toolContract( "analyze_javascript_application", @@ -184,7 +239,7 @@ const fixture = await createAnthropicModelFixture({ await writeFile(join(lab, "claude-evidence.json"), bytes); evidenceSeen = true; } - if (mode === "chat" || evidenceSeen) { + if (mode === "chat" || (sessionStatusSeen && evidenceSeen)) { finalSeen = true; return [{ type: "text", text: "REA_CLIENT_COMPATIBILITY_OK" }]; } @@ -294,6 +349,7 @@ try { "Skill", "WaitForMcpServers", "mcp__rea__analyze_javascript_application", + ...(mode === "call" ? ["mcp__rea__binary_session"] : []), ], }, }), @@ -324,7 +380,7 @@ try { assert.equal(parsed.result, "REA_CLIENT_COMPATIBILITY_OK"); assert.deepEqual(parsed.permission_denials, []); assert(finalSeen); - if (mode === "call") assert(evidenceSeen && skillLoaded); + if (mode === "call") assert(evidenceSeen && skillLoaded && sessionStatusSeen); } catch (cause) { failure = cause; if (typeof cause?.stdout === "string") @@ -355,6 +411,7 @@ try { schemasChecked, pendingServerWaits, evidenceSeen, + sessionStatusSeen, skillLoaded, artifactDigest, finalSeen, diff --git a/scripts/verify-package-capabilities.mjs b/scripts/verify-package-capabilities.mjs index 2137a4c5a..acf848974 100644 --- a/scripts/verify-package-capabilities.mjs +++ b/scripts/verify-package-capabilities.mjs @@ -1,13 +1,19 @@ import { json, run } from "./lib/verify-package-core.mjs"; +import { + expandCapabilitiesOutput, + expandProviderOutput, +} from "./lib/mcp-verifier-results.mjs"; /** Verify packaged capabilities, providers, and non-Linux search results. */ export async function verifyPackageCapabilitiesAndSearch({ cli, environment }) { - const capabilities = json( - await run(cli, ["capabilities", "--json"], environment), + const capabilities = expandCapabilitiesOutput( + json(await run(cli, ["capabilities", "--json"], environment)), ); if (!Array.isArray(capabilities.capabilities)) throw new Error("packaged capabilities CLI failed"); - const providers = json(await run(cli, ["providers", "--json"], environment)); + const providers = expandProviderOutput( + json(await run(cli, ["providers", "--json"], environment)), + ); const candidates = providers.analysis_provider_candidates; if ( !Array.isArray(candidates) || diff --git a/scripts/verify-package-mcp.mjs b/scripts/verify-package-mcp.mjs index 7bb22715d..0d03abab1 100644 --- a/scripts/verify-package-mcp.mjs +++ b/scripts/verify-package-mcp.mjs @@ -1,4 +1,7 @@ -import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpToolError, +} from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { join } from "node:path"; import { execFile } from "node:child_process"; @@ -21,14 +24,14 @@ const verifyMcpToolsAndPrompts = async (client, mcpOptions) => { }; const verifyMcpTargetFree = async (client, mcpOptions) => { - const status = await client.callTool( + const result = await client.callTool( { name: "binary_session", arguments: {} }, mcpOptions, ); - const currentDocument = - status.structuredContent?.result?.tool_availability?.find( - ({ name }) => name === "current_document", - ); + const status = requireMcpLifecycleResult(result, "binary_session"); + const currentDocument = status.tool_availability.find( + ({ name }) => name === "current_document", + ); if ( currentDocument?.available !== false || currentDocument.reason !== "target_required" @@ -67,15 +70,13 @@ const verifyMcpOpenAndBind = async (client, mcpOptions) => { ); if (opened.isError === true) throw new Error("packaged MCP could not open a binary"); - const providerStatusEnvelope = json( - prompts.mcpText( - await client.callTool( - { name: "binary_session", arguments: {} }, - mcpOptions, - ), + const providerStatus = requireMcpLifecycleResult( + await client.callTool( + { name: "binary_session", arguments: {} }, + mcpOptions, ), + "binary_session", ); - const providerStatus = providerStatusEnvelope.result; if ( providerStatus.analysis_provider_binding?.provider?.id !== "hopper" || providerStatus.analysis_provider_binding?.selection_source !== "request" diff --git a/scripts/verify-packaged-ghidra-windows.mjs b/scripts/verify-packaged-ghidra-windows.mjs index b38072c8c..2c5f15e62 100644 --- a/scripts/verify-packaged-ghidra-windows.mjs +++ b/scripts/verify-packaged-ghidra-windows.mjs @@ -1,7 +1,10 @@ #!/usr/bin/env node import assert from "node:assert/strict"; -import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { + expandProviderOutput, + requireMcpOperationResult, +} from "./lib/mcp-verifier-results.mjs"; import { verifyGhidraCompactReferences } from "./lib/real-ghidra-compact-references.mjs"; import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; @@ -230,14 +233,16 @@ const assertCleanup = async () => { assert.equal(entries.length, 0, "Provider runtime cleanup left files."); }; try { - const providers = JSON.parse( - ( - await exec(process.execPath, [entry, "providers", "--json"], { - env: environment, - cwd: callerDirectory, - timeout: 30_000, - }) - ).stdout, + const providers = expandProviderOutput( + JSON.parse( + ( + await exec(process.execPath, [entry, "providers", "--json"], { + env: environment, + cwd: callerDirectory, + timeout: 30_000, + }) + ).stdout, + ), ); assert.equal(providers.analysis_provider_binding, null); assert.ok(Array.isArray(providers.analysis_provider_candidates)); diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs index 12859d4bb..8c0281d56 100644 --- a/scripts/verify-published-package.mjs +++ b/scripts/verify-published-package.mjs @@ -10,6 +10,11 @@ import { pathToFileURL } from "node:url"; import { lt, rcompare } from "semver"; import { CATALOG_IDENTITY } from "../dist/catalogIdentity.js"; +import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; +import { + requireMcpLifecycleResult, + sortVerifierNames, +} from "./lib/mcp-verifier-results.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; const verifierRun = createVerifierRun(); @@ -54,27 +59,39 @@ try { await client.connect(transport); const tools = await client.listTools(); const toolNames = tools.tools.map(({ name }) => name); - const canonicalToolNames = new Set( - CATALOG_IDENTITY.tools.map(({ name }) => name), - ); + const canonicalToolNames = new Set(TOOL_CONTRACTS.map(({ name }) => name)); const unknownToolNames = toolNames.filter( (name) => !canonicalToolNames.has(name), ); + const publishedToolNames = new Set(toolNames); + const missingToolNames = [...canonicalToolNames].filter( + (name) => !publishedToolNames.has(name), + ); publishedToolCount = toolNames.length; if ( publishedToolCount === 0 || - new Set(toolNames).size !== publishedToolCount || - unknownToolNames.length > 0 + publishedToolNames.size !== publishedToolCount || + unknownToolNames.length > 0 || + missingToolNames.length > 0 ) throw new Error( - `Published MCP exposed an invalid capability-scoped tool projection (${String(publishedToolCount)} tools, unknown: ${unknownToolNames.join(", ") || "none"})`, + `Published MCP exposed an invalid canonical tool catalog (${String(publishedToolCount)} tools, unknown: ${unknownToolNames.join(", ") || "none"}, missing: ${missingToolNames.join(", ") || "none"})`, ); - const status = await client.callTool({ + const statusReply = await client.callTool({ name: "binary_session", arguments: {}, }); - if (status.isError === true) - throw new Error("Published target-free MCP session check failed"); + const status = requireMcpLifecycleResult(statusReply, "binary_session"); + const availabilityNames = sortVerifierNames( + status.tool_availability.map(({ name }) => name), + ); + if ( + JSON.stringify(availabilityNames) !== + JSON.stringify(sortVerifierNames(canonicalToolNames)) + ) + throw new Error( + "Published binary_session omitted canonical tool availability", + ); update = await verifyPublishedUpdate({ version, canaryRoot }); } catch (cause) { throw new Error(`Published package verification failed: ${stderr}`, { diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index 97ad4ac9e..12400cb94 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -30,6 +30,7 @@ import { requireMcpOperationResult, requireMcpLifecycleResult, requireMcpEvidenceResult, + sortVerifierNames, requireWorkflowEvidenceProvider, } from "./lib/mcp-verifier-results.mjs"; import { @@ -283,8 +284,10 @@ try { const availableTools = initialStatus.tool_availability; if (!Array.isArray(availableTools)) throw new Error("The real session omitted tool availability"); - const expectedNames = availableTools.map(({ name }) => name).sort(); - const actualNames = listed.tools.map(({ name }) => name).sort(); + const expectedNames = sortVerifierNames( + availableTools.map(({ name }) => name), + ); + const actualNames = sortVerifierNames(listed.tools.map(({ name }) => name)); if (JSON.stringify(actualNames) !== JSON.stringify(expectedNames)) { const expected = new Set(expectedNames); const actual = new Set(actualNames); diff --git a/scripts/verify/android/failure-recovery.mjs b/scripts/verify/android/failure-recovery.mjs index adcf41d74..8e1816aed 100644 --- a/scripts/verify/android/failure-recovery.mjs +++ b/scripts/verify/android/failure-recovery.mjs @@ -1,4 +1,8 @@ -import { requireMcpToolError } from "../../lib/mcp-verifier-results.mjs"; +import { + requireMcpLifecycleResult, + requireMcpToolError, + sortVerifierNames, +} from "../../lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; import { access, @@ -273,22 +277,23 @@ const verifyAndroidReadiness = async ({ try { await client.connect(transport); const catalog = await client.listTools(); - const catalogNames = catalog.tools.map(({ name }) => name).sort(); - const status = await client.callTool({ + const catalogNames = sortVerifierNames( + catalog.tools.map(({ name }) => name), + ); + const statusReply = await client.callTool({ name: "binary_session", arguments: {}, }); - assert.notEqual(status.isError, true, JSON.stringify(status)); - const availability = status.structuredContent?.result?.tool_availability; - assert.ok(Array.isArray(availability)); + const status = requireMcpLifecycleResult(statusReply, "binary_session"); + const availability = status.tool_availability; assert.deepEqual( - availability.map(({ name }) => name).sort(), - TOOL_CONTRACTS.map(({ name }) => name).sort(), + sortVerifierNames(availability.map(({ name }) => name)), + sortVerifierNames(TOOL_CONTRACTS.map(({ name }) => name)), "Readiness must retain one availability record for the complete canonical catalog", ); assert.deepEqual( catalogNames, - TOOL_CONTRACTS.map(({ name }) => name).sort(), + sortVerifierNames(TOOL_CONTRACTS.map(({ name }) => name)), "Readiness must not truncate the SDK tool catalog", ); const byName = new Map(availability.map((item) => [item.name, item])); diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index a267f2b58..6508d3142 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it } from "vitest"; -import { buildCapabilityInventory } from "./CapabilityInventory.js"; +import { + buildCapabilityInventory, + compactToolAvailability, +} from "./CapabilityInventory.js"; +import { TOOL_CONTRACTS } from "../contracts/toolContracts.js"; +import { + expandToolAvailability, + toolAvailabilityInventory, + type ToolAvailability, +} from "../contracts/toolOutputSchemaPrimitives.js"; it("reports module syntax and native resolution availability separately", () => { const input = { open: false, capabilities: [] }; @@ -55,22 +64,25 @@ const status = ( open: options.open ?? false, ...(options.kind === undefined ? {} : { kind: options.kind }), ...(options.format === undefined ? {} : { format: options.format }), - capabilities: (options.capabilities ?? []).map((capability) => ({ - ...capability, - availability_code: capability.available - ? null - : (capability.availability_code ?? null), - effects: { - mutates_artifact: false, - launches_process: false, - may_show_ui: false, - may_access_network: false, - may_write_filesystem: false, - changes_permissions: false, - requires_root: false, - }, - limitations: [], - })), + capabilities: + options.capabilities === undefined || options.capabilities.length === 0 + ? [] + : [ + { + provider: { id: "fixture", name: "Fixture", version: "1" }, + shared_limitations: [], + groups: options.capabilities.map((capability) => ({ + operations: [capability.operation], + effects: [], + limitations: [], + available: capability.available, + reason: capability.reason, + availability_code: capability.available + ? null + : (capability.availability_code ?? null), + })), + }, + ], }); const entry = ( @@ -171,6 +183,152 @@ describe("capability failure reasons", () => { }); }); +describe("compact tool availability", () => { + it("round-trips complete facts, omitted modes, and per-group client requirements", () => { + const full = buildCapabilityInventory(status(), enabledPolicy); + const alternateRequirements = { + required: ["roots"], + optional: [], + missing_required: ["roots"], + missing_optional: [], + } satisfies ToolAvailability["client_requirements"]; + const withOverride = full.map((entry, index) => + index === 1 + ? { + ...entry, + available: false as const, + reason: "client_capability_missing" as const, + remediation: "Use an MCP client with roots support.", + client_requirements: alternateRequirements, + } + : entry, + ); + const compact = compactToolAvailability(withOverride); + const expanded = expandToolAvailability(compact).toSorted((left, right) => + left.name.localeCompare(right.name), + ); + + expect(expanded).toEqual( + withOverride.toSorted((left, right) => + left.name.localeCompare(right.name), + ), + ); + expect(compact.client_requirements).toEqual( + withOverride[0]?.client_requirements, + ); + expect( + compact.groups.find((group) => group.client_requirements !== undefined) + ?.client_requirements, + ).toEqual(alternateRequirements); + expect(compact.groups.every((group) => group.names.length > 0)).toBe(true); + const names = compact.groups.flatMap((group) => group.names).sort(); + expect(names).toEqual(TOOL_CONTRACTS.map(({ name }) => name).sort()); + expect(new Set(names).size).toBe(names.length); + expect(expanded.some((entry) => !Object.hasOwn(entry, "modes"))).toBe(true); + }); + + it("preserves an empty inventory with an explicit empty client baseline", () => { + const compact = compactToolAvailability([]); + expect(compact).toEqual({ + client_requirements: { + required: [], + optional: [], + missing_required: [], + missing_optional: [], + }, + groups: [], + }); + expect(expandToolAvailability(compact)).toEqual([]); + }); + + it("rejects malformed compact facts and accepts an equivalent explicit baseline", () => { + const compact = compactToolAvailability( + buildCapabilityInventory(status(), enabledPolicy), + ); + const firstGroup = compact.groups[0]; + expect(firstGroup).toBeDefined(); + if (firstGroup === undefined) throw new Error("Expected tool groups"); + const availableIndex = compact.groups.findIndex((group) => group.available); + if (availableIndex < 0) throw new Error("Expected an available tool group"); + + const malformedInventories = [ + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 + ? { ...group, effects: [...group.effects, "unknown_effect"] } + : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 + ? { + ...group, + annotations: [...group.annotations, "unknown_hint"], + } + : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 ? { ...group, names: [] } : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === 0 ? { ...group, names: [""] } : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === availableIndex + ? { ...group, reason: "provider_missing" } + : group, + ), + }, + { + ...compact, + groups: compact.groups.map((group, index) => + index === availableIndex + ? { ...group, remediation: "unexpected remediation" } + : group, + ), + }, + ]; + for (const malformed of malformedInventories) + expect(toolAvailabilityInventory.safeParse(malformed).success).toBe( + false, + ); + + const inheritedGroup = compact.groups.find( + (group) => group.client_requirements === undefined, + ); + expect(inheritedGroup).toBeDefined(); + if (inheritedGroup === undefined) + throw new Error("Expected an inherited client-requirements group"); + const explicitBaseline = { + ...compact, + groups: compact.groups.map((group) => + group === inheritedGroup + ? { ...group, client_requirements: compact.client_requirements } + : group, + ), + }; + const parsedOverride = + toolAvailabilityInventory.safeParse(explicitBaseline); + expect(parsedOverride.success).toBe(true); + if (parsedOverride.success) + expect(expandToolAvailability(parsedOverride.data)).toEqual( + expandToolAvailability(compact), + ); + }); +}); + describe("partial navigation availability", () => { it("allows explicit documents when the provider has no current-document getter", () => { const capabilities = [ @@ -258,6 +416,36 @@ describe("partial navigation availability", () => { remediation: "Navigation requires macOS.", }); }); + + it("lists shared missing operations once in combined remediation", () => { + const availability = entry( + "get_navigation_context", + status({ open: true, kind: "executable" }), + ); + + expect(availability).toMatchObject({ + reason: "provider_missing", + remediation: + "Configure a provider for one of: current_document, current_address, resolve_containing_procedure.", + modes: [ + { + name: "current_selection", + missing_operations: [ + "current_document", + "current_address", + "resolve_containing_procedure", + ], + }, + { + name: "explicit_document", + missing_operations: [ + "current_address", + "resolve_containing_procedure", + ], + }, + ], + }); + }); }); it("reports host denial before target mismatch but keeps composed workflow prerequisites", () => { diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index d11a582b2..ec3842376 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -5,9 +5,14 @@ import type { } from "./OptionalObservationProviders.js"; import { - providerCapability, + expandProviderCapabilities, + providerCapabilitySet, + toolAvailabilityInventory, + TOOL_ANNOTATION_NAMES, + TOOL_EFFECT_NAMES, type ClientFeatureAvailability, type ProviderCapability, + type ToolAvailabilityInventory, type ToolAvailability, type ToolAvailabilityMode, type ToolUnavailabilityReason, @@ -26,7 +31,9 @@ const statusSchema = z.object({ open: z.boolean(), kind: z.enum(["executable", "database", "archive", "artifact"]).optional(), format: z.string().optional(), - capabilities: z.array(providerCapability), + capabilities: z + .array(providerCapabilitySet) + .transform(expandProviderCapabilities), }); type ToolAvailabilityReason = "available" | ToolUnavailabilityReason; @@ -214,6 +221,60 @@ export const buildCapabilityInventory = ( }).sort((left, right) => compareUnicodeCodePoints(left.name, right.name)); }; +type ClientRequirements = ToolAvailability["client_requirements"]; +type MutableAvailabilityGroup = Omit< + ToolAvailabilityInventory["groups"][number], + "names" +> & { names: string[] }; + +const EMPTY_CLIENT_REQUIREMENTS: ClientRequirements = { + required: [], + optional: [], + missing_required: [], + missing_optional: [], +}; + +/** Compact repeated metadata while retaining every operation's current availability inline. */ +export const compactToolAvailability = ( + availability: readonly ToolAvailability[], +): ToolAvailabilityInventory => { + const clientRequirements = + availability[0]?.client_requirements ?? EMPTY_CLIENT_REQUIREMENTS; + const groups = new Map(); + + for (const entry of availability) { + const effects = TOOL_EFFECT_NAMES.filter((name) => entry.effects[name]); + const annotations = TOOL_ANNOTATION_NAMES.filter( + (name) => entry.annotations[name], + ); + const groupFacts = { + surface: entry.surface, + effects, + annotations, + available: entry.available, + reason: entry.reason, + remediation: entry.remediation, + ...(entry.default_mode_available === undefined + ? {} + : { default_mode_available: entry.default_mode_available }), + ...(entry.modes === undefined ? {} : { modes: [...entry.modes] }), + ...(JSON.stringify(entry.client_requirements) === + JSON.stringify(clientRequirements) + ? {} + : { client_requirements: entry.client_requirements }), + }; + const key = JSON.stringify(groupFacts); + const existing = groups.get(key); + if (existing) existing.names.push(entry.name); + else groups.set(key, { ...groupFacts, names: [entry.name] }); + } + + return toolAvailabilityInventory.parse({ + client_requirements: clientRequirements, + groups: [...groups.values()], + }); +}; + const availabilityFor = (context: AvailabilityContext): Availability => { // These built-in workflows accept a caller-selected endpoint; they do not load the optional passive provider. if ( @@ -626,14 +687,15 @@ const navigationContextAvailability = ( }; if (availableMode !== undefined) return { ...facts, reason: "available", remediation: null }; + const missingOperations = [ + ...new Set(modes.flatMap((mode) => mode.missing_operations)), + ]; return { ...facts, reason: failure?.reason ?? "provider_missing", remediation: specificFailureRemediation ?? - `Configure a provider for one of: ${modes - .flatMap((mode) => mode.missing_operations) - .join(", ")}.`, + `Configure a provider for one of: ${missingOperations.join(", ")}.`, }; }; diff --git a/src/application/binary/BinarySessionStatus.test.ts b/src/application/binary/BinarySessionStatus.test.ts new file mode 100644 index 000000000..a72b7ed80 --- /dev/null +++ b/src/application/binary/BinarySessionStatus.test.ts @@ -0,0 +1,391 @@ +import { describe, expect, it } from "vitest"; + +import type { + CapabilityDescriptor, + ProviderIdentity, +} from "../AnalysisProvider.js"; +import { + expandProviderCapabilities, + type ProviderCapabilitySet, +} from "../../contracts/toolOutputSchemaPrimitives.js"; +import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js"; +import type { AnalysisProviderCandidateStatus } from "./AnalysisProviderRegistry.js"; +import type { + SessionProviderRoute, + SessionProviderRouter, +} from "./SessionProviderRouter.js"; +import { binarySessionStatus } from "./BinarySessionStatus.js"; + +type ProjectedStatus = { + capabilities: ProviderCapabilitySet[]; + analysis_provider_candidates: { + provider: ProviderIdentity; + availability: unknown; + target_support: unknown; + selected: boolean; + capabilities: ProviderCapabilitySet[]; + }[]; +}; + +const providerA: ProviderIdentity = { + id: "provider-a", + name: "Provider A", + version: "1.0", +}; +const providerB: ProviderIdentity = { + id: "provider-b", + name: "Provider B", + version: "2.0", +}; + +const MUTATING_EFFECTS = { + mutates_artifact: true, + launches_process: false, + may_show_ui: false, + may_access_network: false, + may_write_filesystem: false, + changes_permissions: false, + requires_root: false, +}; + +type DescriptorOptions = { + readonly launchesProcess?: boolean; +} & ( + | { + readonly available?: true; + readonly reason?: never; + readonly availabilityCode?: never; + } + | { + readonly available: false; + readonly reason?: string; + readonly availabilityCode?: "architecture_unsupported"; + } +); + +const descriptor = ( + provider: ProviderIdentity, + operation: CapabilityDescriptor["operation"], + limitations: readonly string[], + options: DescriptorOptions = {}, +): CapabilityDescriptor => { + const availability = + options.available === false + ? { + available: false as const, + reason: options.reason ?? "Unavailable", + ...(options.availabilityCode === undefined + ? {} + : { availabilityCode: options.availabilityCode }), + } + : { available: true as const, reason: null }; + return { + provider, + operation, + ...availability, + effects: { + mutatesArtifact: true, + launchesProcess: options.launchesProcess ?? false, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: false, + changesPermissions: false, + requiresRoot: false, + }, + limitations, + }; +}; + +const candidate = ( + provider: ProviderIdentity, + capabilities: readonly CapabilityDescriptor[], +): AnalysisProviderCandidateStatus => ({ + provider, + availability: { + status: "available", + code: null, + reason: null, + diagnostics: {}, + }, + targetSupport: { + status: "unknown", + code: null, + reason: "No target is open.", + diagnostics: {}, + }, + selected: true, + capabilities, +}); + +const project = ( + routeCapabilities: readonly CapabilityDescriptor[], + candidates: readonly AnalysisProviderCandidateStatus[], + runtimeUnavailability: ReadonlyMap< + string, + { readonly reason: string } + > = new Map(), +): ProjectedStatus => { + const route = { + capabilities: new Map( + routeCapabilities.map((capability) => [capability.operation, capability]), + ), + binding: null, + } as unknown as SessionProviderRoute; + const router = { + candidateStatuses: () => candidates, + } as unknown as SessionProviderRouter; + return binarySessionStatus({ + target: undefined, + route, + router, + runtimeUnavailability, + runId: undefined, + runtimeLineageSnapshots: [], + requestActivitySnapshots: [], + }) as unknown as ProjectedStatus; +}; + +const expandCanonicalCapabilities = (sets: readonly ProviderCapabilitySet[]) => + [...expandProviderCapabilities(sets)].sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ); + +const selectedCapabilities = [ + descriptor(providerA, "xrefs", ["Shared limitation"]), + descriptor(providerA, "procedure_info", ["Shared limitation"]), + descriptor(providerA, "read_bytes", ["Shared limitation", "beta detail"]), +]; +const otherCapabilities = [ + descriptor(providerB, "read_bytes", ["Different limitation"], { + launchesProcess: true, + available: false, + reason: "Architecture unsupported", + availabilityCode: "architecture_unsupported", + }), +]; + +const expectedRouteCapabilities: ProviderCapabilitySet[] = [ + { + provider: providerA, + shared_limitations: ["Shared limitation"], + groups: [ + { + operations: ["procedure_info"], + effects: ["mutates_artifact"], + limitations: [], + available: false, + reason: "Runtime operation failure", + availability_code: null, + }, + { + operations: ["read_bytes"], + effects: ["mutates_artifact"], + limitations: ["beta detail"], + available: true, + reason: null, + availability_code: null, + }, + { + operations: ["xrefs"], + effects: ["mutates_artifact"], + limitations: [], + available: true, + reason: null, + availability_code: null, + }, + ], + }, +]; +const expectedExpandedRouteCapabilities = [ + { + provider: providerA, + operation: "procedure_info", + available: false, + reason: "Runtime operation failure", + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, + { + provider: providerA, + operation: "read_bytes", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation", "beta detail"], + }, + { + provider: providerA, + operation: "xrefs", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, +]; +const expectedCandidateCapabilities: ProviderCapabilitySet[] = [ + { + provider: providerA, + shared_limitations: ["Shared limitation"], + groups: [ + { + operations: ["procedure_info", "xrefs"], + effects: ["mutates_artifact"], + limitations: [], + available: true, + reason: null, + availability_code: null, + }, + { + operations: ["read_bytes"], + effects: ["mutates_artifact"], + limitations: ["beta detail"], + available: true, + reason: null, + availability_code: null, + }, + ], + }, + { + provider: providerB, + shared_limitations: ["Different limitation"], + groups: [ + { + operations: ["read_bytes"], + effects: ["mutates_artifact", "launches_process"], + limitations: [], + available: false, + reason: "Architecture unsupported", + availability_code: "architecture_unsupported", + }, + ], + }, +]; +const expectedCandidateStatusFacts = [ + expect.objectContaining({ + provider: providerA, + target_support: { + status: "unknown", + code: null, + reason: "No target is open.", + diagnostics: {}, + }, + capabilities: [expectedCandidateCapabilities[0]], + }), + expect.objectContaining({ + provider: providerB, + capabilities: [expectedCandidateCapabilities[1]], + }), +]; +const expectedExpandedCandidateCapabilities = [ + [ + { + provider: providerA, + operation: "procedure_info", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, + { + provider: providerA, + operation: "read_bytes", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation", "beta detail"], + }, + { + provider: providerA, + operation: "xrefs", + available: true, + reason: null, + availability_code: null, + effects: MUTATING_EFFECTS, + limitations: ["Shared limitation"], + }, + ], + [ + { + provider: providerB, + operation: "read_bytes", + available: false, + reason: "Architecture unsupported", + availability_code: "architecture_unsupported", + effects: { ...MUTATING_EFFECTS, launches_process: true }, + limitations: ["Different limitation"], + }, + ], +]; + +describe("binary session provider capability projection", () => { + it("preserves provider scope and route overlays while factoring exact prefixes", () => { + const selectedBefore = structuredClone(selectedCapabilities); + const otherBefore = structuredClone(otherCapabilities); + const status = project( + selectedCapabilities, + [ + candidate(providerA, selectedCapabilities), + candidate(providerB, otherCapabilities), + ], + new Map([["procedure_info", { reason: "Runtime operation failure" }]]), + ); + expect(selectedCapabilities).toEqual(selectedBefore); + expect(otherCapabilities).toEqual(otherBefore); + expect(status.capabilities).toEqual(expectedRouteCapabilities); + expect(expandCanonicalCapabilities(status.capabilities)).toEqual( + expectedExpandedRouteCapabilities, + ); + expect(status.analysis_provider_candidates).toEqual( + expectedCandidateStatusFacts, + ); + const selectedCandidate = status.analysis_provider_candidates[0]; + const otherCandidate = status.analysis_provider_candidates[1]; + expect(selectedCandidate).toBeDefined(); + expect(otherCandidate).toBeDefined(); + expect( + expandCanonicalCapabilities(selectedCandidate?.capabilities ?? []), + ).toEqual(expectedExpandedCandidateCapabilities[0]); + expect( + expandCanonicalCapabilities(otherCandidate?.capabilities ?? []), + ).toEqual(expectedExpandedCandidateCapabilities[1]); + }); + + it("keeps empty route and candidate capability collections empty", () => { + const status = project([], [candidate(providerA, [])]); + expect(status.capabilities).toEqual([]); + expect(status.analysis_provider_candidates[0]?.capabilities).toEqual([]); + }); + + it("preserves repeated limitations in their original order", () => { + const capabilities = [ + descriptor(providerA, "procedure_info", [ + "Repeated", + "Repeated", + "left detail", + ]), + descriptor(providerA, "xrefs", ["Repeated", "Repeated", "right detail"]), + ]; + const status = project(capabilities, []); + + expect(status.capabilities[0]).toMatchObject({ + shared_limitations: ["Repeated", "Repeated"], + groups: [ + { operations: ["procedure_info"], limitations: ["left detail"] }, + { operations: ["xrefs"], limitations: ["right detail"] }, + ], + }); + expect( + expandCanonicalCapabilities(status.capabilities).map( + ({ limitations }) => limitations, + ), + ).toEqual([ + ["Repeated", "Repeated", "left detail"], + ["Repeated", "Repeated", "right detail"], + ]); + }); +}); diff --git a/src/application/binary/BinarySessionStatus.ts b/src/application/binary/BinarySessionStatus.ts index 8fbe4aa2b..4794bed49 100644 --- a/src/application/binary/BinarySessionStatus.ts +++ b/src/application/binary/BinarySessionStatus.ts @@ -4,9 +4,14 @@ import type { JsonValue } from "../../domain/jsonValue.js"; import type { ProviderOperationHealth } from "../../domain/providerOperationHealth.js"; import type { CapabilityDescriptor, + ProviderIdentity, ProviderRequestActivitySnapshot, ProviderRuntimeLineageSnapshot, } from "../AnalysisProvider.js"; +import { + PROVIDER_EFFECT_NAMES, + type ProviderCapabilitySet, +} from "../../contracts/toolOutputSchemaPrimitives.js"; import type { SessionProviderRoute, SessionProviderRouter, @@ -38,7 +43,7 @@ export const binarySessionStatus = ({ requestActivitySnapshots, providerOperationHealth, }: BinarySessionStatusInput): JsonValue => { - const capabilities = [...route.capabilities.values()] + const routeCapabilities = [...route.capabilities.values()] .filter(({ operation }) => operation !== "inventory_artifact") .sort((left, right) => compareUnicodeCodePoints(left.operation, right.operation), @@ -52,8 +57,8 @@ export const binarySessionStatus = ({ available: false as const, reason: unavailable.reason, }; - }) - .map(capabilityStatus); + }); + const capabilities = providerCapabilitySets(routeCapabilities); const analysisProviderBinding = route.binding === null ? null @@ -69,13 +74,14 @@ export const binarySessionStatus = ({ availability: structuredClone(candidate.availability), target_support: structuredClone(candidate.targetSupport), selected: candidate.selected, - capabilities: candidate.capabilities - .slice() - .filter(({ operation }) => operation !== "inventory_artifact") - .sort((left, right) => - compareUnicodeCodePoints(left.operation, right.operation), - ) - .map(capabilityStatus), + capabilities: providerCapabilitySets( + candidate.capabilities + .slice() + .filter(({ operation }) => operation !== "inventory_artifact") + .sort((left, right) => + compareUnicodeCodePoints(left.operation, right.operation), + ), + ), })); const common = { capabilities, @@ -187,17 +193,109 @@ const processLineageStatus = ( })), }; -const capabilityStatus = ( - descriptor: Omit & { - readonly available: boolean; - readonly reason: string | null; - }, -) => ({ - operation: descriptor.operation, - available: descriptor.available, - reason: descriptor.reason, - availability_code: descriptor.availabilityCode ?? null, - effects: { +type CapabilityEffectName = (typeof PROVIDER_EFFECT_NAMES)[number]; +type CapabilityAvailabilityCode = Exclude< + CapabilityDescriptor["availabilityCode"], + undefined +>; +type CapabilityProjection = Omit & + ( + | { readonly available: true; readonly reason: null } + | { readonly available: false; readonly reason: string } + ); +type ProviderCapabilityGroup = ProviderCapabilitySet["groups"][number]; + +/** Group inline capability facts without merging distinct providers or states. */ +const providerCapabilitySets = ( + descriptors: readonly CapabilityProjection[], +): ProviderCapabilitySet[] => { + const providers = new Map< + string, + { + readonly identity: ProviderIdentity; + readonly descriptors: CapabilityProjection[]; + } + >(); + for (const descriptor of descriptors) { + const key = JSON.stringify([ + descriptor.provider.id, + descriptor.provider.name, + descriptor.provider.version, + ]); + const provider = providers.get(key); + if (provider === undefined) + providers.set(key, { + identity: descriptor.provider, + descriptors: [descriptor], + }); + else provider.descriptors.push(descriptor); + } + return [...providers.values()].map(({ identity, descriptors: owned }) => { + const sharedLimitations = commonLimitationsPrefix(owned); + const groups = new Map(); + for (const descriptor of owned) { + const effects = enabledCapabilityEffects(descriptor); + const limitations = descriptor.limitations.slice( + sharedLimitations.length, + ); + const availabilityCode: CapabilityAvailabilityCode | null = + descriptor.availabilityCode ?? null; + const status = descriptor.available + ? { + available: true as const, + reason: null, + availability_code: null, + } + : { + available: false as const, + reason: descriptor.reason, + availability_code: availabilityCode, + }; + const key = JSON.stringify([ + effects, + limitations, + status.available, + status.reason, + status.availability_code, + ]); + const group = groups.get(key); + if (group === undefined) + groups.set(key, { + operations: [descriptor.operation], + effects, + limitations, + ...status, + }); + else group.operations.push(descriptor.operation); + } + return { + provider: providerSummary(identity), + shared_limitations: sharedLimitations, + groups: [...groups.values()], + }; + }); +}; + +const commonLimitationsPrefix = ( + descriptors: readonly CapabilityProjection[], +): string[] => { + const first = descriptors[0]?.limitations; + if (first === undefined) return []; + let length = first.length; + for (const descriptor of descriptors.slice(1)) { + length = Math.min(length, descriptor.limitations.length); + let index = 0; + while (index < length && first[index] === descriptor.limitations[index]) + index++; + length = index; + } + return first.slice(0, length); +}; + +const enabledCapabilityEffects = ( + descriptor: CapabilityProjection, +): CapabilityEffectName[] => { + const effectValues = { mutates_artifact: descriptor.effects.mutatesArtifact, launches_process: descriptor.effects.launchesProcess, may_show_ui: descriptor.effects.mayShowUi, @@ -205,9 +303,9 @@ const capabilityStatus = ( may_write_filesystem: descriptor.effects.mayWriteFilesystem, changes_permissions: descriptor.effects.changesPermissions, requires_root: descriptor.effects.requiresRoot, - }, - limitations: [...descriptor.limitations], -}); + } satisfies Record; + return PROVIDER_EFFECT_NAMES.filter((name) => effectValues[name]); +}; const providerOperationHealthStatus = ( health: ProviderOperationHealth | null | undefined, diff --git a/src/catalogIdentity.ts b/src/catalogIdentity.ts index 408ec68e0..050f5c7dc 100644 --- a/src/catalogIdentity.ts +++ b/src/catalogIdentity.ts @@ -74,15 +74,4 @@ export const CATALOG_IDENTITY = { get digests() { return (catalogDigests ??= createCatalogDigests()); }, - tools: sortedToolContracts.map(({ name, kind, effects, annotations }) => ({ - name, - surface: kind, - effects: { ...effects }, - annotations: { - read_only: annotations.readOnlyHint ?? false, - destructive: annotations.destructiveHint ?? false, - idempotent: annotations.idempotentHint ?? false, - open_world: annotations.openWorldHint ?? true, - }, - })), } as const; diff --git a/src/contracts/toolOutputSchemaPrimitives.ts b/src/contracts/toolOutputSchemaPrimitives.ts index b8d7815f3..618301797 100644 --- a/src/contracts/toolOutputSchemaPrimitives.ts +++ b/src/contracts/toolOutputSchemaPrimitives.ts @@ -108,12 +108,73 @@ export const providerCapability = z.discriminatedUnion("available", [ ]); export type ProviderCapability = z.output; +/** Listed effects are true; all other members of this closed vocabulary are false. */ +export const PROVIDER_EFFECT_NAMES = Object.keys( + providerCapabilityFacts.effects.shape, +) as (keyof z.output)[]; + export const providerIdentity = z.object({ id: z.string(), name: z.string(), version: z.string().nullable(), }); +const providerCapabilityGroupFacts = { + operations: z.array(z.string().min(1)).min(1), + effects: z.array(providerCapabilityFacts.effects.keyof()), + limitations: z.array(z.string()), +}; + +const providerCapabilityGroup = z.discriminatedUnion("available", [ + z.strictObject({ + ...providerCapabilityGroupFacts, + available: z.literal(true), + reason: z.null(), + availability_code: z.null(), + }), + z.strictObject({ + ...providerCapabilityGroupFacts, + available: z.literal(false), + reason: z.string().nullable(), + availability_code: z.enum(PROVIDER_REJECTION_CODES).nullable(), + }), +]); + +/** Shared limitations precede each group's suffix, preserving exact source order. */ +export const providerCapabilitySet = z.strictObject({ + provider: providerIdentity, + shared_limitations: z.array(z.string()), + groups: z.array(providerCapabilityGroup), +}); +export type ProviderCapabilitySet = z.output; + +/** Decode one status scope without losing provider identity or limitation order. */ +export const expandProviderCapabilities = ( + sets: readonly ProviderCapabilitySet[], +): readonly (ProviderCapability & { + readonly provider: z.output; +})[] => + sets.flatMap((set) => + set.groups.flatMap((group) => + group.operations.map((operation) => ({ + ...providerCapability.parse({ + operation, + available: group.available, + reason: group.reason, + availability_code: group.availability_code, + effects: Object.fromEntries( + PROVIDER_EFFECT_NAMES.map((name) => [ + name, + group.effects.includes(name), + ]), + ), + limitations: [...set.shared_limitations, ...group.limitations], + }), + provider: set.provider, + })), + ), + ); + const processCoordinates = { launcher_pid: z.number().int().min(1), process_group_id: z.number().int().min(1), @@ -334,9 +395,77 @@ export type ToolUnavailabilityReason = z.output< typeof toolUnavailabilityReason >; +export const TOOL_EFFECT_NAMES = Object.keys( + toolAvailabilityFacts.effects.shape, +) as (keyof z.output)[]; +export const TOOL_ANNOTATION_NAMES = Object.keys( + toolAvailabilityFacts.annotations.shape, +) as (keyof z.output)[]; + +const toolAvailabilityGroupFacts = { + names: z.array(z.string().min(1)).min(1), + surface: toolAvailabilityFacts.surface, + default_mode_available: toolAvailabilityFacts.default_mode_available, + modes: toolAvailabilityFacts.modes, + client_requirements: toolAvailabilityFacts.client_requirements.optional(), + effects: z.array(toolAvailabilityFacts.effects.keyof()), + annotations: z.array(toolAvailabilityFacts.annotations.keyof()), +}; + +const toolAvailabilityGroup = z.discriminatedUnion("available", [ + z.strictObject({ + ...toolAvailabilityGroupFacts, + available: z.literal(true), + reason: z.literal("available"), + remediation: z.null(), + }), + z.strictObject({ + ...toolAvailabilityGroupFacts, + available: z.literal(false), + reason: toolUnavailabilityReason, + remediation: z.string().nullable(), + }), +]); + +/** Complete inventory with explicit inherited client requirements and enabled flags. */ +export const toolAvailabilityInventory = z.strictObject({ + client_requirements: toolAvailabilityFacts.client_requirements, + groups: z.array(toolAvailabilityGroup), +}); +export type ToolAvailabilityInventory = z.output< + typeof toolAvailabilityInventory +>; + +/** Expand the wire representation once at the consuming boundary. */ +export const expandToolAvailability = ( + inventory: ToolAvailabilityInventory, +): readonly ToolAvailability[] => + inventory.groups.flatMap((group) => + group.names.map((name) => + toolAvailability.parse({ + ...group, + name, + client_requirements: + group.client_requirements ?? inventory.client_requirements, + effects: Object.fromEntries( + TOOL_EFFECT_NAMES.map((effect) => [ + effect, + group.effects.includes(effect), + ]), + ), + annotations: Object.fromEntries( + TOOL_ANNOTATION_NAMES.map((annotation) => [ + annotation, + group.annotations.includes(annotation), + ]), + ), + }), + ), + ); + export const sessionProvider = z .object({ - capabilities: z.array(providerCapability), + capabilities: z.array(providerCapabilitySet), analysis_run: analysisRun, analysis_activity: analysisActivity, provider_operation_health: providerOperationHealth, @@ -357,10 +486,10 @@ export const sessionProvider = z availability: providerAvailability, target_support: providerTargetSupport, selected: z.boolean(), - capabilities: z.array(providerCapability), + capabilities: z.array(providerCapabilitySet), }), ), - tool_availability: z.array(toolAvailability), + tool_availability: toolAvailabilityInventory, client_features: z.object({ elicitation_form: z.boolean(), elicitation_url: z.boolean(), @@ -388,7 +517,18 @@ export const sessionProvider = z negotiated_protocol_version: z.string().nullable(), client: z.object({ name: z.string(), version: z.string() }).nullable(), skill: z.object({ name: z.string(), expected_version: z.string() }), - catalog: z.record(z.string(), jsonValueSchema), + catalog: z.strictObject({ + counts: z.strictObject({ + cli_commands: z.number().int().min(0), + mcp_tools: z.number().int().min(0), + mcp_prompts: z.number().int().min(0), + }), + digests: z.strictObject({ + tools_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + prompts_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + combined_sha256: z.string().regex(/^[a-f0-9]{64}$/u), + }), + }), protocol_features: z.object({ progress: z.boolean(), cancellation: z.boolean(), diff --git a/src/server/registerSessionStatusTool.ts b/src/server/registerSessionStatusTool.ts index c20170647..563e7528b 100644 --- a/src/server/registerSessionStatusTool.ts +++ b/src/server/registerSessionStatusTool.ts @@ -3,9 +3,15 @@ import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; import type { BinarySessionPort } from "../application/binary/BinarySessionPort.js"; import type { ProviderAvailability } from "../application/AnalysisProvider.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; -import { buildCapabilityInventory } from "../application/CapabilityInventory.js"; +import { + buildCapabilityInventory, + compactToolAvailability, +} from "../application/CapabilityInventory.js"; import { toolContract } from "../contracts/toolContracts.js"; -import type { ClientFeatureAvailability } from "../contracts/toolOutputSchemaPrimitives.js"; +import type { + ClientFeatureAvailability, + ToolAvailability, +} from "../contracts/toolOutputSchemaPrimitives.js"; import { jsonObjectSchema } from "../domain/jsonValue.js"; import { createServerIdentity } from "../serverIdentity.js"; import { mcpClientMetadata } from "./mcpClientMetadata.js"; @@ -13,8 +19,6 @@ import type { AvailabilityPolicy } from "../application/CapabilityInventory.js"; import { toolRegistrationOptions } from "./toolRegistrationOptions.js"; import { err } from "../domain/result.js"; -type ToolAvailability = ReturnType[number]; - /** Inputs required to register the binary session status tool. */ export interface SessionStatusToolOptions { readonly server: EvidenceMcpServer; @@ -111,7 +115,7 @@ const projectSessionStatus = (options: { }) => jsonObjectSchema.parse({ ...options.status, - tool_availability: options.toolAvailability, + tool_availability: compactToolAvailability(options.toolAvailability), client_features: options.clientFeatures, server_identity: options.serverIdentity, }); diff --git a/tests/acceptance/applications/runtime.test.ts b/tests/acceptance/applications/runtime.test.ts index 3c454258f..cd9f2c312 100644 --- a/tests/acceptance/applications/runtime.test.ts +++ b/tests/acceptance/applications/runtime.test.ts @@ -7,7 +7,11 @@ import { z } from "zod"; import { CATALOG_IDENTITY } from "../../../src/catalogIdentity.js"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; -import { evidenceResultOf } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { + evidenceResultOf, + expandToolAvailability, + toolAvailabilityInventory, +} from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { parseEvidence } from "../../../src/domain/evidence.js"; const mainPath = fileURLToPath( @@ -24,15 +28,14 @@ const expectAvailableToolInventory = async (client: Client) => { arguments: {}, }); expect(status.isError).not.toBe(true); - const availability = z + const inventory = z .object({ result: z.object({ - tool_availability: z.array( - z.object({ name: z.string(), available: z.boolean() }), - ), + tool_availability: toolAvailabilityInventory, }), }) .parse(status.structuredContent).result.tool_availability; + const availability = expandToolAvailability(inventory); // This asserts the compiled entrypoint matches the source catalog. When it // fails, check that `dist/` is current (`npm run build:cached`) before // suspecting a registration bug -- a stale build is by far the likeliest diff --git a/tests/boundary/mcp/browserMcp.test.ts b/tests/boundary/mcp/browserMcp.test.ts index d97376c55..a8a780d61 100644 --- a/tests/boundary/mcp/browserMcp.test.ts +++ b/tests/boundary/mcp/browserMcp.test.ts @@ -10,7 +10,11 @@ import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; import { webPageInspectionSchema } from "../../../src/domain/browserObservationSchemas.js"; import { createWebScreenshotArtifact } from "../../../src/domain/webScreenshot.js"; import { JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE } from "../../../src/contracts/javascript/javascriptRuntimeReconciliationExample.js"; -import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { createServer } from "../../../src/server/createServer.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { runCliJson } from "../../fixtures/cliJsonProcess.js"; @@ -59,16 +63,18 @@ it( name: "binary_session", arguments: {}, }); - expect(status.structuredContent).toMatchObject({ - result: { - tool_availability: expect.arrayContaining([ - expect.objectContaining({ - name: "inspect_web_page", - reason: expect.any(String), - }), - ]), - }, - }); + const sessionStatus = toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ); + const availability = expandToolAvailability( + sessionStatus.result.tool_availability, + ); + expect(availability).toContainEqual( + expect.objectContaining({ + name: "inspect_web_page", + reason: expect.any(String), + }), + ); const listed = await connected.client.callTool({ name: "list_browser_targets", arguments: { diff --git a/tests/boundary/mcp/clientCapabilities.test.ts b/tests/boundary/mcp/clientCapabilities.test.ts index a8acc0048..c741045b2 100644 --- a/tests/boundary/mcp/clientCapabilities.test.ts +++ b/tests/boundary/mcp/clientCapabilities.test.ts @@ -7,11 +7,25 @@ import { import { describe, expect, it } from "vitest"; import { createServer } from "../../../src/server/createServer.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { createCacheProvider, createTestBinarySession, } from "../../fixtures/binarySession.js"; +const assertCompleteToolAvailability = (structuredContent: unknown): void => { + const status = + toolContract("binary_session").outputSchema.parse(structuredContent); + const availability = expandToolAvailability(status.result.tool_availability); + expect(availability.map(({ name }) => name).sort()).toEqual( + TOOL_CONTRACTS.map(({ name }) => name).sort(), + ); +}; + describe("per-request client capability reporting", () => { it.each([ [{ elicitation: {} }, true, false], @@ -39,6 +53,7 @@ describe("per-request client capability reporting", () => { [CLIENT_CAPABILITIES_META_KEY]: capabilities, }, }); + assertCompleteToolAvailability(status.structuredContent); expect(status.structuredContent).toMatchObject({ result: { client_features: { elicitation_form: form, elicitation_url: url }, @@ -71,13 +86,15 @@ describe("initialize-scoped client metadata", () => { try { await server.connect(serverTransport); await client.connect(clientTransport); - return ( + const result = ( await client.callTool({ name: "binary_session", arguments: {}, ...(meta === undefined ? {} : { _meta: meta }), }) ).structuredContent; + assertCompleteToolAvailability(result); + return result; } finally { await Promise.allSettled([ client.close(), diff --git a/tests/boundary/mcp/ghidraRoutingMcp.test.ts b/tests/boundary/mcp/ghidraRoutingMcp.test.ts index f4ed96938..e9fa4d722 100644 --- a/tests/boundary/mcp/ghidraRoutingMcp.test.ts +++ b/tests/boundary/mcp/ghidraRoutingMcp.test.ts @@ -1,6 +1,8 @@ import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; import { describe, expect, it } from "vitest"; import { connectGhidraMcp } from "./ghidraMcpHarness.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; describe("Ghidra MCP capability routing", () => { it("routes composed and capability tools", async () => { @@ -41,44 +43,35 @@ describe("Ghidra MCP capability routing", () => { name: "binary_session", arguments: {}, }); - expect(status.structuredContent).toMatchObject({ - result: { - open: true, - analysis_provider_binding: { - provider: { id: "ghidra", version: "12.1.4" }, - }, - tool_availability: expect.arrayContaining([ - expect.objectContaining({ - name: "binary_overview", - available: true, - }), - expect.objectContaining({ - name: "get_objc_classes", - available: true, - }), - expect.objectContaining({ - name: "batch_decompile", - available: true, - }), - expect.objectContaining({ - name: "trace_feature", - available: true, - }), - expect.objectContaining({ - name: "get_call_graph", - available: true, - }), - expect.objectContaining({ - name: "find_xrefs_to_name", - available: true, - }), - expect.objectContaining({ - name: "set_comment", - available: false, - }), - ]), + const sessionStatus = toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ); + expect(sessionStatus.result).toMatchObject({ + open: true, + analysis_provider_binding: { + provider: { id: "ghidra", version: "12.1.4" }, }, }); + const availability = expandToolAvailability( + sessionStatus.result.tool_availability, + ); + expect(availability).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "binary_overview", available: true }), + expect.objectContaining({ + name: "get_objc_classes", + available: true, + }), + expect.objectContaining({ name: "batch_decompile", available: true }), + expect.objectContaining({ name: "trace_feature", available: true }), + expect.objectContaining({ name: "get_call_graph", available: true }), + expect.objectContaining({ + name: "find_xrefs_to_name", + available: true, + }), + expect.objectContaining({ name: "set_comment", available: false }), + ]), + ); } finally { await harness.close(); } diff --git a/tests/boundary/mcp/optionalProviderIsolation.test.ts b/tests/boundary/mcp/optionalProviderIsolation.test.ts index eb8f64fd1..5986c7894 100644 --- a/tests/boundary/mcp/optionalProviderIsolation.test.ts +++ b/tests/boundary/mcp/optionalProviderIsolation.test.ts @@ -14,6 +14,7 @@ import { TOOL_CONTRACTS, toolContract, } from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { run } from "../../../src/main.js"; import { createServer } from "../../../src/server/createServer.js"; import { optionalObservationFactories } from "../../fixtures/optionalObservationProviders.js"; @@ -154,33 +155,37 @@ mcpTest.for(failedAdapters)( name: "binary_session", arguments: {}, }); - expect(status.structuredContent).toMatchObject({ - result: { - tool_availability: expect.arrayContaining([ - expect.objectContaining({ - name: operation, - available: false, - reason: "provider_unavailable", - remediation: expect.stringContaining(failure), - }), - expect.objectContaining({ - name: "analyze_javascript_application", - available: true, - }), - expect.objectContaining({ - name: "reconcile_javascript_runtime", - available: true, - }), - expect.objectContaining({ - name: - port === "electronActiveObservation" - ? "list_electron_targets" - : "capture_electron_scenario", - available: true, - }), - ]), - }, - }); + const sessionStatus = toolContract("binary_session").outputSchema.parse( + status.structuredContent, + ); + const availability = expandToolAvailability( + sessionStatus.result.tool_availability, + ); + expect(availability).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + name: operation, + available: false, + reason: "provider_unavailable", + remediation: expect.stringContaining(failure), + }), + expect.objectContaining({ + name: "analyze_javascript_application", + available: true, + }), + expect.objectContaining({ + name: "reconcile_javascript_runtime", + available: true, + }), + expect.objectContaining({ + name: + port === "electronActiveObservation" + ? "list_electron_targets" + : "capture_electron_scenario", + available: true, + }), + ]), + ); const healthy = port === "javascriptRuntimeObservation" ? "list_browser_targets" diff --git a/tests/boundary/mcp/serverIdentity.test.ts b/tests/boundary/mcp/serverIdentity.test.ts index 09ace062d..a570a5caf 100644 --- a/tests/boundary/mcp/serverIdentity.test.ts +++ b/tests/boundary/mcp/serverIdentity.test.ts @@ -15,6 +15,10 @@ import { CLI_COMMAND_NAMES } from "../../../src/cliCommandNames.js"; import { PACKAGE_METADATA } from "../../../src/generatedPackageMetadata.js"; import { PRODUCT_IDENTITY, SDK_IDENTITY } from "../../../src/identity.js"; import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; +import { + expandToolAvailability, + sessionProvider, +} from "../../../src/contracts/toolOutputSchemaPrimitives.js"; import { PROMPT_CONTRACTS } from "../../../src/contracts/promptContracts.js"; import { createServer } from "../../../src/server/createServer.js"; import { createServerIdentity } from "../../../src/serverIdentity.js"; @@ -70,20 +74,18 @@ const statusCapability = ( readonly reason: string; } = { available: true }, ) => ({ - operation, - effects: { - mutates_artifact: false, - launches_process: false, - may_show_ui: false, - may_access_network: false, - may_write_filesystem: false, - changes_permissions: false, - requires_root: false, - }, - limitations: [], - ...(availability.available - ? { available: true as const, reason: null, availability_code: null } - : availability), + provider: { id: "status-fixture", name: "Status fixture", version: null }, + shared_limitations: [], + groups: [ + { + operations: [operation], + effects: [], + limitations: [], + ...(availability.available + ? { available: true as const, reason: null, availability_code: null } + : availability), + }, + ], }); describe("server and catalog identity", () => { @@ -326,15 +328,10 @@ const activeTargetReasons = async ( name: "binary_session", arguments: {}, }); - const availability = z - .object({ - result: z.object({ - tool_availability: z.array( - z.object({ name: z.string(), reason: z.string() }).loose(), - ), - }), - }) - .parse(status.structuredContent).result.tool_availability; + const availability = expandToolAvailability( + z.object({ result: sessionProvider }).parse(status.structuredContent).result + .tool_availability, + ); return Object.fromEntries( ["inspect_macho", "inspect_artifact", "inspect_managed_artifact"].map( (name) => [name, availability.find((tool) => tool.name === name)?.reason], @@ -347,6 +344,32 @@ const assertLiveIdentity = async (client: Client): Promise => { name: "binary_session", arguments: {}, }); + const status = z + .object({ result: sessionProvider }) + .parse(identity.structuredContent).result; + expect(Object.keys(status.server_identity.catalog).sort()).toEqual([ + "counts", + "digests", + ]); + const availability = expandToolAvailability(status.tool_availability); + expect(availability.map(({ name }) => name).sort()).toEqual( + TOOL_CONTRACTS.map(({ name }) => name).sort(), + ); + for (const contract of TOOL_CONTRACTS) { + expect(availability).toContainEqual( + expect.objectContaining({ + name: contract.name, + surface: contract.kind, + effects: contract.effects, + annotations: { + read_only: contract.annotations.readOnlyHint ?? false, + destructive: contract.annotations.destructiveHint ?? false, + idempotent: contract.annotations.idempotentHint ?? false, + open_world: contract.annotations.openWorldHint ?? true, + }, + }), + ); + } expect(identity.structuredContent).toMatchObject({ result: { server_identity: { @@ -370,7 +393,17 @@ const assertSessionIdentity = async (client: Client): Promise => { name: "binary_session", arguments: { expected_package_version: "1.2.0" }, }); - expect(status.structuredContent).toMatchObject({ + const parsed = z + .object({ result: sessionProvider }) + .parse(status.structuredContent); + expect({ + result: { + ...parsed.result, + tool_availability: expandToolAvailability( + parsed.result.tool_availability, + ), + }, + }).toMatchObject({ result: { server_identity: { catalog: { diff --git a/tests/boundary/mcp/serverWorkflowMcp.test.ts b/tests/boundary/mcp/serverWorkflowMcp.test.ts index 4651de6e8..c72fca5cb 100644 --- a/tests/boundary/mcp/serverWorkflowMcp.test.ts +++ b/tests/boundary/mcp/serverWorkflowMcp.test.ts @@ -3,7 +3,6 @@ import { ok as resultOk } from "../../../src/domain/result.js"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; import type { CallToolResult } from "@modelcontextprotocol/server"; import { afterEach, expect, it } from "vitest"; -import { z } from "zod"; import { createTestBinarySession } from "../../fixtures/binarySession.js"; import type { @@ -12,8 +11,11 @@ import type { } from "../../../src/application/AnalysisProvider.js"; import { observed as ok } from "../../fixtures/analysisExecution.js"; import { createServer } from "../../../src/server/createServer.js"; -import { TOOL_CONTRACTS } from "../../../src/contracts/toolContracts.js"; -import { toolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { + TOOL_CONTRACTS, + toolContract, +} from "../../../src/contracts/toolContracts.js"; +import { expandToolAvailability } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; const resources: Array<{ close(): Promise }> = []; @@ -126,11 +128,10 @@ it("advertises the complete currently available inventory with a session", async arguments: {}, }), ); - const inventory = z - .object({ - result: z.object({ tool_availability: z.array(toolAvailability) }), - }) - .parse(status).result.tool_availability; + const inventory = expandToolAvailability( + toolContract("binary_session").outputSchema.parse(status).result + .tool_availability, + ); for (const availability of inventory) { if (availability.available) continue; expect(availability.reason).not.toBe("available"); diff --git a/tests/boundary/mcp/sessionMcp.test.ts b/tests/boundary/mcp/sessionMcp.test.ts index 302f91ad7..42e92e501 100644 --- a/tests/boundary/mcp/sessionMcp.test.ts +++ b/tests/boundary/mcp/sessionMcp.test.ts @@ -26,6 +26,8 @@ import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { MAX_JSON_DEPTH } from "../../../src/domain/jsonValue.js"; import { parseAnalysisSnapshot } from "../../../src/domain/analysisSnapshot.js"; import { INVESTIGATION_EXAMPLES } from "../../../src/contracts/investigationExamples.js"; +import { expandProviderCapabilities } from "../../../src/contracts/toolOutputSchemaPrimitives.js"; +import { toolContract } from "../../../src/contracts/toolContracts.js"; import { ok as resultOk } from "../../../src/domain/result.js"; import { createSessionMcpHarness, @@ -294,15 +296,12 @@ describe("target-free MCP workflow", () => { expect(openedStatus).toMatchObject({ result: { path: await realpath(first) }, }); + const sessionStatus = + toolContract("binary_session").outputSchema.parse(openedStatus); expect( - z - .object({ - result: z.object({ - capabilities: z.array(z.object({ operation: z.string() })), - }), - }) - .parse(openedStatus) - .result.capabilities.map(({ operation }) => operation), + expandProviderCapabilities(sessionStatus.result.capabilities).map( + ({ operation }) => operation, + ), ).not.toContain("inventory_artifact"); expect( text(await mcp.callTool({ name: "current_document", arguments: {} })), diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index aa809b6fb..302d51d50 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -597,10 +597,19 @@ describe("provider health and availability", () => { expect(session.status()).toMatchObject({ capabilities: expect.arrayContaining([ expect.objectContaining({ - operation: "address_name", - available: false, - reason: - "ProviderAdapterError: Provider fixture adapter failed during address_name", + provider: { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }, + groups: expect.arrayContaining([ + expect.objectContaining({ + operations: expect.arrayContaining(["address_name"]), + available: false, + reason: + "ProviderAdapterError: Provider fixture adapter failed during address_name", + }), + ]), }), ]), }); @@ -609,9 +618,18 @@ describe("provider health and availability", () => { expect(session.status()).toMatchObject({ capabilities: expect.arrayContaining([ expect.objectContaining({ - operation: "address_name", - available: true, - reason: null, + provider: { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }, + groups: expect.arrayContaining([ + expect.objectContaining({ + operations: expect.arrayContaining(["address_name"]), + available: true, + reason: null, + }), + ]), }), ]), }); @@ -657,9 +675,18 @@ describe("provider health and availability", () => { expect(session.status()).toMatchObject({ capabilities: expect.arrayContaining([ expect.objectContaining({ - operation: "address_name", - available: true, - reason: null, + provider: { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + }, + groups: expect.arrayContaining([ + expect.objectContaining({ + operations: expect.arrayContaining(["address_name"]), + available: true, + reason: null, + }), + ]), }), ]), });