import { describe, expect, it } from "vitest"; import { z } from "zod"; import { ENHANCED_TOOL_CONTRACTS, OFFICIAL_TOOL_CONTRACTS, SESSION_TOOL_CONTRACTS, TOOL_CONTRACTS, } from "../src/contracts/toolContracts.js"; import { NATIVE_TOOL_CONTRACTS } from "../src/contracts/nativeToolContracts.js"; import { ARTIFACT_TOOL_CONTRACTS } from "../src/contracts/artifactToolContracts.js"; import { MANAGED_TOOL_CONTRACTS } from "../src/contracts/managedToolContracts.js"; import { MANAGED_WORKFLOW_TOOL_CONTRACTS } from "../src/contracts/managedWorkflowToolContracts.js"; import { BROWSER_TOOL_CONTRACTS } from "../src/contracts/browserToolContracts.js"; import { ELECTRON_TOOL_CONTRACTS } from "../src/contracts/electronToolContracts.js"; import { APPLICATION_TOOL_CONTRACTS } from "../src/contracts/applicationToolContracts.js"; import { enhancedOutputSchemas, managedOutputSchemas, managedWorkflowOutputSchemas, officialOutputSchemas, sessionOutputSchemas, } from "../src/contracts/toolOutputSchemas.js"; import { annotationsFromEffects, TOOL_EFFECTS, } from "../src/contracts/toolEffects.js"; import { toolRegistrationOptions } from "../src/server/toolRegistrationOptions.js"; const convertContractJsonSchema = (schema: z.ZodType) => z.toJSONSchema(schema, { target: "draft-07", unrepresentable: "any", }); const jsonSchemaCache = new WeakMap< z.ZodType, ReturnType >(); const contractJsonSchema = (schema: z.ZodType) => { const cached = jsonSchemaCache.get(schema); if (cached !== undefined) return cached; const converted = convertContractJsonSchema(schema); jsonSchemaCache.set(schema, converted); return converted; }; const emptySchemaPaths = (value: unknown, path = "$"): string[] => { if (Array.isArray(value)) return value.flatMap((item, index) => emptySchemaPaths(item, `${path}[${String(index)}]`), ); if (typeof value !== "object" || value === null) return []; const entries = Object.entries(value); if (entries.length === 0) return [path]; return entries.flatMap(([key, item]) => emptySchemaPaths(item, `${path}.${key}`), ); }; describe("tool contract surface", () => { it("gives fallback parameter guidance operational meaning", () => { const contract = TOOL_CONTRACTS.find( ({ name }) => name === "analyze_function", ); expect(contract).toBeDefined(); if (contract === undefined) return; const advertisedSchema = toolRegistrationOptions(contract).inputSchema[ "~standard" ].jsonSchema.input(); const properties = Object.fromEntries( Object.entries(advertisedSchema.properties ?? {}), ); expect(properties.include_assembly).toMatchObject({ description: "Whether to include assembly in the result.", }); expect(properties.max_instructions).toMatchObject({ description: "Maximum permitted instructions for this operation.", }); expect(properties.pseudocode_offset).toMatchObject({ description: "Zero-based index of the first pseudocode to return.", }); }); it("advertises complete typed schemas and annotations for all analysis tools", () => { const contracts = [ ...OFFICIAL_TOOL_CONTRACTS, ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, ...MANAGED_WORKFLOW_TOOL_CONTRACTS, ...BROWSER_TOOL_CONTRACTS, ...ELECTRON_TOOL_CONTRACTS, ...APPLICATION_TOOL_CONTRACTS, ]; expect(contracts).toHaveLength(77); for (const contract of contracts) { const inputSchema = contractJsonSchema(contract.inputSchema); const outputSchema = contractJsonSchema(contract.outputSchema); expect(inputSchema.type).toBe("object"); expect(outputSchema.type).toBe("object"); expect(contract.title.length).toBeGreaterThan(2); expect(contract.annotations).toEqual( annotationsFromEffects(contract.effects), ); expect(typeof contract.annotations.idempotentHint).toBe("boolean"); expect(typeof contract.annotations.openWorldHint).toBe("boolean"); expect(typeof contract.annotations.readOnlyHint).toBe("boolean"); expect(typeof contract.annotations.destructiveHint).toBe("boolean"); expect(contract.examples).toHaveLength(1); for (const example of contract.examples) { expect(example.title.length).toBeGreaterThan(10); expect(contract.inputSchema.safeParse(example.input).success).toBe( true, ); } } }); it("audits every tool effect explicitly with no heuristic fallback", () => { const names = [ ...OFFICIAL_TOOL_CONTRACTS, ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, ...MANAGED_WORKFLOW_TOOL_CONTRACTS, ...BROWSER_TOOL_CONTRACTS, ...ELECTRON_TOOL_CONTRACTS, ...APPLICATION_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ].map(({ name }) => name); expect(Object.keys(TOOL_EFFECTS).sort()).toEqual(names.sort()); }); it("keeps exactly eighteen additive session contracts", () => { expect( SESSION_TOOL_CONTRACTS.map(({ name, kind }) => ({ name, kind })), ).toEqual([ { name: "open_binary", kind: "session" }, { name: "close_binary", kind: "session" }, { name: "binary_session", kind: "session" }, { name: "export_evidence_bundle", kind: "session" }, { name: "import_evidence_bundle", kind: "session" }, { name: "capture_process_scenario", kind: "session" }, { name: "compare_process_captures", kind: "session" }, { name: "compare_artifacts", kind: "session" }, { name: "compare_functions", kind: "session" }, { name: "compare_bundles", kind: "session" }, { name: "find_changed_behavior", kind: "session" }, { name: "build_call_path", kind: "session" }, { name: "correlate_static_and_runtime", kind: "session" }, { name: "verify_reconstruction", kind: "session" }, { name: "list_unknowns", kind: "session" }, { name: "record_unknown", kind: "session" }, { name: "update_unknown", kind: "session" }, { name: "verify_unknown_resolution", kind: "session" }, ]); expect( SESSION_TOOL_CONTRACTS.find( ({ name }) => name === "capture_process_scenario", )?.annotations.openWorldHint, ).toBe(true); }); it("advertises evidence filesystem effects conservatively", () => { const exported = SESSION_TOOL_CONTRACTS.find( ({ name }) => name === "export_evidence_bundle", ); const imported = SESSION_TOOL_CONTRACTS.find( ({ name }) => name === "import_evidence_bundle", ); expect(exported?.annotations).toMatchObject({ readOnlyHint: false, destructiveHint: true, }); expect(imported?.annotations).toMatchObject({ readOnlyHint: false, destructiveHint: false, }); }); it("publishes a dedicated output schema and agent guidance for every tool", () => { expect(Object.keys(officialOutputSchemas).sort()).toEqual( OFFICIAL_TOOL_CONTRACTS.map(({ name }) => name).sort(), ); expect(Object.keys(enhancedOutputSchemas).sort()).toEqual( ENHANCED_TOOL_CONTRACTS.map(({ name }) => name).sort(), ); expect(Object.keys(sessionOutputSchemas).sort()).toEqual( SESSION_TOOL_CONTRACTS.map(({ name }) => name).sort(), ); expect(Object.keys(managedOutputSchemas).sort()).toEqual( MANAGED_TOOL_CONTRACTS.map(({ name }) => name).sort(), ); expect(Object.keys(managedWorkflowOutputSchemas).sort()).toEqual( MANAGED_WORKFLOW_TOOL_CONTRACTS.map(({ name }) => name).sort(), ); for (const contract of [ ...OFFICIAL_TOOL_CONTRACTS, ...ENHANCED_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, ...MANAGED_WORKFLOW_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ]) { const schema = contractJsonSchema(contract.outputSchema); expect(JSON.stringify(schema)).not.toContain('"result":{}'); expect(contract.description.length).toBeGreaterThan(100); expect(contract.description).toMatch(/[.;]/u); } }); it("publishes no unconstrained output-schema holes across all tools", () => { const contracts = [ ...OFFICIAL_TOOL_CONTRACTS, ...ENHANCED_TOOL_CONTRACTS, ...NATIVE_TOOL_CONTRACTS, ...ARTIFACT_TOOL_CONTRACTS, ...MANAGED_TOOL_CONTRACTS, ...MANAGED_WORKFLOW_TOOL_CONTRACTS, ...BROWSER_TOOL_CONTRACTS, ...ELECTRON_TOOL_CONTRACTS, ...APPLICATION_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ]; expect(contracts).toHaveLength(TOOL_CONTRACTS.length); for (const contract of contracts) { const schema = contractJsonSchema(contract.outputSchema); expect(emptySchemaPaths(schema), contract.name).toEqual([]); } }); });