diff --git a/src/cli/browserAdvancedCommands.ts b/src/cli/browserAdvancedCommands.ts index 1eff33f4b..50c9ce446 100644 --- a/src/cli/browserAdvancedCommands.ts +++ b/src/cli/browserAdvancedCommands.ts @@ -14,7 +14,9 @@ import { AnalysisInputError, type AnalysisInputIssue, } from "../domain/analysisErrorCore.js"; -import { browserCaptureComparisonInputSchema } from "../domain/browserCaptureComparison.js"; +import { compareBrowserScenariosInputSchema } from "../domain/browserScenarioDiffValues.js"; +import { browserScenarioCaptureSchema } from "../domain/browserScenarioCapture.js"; +import { compareWebCapturesInputSchema } from "../domain/webCaptureDiffSchemas.js"; import { projectInputIssues } from "../domain/inputIssueProjection.js"; import { discoverWebMcpToolsInputSchema } from "../domain/webMcpDiscovery.js"; import { @@ -59,13 +61,18 @@ const registerWebMcp = ( run: ({ args, options }) => logCliCommand(logger, "discover-webmcp-tools", async () => { const context = browserContext(); - const parsed = discoverWebMcpToolsInputSchema.safeParse({ + const input = { cdp_endpoint: args.endpoint, allowed_origins: options.allowedOrigins, target_id: args.targetId, observation_ms: options.observationMs, - }); - if (!parsed.success) return inputError("discover_webmcp_tools"); + }; + const parsed = discoverWebMcpToolsInputSchema.safeParse(input); + if (!parsed.success) + return inputError( + "discover_webmcp_tools", + projectInputIssues(parsed.error.issues, input), + ); const result = await discoverWebMcpTools(context.provider, parsed.data); return result.ok ? result.value : browserCliError(result.error); }), @@ -104,16 +111,28 @@ const registerCaptureDiff = ( }); if (!inputs.ok) return inputs.error; const { before, after, normalization } = inputs.value; - const scenarioComparison = - browserCaptureComparisonInputSchema.safeParse({ - before_scenario: before, - after_scenario: after, - normalization, - }); - const parsed = scenarioComparison.success - ? scenarioComparison - : browserCaptureComparisonInputSchema.safeParse({ before, after }); - if (!parsed.success) return inputError("compare_web_captures"); + // Passive snapshots own the inspection wrapper. Otherwise recognize + // scenario-owned fields even when the scenario manifest is missing. + const scenarioInput = [before, after].some( + (capture) => + typeof capture === "object" && + capture !== null && + !("inspection" in capture) && + Object.keys(browserScenarioCaptureSchema.shape).some( + (field) => field in capture, + ), + ); + const input = scenarioInput + ? { before_scenario: before, after_scenario: after, normalization } + : { before, after }; + const parsed = scenarioInput + ? compareBrowserScenariosInputSchema.safeParse(input) + : compareWebCapturesInputSchema.safeParse(input); + if (!parsed.success) + return inputError( + "compare_web_captures", + projectInputIssues(parsed.error.issues, input), + ); const result = await compareWebCaptureEvidence( new CdpBrowserProvider(), parsed.data, @@ -140,12 +159,17 @@ const registerScreenshot = ( run: ({ args, options }) => logCliCommand(logger, "capture-web-screenshot", async () => { const context = browserContext(); - const parsed = captureWebScreenshotInputSchema.safeParse({ + const input = { cdp_endpoint: args.endpoint, allowed_origins: options.allowedOrigins, target_id: args.targetId, - }); - if (!parsed.success) return inputError("capture_web_screenshot"); + }; + const parsed = captureWebScreenshotInputSchema.safeParse(input); + if (!parsed.success) + return inputError( + "capture_web_screenshot", + projectInputIssues(parsed.error.issues, input), + ); const result = await captureWebScreenshot( context.provider, parsed.data, diff --git a/src/domain/browserObservation.ts b/src/domain/browserObservation.ts index d32a2adc1..0a9fc8e32 100644 --- a/src/domain/browserObservation.ts +++ b/src/domain/browserObservation.ts @@ -105,7 +105,7 @@ const inspectWebPageInputFacts = { const inspectWebPageWithoutSourceSchema = z.strictObject({ ...inspectWebPageInputFacts, - include_script_sources: z.literal(false).default(false), + include_script_sources: z.literal(false).optional().default(false), }); const inspectWebPageWithSourceShapeSchema = z.strictObject({ @@ -134,10 +134,17 @@ export const inspectWebPageWithSourceInputSchema = inspectWebPageWithSourceShapeSchema.superRefine(refineInspectWebPageInput); /** Caller-visible schema for one passive inspection. */ -export const inspectWebPageInputSchema = z.union([ - inspectWebPageWithoutSourceSchema.superRefine(refineInspectWebPageInput), - inspectWebPageWithSourceInputSchema, -]); +export const inspectWebPageInputSchema = z.discriminatedUnion( + "include_script_sources", + [ + inspectWebPageWithoutSourceSchema.superRefine(refineInspectWebPageInput), + inspectWebPageWithSourceShapeSchema + .extend({ + include_script_sources: z.literal(true), + }) + .superRefine(refineInspectWebPageInput), + ], +); export type ListBrowserTargetsInput = z.infer< typeof listBrowserTargetsInputSchema diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.ts index 7b2cb93ba..5f8193fce 100644 --- a/tests/boundary/cli/browserCli.test.ts +++ b/tests/boundary/cli/browserCli.test.ts @@ -93,7 +93,7 @@ it( const compared = await runCli( [ "compare-web-captures", - JSON.stringify({ inspection }), + JSON.stringify({ inspection, scenario: { caller_metadata: true } }), JSON.stringify({ inspection }), "--json", ], @@ -183,6 +183,119 @@ it( INTEGRATION_TEST_TIMEOUT_MS, ); +describe("scenario comparison CLI diagnostics", () => { + const rule = { + rule_id: "domain", + artifacts: ["url"], + match: "app.example.test", + replacement: "normalized.test", + }; + it.each([ + { + name: "duplicate rule IDs", + normalization: { rules: [rule, rule] }, + issue: { + path: ["normalization", "rules", 1, "rule_id"], + reason: "invalid_value", + message: "Normalization rule IDs must be unique", + }, + }, + { + name: "duplicate artifact kinds", + normalization: { rules: [{ ...rule, artifacts: ["url", "url"] }] }, + issue: { + path: ["normalization", "rules", 0, "artifacts"], + reason: "invalid_value", + message: "Normalization rule artifact kinds must be unique", + }, + }, + { + name: "structurally invalid rules", + normalization: { rules: "invalid" }, + issue: { + path: ["normalization", "rules"], + reason: "invalid_type", + expected: "array", + }, + }, + ])( + "retains scenario normalization diagnostics for $name", + async ({ normalization, issue }) => { + const capture = JSON.stringify(completeScenarioCapture()); + const result = await runCli([ + "compare-web-captures", + capture, + capture, + "--normalization-json", + JSON.stringify(normalization), + "--json", + ]); + expect(result).toMatchObject({ + code: "invalid_request", + details: { operation: "compare_web_captures", issues: [issue] }, + }); + }, + ); + + it("keeps malformed scenario diagnostics in the selected capture group", async () => { + const capture = completeScenarioCapture(); + const result = await runCli([ + "compare-web-captures", + JSON.stringify({ ...capture, duration_ms: "invalid" }), + JSON.stringify(capture), + "--json", + ]); + expect(result).toMatchObject({ + code: "invalid_request", + details: { + operation: "compare_web_captures", + issues: [ + { + path: ["before_scenario", "duration_ms"], + reason: "invalid_type", + expected: "number", + }, + ], + }, + }); + }); + + it.each(["omitted", "misspelled"])( + "reports the missing scenario manifest when it is %s on both captures", + async (kind) => { + const { scenario, ...capture } = completeScenarioCapture(); + const input = JSON.stringify( + kind === "omitted" ? capture : { ...capture, scneario: scenario }, + ); + const result = await runCli([ + "compare-web-captures", + input, + input, + "--json", + ]); + expect(result).toMatchObject({ + code: "invalid_request", + details: { + operation: "compare_web_captures", + issues: expect.arrayContaining([ + { + path: ["before_scenario", "scenario"], + reason: "missing_argument", + expected: "object", + }, + { + path: ["after_scenario", "scenario"], + reason: "missing_argument", + expected: "object", + }, + ]), + }, + }); + expect(JSON.stringify(result)).not.toContain('"inspection"'); + }, + ); +}); + describe("browser CLI capture parity", () => { it( "runs analysis, session, WebMCP, screenshot, and comparison from explicit request scope", diff --git a/tests/boundary/cli/browserInputDiagnostics.test.ts b/tests/boundary/cli/browserInputDiagnostics.test.ts index e6b1646ae..745948e92 100644 --- a/tests/boundary/cli/browserInputDiagnostics.test.ts +++ b/tests/boundary/cli/browserInputDiagnostics.test.ts @@ -61,8 +61,59 @@ const invalidInputs = [ error: "JavaScript runtime observation failed", issue: { path: ["target_id"], reason: "out_of_range", minimum: 1 }, }, + { + command: "discover-webmcp-tools", + arguments: [malformedEndpoint, "target"], + operation: "discover_webmcp_tools", + error: "Browser observation failed", + issue: { + path: ["cdp_endpoint"], + reason: "invalid_value", + message: "Invalid CDP endpoint URL", + }, + }, + { + command: "capture-web-screenshot", + arguments: [browserEndpoint, ""], + operation: "capture_web_screenshot", + error: "Browser observation failed", + issue: { path: ["target_id"], reason: "out_of_range", minimum: 1 }, + }, ] as const; +describe("inspection branch diagnostics", () => { + cliTest.for([[], ["--include-script-sources"]])( + "reports only the selected inspection branch's endpoint error (%j)", + async (flags, { cli, workspace }) => { + const result = await cli.run({ + arguments: [ + "inspect-web-page", + malformedEndpoint, + "target", + ...flags, + "--json", + ], + cwd: workspace.root, + environment: { ...workspace.environment, REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + operation: "inspect_web_page", + issues: [ + { + path: ["cdp_endpoint"], + reason: "invalid_value", + message: "Invalid CDP endpoint URL", + }, + ], + }, + }); + }, + ); +}); + describe("browser and Inspector CLI input diagnostics", () => { cliTest.for(invalidInputs)( "$command retains the named contract's correction details", @@ -159,6 +210,28 @@ describe("browser and Inspector CLI input diagnostics", () => { }, ); + cliTest( + "retains correction details for invalid capture comparison inputs", + async ({ cli, workspace }) => { + const result = await cli.run({ + arguments: ["compare-web-captures", "{}", "{}", "--json"], + cwd: workspace.root, + environment: { ...workspace.environment, REA_LOG_LEVEL: "silent" }, + }); + + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + operation: "compare_web_captures", + issues: expect.arrayContaining([ + expect.objectContaining({ reason: expect.any(String) }), + ]), + }, + }); + }, + ); + cliTest( "preserves the parser diagnostic for an omitted endpoint", async ({ cli, workspace }) => { diff --git a/tests/boundary/mcp/toolSchemaValidity.test.ts b/tests/boundary/mcp/toolSchemaValidity.test.ts index b833310c3..5a94d27bf 100644 --- a/tests/boundary/mcp/toolSchemaValidity.test.ts +++ b/tests/boundary/mcp/toolSchemaValidity.test.ts @@ -454,6 +454,35 @@ describe("MCP root input schemas", () => { expectStrictInputSchemaParity(tools, ajv); + for (const name of ["inspect_web_page", "analyze_web_bundle"]) { + const contract = TOOL_CONTRACTS.find((tool) => tool.name === name); + const tool = advertised.get(name); + if (contract === undefined || tool === undefined) + throw new Error(`${name} was not advertised`); + const validate = ajv.compile(tool.inputSchema); + const base = { + cdp_endpoint: "http://127.0.0.1:9222", + target_id: "target", + }; + for (const flag of [undefined, false, true, "false", null]) { + const input = + flag === undefined + ? base + : { ...base, include_script_sources: flag }; + const accepted = + flag === undefined || + flag === true || + (name === "inspect_web_page" && flag === false); + const parsed = contract.inputSchema.safeParse(input); + expect(parsed.success, `${name}: ${String(flag)}`).toBe(accepted); + expect(validate(input), `${name}: ${String(flag)}`).toBe(accepted); + if (parsed.success && flag === undefined) + expect(parsed.data).toMatchObject({ + include_script_sources: name === "analyze_web_bundle", + }); + } + } + const nativeObservation = TOOL_CONTRACTS.find( ({ name }) => name === "observe_native_ui", );