fix(browser): retain scenario observations across failures

This commit is contained in:
morluto
2026-10-09 22:46:08 +08:00
parent 73e98b2516
commit 614e0c2e74
9 changed files with 622 additions and 76 deletions
@@ -7,6 +7,10 @@ export async function startBrowserVerifierSite() {
let port = 0;
let sessionGeneration = 0;
const server = createServer((request, response) => {
if (request.url === "/failed-navigation") {
response.destroy();
return;
}
if (request.url === "/slow-json") {
response.setHeader("content-type", "application/json");
response.write('{"pending":');
+72
View File
@@ -1,9 +1,14 @@
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { readdir } from "node:fs/promises";
import { tmpdir } from "node:os";
import { promisify } from "node:util";
import { fileURLToPath } from "node:url";
import { Client } from "@modelcontextprotocol/client";
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
const execute = promisify(execFile);
@@ -142,6 +147,73 @@ export async function runScenarioCli(scenario) {
return JSON.parse(stdout);
}
/** Verify failed real navigation retains observations through both public adapters. */
export async function verifyScenarioFailureEvidence(executable, origin) {
const scenario = browserScenarioSchema.parse({
browser: { mode: "launch", executable_path: executable },
start_url: { url: `${origin}/failed-navigation` },
actions: [
{ step_id: "unused", action: "wait_for_timeout", duration_ms: 1 },
],
capture: { events: ["network"] },
});
const profilesBefore = await scenarioProfiles();
let cliFailure;
try {
await runScenarioCli(scenario);
assert.fail("The fixture must fail before initial navigation completes");
} catch (cause) {
assert.equal(typeof cause.stdout, "string");
cliFailure = JSON.parse(cause.stdout);
}
assertFailedNavigation(cliFailure, scenario.start_url.url);
const transport = new StdioClientTransport({
command: process.execPath,
args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
env: process.env,
stderr: "pipe",
});
const client = new Client({
name: "browser-scenario-failure-e2e",
version: "1",
});
try {
await client.connect(transport);
const result = await client.callTool({
name: "capture_browser_scenario",
arguments: scenario,
});
assert.equal(result.isError, true);
assertFailedNavigation(requireMcpToolError(result), scenario.start_url.url);
} finally {
await client.close();
await transport.close();
}
const profilesAfter = await scenarioProfiles();
assert.ok([...profilesAfter].every((profile) => profilesBefore.has(profile)));
return {
mocked: false,
cli: true,
stdio_mcp: true,
failed_navigation_events_retained: true,
profile_cleanup: true,
};
}
const assertFailedNavigation = (error, requestedUrl) => {
assert.equal(error?.code, "execution_failure");
const observation = error.details?.partial_observation;
assert.equal(observation?.kind, "browser-scenario-observation");
assert.equal(observation.capture.browser.cleanup, "terminated-owned-process");
assert.deepEqual(observation.capture.steps, []);
assert.ok(
observation.capture.events.items.some(
(event) =>
event.kind === "request-failed" && event.url.url === requestedUrl,
),
);
};
/** Snapshot provider-owned profile names for cleanup comparison. */
export async function scenarioProfiles() {
return new Set(
+7 -1
View File
@@ -16,7 +16,7 @@ import {
listBrowserTargetsInputSchema,
} from "../dist/domain/browserObservation.js";
import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js";
import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js";
import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiffSchemas.js";
import {
captureWebScreenshotInputSchema,
compareWebScreenshotsInputSchema,
@@ -33,6 +33,7 @@ import {
browserScenario,
runScenarioCli,
scenarioProfiles,
verifyScenarioFailureEvidence,
} from "./lib/browser-scenario-verifier.mjs";
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
@@ -294,6 +295,10 @@ try {
if ([...profilesAfter].some((entry) => !profilesBefore.has(entry)))
throw new Error("Scenario launch retained a temporary browser profile");
assertScenarioCapture(launchedScenario.value);
const scenarioFailure = await verifyScenarioFailureEvidence(
executable,
site.origin,
);
const scenarioResults = JSON.stringify([
attachedScenario,
launchedScenario.value,
@@ -360,6 +365,7 @@ try {
browserScenarioCli: true,
browserScenarioAttachCleanup: "disconnected-external",
browserScenarioLaunchCleanup: "terminated-owned-process",
scenarioFailure,
verified: true,
};
} finally {