fix(browser): retain scenario observations across failures
This commit is contained in:
@@ -7,6 +7,10 @@ export async function startBrowserVerifierSite() {
|
||||
let port = 0;
|
||||
let sessionGeneration = 0;
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/failed-navigation") {
|
||||
response.destroy();
|
||||
return;
|
||||
}
|
||||
if (request.url === "/slow-json") {
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.write('{"pending":');
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
|
||||
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
|
||||
@@ -142,6 +147,73 @@ export async function runScenarioCli(scenario) {
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
/** Verify failed real navigation retains observations through both public adapters. */
|
||||
export async function verifyScenarioFailureEvidence(executable, origin) {
|
||||
const scenario = browserScenarioSchema.parse({
|
||||
browser: { mode: "launch", executable_path: executable },
|
||||
start_url: { url: `${origin}/failed-navigation` },
|
||||
actions: [
|
||||
{ step_id: "unused", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
capture: { events: ["network"] },
|
||||
});
|
||||
const profilesBefore = await scenarioProfiles();
|
||||
let cliFailure;
|
||||
try {
|
||||
await runScenarioCli(scenario);
|
||||
assert.fail("The fixture must fail before initial navigation completes");
|
||||
} catch (cause) {
|
||||
assert.equal(typeof cause.stdout, "string");
|
||||
cliFailure = JSON.parse(cause.stdout);
|
||||
}
|
||||
assertFailedNavigation(cliFailure, scenario.start_url.url);
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
|
||||
env: process.env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({
|
||||
name: "browser-scenario-failure-e2e",
|
||||
version: "1",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const result = await client.callTool({
|
||||
name: "capture_browser_scenario",
|
||||
arguments: scenario,
|
||||
});
|
||||
assert.equal(result.isError, true);
|
||||
assertFailedNavigation(requireMcpToolError(result), scenario.start_url.url);
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
}
|
||||
const profilesAfter = await scenarioProfiles();
|
||||
assert.ok([...profilesAfter].every((profile) => profilesBefore.has(profile)));
|
||||
return {
|
||||
mocked: false,
|
||||
cli: true,
|
||||
stdio_mcp: true,
|
||||
failed_navigation_events_retained: true,
|
||||
profile_cleanup: true,
|
||||
};
|
||||
}
|
||||
|
||||
const assertFailedNavigation = (error, requestedUrl) => {
|
||||
assert.equal(error?.code, "execution_failure");
|
||||
const observation = error.details?.partial_observation;
|
||||
assert.equal(observation?.kind, "browser-scenario-observation");
|
||||
assert.equal(observation.capture.browser.cleanup, "terminated-owned-process");
|
||||
assert.deepEqual(observation.capture.steps, []);
|
||||
assert.ok(
|
||||
observation.capture.events.items.some(
|
||||
(event) =>
|
||||
event.kind === "request-failed" && event.url.url === requestedUrl,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
/** Snapshot provider-owned profile names for cleanup comparison. */
|
||||
export async function scenarioProfiles() {
|
||||
return new Set(
|
||||
|
||||
@@ -16,7 +16,7 @@ import {
|
||||
listBrowserTargetsInputSchema,
|
||||
} from "../dist/domain/browserObservation.js";
|
||||
import { observeWebSessionInputSchema } from "../dist/domain/browserSession.js";
|
||||
import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiff.js";
|
||||
import { compareWebCapturesInputSchema } from "../dist/domain/webCaptureDiffSchemas.js";
|
||||
import {
|
||||
captureWebScreenshotInputSchema,
|
||||
compareWebScreenshotsInputSchema,
|
||||
@@ -33,6 +33,7 @@ import {
|
||||
browserScenario,
|
||||
runScenarioCli,
|
||||
scenarioProfiles,
|
||||
verifyScenarioFailureEvidence,
|
||||
} from "./lib/browser-scenario-verifier.mjs";
|
||||
import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { verifyLargeScreenshotE2e } from "./lib/browser-screenshot-e2e.mjs";
|
||||
@@ -294,6 +295,10 @@ try {
|
||||
if ([...profilesAfter].some((entry) => !profilesBefore.has(entry)))
|
||||
throw new Error("Scenario launch retained a temporary browser profile");
|
||||
assertScenarioCapture(launchedScenario.value);
|
||||
const scenarioFailure = await verifyScenarioFailureEvidence(
|
||||
executable,
|
||||
site.origin,
|
||||
);
|
||||
const scenarioResults = JSON.stringify([
|
||||
attachedScenario,
|
||||
launchedScenario.value,
|
||||
@@ -360,6 +365,7 @@ try {
|
||||
browserScenarioCli: true,
|
||||
browserScenarioAttachCleanup: "disconnected-external",
|
||||
browserScenarioLaunchCleanup: "terminated-owned-process",
|
||||
scenarioFailure,
|
||||
verified: true,
|
||||
};
|
||||
} finally {
|
||||
|
||||
Reference in New Issue
Block a user