fix(browser): retain scenario observations across failures
This commit is contained in:
@@ -1,9 +1,14 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "@modelcontextprotocol/client";
|
||||
import { StdioClientTransport } from "@modelcontextprotocol/client/stdio";
|
||||
|
||||
import { browserScenarioSchema } from "../../dist/domain/browserScenario.js";
|
||||
import { requireMcpToolError } from "./mcp-verifier-results.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
|
||||
@@ -142,6 +147,73 @@ export async function runScenarioCli(scenario) {
|
||||
return JSON.parse(stdout);
|
||||
}
|
||||
|
||||
/** Verify failed real navigation retains observations through both public adapters. */
|
||||
export async function verifyScenarioFailureEvidence(executable, origin) {
|
||||
const scenario = browserScenarioSchema.parse({
|
||||
browser: { mode: "launch", executable_path: executable },
|
||||
start_url: { url: `${origin}/failed-navigation` },
|
||||
actions: [
|
||||
{ step_id: "unused", action: "wait_for_timeout", duration_ms: 1 },
|
||||
],
|
||||
capture: { events: ["network"] },
|
||||
});
|
||||
const profilesBefore = await scenarioProfiles();
|
||||
let cliFailure;
|
||||
try {
|
||||
await runScenarioCli(scenario);
|
||||
assert.fail("The fixture must fail before initial navigation completes");
|
||||
} catch (cause) {
|
||||
assert.equal(typeof cause.stdout, "string");
|
||||
cliFailure = JSON.parse(cause.stdout);
|
||||
}
|
||||
assertFailedNavigation(cliFailure, scenario.start_url.url);
|
||||
const transport = new StdioClientTransport({
|
||||
command: process.execPath,
|
||||
args: [fileURLToPath(new URL("../rea.mjs", import.meta.url)), "mcp"],
|
||||
env: process.env,
|
||||
stderr: "pipe",
|
||||
});
|
||||
const client = new Client({
|
||||
name: "browser-scenario-failure-e2e",
|
||||
version: "1",
|
||||
});
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const result = await client.callTool({
|
||||
name: "capture_browser_scenario",
|
||||
arguments: scenario,
|
||||
});
|
||||
assert.equal(result.isError, true);
|
||||
assertFailedNavigation(requireMcpToolError(result), scenario.start_url.url);
|
||||
} finally {
|
||||
await client.close();
|
||||
await transport.close();
|
||||
}
|
||||
const profilesAfter = await scenarioProfiles();
|
||||
assert.ok([...profilesAfter].every((profile) => profilesBefore.has(profile)));
|
||||
return {
|
||||
mocked: false,
|
||||
cli: true,
|
||||
stdio_mcp: true,
|
||||
failed_navigation_events_retained: true,
|
||||
profile_cleanup: true,
|
||||
};
|
||||
}
|
||||
|
||||
const assertFailedNavigation = (error, requestedUrl) => {
|
||||
assert.equal(error?.code, "execution_failure");
|
||||
const observation = error.details?.partial_observation;
|
||||
assert.equal(observation?.kind, "browser-scenario-observation");
|
||||
assert.equal(observation.capture.browser.cleanup, "terminated-owned-process");
|
||||
assert.deepEqual(observation.capture.steps, []);
|
||||
assert.ok(
|
||||
observation.capture.events.items.some(
|
||||
(event) =>
|
||||
event.kind === "request-failed" && event.url.url === requestedUrl,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
/** Snapshot provider-owned profile names for cleanup comparison. */
|
||||
export async function scenarioProfiles() {
|
||||
return new Set(
|
||||
|
||||
Reference in New Issue
Block a user