From 0043d9b134c0a353e8bf85c0bde4492275f1f720 Mon Sep 17 00:00:00 2001 From: LouisChen <65096591@qq.com> Date: Tue, 13 Oct 2026 02:36:45 +0800 Subject: [PATCH] fix(browser): report mismatched event family selection as unknown in capture comparison (#2239) * fix(browser): report mismatched event family selection as unknown in capture comparison * fix(browser): retain selected event families and unknown coverage Fold the complementary implementation from #2253 into the earlier #2239 proposal, retaining the original contribution history. Co-authored-by: itsF4LCON <107556180+itsF4LCON@users.noreply.github.com> --------- Co-authored-by: LouisChen <3361008+chqcjq@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: itsF4LCON <107556180+itsF4LCON@users.noreply.github.com> --- docs/browser-scenario-contract.md | 6 ++ .../BrowserScenarioPartialObservation.ts | 1 + .../PlaywrightBrowserScenarioProvider.test.ts | 53 +++++++++++ src/contracts/browserToolContracts.ts | 1 + src/domain/browserScenarioCapture.ts | 2 + src/domain/browserScenarioCaptureValues.ts | 15 ++++ src/domain/browserScenarioDiff.test.ts | 90 +++++++++++++++++++ src/domain/browserScenarioDiff.ts | 14 ++- src/domain/browserScenarioDiffEvidence.ts | 20 +++++ src/domain/browserScenarioValues.ts | 16 +--- tests/boundary/cli/browserCli.test.ts | 1 + tests/boundary/mcp/browserScenarioMcp.test.ts | 1 + tests/fixtures/webScriptCapture.ts | 1 + 13 files changed, 206 insertions(+), 15 deletions(-) diff --git a/docs/browser-scenario-contract.md b/docs/browser-scenario-contract.md index b994d2936..403865be8 100644 --- a/docs/browser-scenario-contract.md +++ b/docs/browser-scenario-contract.md @@ -312,3 +312,9 @@ artifacts carry their normalized before/after digests and capture states. Every changed or unknown artifact record is returned inline for each aligned step. Missing, truncated, or mismatched capture coverage remains unknown instead of being treated as equality. + +`scenario.event_families` records the sorted, de-duplicated `capture.events` +selection. Captures with different selections report event evidence as unknown +with a recovery instruction, even when no selected family emitted an event. +Captures written before this field existed omit it, so their event evidence is +also unknown. diff --git a/src/browser/BrowserScenarioPartialObservation.ts b/src/browser/BrowserScenarioPartialObservation.ts index 1fd395a64..ca0df4273 100644 --- a/src/browser/BrowserScenarioPartialObservation.ts +++ b/src/browser/BrowserScenarioPartialObservation.ts @@ -89,6 +89,7 @@ export const browserScenarioCaptureData = (input: { .map(({ secret_id: id }) => id) .sort(), network_content: input.scenario.capture.network, + event_families: [...new Set(input.scenario.capture.events)].sort(), }, duration_ms: Date.now() - input.startedAt, steps: [...input.steps], diff --git a/src/browser/PlaywrightBrowserScenarioProvider.test.ts b/src/browser/PlaywrightBrowserScenarioProvider.test.ts index 6d4e284db..aacf766ad 100644 --- a/src/browser/PlaywrightBrowserScenarioProvider.test.ts +++ b/src/browser/PlaywrightBrowserScenarioProvider.test.ts @@ -14,10 +14,13 @@ import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js"; import { browserStepArtifactsSchema, type BrowserScenarioEvent, + type BrowserScenarioEventFamily, type BrowserStepArtifacts, } from "../domain/browserScenarioCaptureValues.js"; import { PLAYWRIGHT_RESPONSE_BODY_PROVENANCE_LIMITATION } from "../domain/browserNetworkEvidence.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; +import { compareBrowserScenarios } from "../domain/browserScenarioDiff.js"; +import { compareBrowserScenariosInputSchema } from "../domain/browserScenarioDiffValues.js"; import { AnalysisCancelledError, AnalysisOutputError, @@ -373,6 +376,56 @@ describe("PlaywrightBrowserScenarioProvider", () => { }); }); +describe("PlaywrightBrowserScenarioProvider comparison coverage", () => { + it("keeps quiet captures with different event families from comparing unchanged", async () => { + const capture = async (events: readonly BrowserScenarioEventFamily[]) => { + const provider = new PlaywrightBrowserScenarioProvider(() => + Promise.resolve(new FakeSession("launch")), + ); + const result = await provider.captureScenario(scenario({ events })); + if (!result.ok) throw result.error; + return result.value; + }; + const compare = async ( + before: readonly BrowserScenarioEventFamily[], + after: readonly BrowserScenarioEventFamily[], + ) => + compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: await capture(before), + after_scenario: await capture(after), + }), + ); + + const recorded = await capture(["page-errors", "console", "console"]); + expect(recorded.completeness.equality_eligible).toBe(true); + expect(recorded.scenario.event_families).toEqual([ + "console", + "page-errors", + ]); + for (const [before, after] of [ + [[], ["console"]], + [["console"], ["page-errors"]], + ] as const) { + const compared = await compare(before, after); + expect(compared.overall_status).toBe("unknown"); + expect( + compared.steps.flatMap((step) => step.artifact_diffs), + ).toContainEqual( + expect.objectContaining({ + artifact: "events", + status: "unknown", + reason: expect.stringContaining("Event family selections differ"), + }), + ); + } + expect( + (await compare(["console", "page-errors"], ["page-errors", "console"])) + .overall_status, + ).toBe("unchanged"); + }); +}); + describe("PlaywrightBrowserScenarioProvider cleanup", () => { it("returns a cleanup failure with completed observations when close fails", async () => { const session = new FakeSession("launch", { diff --git a/src/contracts/browserToolContracts.ts b/src/contracts/browserToolContracts.ts index 7d5a23313..70111640a 100644 --- a/src/contracts/browserToolContracts.ts +++ b/src/contracts/browserToolContracts.ts @@ -84,6 +84,7 @@ const exampleScenarioCapture = () => ({ response_body: false, header_values: false, }, + event_families: [], }, duration_ms: 10, steps: [ diff --git a/src/domain/browserScenarioCapture.ts b/src/domain/browserScenarioCapture.ts index 8ad389588..61c86b431 100644 --- a/src/domain/browserScenarioCapture.ts +++ b/src/domain/browserScenarioCapture.ts @@ -3,6 +3,7 @@ import { browserNetworkContentSelectionValuesSchema } from "./browserNetworkEvid import { browserScenarioCompletenessSchema, + browserScenarioEventFamilySchema, browserScenarioEventSchema, browserScenarioStepSchema, } from "./browserScenarioCaptureValues.js"; @@ -22,6 +23,7 @@ export const browserScenarioCaptureSchema = z action_count: z.number().int().min(1), secret_references: z.array(z.string().min(1)), network_content: browserNetworkContentSelectionValuesSchema, + event_families: z.array(browserScenarioEventFamilySchema).optional(), }), duration_ms: z.number().int().min(0), steps: z.array(browserScenarioStepSchema).min(1), diff --git a/src/domain/browserScenarioCaptureValues.ts b/src/domain/browserScenarioCaptureValues.ts index 81fef0c15..26c1559bb 100644 --- a/src/domain/browserScenarioCaptureValues.ts +++ b/src/domain/browserScenarioCaptureValues.ts @@ -119,6 +119,21 @@ export const browserStepArtifactsSchema = z.strictObject({ }); export type BrowserStepArtifacts = z.infer; +/** Event families a scenario can select for capture. */ +export const browserScenarioEventFamilySchema = z.enum([ + "console", + "page-errors", + "network", + "websockets", + "frames", + "workers", + "popups", + "downloads", +]); +export type BrowserScenarioEventFamily = z.infer< + typeof browserScenarioEventFamilySchema +>; + const eventBase = { sequence: z.number().int().min(1), step_index: z.number().int().min(0), diff --git a/src/domain/browserScenarioDiff.test.ts b/src/domain/browserScenarioDiff.test.ts index 4c5272b9c..f15bd9da0 100644 --- a/src/domain/browserScenarioDiff.test.ts +++ b/src/domain/browserScenarioDiff.test.ts @@ -62,6 +62,7 @@ const networkCapture = (sourceSequence: number, receiptStep: number) => { response_body: false, header_values: false, }, + event_families: ["network"], }, events: { items: events, retained: 2, dropped: 0 }, }); @@ -146,6 +147,94 @@ it("reports content-selection mismatch as unknown instead of a website change", ).toBe(false); }); +it("reports event-family mismatch and legacy selection as unknown", () => { + const consoleOnly = scenarioCapture({}); + const withFamilies = ( + capture: BrowserScenarioCapture, + families: BrowserScenarioCapture["scenario"]["event_families"], + ) => + browserScenarioCaptureSchema.parse({ + ...capture, + scenario: { ...capture.scenario, event_families: families }, + }); + const compare = ( + before: BrowserScenarioCapture, + after: BrowserScenarioCapture, + ) => + compareBrowserScenarios( + compareBrowserScenariosInputSchema.parse({ + before_scenario: before, + after_scenario: after, + }), + ); + const eventDiffs = (compared: ReturnType) => + compared.steps + .flatMap((step) => step.artifact_diffs) + .filter(({ artifact }) => artifact === "events"); + + const widened = withFamilies(consoleOnly, ["page-errors", "console"]); + const mismatched = compare(consoleOnly, widened); + expect(mismatched.overall_status).toBe("unknown"); + expect(eventDiffs(mismatched)).toContainEqual( + expect.objectContaining({ + status: "unknown", + before_state: "captured", + after_state: "captured", + reason: expect.stringContaining( + "Event family selections differ (console before, console, page-errors after)", + ), + }), + ); + expect( + compare( + withFamilies(consoleOnly, ["console", "page-errors"]), + withFamilies(consoleOnly, ["page-errors", "console"]), + ).overall_status, + ).toBe("unchanged"); + + const legacy = withFamilies(consoleOnly, undefined); + expect(legacy.scenario.event_families).toBeUndefined(); + for (const [before, after] of [ + [legacy, consoleOnly], + [legacy, legacy], + ] as const) + expect(eventDiffs(compare(before, after))).toContainEqual( + expect.objectContaining({ + status: "unknown", + reason: expect.stringContaining("Event family selection is unknown"), + }), + ); + expect( + compareBrowserScenariosInputSchema.safeParse({ + before_scenario: { + ...consoleOnly, + scenario: { ...consoleOnly.scenario, event_families: ["dialogs"] }, + }, + after_scenario: consoleOnly, + }).success, + ).toBe(false); + + const quiet = browserScenarioCaptureSchema.parse({ + ...withFamilies(consoleOnly, []), + events: { retained: 0, dropped: 0, items: [] }, + }); + const differentEvents = compare(quiet, widened); + expect(differentEvents.overall_status).toBe("unknown"); + expect(eventDiffs(differentEvents).map(({ status }) => status)).toEqual([ + "unknown", + "unknown", + ]); + + const both = eventDiffs(compare(networkCapture(1, 0), consoleOnly)); + expect(both).toContainEqual( + expect.objectContaining({ + reason: expect.stringMatching( + /Event family selections differ.*content selections differ/u, + ), + }), + ); +}); + describe("browser scenario comparison", () => { it("records canonical literal normalization and ignores declared volatile fields", () => { const before = scenarioCapture({ @@ -485,6 +574,7 @@ const scenarioCapture = ( response_body: false, header_values: false, }, + event_families: ["console"], }, duration_ms: options.elapsedMs ?? 10, steps: [ diff --git a/src/domain/browserScenarioDiff.ts b/src/domain/browserScenarioDiff.ts index acfe3dd63..2cb761a76 100644 --- a/src/domain/browserScenarioDiff.ts +++ b/src/domain/browserScenarioDiff.ts @@ -5,6 +5,7 @@ import { artifactEvidenceState, captured, eventEvidenceState, + eventFamilyCoverageReason, eventsFor, networkContentCoverageReason, screenshotProjection, @@ -332,7 +333,7 @@ const compareStepArtifacts = ( ), beforeState: eventEvidenceState(input.beforeCapture, input.beforeStep), afterState: eventEvidenceState(input.afterCapture, input.afterStep), - incomparableReason: networkContentCoverageReason( + incomparableReason: eventCoverageReason( input.beforeCapture, input.afterCapture, ), @@ -340,6 +341,17 @@ const compareStepArtifacts = ( ]; }; +const eventCoverageReason = ( + before: BrowserScenarioCapture, + after: BrowserScenarioCapture, +): string | undefined => { + const reasons = [ + eventFamilyCoverageReason(before, after), + networkContentCoverageReason(before, after), + ].filter((reason) => reason !== undefined); + return reasons.length === 0 ? undefined : reasons.join(" "); +}; + interface CompareCapturedValuesInput { readonly artifact: BrowserScenarioArtifactKind; readonly before: CaptureState; diff --git a/src/domain/browserScenarioDiffEvidence.ts b/src/domain/browserScenarioDiffEvidence.ts index 015da4bff..f7f4b45e3 100644 --- a/src/domain/browserScenarioDiffEvidence.ts +++ b/src/domain/browserScenarioDiffEvidence.ts @@ -130,6 +130,26 @@ export const networkContentCoverageReason = ( : undefined; }; +/** Different or unrecorded event-family selection is missing comparability, not an observed website change. */ +export const eventFamilyCoverageReason = ( + before: BrowserScenarioCapture, + after: BrowserScenarioCapture, +): string | undefined => { + const left = before.scenario.event_families; + const right = after.scenario.event_families; + if (left === undefined || right === undefined) + return "Event family selection is unknown in a legacy capture; capture both sides with matching event families before comparing event evidence."; + const leftFamilies = new Set(left); + const rightFamilies = new Set(right); + return leftFamilies.size === rightFamilies.size && + [...leftFamilies].every((family) => rightFamilies.has(family)) + ? undefined + : `Event family selections differ (${familyList(leftFamilies)} before, ${familyList(rightFamilies)} after); capture both sides with matching event families before comparing event evidence.`; +}; + +const familyList = (families: ReadonlySet): string => + families.size === 0 ? "none" : [...families].sort().join(", "); + /** Wrap one value as captured comparison evidence. */ export const captured = (value: Value): CaptureState => ({ state: "captured", diff --git a/src/domain/browserScenarioValues.ts b/src/domain/browserScenarioValues.ts index 544d4c222..b0344ae0e 100644 --- a/src/domain/browserScenarioValues.ts +++ b/src/domain/browserScenarioValues.ts @@ -5,6 +5,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js"; import { localPathStringSchema } from "./localPath.js"; import { browserNetworkContentSelectionSchema } from "./browserNetworkEvidence.js"; +import { browserScenarioEventFamilySchema } from "./browserScenarioCaptureValues.js"; import { browserEndpointSchema, @@ -289,20 +290,7 @@ export const browserScenarioCaptureSchema = z .strictObject({ after_each_step: z.array(snapshotKindSchema).default([]), at_end: z.array(snapshotKindSchema).default(["url"]), - events: z - .array( - z.enum([ - "console", - "page-errors", - "network", - "websockets", - "frames", - "workers", - "popups", - "downloads", - ]), - ) - .default([]), + events: z.array(browserScenarioEventFamilySchema).default([]), network: browserNetworkContentSelectionSchema, }) .default({ diff --git a/tests/boundary/cli/browserCli.test.ts b/tests/boundary/cli/browserCli.test.ts index 9084e87a9..7b2cb93ba 100644 --- a/tests/boundary/cli/browserCli.test.ts +++ b/tests/boundary/cli/browserCli.test.ts @@ -352,6 +352,7 @@ const completeScenarioCapture = () => { response_body: false, header_values: false, }, + event_families: [], }, duration_ms: 1, steps: [ diff --git a/tests/boundary/mcp/browserScenarioMcp.test.ts b/tests/boundary/mcp/browserScenarioMcp.test.ts index c6bfdd6e1..8e086ed06 100644 --- a/tests/boundary/mcp/browserScenarioMcp.test.ts +++ b/tests/boundary/mcp/browserScenarioMcp.test.ts @@ -85,6 +85,7 @@ const captureFor = (scenario: BrowserScenario): BrowserScenarioCapture => { action_count: scenario.actions.length, secret_references: scenario.secrets.map(({ secret_id: id }) => id), network_content: scenario.capture.network, + event_families: scenario.capture.events, }, duration_ms: 2, steps: [ diff --git a/tests/fixtures/webScriptCapture.ts b/tests/fixtures/webScriptCapture.ts index 4d3a2ff64..0c6318531 100644 --- a/tests/fixtures/webScriptCapture.ts +++ b/tests/fixtures/webScriptCapture.ts @@ -105,6 +105,7 @@ export const scriptScenarioFixture = ( response_body: true, header_values: false, }, + event_families: ["network"], }, duration_ms: 2, steps: [step(0), step(1)],