fix(browser): report mismatched event family selection as unknown in capture comparison (#2239)

* fix(browser): report mismatched event family selection as unknown in capture comparison

* fix(browser): retain selected event families and unknown coverage

Fold the complementary implementation from #2253 into the earlier #2239 proposal, retaining the original contribution history.

Co-authored-by: itsF4LCON <107556180+itsF4LCON@users.noreply.github.com>

---------

Co-authored-by: LouisChen <3361008+chqcjq@users.noreply.github.com>
Co-authored-by: morluto <76467478+morluto@users.noreply.github.com>
Co-authored-by: itsF4LCON <107556180+itsF4LCON@users.noreply.github.com>
This commit is contained in:
LouisChen
2026-10-13 02:36:45 +08:00
committed by GitHub
co-authored by LouisChen morluto itsF4LCON
parent aeee5074d1
commit 0043d9b134
13 changed files with 206 additions and 15 deletions
+6
View File
@@ -312,3 +312,9 @@ artifacts carry their normalized before/after digests and capture states.
Every changed or unknown artifact record is returned inline for each aligned
step. Missing, truncated, or mismatched capture coverage remains unknown
instead of being treated as equality.
`scenario.event_families` records the sorted, de-duplicated `capture.events`
selection. Captures with different selections report event evidence as unknown
with a recovery instruction, even when no selected family emitted an event.
Captures written before this field existed omit it, so their event evidence is
also unknown.
@@ -89,6 +89,7 @@ export const browserScenarioCaptureData = (input: {
.map(({ secret_id: id }) => id)
.sort(),
network_content: input.scenario.capture.network,
event_families: [...new Set(input.scenario.capture.events)].sort(),
},
duration_ms: Date.now() - input.startedAt,
steps: [...input.steps],
@@ -14,10 +14,13 @@ import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
import {
browserStepArtifactsSchema,
type BrowserScenarioEvent,
type BrowserScenarioEventFamily,
type BrowserStepArtifacts,
} from "../domain/browserScenarioCaptureValues.js";
import { PLAYWRIGHT_RESPONSE_BODY_PROVENANCE_LIMITATION } from "../domain/browserNetworkEvidence.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { compareBrowserScenarios } from "../domain/browserScenarioDiff.js";
import { compareBrowserScenariosInputSchema } from "../domain/browserScenarioDiffValues.js";
import {
AnalysisCancelledError,
AnalysisOutputError,
@@ -373,6 +376,56 @@ describe("PlaywrightBrowserScenarioProvider", () => {
});
});
describe("PlaywrightBrowserScenarioProvider comparison coverage", () => {
it("keeps quiet captures with different event families from comparing unchanged", async () => {
const capture = async (events: readonly BrowserScenarioEventFamily[]) => {
const provider = new PlaywrightBrowserScenarioProvider(() =>
Promise.resolve(new FakeSession("launch")),
);
const result = await provider.captureScenario(scenario({ events }));
if (!result.ok) throw result.error;
return result.value;
};
const compare = async (
before: readonly BrowserScenarioEventFamily[],
after: readonly BrowserScenarioEventFamily[],
) =>
compareBrowserScenarios(
compareBrowserScenariosInputSchema.parse({
before_scenario: await capture(before),
after_scenario: await capture(after),
}),
);
const recorded = await capture(["page-errors", "console", "console"]);
expect(recorded.completeness.equality_eligible).toBe(true);
expect(recorded.scenario.event_families).toEqual([
"console",
"page-errors",
]);
for (const [before, after] of [
[[], ["console"]],
[["console"], ["page-errors"]],
] as const) {
const compared = await compare(before, after);
expect(compared.overall_status).toBe("unknown");
expect(
compared.steps.flatMap((step) => step.artifact_diffs),
).toContainEqual(
expect.objectContaining({
artifact: "events",
status: "unknown",
reason: expect.stringContaining("Event family selections differ"),
}),
);
}
expect(
(await compare(["console", "page-errors"], ["page-errors", "console"]))
.overall_status,
).toBe("unchanged");
});
});
describe("PlaywrightBrowserScenarioProvider cleanup", () => {
it("returns a cleanup failure with completed observations when close fails", async () => {
const session = new FakeSession("launch", {
+1
View File
@@ -84,6 +84,7 @@ const exampleScenarioCapture = () => ({
response_body: false,
header_values: false,
},
event_families: [],
},
duration_ms: 10,
steps: [
+2
View File
@@ -3,6 +3,7 @@ import { browserNetworkContentSelectionValuesSchema } from "./browserNetworkEvid
import {
browserScenarioCompletenessSchema,
browserScenarioEventFamilySchema,
browserScenarioEventSchema,
browserScenarioStepSchema,
} from "./browserScenarioCaptureValues.js";
@@ -22,6 +23,7 @@ export const browserScenarioCaptureSchema = z
action_count: z.number().int().min(1),
secret_references: z.array(z.string().min(1)),
network_content: browserNetworkContentSelectionValuesSchema,
event_families: z.array(browserScenarioEventFamilySchema).optional(),
}),
duration_ms: z.number().int().min(0),
steps: z.array(browserScenarioStepSchema).min(1),
@@ -119,6 +119,21 @@ export const browserStepArtifactsSchema = z.strictObject({
});
export type BrowserStepArtifacts = z.infer<typeof browserStepArtifactsSchema>;
/** Event families a scenario can select for capture. */
export const browserScenarioEventFamilySchema = z.enum([
"console",
"page-errors",
"network",
"websockets",
"frames",
"workers",
"popups",
"downloads",
]);
export type BrowserScenarioEventFamily = z.infer<
typeof browserScenarioEventFamilySchema
>;
const eventBase = {
sequence: z.number().int().min(1),
step_index: z.number().int().min(0),
+90
View File
@@ -62,6 +62,7 @@ const networkCapture = (sourceSequence: number, receiptStep: number) => {
response_body: false,
header_values: false,
},
event_families: ["network"],
},
events: { items: events, retained: 2, dropped: 0 },
});
@@ -146,6 +147,94 @@ it("reports content-selection mismatch as unknown instead of a website change",
).toBe(false);
});
it("reports event-family mismatch and legacy selection as unknown", () => {
const consoleOnly = scenarioCapture({});
const withFamilies = (
capture: BrowserScenarioCapture,
families: BrowserScenarioCapture["scenario"]["event_families"],
) =>
browserScenarioCaptureSchema.parse({
...capture,
scenario: { ...capture.scenario, event_families: families },
});
const compare = (
before: BrowserScenarioCapture,
after: BrowserScenarioCapture,
) =>
compareBrowserScenarios(
compareBrowserScenariosInputSchema.parse({
before_scenario: before,
after_scenario: after,
}),
);
const eventDiffs = (compared: ReturnType<typeof compare>) =>
compared.steps
.flatMap((step) => step.artifact_diffs)
.filter(({ artifact }) => artifact === "events");
const widened = withFamilies(consoleOnly, ["page-errors", "console"]);
const mismatched = compare(consoleOnly, widened);
expect(mismatched.overall_status).toBe("unknown");
expect(eventDiffs(mismatched)).toContainEqual(
expect.objectContaining({
status: "unknown",
before_state: "captured",
after_state: "captured",
reason: expect.stringContaining(
"Event family selections differ (console before, console, page-errors after)",
),
}),
);
expect(
compare(
withFamilies(consoleOnly, ["console", "page-errors"]),
withFamilies(consoleOnly, ["page-errors", "console"]),
).overall_status,
).toBe("unchanged");
const legacy = withFamilies(consoleOnly, undefined);
expect(legacy.scenario.event_families).toBeUndefined();
for (const [before, after] of [
[legacy, consoleOnly],
[legacy, legacy],
] as const)
expect(eventDiffs(compare(before, after))).toContainEqual(
expect.objectContaining({
status: "unknown",
reason: expect.stringContaining("Event family selection is unknown"),
}),
);
expect(
compareBrowserScenariosInputSchema.safeParse({
before_scenario: {
...consoleOnly,
scenario: { ...consoleOnly.scenario, event_families: ["dialogs"] },
},
after_scenario: consoleOnly,
}).success,
).toBe(false);
const quiet = browserScenarioCaptureSchema.parse({
...withFamilies(consoleOnly, []),
events: { retained: 0, dropped: 0, items: [] },
});
const differentEvents = compare(quiet, widened);
expect(differentEvents.overall_status).toBe("unknown");
expect(eventDiffs(differentEvents).map(({ status }) => status)).toEqual([
"unknown",
"unknown",
]);
const both = eventDiffs(compare(networkCapture(1, 0), consoleOnly));
expect(both).toContainEqual(
expect.objectContaining({
reason: expect.stringMatching(
/Event family selections differ.*content selections differ/u,
),
}),
);
});
describe("browser scenario comparison", () => {
it("records canonical literal normalization and ignores declared volatile fields", () => {
const before = scenarioCapture({
@@ -485,6 +574,7 @@ const scenarioCapture = (
response_body: false,
header_values: false,
},
event_families: ["console"],
},
duration_ms: options.elapsedMs ?? 10,
steps: [
+13 -1
View File
@@ -5,6 +5,7 @@ import {
artifactEvidenceState,
captured,
eventEvidenceState,
eventFamilyCoverageReason,
eventsFor,
networkContentCoverageReason,
screenshotProjection,
@@ -332,7 +333,7 @@ const compareStepArtifacts = (
),
beforeState: eventEvidenceState(input.beforeCapture, input.beforeStep),
afterState: eventEvidenceState(input.afterCapture, input.afterStep),
incomparableReason: networkContentCoverageReason(
incomparableReason: eventCoverageReason(
input.beforeCapture,
input.afterCapture,
),
@@ -340,6 +341,17 @@ const compareStepArtifacts = (
];
};
const eventCoverageReason = (
before: BrowserScenarioCapture,
after: BrowserScenarioCapture,
): string | undefined => {
const reasons = [
eventFamilyCoverageReason(before, after),
networkContentCoverageReason(before, after),
].filter((reason) => reason !== undefined);
return reasons.length === 0 ? undefined : reasons.join(" ");
};
interface CompareCapturedValuesInput<Value> {
readonly artifact: BrowserScenarioArtifactKind;
readonly before: CaptureState<Value>;
+20
View File
@@ -130,6 +130,26 @@ export const networkContentCoverageReason = (
: undefined;
};
/** Different or unrecorded event-family selection is missing comparability, not an observed website change. */
export const eventFamilyCoverageReason = (
before: BrowserScenarioCapture,
after: BrowserScenarioCapture,
): string | undefined => {
const left = before.scenario.event_families;
const right = after.scenario.event_families;
if (left === undefined || right === undefined)
return "Event family selection is unknown in a legacy capture; capture both sides with matching event families before comparing event evidence.";
const leftFamilies = new Set(left);
const rightFamilies = new Set(right);
return leftFamilies.size === rightFamilies.size &&
[...leftFamilies].every((family) => rightFamilies.has(family))
? undefined
: `Event family selections differ (${familyList(leftFamilies)} before, ${familyList(rightFamilies)} after); capture both sides with matching event families before comparing event evidence.`;
};
const familyList = (families: ReadonlySet<string>): string =>
families.size === 0 ? "none" : [...families].sort().join(", ");
/** Wrap one value as captured comparison evidence. */
export const captured = <Value>(value: Value): CaptureState<Value> => ({
state: "captured",
+2 -14
View File
@@ -5,6 +5,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js";
import { localPathStringSchema } from "./localPath.js";
import { browserNetworkContentSelectionSchema } from "./browserNetworkEvidence.js";
import { browserScenarioEventFamilySchema } from "./browserScenarioCaptureValues.js";
import {
browserEndpointSchema,
@@ -289,20 +290,7 @@ export const browserScenarioCaptureSchema = z
.strictObject({
after_each_step: z.array(snapshotKindSchema).default([]),
at_end: z.array(snapshotKindSchema).default(["url"]),
events: z
.array(
z.enum([
"console",
"page-errors",
"network",
"websockets",
"frames",
"workers",
"popups",
"downloads",
]),
)
.default([]),
events: z.array(browserScenarioEventFamilySchema).default([]),
network: browserNetworkContentSelectionSchema,
})
.default({
+1
View File
@@ -352,6 +352,7 @@ const completeScenarioCapture = () => {
response_body: false,
header_values: false,
},
event_families: [],
},
duration_ms: 1,
steps: [
@@ -85,6 +85,7 @@ const captureFor = (scenario: BrowserScenario): BrowserScenarioCapture => {
action_count: scenario.actions.length,
secret_references: scenario.secrets.map(({ secret_id: id }) => id),
network_content: scenario.capture.network,
event_families: scenario.capture.events,
},
duration_ms: 2,
steps: [
+1
View File
@@ -105,6 +105,7 @@ export const scriptScenarioFixture = (
response_body: true,
header_values: false,
},
event_families: ["network"],
},
duration_ms: 2,
steps: [step(0), step(1)],