fix(browser): report mismatched event family selection as unknown in capture comparison (#2239)
* fix(browser): report mismatched event family selection as unknown in capture comparison * fix(browser): retain selected event families and unknown coverage Fold the complementary implementation from #2253 into the earlier #2239 proposal, retaining the original contribution history. Co-authored-by: itsF4LCON <107556180+itsF4LCON@users.noreply.github.com> --------- Co-authored-by: LouisChen <3361008+chqcjq@users.noreply.github.com> Co-authored-by: morluto <76467478+morluto@users.noreply.github.com> Co-authored-by: itsF4LCON <107556180+itsF4LCON@users.noreply.github.com>
This commit is contained in:
co-authored by
LouisChen
morluto
itsF4LCON
parent
aeee5074d1
commit
0043d9b134
@@ -312,3 +312,9 @@ artifacts carry their normalized before/after digests and capture states.
|
||||
Every changed or unknown artifact record is returned inline for each aligned
|
||||
step. Missing, truncated, or mismatched capture coverage remains unknown
|
||||
instead of being treated as equality.
|
||||
|
||||
`scenario.event_families` records the sorted, de-duplicated `capture.events`
|
||||
selection. Captures with different selections report event evidence as unknown
|
||||
with a recovery instruction, even when no selected family emitted an event.
|
||||
Captures written before this field existed omit it, so their event evidence is
|
||||
also unknown.
|
||||
|
||||
@@ -89,6 +89,7 @@ export const browserScenarioCaptureData = (input: {
|
||||
.map(({ secret_id: id }) => id)
|
||||
.sort(),
|
||||
network_content: input.scenario.capture.network,
|
||||
event_families: [...new Set(input.scenario.capture.events)].sort(),
|
||||
},
|
||||
duration_ms: Date.now() - input.startedAt,
|
||||
steps: [...input.steps],
|
||||
|
||||
@@ -14,10 +14,13 @@ import type { BrowserScenarioAction } from "../domain/browserScenarioValues.js";
|
||||
import {
|
||||
browserStepArtifactsSchema,
|
||||
type BrowserScenarioEvent,
|
||||
type BrowserScenarioEventFamily,
|
||||
type BrowserStepArtifacts,
|
||||
} from "../domain/browserScenarioCaptureValues.js";
|
||||
import { PLAYWRIGHT_RESPONSE_BODY_PROVENANCE_LIMITATION } from "../domain/browserNetworkEvidence.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { compareBrowserScenarios } from "../domain/browserScenarioDiff.js";
|
||||
import { compareBrowserScenariosInputSchema } from "../domain/browserScenarioDiffValues.js";
|
||||
import {
|
||||
AnalysisCancelledError,
|
||||
AnalysisOutputError,
|
||||
@@ -373,6 +376,56 @@ describe("PlaywrightBrowserScenarioProvider", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("PlaywrightBrowserScenarioProvider comparison coverage", () => {
|
||||
it("keeps quiet captures with different event families from comparing unchanged", async () => {
|
||||
const capture = async (events: readonly BrowserScenarioEventFamily[]) => {
|
||||
const provider = new PlaywrightBrowserScenarioProvider(() =>
|
||||
Promise.resolve(new FakeSession("launch")),
|
||||
);
|
||||
const result = await provider.captureScenario(scenario({ events }));
|
||||
if (!result.ok) throw result.error;
|
||||
return result.value;
|
||||
};
|
||||
const compare = async (
|
||||
before: readonly BrowserScenarioEventFamily[],
|
||||
after: readonly BrowserScenarioEventFamily[],
|
||||
) =>
|
||||
compareBrowserScenarios(
|
||||
compareBrowserScenariosInputSchema.parse({
|
||||
before_scenario: await capture(before),
|
||||
after_scenario: await capture(after),
|
||||
}),
|
||||
);
|
||||
|
||||
const recorded = await capture(["page-errors", "console", "console"]);
|
||||
expect(recorded.completeness.equality_eligible).toBe(true);
|
||||
expect(recorded.scenario.event_families).toEqual([
|
||||
"console",
|
||||
"page-errors",
|
||||
]);
|
||||
for (const [before, after] of [
|
||||
[[], ["console"]],
|
||||
[["console"], ["page-errors"]],
|
||||
] as const) {
|
||||
const compared = await compare(before, after);
|
||||
expect(compared.overall_status).toBe("unknown");
|
||||
expect(
|
||||
compared.steps.flatMap((step) => step.artifact_diffs),
|
||||
).toContainEqual(
|
||||
expect.objectContaining({
|
||||
artifact: "events",
|
||||
status: "unknown",
|
||||
reason: expect.stringContaining("Event family selections differ"),
|
||||
}),
|
||||
);
|
||||
}
|
||||
expect(
|
||||
(await compare(["console", "page-errors"], ["page-errors", "console"]))
|
||||
.overall_status,
|
||||
).toBe("unchanged");
|
||||
});
|
||||
});
|
||||
|
||||
describe("PlaywrightBrowserScenarioProvider cleanup", () => {
|
||||
it("returns a cleanup failure with completed observations when close fails", async () => {
|
||||
const session = new FakeSession("launch", {
|
||||
|
||||
@@ -84,6 +84,7 @@ const exampleScenarioCapture = () => ({
|
||||
response_body: false,
|
||||
header_values: false,
|
||||
},
|
||||
event_families: [],
|
||||
},
|
||||
duration_ms: 10,
|
||||
steps: [
|
||||
|
||||
@@ -3,6 +3,7 @@ import { browserNetworkContentSelectionValuesSchema } from "./browserNetworkEvid
|
||||
|
||||
import {
|
||||
browserScenarioCompletenessSchema,
|
||||
browserScenarioEventFamilySchema,
|
||||
browserScenarioEventSchema,
|
||||
browserScenarioStepSchema,
|
||||
} from "./browserScenarioCaptureValues.js";
|
||||
@@ -22,6 +23,7 @@ export const browserScenarioCaptureSchema = z
|
||||
action_count: z.number().int().min(1),
|
||||
secret_references: z.array(z.string().min(1)),
|
||||
network_content: browserNetworkContentSelectionValuesSchema,
|
||||
event_families: z.array(browserScenarioEventFamilySchema).optional(),
|
||||
}),
|
||||
duration_ms: z.number().int().min(0),
|
||||
steps: z.array(browserScenarioStepSchema).min(1),
|
||||
|
||||
@@ -119,6 +119,21 @@ export const browserStepArtifactsSchema = z.strictObject({
|
||||
});
|
||||
export type BrowserStepArtifacts = z.infer<typeof browserStepArtifactsSchema>;
|
||||
|
||||
/** Event families a scenario can select for capture. */
|
||||
export const browserScenarioEventFamilySchema = z.enum([
|
||||
"console",
|
||||
"page-errors",
|
||||
"network",
|
||||
"websockets",
|
||||
"frames",
|
||||
"workers",
|
||||
"popups",
|
||||
"downloads",
|
||||
]);
|
||||
export type BrowserScenarioEventFamily = z.infer<
|
||||
typeof browserScenarioEventFamilySchema
|
||||
>;
|
||||
|
||||
const eventBase = {
|
||||
sequence: z.number().int().min(1),
|
||||
step_index: z.number().int().min(0),
|
||||
|
||||
@@ -62,6 +62,7 @@ const networkCapture = (sourceSequence: number, receiptStep: number) => {
|
||||
response_body: false,
|
||||
header_values: false,
|
||||
},
|
||||
event_families: ["network"],
|
||||
},
|
||||
events: { items: events, retained: 2, dropped: 0 },
|
||||
});
|
||||
@@ -146,6 +147,94 @@ it("reports content-selection mismatch as unknown instead of a website change",
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("reports event-family mismatch and legacy selection as unknown", () => {
|
||||
const consoleOnly = scenarioCapture({});
|
||||
const withFamilies = (
|
||||
capture: BrowserScenarioCapture,
|
||||
families: BrowserScenarioCapture["scenario"]["event_families"],
|
||||
) =>
|
||||
browserScenarioCaptureSchema.parse({
|
||||
...capture,
|
||||
scenario: { ...capture.scenario, event_families: families },
|
||||
});
|
||||
const compare = (
|
||||
before: BrowserScenarioCapture,
|
||||
after: BrowserScenarioCapture,
|
||||
) =>
|
||||
compareBrowserScenarios(
|
||||
compareBrowserScenariosInputSchema.parse({
|
||||
before_scenario: before,
|
||||
after_scenario: after,
|
||||
}),
|
||||
);
|
||||
const eventDiffs = (compared: ReturnType<typeof compare>) =>
|
||||
compared.steps
|
||||
.flatMap((step) => step.artifact_diffs)
|
||||
.filter(({ artifact }) => artifact === "events");
|
||||
|
||||
const widened = withFamilies(consoleOnly, ["page-errors", "console"]);
|
||||
const mismatched = compare(consoleOnly, widened);
|
||||
expect(mismatched.overall_status).toBe("unknown");
|
||||
expect(eventDiffs(mismatched)).toContainEqual(
|
||||
expect.objectContaining({
|
||||
status: "unknown",
|
||||
before_state: "captured",
|
||||
after_state: "captured",
|
||||
reason: expect.stringContaining(
|
||||
"Event family selections differ (console before, console, page-errors after)",
|
||||
),
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
compare(
|
||||
withFamilies(consoleOnly, ["console", "page-errors"]),
|
||||
withFamilies(consoleOnly, ["page-errors", "console"]),
|
||||
).overall_status,
|
||||
).toBe("unchanged");
|
||||
|
||||
const legacy = withFamilies(consoleOnly, undefined);
|
||||
expect(legacy.scenario.event_families).toBeUndefined();
|
||||
for (const [before, after] of [
|
||||
[legacy, consoleOnly],
|
||||
[legacy, legacy],
|
||||
] as const)
|
||||
expect(eventDiffs(compare(before, after))).toContainEqual(
|
||||
expect.objectContaining({
|
||||
status: "unknown",
|
||||
reason: expect.stringContaining("Event family selection is unknown"),
|
||||
}),
|
||||
);
|
||||
expect(
|
||||
compareBrowserScenariosInputSchema.safeParse({
|
||||
before_scenario: {
|
||||
...consoleOnly,
|
||||
scenario: { ...consoleOnly.scenario, event_families: ["dialogs"] },
|
||||
},
|
||||
after_scenario: consoleOnly,
|
||||
}).success,
|
||||
).toBe(false);
|
||||
|
||||
const quiet = browserScenarioCaptureSchema.parse({
|
||||
...withFamilies(consoleOnly, []),
|
||||
events: { retained: 0, dropped: 0, items: [] },
|
||||
});
|
||||
const differentEvents = compare(quiet, widened);
|
||||
expect(differentEvents.overall_status).toBe("unknown");
|
||||
expect(eventDiffs(differentEvents).map(({ status }) => status)).toEqual([
|
||||
"unknown",
|
||||
"unknown",
|
||||
]);
|
||||
|
||||
const both = eventDiffs(compare(networkCapture(1, 0), consoleOnly));
|
||||
expect(both).toContainEqual(
|
||||
expect.objectContaining({
|
||||
reason: expect.stringMatching(
|
||||
/Event family selections differ.*content selections differ/u,
|
||||
),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
describe("browser scenario comparison", () => {
|
||||
it("records canonical literal normalization and ignores declared volatile fields", () => {
|
||||
const before = scenarioCapture({
|
||||
@@ -485,6 +574,7 @@ const scenarioCapture = (
|
||||
response_body: false,
|
||||
header_values: false,
|
||||
},
|
||||
event_families: ["console"],
|
||||
},
|
||||
duration_ms: options.elapsedMs ?? 10,
|
||||
steps: [
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
artifactEvidenceState,
|
||||
captured,
|
||||
eventEvidenceState,
|
||||
eventFamilyCoverageReason,
|
||||
eventsFor,
|
||||
networkContentCoverageReason,
|
||||
screenshotProjection,
|
||||
@@ -332,7 +333,7 @@ const compareStepArtifacts = (
|
||||
),
|
||||
beforeState: eventEvidenceState(input.beforeCapture, input.beforeStep),
|
||||
afterState: eventEvidenceState(input.afterCapture, input.afterStep),
|
||||
incomparableReason: networkContentCoverageReason(
|
||||
incomparableReason: eventCoverageReason(
|
||||
input.beforeCapture,
|
||||
input.afterCapture,
|
||||
),
|
||||
@@ -340,6 +341,17 @@ const compareStepArtifacts = (
|
||||
];
|
||||
};
|
||||
|
||||
const eventCoverageReason = (
|
||||
before: BrowserScenarioCapture,
|
||||
after: BrowserScenarioCapture,
|
||||
): string | undefined => {
|
||||
const reasons = [
|
||||
eventFamilyCoverageReason(before, after),
|
||||
networkContentCoverageReason(before, after),
|
||||
].filter((reason) => reason !== undefined);
|
||||
return reasons.length === 0 ? undefined : reasons.join(" ");
|
||||
};
|
||||
|
||||
interface CompareCapturedValuesInput<Value> {
|
||||
readonly artifact: BrowserScenarioArtifactKind;
|
||||
readonly before: CaptureState<Value>;
|
||||
|
||||
@@ -130,6 +130,26 @@ export const networkContentCoverageReason = (
|
||||
: undefined;
|
||||
};
|
||||
|
||||
/** Different or unrecorded event-family selection is missing comparability, not an observed website change. */
|
||||
export const eventFamilyCoverageReason = (
|
||||
before: BrowserScenarioCapture,
|
||||
after: BrowserScenarioCapture,
|
||||
): string | undefined => {
|
||||
const left = before.scenario.event_families;
|
||||
const right = after.scenario.event_families;
|
||||
if (left === undefined || right === undefined)
|
||||
return "Event family selection is unknown in a legacy capture; capture both sides with matching event families before comparing event evidence.";
|
||||
const leftFamilies = new Set(left);
|
||||
const rightFamilies = new Set(right);
|
||||
return leftFamilies.size === rightFamilies.size &&
|
||||
[...leftFamilies].every((family) => rightFamilies.has(family))
|
||||
? undefined
|
||||
: `Event family selections differ (${familyList(leftFamilies)} before, ${familyList(rightFamilies)} after); capture both sides with matching event families before comparing event evidence.`;
|
||||
};
|
||||
|
||||
const familyList = (families: ReadonlySet<string>): string =>
|
||||
families.size === 0 ? "none" : [...families].sort().join(", ");
|
||||
|
||||
/** Wrap one value as captured comparison evidence. */
|
||||
export const captured = <Value>(value: Value): CaptureState<Value> => ({
|
||||
state: "captured",
|
||||
|
||||
@@ -5,6 +5,7 @@ import { IDENTIFIER_PATTERN } from "./stringPatterns.js";
|
||||
|
||||
import { localPathStringSchema } from "./localPath.js";
|
||||
import { browserNetworkContentSelectionSchema } from "./browserNetworkEvidence.js";
|
||||
import { browserScenarioEventFamilySchema } from "./browserScenarioCaptureValues.js";
|
||||
|
||||
import {
|
||||
browserEndpointSchema,
|
||||
@@ -289,20 +290,7 @@ export const browserScenarioCaptureSchema = z
|
||||
.strictObject({
|
||||
after_each_step: z.array(snapshotKindSchema).default([]),
|
||||
at_end: z.array(snapshotKindSchema).default(["url"]),
|
||||
events: z
|
||||
.array(
|
||||
z.enum([
|
||||
"console",
|
||||
"page-errors",
|
||||
"network",
|
||||
"websockets",
|
||||
"frames",
|
||||
"workers",
|
||||
"popups",
|
||||
"downloads",
|
||||
]),
|
||||
)
|
||||
.default([]),
|
||||
events: z.array(browserScenarioEventFamilySchema).default([]),
|
||||
network: browserNetworkContentSelectionSchema,
|
||||
})
|
||||
.default({
|
||||
|
||||
@@ -352,6 +352,7 @@ const completeScenarioCapture = () => {
|
||||
response_body: false,
|
||||
header_values: false,
|
||||
},
|
||||
event_families: [],
|
||||
},
|
||||
duration_ms: 1,
|
||||
steps: [
|
||||
|
||||
@@ -85,6 +85,7 @@ const captureFor = (scenario: BrowserScenario): BrowserScenarioCapture => {
|
||||
action_count: scenario.actions.length,
|
||||
secret_references: scenario.secrets.map(({ secret_id: id }) => id),
|
||||
network_content: scenario.capture.network,
|
||||
event_families: scenario.capture.events,
|
||||
},
|
||||
duration_ms: 2,
|
||||
steps: [
|
||||
|
||||
Vendored
+1
@@ -105,6 +105,7 @@ export const scriptScenarioFixture = (
|
||||
response_body: true,
|
||||
header_values: false,
|
||||
},
|
||||
event_families: ["network"],
|
||||
},
|
||||
duration_ms: 2,
|
||||
steps: [step(0), step(1)],
|
||||
|
||||
Reference in New Issue
Block a user