fix(browser): preserve authorized redirect hops (#986)

* fix(browser): preserve authorized redirect hops

* fix(browser): compare redirect semantics without event times

* fix(browser): classify missing redirect predecessors

* fix(browser): distinguish malformed redirect URLs

* fix(browser): preserve evidence on malformed redirect envelopes

* fix(browser): freeze evidence after malformed redirect

* docs: regenerate catalogs after main integration

* fix(browser): account for skipped redirect body shapes

* fix(browser): mark skipped redirect response shapes unavailable

When a malformed redirect keeps a JSON predecessor, record that its response body shape was not captured.

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
morluto
2026-10-08 01:08:31 +08:00
committed by GitHub
co-authored by Cursor Agent
parent a6132be054
commit ae0f725fad
16 changed files with 798 additions and 12 deletions
+1 -1
View File
@@ -133,7 +133,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex
- URL observations preserve the complete local URL, including query values, fragments, duplicate parameters, and their order. Only username/password userinfo is removed; malformed original URL strings remain available with `origin: null`.
- DOM snapshots retain node types, node names, value lengths, and attribute names, but not text or attribute values.
- Accessibility structure and roles are retained by default, while names and descriptions require `include_accessibility_text: true`.
- Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained.
- Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Redirects reported by CDP are attached as ordered `redirects` on the one request ID, with the prior request URL, CDP response URL, method/type, response status/MIME/encoded length, request timestamp, and the next `requestWillBeSent` event timestamp. CDP delivers `redirectResponse` with that next request event, so the latter is an event boundary rather than an exact response timestamp. Redirect response headers and bodies are discarded; hops outside the selected origins are discarded with the request chain. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained.
- Request/response bodies are not requested or parsed by default. When selected, only allowed-origin JSON media types are read, converted immediately to complete value-free property paths/types, and discarded. `Network.getResponseBody` is never sent unless JSON body shape capture is selected.
- Console observations with a stack source retain call type, argument types, timestamp, and source location. When console text capture is selected, already-delivered primitive values are retained verbatim; objects, getters, and remote properties are never expanded.
- WebSocket observations retain direction, opcode, and payload byte length. When shape capture is selected, text frames are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw frames are never retained.
+2 -2
View File
@@ -675,9 +675,9 @@
"mcp_prompts": 6
},
"digests": {
"tools_sha256": "c77a3b4740fc2e78172e1d6757b1915d2b638185d8e87d2f1885bb41ce538813",
"tools_sha256": "4baede40da9b3bb792e6c35b9b461745ac302973931281be26c8ad7a99f72872",
"prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d",
"combined_sha256": "e652c2ec4cde078a23735789c1d01cb1b9d316c9af7708e86f2c5c31b37b0e51",
"combined_sha256": "ed0b788dd5a6bf9446f840e388822db94b54589a5e294816a3e13bf2bec9f48c",
"providers_sha256": "aea15f517c822d80918ccaba57a4fa06557f484cf27b8acbaed4e1a660fac896"
}
}
@@ -1,5 +1,5 @@
{
"manifest_id": "ecm_aa4a1c78be657e901c87cc215a703938cfe54d7a54411cbfd64872de3ce78b93",
"manifest_id": "ecm_4d1cf621854fe1f40698ef60f340fb99263edb0e99218c865b09cb26112d2547",
"verifier": {
"id": "rea-managed-conformance",
"version": "1"
@@ -13,7 +13,7 @@
"skill_digests": [
{
"skill_id": "reverse-engineer-anything",
"sha256": "57f28ace603b9eb5d6d9ea64d68f563ea8a4833e6a53dd7b4b40ce5fb3ac56e1"
"sha256": "1d816388b86e50fff47db599200968788a4bf2272f40d5bdd3bb51aae7f5a9b4"
}
],
"claims": [
+1 -1
View File
@@ -4,7 +4,7 @@ description: Reverse engineer native, managed, Electron/JavaScript, packaged, fi
metadata:
version: "27"
tool_count: 133
catalog_digest: "e652c2ec4cde078a23735789c1d01cb1b9d316c9af7708e86f2c5c31b37b0e51"
catalog_digest: "ed0b788dd5a6bf9446f840e388822db94b54589a5e294816a3e13bf2bec9f48c"
---
# REA
+102 -2
View File
@@ -10,7 +10,10 @@ import {
cdpStringValue,
type UnknownRecord,
} from "./CdpCaptureValues.js";
import { requestBodyShape } from "./CdpCaptureEventBodyShapes.js";
import {
requestBodyShape,
updateResponseBodyShape,
} from "./CdpCaptureEventBodyShapes.js";
import {
consolePrimitive,
decodeBase64,
@@ -19,10 +22,11 @@ import {
firstCallFrame,
initiatorLocation,
integerOrNull,
isJsonMediaType,
isMainFrameNavigation,
} from "./CdpCaptureEventHelpers.js";
import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js";
import type { CapturedScript } from "./CdpCaptureEventTypes.js";
import type { CapturedScript, NetworkState } from "./CdpCaptureEventTypes.js";
export const handleExecutionContextCreated = (
state: CdpCaptureEventsState,
@@ -120,9 +124,11 @@ export const handleRequestWillBeSent = (
state.completeness.exclude("network_requests", "invalid_protocol_value");
return;
}
if (state.malformedRedirectRequestIds.has(requestId)) return;
if (request === undefined) {
state.completeness.exclude("network_requests", "invalid_protocol_value");
state.network.delete(requestId);
state.networkRequestTimestamps.delete(requestId);
return;
}
const sanitized = allowedSanitizedUrl(request.url, state.allowedOrigins);
@@ -132,8 +138,60 @@ export const handleRequestWillBeSent = (
exclusionReasonForUrl(cdpStringValue(request.url)),
);
state.network.delete(requestId);
state.networkRequestTimestamps.delete(requestId);
return;
}
const previous = state.network.get(requestId);
const redirectResponse = recordValue(params.redirectResponse);
if (
Object.hasOwn(params, "redirectResponse") &&
redirectResponse === undefined &&
preserveMalformedRedirectEvidence(state, requestId, previous)
)
return;
const redirects = [...(previous?.redirects ?? [])];
if (redirectResponse !== undefined) {
const rawResponseUrl = cdpStringValue(redirectResponse.url);
const responseUrl =
rawResponseUrl === undefined || rawResponseUrl === ""
? undefined
: allowedSanitizedUrl(rawResponseUrl, state.allowedOrigins);
const responseUrlReason = redirectResponseUrlReason(rawResponseUrl);
if (previous === undefined) {
if (responseUrl === undefined)
state.completeness.exclude("network_requests", responseUrlReason);
else
// CDP may deliver the continuation after its predecessor was excluded
// by origin policy or before capture began. Keep the final request but
// report missing prior coverage without calling it malformed protocol.
state.completeness.attachLimited("network_requests");
} else if (responseUrl === undefined) {
// redirectResponse belongs to the same request ID and can contain an
// excluded origin. Drop the entire chain before retaining any details.
state.completeness.exclude("network_requests", responseUrlReason);
state.network.delete(requestId);
state.networkRequestTimestamps.delete(requestId);
return;
} else {
redirects.push({
url: previous.url,
response_url: responseUrl.url,
method: previous.method,
resource_type: previous.resource_type,
status: numberValue(redirectResponse.status) ?? null,
mime_type: cdpStringValue(redirectResponse.mimeType) ?? null,
encoded_data_length: (() => {
const length = numberValue(redirectResponse.encodedDataLength);
return length === undefined ? null : Math.max(0, length);
})(),
request_timestamp:
state.networkRequestTimestamps.get(requestId) ?? null,
// CDP reports redirectResponse on the next requestWillBeSent event.
// This is the event boundary, not a separately observed response time.
redirect_event_timestamp: numberValue(params.timestamp) ?? null,
});
}
}
const initiator = recordValue(params.initiator);
const initiatorFrame = initiatorLocation(initiator);
const rawInitiatorUrl = cdpStringValue(initiatorFrame?.url);
@@ -155,6 +213,7 @@ export const handleRequestWillBeSent = (
status: null,
mime_type: null,
encoded_data_length: null,
redirects,
initiator: {
type: cdpStringValue(initiator?.type) ?? "other",
url: initiatorUrl?.url ?? null,
@@ -163,6 +222,44 @@ export const handleRequestWillBeSent = (
},
body_shapes: requestBodyShape(state, request),
});
const timestamp = numberValue(params.timestamp);
if (timestamp === undefined) state.networkRequestTimestamps.delete(requestId);
else state.networkRequestTimestamps.set(requestId, timestamp);
};
const preserveMalformedRedirectEvidence = (
state: CdpCaptureEventsState,
requestId: string,
previous: NetworkState | undefined,
): boolean => {
state.completeness.exclude("network_requests", "invalid_protocol_value");
if (previous === undefined) return false;
state.malformedRedirectRequestIds.add(requestId);
if (
state.input.include_json_body_shapes &&
isJsonMediaType(previous.mime_type) &&
previous.body_shapes.response === null
) {
updateResponseBodyShape(state, requestId, null);
state.completeness.exclude("json_body_shapes", "invalid_protocol_value");
}
return true;
};
const redirectResponseUrlReason = (
value: string | undefined,
):
| "invalid_protocol_value"
| "disallowed_origin"
| "unsupported_url"
| "unattributed_origin" => {
if (value === undefined || value === "") return "invalid_protocol_value";
try {
new URL(value);
} catch {
return "invalid_protocol_value";
}
return exclusionReasonForUrl(value);
};
export const handleResponseReceived = (
@@ -174,6 +271,7 @@ export const handleResponseReceived = (
state.completeness.exclude("network_requests", "invalid_protocol_value");
return;
}
if (state.malformedRedirectRequestIds.has(requestId)) return;
const current = state.network.get(requestId);
const response = recordValue(params.response);
if (current === undefined) return;
@@ -186,6 +284,7 @@ export const handleResponseReceived = (
: exclusionReasonForUrl(cdpStringValue(response.url)),
);
state.network.delete(requestId);
state.networkRequestTimestamps.delete(requestId);
return;
}
state.network.set(requestId, {
@@ -209,6 +308,7 @@ export const handleLoadingFinished = (
): void => {
const requestId = cdpStringValue(params.requestId);
if (requestId === undefined) return;
if (state.malformedRedirectRequestIds.has(requestId)) return;
const current = state.network.get(requestId);
if (current === undefined) return;
state.network.set(requestId, {
+2
View File
@@ -11,6 +11,8 @@ export interface CdpCaptureEventsState {
readonly scripts: Map<string, CapturedScript>;
readonly executionContextFrames: Map<string, string>;
readonly network: Map<string, NetworkState>;
readonly networkRequestTimestamps: Map<string, number>;
readonly malformedRedirectRequestIds: Set<string>;
readonly allowedWebSockets: Set<string>;
console: WebPageInspection["console"]["events"];
websockets: WebPageInspection["network"]["websocket_events"];
+9
View File
@@ -18,6 +18,8 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
readonly scripts = new Map<string, CapturedScript>();
readonly executionContextFrames = new Map<string, string>();
readonly network = new Map<string, NetworkState>();
readonly networkRequestTimestamps = new Map<string, number>();
readonly malformedRedirectRequestIds = new Set<string>();
readonly allowedWebSockets = new Set<string>();
console: WebPageInspection["console"]["events"] = [];
websockets: WebPageInspection["network"]["websocket_events"] = [];
@@ -51,6 +53,8 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
this.scripts.clear();
this.executionContextFrames.clear();
this.network.clear();
this.networkRequestTimestamps.clear();
this.malformedRedirectRequestIds.clear();
this.allowedWebSockets.clear();
this.console.length = 0;
this.websockets.length = 0;
@@ -135,16 +139,21 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
responseBodyRequestIds(): readonly string[] {
if (!this.input.include_json_body_shapes) return [];
return [...this.network.values()]
.filter(
(request) => !this.malformedRedirectRequestIds.has(request.request_id),
)
.filter((request) => isJsonMediaType(request.mime_type))
.map((request) => request.request_id);
}
responseBodyUnavailable(requestId: string): void {
if (this.malformedRedirectRequestIds.has(requestId)) return;
bodyShapes.updateResponseBodyShape(this, requestId, null);
this.completeness.unavailable("json_body_shapes");
}
ingestResponseBody(requestId: string, value: unknown): void {
if (this.malformedRedirectRequestIds.has(requestId)) return;
bodyShapes.ingestResponseBodyShape(this, requestId, value);
}
}
+15
View File
@@ -114,6 +114,21 @@ const browserNetworkRequestSchema = z.object({
status: z.number().nullable(),
mime_type: z.string().nullable(),
encoded_data_length: z.number().min(0).nullable(),
redirects: z
.array(
z.object({
url: z.string(),
response_url: z.string(),
method: z.string(),
resource_type: z.string().nullable(),
status: z.number().nullable(),
mime_type: z.string().nullable(),
encoded_data_length: z.number().min(0).nullable(),
request_timestamp: z.number().nullable(),
redirect_event_timestamp: z.number().nullable(),
}),
)
.optional(),
initiator: z.object({
type: z.string(),
url: z.string().nullable(),
+7
View File
@@ -249,6 +249,13 @@ const networkMap = (
status: request.status,
mime_type: request.mime_type,
encoded_data_length: request.encoded_data_length,
redirects: (request.redirects ?? []).map(
({
request_timestamp: _requestTimestamp,
redirect_event_timestamp: _redirectEventTimestamp,
...redirect
}) => redirect,
),
initiator: request.initiator,
body_shapes: request.body_shapes,
});
@@ -6,6 +6,59 @@ import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
import { describeBrowser, trackBrowser } from "./cdpBrowserProvider.support.js";
describeBrowser("CdpBrowserProvider: network 1", () => {
it("retains same-origin redirect hops on the one final request", async () => {
const browser = await startFakeCdpBrowser({ redirectWithinOrigin: true });
trackBrowser(browser);
const result = await new CdpBrowserProvider().inspectPage(
inspectWebPageInputSchema.parse({
cdp_endpoint: browser.endpoint,
allowed_origins: [browser.allowedOrigin],
target_id: "allowed-page",
observation_ms: 0,
}),
);
if (!result.ok) throw result.error;
expect(result.value.network.requests).toHaveLength(1);
expect(result.value.network.requests[0]).toMatchObject({
request_id: "request-1",
url: `${browser.allowedOrigin}/redirected?token=final-secret`,
status: 200,
redirects: [
{
url: `${browser.allowedOrigin}/api?token=network-secret`,
response_url: `${browser.allowedOrigin}/api?token=network-secret`,
method: "POST",
resource_type: "Fetch",
status: 301,
mime_type: "text/plain",
encoded_data_length: 23,
request_timestamp: 7,
redirect_event_timestamp: 8,
},
{
url: `${browser.allowedOrigin}/intermediate`,
response_url: `${browser.allowedOrigin}/intermediate`,
method: "GET",
resource_type: "Fetch",
status: 302,
mime_type: "text/plain",
encoded_data_length: 27,
request_timestamp: 8,
redirect_event_timestamp: 9,
},
],
});
expect(JSON.stringify(result.value.network.requests)).not.toContain(
"redirect-header-secret",
);
expect(result.value.completeness.excluded).not.toContainEqual({
section: "network_requests",
reason: "invalid_protocol_value",
count: expect.any(Number),
});
});
it("drops network evidence when a request redirects outside the approved origin", async () => {
const browser = await startFakeCdpBrowser({
redirectToDisallowedOrigin: true,
@@ -23,4 +76,42 @@ describeBrowser("CdpBrowserProvider: network 1", () => {
if (!result.ok) throw result.error;
expect(result.value.network.requests).toEqual([]);
});
it("does not retain excluded prior-hop data when a redirect returns in scope", async () => {
const browser = await startFakeCdpBrowser({
redirectFromDisallowedOrigin: true,
});
trackBrowser(browser);
const result = await new CdpBrowserProvider().inspectPage(
inspectWebPageInputSchema.parse({
cdp_endpoint: browser.endpoint,
allowed_origins: [browser.allowedOrigin],
target_id: "allowed-page",
observation_ms: 0,
}),
);
if (!result.ok) throw result.error;
expect(result.value.network.requests).toHaveLength(1);
expect(result.value.network.requests[0]).toMatchObject({
url: `${browser.allowedOrigin}/returned`,
redirects: [],
});
const output = JSON.stringify(result.value.network.requests);
expect(output).not.toContain("private.example.test");
expect(output).not.toContain("redirect-header-secret");
expect(output).not.toContain("redirect-body-secret");
expect(result.value.completeness.policy_filtered_sections).toContain(
"network_requests",
);
expect(result.value.completeness.status).toBe("policy_filtered");
expect(result.value.completeness.unavailable_sections).not.toContain(
"network_requests",
);
expect(result.value.completeness.excluded).not.toContainEqual({
section: "network_requests",
reason: "invalid_protocol_value",
count: expect.any(Number),
});
});
});
@@ -0,0 +1,66 @@
import { expect, it } from "vitest";
import { CdpCaptureEvents } from "../../../src/browser/CdpCaptureEvents.js";
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
it("preserves an already captured response shape without marking it unavailable", () => {
const origin = "http://127.0.0.1:43127";
const events = new CdpCaptureEvents(
inspectWebPageInputSchema.parse({
cdp_endpoint: origin,
allowed_origins: [origin],
target_id: "allowed-page",
include_json_body_shapes: true,
}),
new Set([origin]),
);
events.ingest({
method: "Network.requestWillBeSent",
params: {
requestId: "request-1",
request: {
url: `${origin}/prior`,
method: "POST",
headers: { "Content-Type": "application/json" },
postData: '{"request":true}',
},
timestamp: 1,
},
});
events.ingest({
method: "Network.responseReceived",
params: {
requestId: "request-1",
response: {
url: `${origin}/prior`,
status: 200,
mimeType: "application/json",
},
},
});
events.ingestResponseBody("request-1", {
body: '{"response":true}',
base64Encoded: false,
});
const capturedShape = events.network.get("request-1")?.body_shapes.response;
expect(capturedShape).not.toBeNull();
events.ingest({
method: "Network.requestWillBeSent",
params: {
requestId: "request-1",
request: { url: `${origin}/continuation`, method: "GET" },
redirectResponse: null,
timestamp: 2,
},
});
expect(events.network.get("request-1")?.body_shapes.response).toEqual(
capturedShape,
);
expect(events.network.get("request-1")?.body_shapes.status).toBe("included");
expect(events.responseBodyRequestIds()).toEqual([]);
expect(events.completeness.snapshot().unavailable_sections).not.toContain(
"json_body_shapes",
);
});
@@ -0,0 +1,150 @@
import { expect, it } from "vitest";
import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
import type { FakeOptions } from "../../fixtures/fakeCdpBrowserTypes.js";
import { describeBrowser, trackBrowser } from "./cdpBrowserProvider.support.js";
const inspectNetwork = async (options: FakeOptions) => {
const browser = await startFakeCdpBrowser(options);
trackBrowser(browser);
const result = await new CdpBrowserProvider().inspectPage(
inspectWebPageInputSchema.parse({
cdp_endpoint: browser.endpoint,
allowed_origins: [browser.allowedOrigin],
target_id: "allowed-page",
observation_ms: 0,
include_json_body_shapes: true,
}),
);
if (!result.ok) throw result.error;
return { browser, inspection: result.value };
};
describeBrowser("CdpBrowserProvider: redirect errors", () => {
it.each([
{ label: "missing", url: undefined, reason: "invalid_protocol_value" },
{ label: "malformed", url: "http://%", reason: "invalid_protocol_value" },
{
label: "unsupported scheme",
url: "file:///private/redirect",
reason: "unsupported_url",
},
])("classifies $label redirect response URLs", async ({ url, reason }) => {
const { inspection } = await inspectNetwork({
malformedRedirectResponse: true,
...(url === undefined ? {} : { redirectResponseUrl: url }),
});
expect(inspection.network.requests).toEqual([]);
expect(inspection.completeness.excluded).toContainEqual({
section: "network_requests",
reason,
count: expect.any(Number),
});
const section =
reason === "unsupported_url"
? inspection.completeness.policy_filtered_sections
: inspection.completeness.unavailable_sections;
expect(section).toContain("network_requests");
});
it.each([
{ label: "null", envelope: null },
{ label: "array", envelope: [] },
{ label: "scalar", envelope: "malformed redirect" },
])(
"preserves prior request for a $label redirect envelope",
async ({ envelope }) => {
const { browser, inspection } = await inspectNetwork({
malformedRedirectResponse: true,
redirectResponseEnvelope: envelope,
});
expect(inspection.network.requests).toHaveLength(1);
expect(inspection.network.requests[0]).toMatchObject({
request_id: "request-1",
url: `${browser.allowedOrigin}/malformed-redirect-prior`,
status: 201,
mime_type: "application/json",
encoded_data_length: null,
redirects: [
{
url: `${browser.allowedOrigin}/api?token=network-secret`,
response_url: `${browser.allowedOrigin}/api?token=network-secret`,
request_timestamp: 7,
redirect_event_timestamp: 8,
},
],
});
expect(
inspection.network.requests[0]?.body_shapes.request,
).not.toBeNull();
expect(inspection.network.requests[0]?.body_shapes.response).toBeNull();
expect(inspection.network.requests[0]?.body_shapes.status).toBe(
"partial",
);
expect(inspection.metadata.responses).toHaveLength(1);
expect(inspection.metadata.responses[0]).toMatchObject({
url: `${browser.allowedOrigin}/malformed-redirect-prior`,
mime_type: "application/json",
content_length: 123,
content_encoding: "gzip",
});
expect(inspection.completeness.unavailable_sections).toContain(
"network_requests",
);
expect(inspection.completeness.unavailable_sections).toContain(
"json_body_shapes",
);
expect(inspection.completeness.excluded).toContainEqual({
section: "network_requests",
reason: "invalid_protocol_value",
count: expect.any(Number),
});
expect(inspection.completeness.excluded).toContainEqual({
section: "json_body_shapes",
reason: "invalid_protocol_value",
count: expect.any(Number),
});
expect(JSON.stringify(inspection.network.requests)).not.toContain(
"malformed-redirect-final",
);
},
);
it("keeps authorized prior evidence when the later response URL is disallowed", async () => {
const { browser, inspection } = await inspectNetwork({
malformedRedirectResponse: true,
redirectResponseEnvelope: null,
responseAfterMalformedUrl:
"https://private.example.test/late?token=unauthorized-secret",
});
expect(inspection.network.requests).toHaveLength(1);
expect(inspection.network.requests[0]).toMatchObject({
url: `${browser.allowedOrigin}/malformed-redirect-prior`,
status: 201,
mime_type: "application/json",
encoded_data_length: null,
body_shapes: { status: "partial", response: null },
});
expect(inspection.metadata.responses).toHaveLength(1);
expect(inspection.metadata.responses[0]?.url).toBe(
`${browser.allowedOrigin}/malformed-redirect-prior`,
);
expect(inspection.completeness.unavailable_sections).toContain(
"json_body_shapes",
);
expect(inspection.completeness.excluded).toContainEqual({
section: "network_requests",
reason: "invalid_protocol_value",
count: expect.any(Number),
});
const output = JSON.stringify(inspection);
expect(output).not.toContain("private.example.test");
expect(output).not.toContain("unauthorized-secret");
expect(output).not.toContain("header-secret");
});
});
+78 -1
View File
@@ -48,7 +48,22 @@ describe("web capture diff", () => {
const after = structuredClone(captured.value);
after.scripts.items = [];
const request = after.network.requests[0];
if (request !== undefined) request.status = 204;
if (request !== undefined) {
request.status = 204;
request.redirects = [
{
url: `${browser.allowedOrigin}/prior`,
response_url: `${browser.allowedOrigin}/prior`,
method: "GET",
resource_type: "Fetch",
status: 302,
mime_type: "text/plain",
encoded_data_length: 12,
request_timestamp: 1,
redirect_event_timestamp: 2,
},
];
}
const result = compareWebCaptures(
compareWebCapturesInputSchema.parse({
@@ -163,6 +178,68 @@ describe("web capture diff", () => {
});
});
describe("web capture redirect comparison", () => {
it("ignores redirect event times but compares redirect semantics", async () => {
const browser = await startFakeCdpBrowser();
browsers.push(browser);
const captured = await new CdpBrowserProvider().inspectPage(
inspectWebPageInputSchema.parse({
cdp_endpoint: browser.endpoint,
allowed_origins: [browser.allowedOrigin],
target_id: "allowed-page",
observation_ms: 0,
}),
);
if (!captured.ok) throw captured.error;
const before = structuredClone(captured.value);
const beforeRequest = before.network.requests[0];
if (beforeRequest === undefined) throw new Error("Missing network request");
beforeRequest.redirects = [
{
url: `${browser.allowedOrigin}/prior`,
response_url: `${browser.allowedOrigin}/prior`,
method: "GET",
resource_type: "Fetch",
status: 302,
mime_type: "text/plain",
encoded_data_length: 12,
request_timestamp: 1,
redirect_event_timestamp: 2,
},
];
markSectionsComplete(before, ["network_requests"]);
const after = structuredClone(before);
const afterRequest = after.network.requests[0];
if (afterRequest === undefined) throw new Error("Missing network request");
const hop = afterRequest.redirects?.[0];
if (hop === undefined) throw new Error("Missing redirect hop");
hop.request_timestamp = 10;
hop.redirect_event_timestamp = 20;
const compare = (inspection: typeof before) =>
compareWebCaptures(
compareWebCapturesInputSchema.parse({
before: { inspection: before },
after: { inspection },
}),
).dimensions.network;
expect(compare(after)).toMatchObject({
status: "unchanged",
total_changes: 0,
});
const changed = structuredClone(after);
const changedHop = changed.network.requests[0]?.redirects?.[0];
if (changedHop === undefined) throw new Error("Missing redirect hop");
changedHop.status = 307;
expect(compare(changed)).toMatchObject({
status: "changed",
changes: [expect.objectContaining({ change: "modified" })],
});
});
});
describe("web capture diff incomplete inventories", () => {
it.each(["truncated_sections", "unavailable_sections"] as const)(
"does not infer script additions or removals from %s",
@@ -0,0 +1,113 @@
import { execFile } from "node:child_process";
import { promisify } from "node:util";
import { afterEach, expect, it } from "vitest";
import {
startFakeCdpBrowser,
type FakeCdpBrowser,
} from "../../fixtures/fakeCdpBrowser.js";
const execute = promisify(execFile);
const browsers: FakeCdpBrowser[] = [];
afterEach(async () => {
await Promise.all(browsers.splice(0).map(async (browser) => browser.close()));
});
it("reports missing JSON shapes only when body shapes are selected", async () => {
const browser = await startFakeCdpBrowser({
malformedRedirectResponse: true,
redirectResponseEnvelope: null,
});
browsers.push(browser);
const commonArgs = [
"inspect-web-page",
browser.endpoint,
"allowed-page",
"--allowed-origins",
browser.allowedOrigin,
"--observation-ms",
"0",
];
const selected = normalizedResult(
await runCli([...commonArgs, "--include-json-body-shapes", "--json"]),
);
expect(selected).toMatchObject({
network: {
requests: [
{
body_shapes: {
status: "partial",
request: expect.any(Object),
response: null,
},
},
],
},
completeness: {
unavailable_sections: expect.arrayContaining([
"network_requests",
"json_body_shapes",
]),
excluded: expect.arrayContaining([
{
section: "json_body_shapes",
reason: "invalid_protocol_value",
count: expect.any(Number),
},
]),
},
});
const defaulted = normalizedResult(await runCli([...commonArgs, "--json"]));
expect(defaulted).toMatchObject({
network: {
requests: [
{
body_shapes: {
status: "not_approved",
request: null,
response: null,
},
},
],
},
});
expect(defaulted.completeness).toMatchObject({
unavailable_sections: expect.not.arrayContaining(["json_body_shapes"]),
});
});
const runCli = async (arguments_: readonly string[]): Promise<unknown> => {
try {
const { stdout } = await execute(
process.execPath,
["scripts/rea.mjs", ...arguments_],
{
cwd: process.cwd(),
env: process.env,
maxBuffer: 16 * 1_024 * 1_024,
},
);
return JSON.parse(stdout);
} catch (cause: unknown) {
if (
typeof cause === "object" &&
cause !== null &&
"stdout" in cause &&
typeof cause.stdout === "string"
)
return JSON.parse(cause.stdout);
throw cause;
}
};
const normalizedResult = (value: unknown): Record<string, unknown> => {
if (!isRecord(value) || !isRecord(value.normalized_result))
throw new TypeError("Missing CLI normalized result");
return value.normalized_result;
};
const isRecord = (value: unknown): value is Record<string, unknown> =>
typeof value === "object" && value !== null && !Array.isArray(value);
+153 -3
View File
@@ -282,6 +282,133 @@ const emitSessionTimeline = (
}
};
const emitSameOriginRedirectEvents = (
socket: WebSocket,
command: FakeCdpCommand,
port: number,
url: string,
): string => {
const intermediate = `http://127.0.0.1:${String(port)}/intermediate`;
const finalUrl = `http://127.0.0.1:${String(port)}/redirected?token=final-secret`;
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
type: "Fetch",
request: { url: intermediate, method: "GET" },
redirectResponse: {
url,
status: 301,
mimeType: "text/plain",
encodedDataLength: 23,
headers: { "X-Private": "redirect-header-secret" },
},
timestamp: 8,
});
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
type: "Fetch",
request: { url: finalUrl, method: "GET" },
redirectResponse: {
url: intermediate,
status: 302,
mimeType: "text/plain",
encodedDataLength: 27,
},
timestamp: 9,
});
return finalUrl;
};
const emitReturnFromDisallowedRedirect = (
socket: WebSocket,
command: FakeCdpCommand,
port: number,
): string => {
const finalUrl = `http://127.0.0.1:${String(port)}/returned`;
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
type: "Fetch",
request: { url: finalUrl, method: "GET" },
redirectResponse: {
url: "https://private.example.test/outside?token=redirect-url-secret",
status: 302,
mimeType: "text/plain",
encodedDataLength: 31,
headers: { "X-Private": "redirect-header-secret" },
postData: "redirect-body-secret",
},
timestamp: 8,
});
return finalUrl;
};
const emitMalformedRedirectPriorResponse = (
socket: WebSocket,
command: FakeCdpCommand,
port: number,
initialUrl: string,
): void => {
const priorUrl = `http://127.0.0.1:${String(port)}/malformed-redirect-prior`;
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
type: "Fetch",
request: {
url: priorUrl,
method: "POST",
headers: { "Content-Type": "application/json; charset=utf-8" },
postData: JSON.stringify({
operation: "preserve-prior",
token: "prior-body",
}),
},
redirectResponse: {
url: initialUrl,
status: 307,
mimeType: "text/plain",
encodedDataLength: 23,
},
timestamp: 8,
});
event(socket, "Network.responseReceived", command.sessionId, {
requestId: "request-1",
response: {
url: priorUrl,
status: 201,
mimeType: "application/json",
headers: {
"Content-Length": "123",
"Content-Encoding": "gzip",
"Content-Security-Policy": "default-src 'self'",
"X-Model-Context": "prior-context-hint",
},
},
});
};
const emitMalformedRedirect = (
socket: WebSocket,
command: FakeCdpCommand,
port: number,
options: FakeOptions,
): string => {
const finalUrl = `http://127.0.0.1:${String(port)}/malformed-redirect-final`;
const redirectResponse = Object.hasOwn(options, "redirectResponseEnvelope")
? options.redirectResponseEnvelope
: {
...(options.redirectResponseUrl === undefined
? {}
: { url: options.redirectResponseUrl }),
status: 302,
};
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
type: "Fetch",
request: { url: finalUrl, method: "GET" },
redirectResponse,
timestamp: 9,
});
return finalUrl;
};
const emitNetworkEvents = (
socket: WebSocket,
command: FakeCdpCommand,
@@ -297,11 +424,15 @@ const emitNetworkEvents = (
},
});
const url = `http://127.0.0.1:${String(port)}/api?token=network-secret`;
const requestUrl =
options.redirectFromDisallowedOrigin === true
? "https://private.example.test/start?token=initial-url-secret"
: url;
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
type: "Fetch",
request: {
url,
url: requestUrl,
method: "POST",
headers: {
Authorization: "Bearer request-secret",
@@ -313,6 +444,7 @@ const emitNetworkEvents = (
filters: { active: true },
}),
},
timestamp: 7,
initiator: {
type: "script",
stack: {
@@ -326,6 +458,16 @@ const emitNetworkEvents = (
},
},
});
if (options.malformedRedirectResponse === true)
emitMalformedRedirectPriorResponse(socket, command, port, url);
const responseUrl =
options.redirectFromDisallowedOrigin === true
? emitReturnFromDisallowedRedirect(socket, command, port)
: options.malformedRedirectResponse === true
? emitMalformedRedirect(socket, command, port, options)
: options.redirectWithinOrigin === true
? emitSameOriginRedirectEvents(socket, command, port, url)
: url;
if (options.redirectToDisallowedOrigin === true)
event(socket, "Network.requestWillBeSent", command.sessionId, {
requestId: "request-1",
@@ -334,14 +476,22 @@ const emitNetworkEvents = (
url: "https://private.example.test/redirected",
method: "GET",
},
redirectResponse: {
url,
status: 302,
mimeType: "text/plain",
encodedDataLength: 23,
headers: { "X-Private": "redirect-header-secret" },
},
});
event(socket, "Network.responseReceived", command.sessionId, {
requestId: "request-1",
response: {
url:
options.redirectToDisallowedOrigin === true
options.responseAfterMalformedUrl ??
(options.redirectToDisallowedOrigin === true
? "https://private.example.test/redirected"
: url,
: responseUrl),
status: 200,
mimeType: "application/json",
headers: {
+6
View File
@@ -75,6 +75,12 @@ export interface FakeOptions {
readonly indexedDbDateKeys?: boolean;
readonly foreignSessionEvents?: boolean;
readonly redirectToDisallowedOrigin?: boolean;
readonly redirectFromDisallowedOrigin?: boolean;
readonly redirectWithinOrigin?: boolean;
readonly malformedRedirectResponse?: boolean;
readonly redirectResponseUrl?: string;
readonly redirectResponseEnvelope?: unknown;
readonly responseAfterMalformedUrl?: string;
readonly unrelatedWorker?: boolean;
readonly binaryWebSocketEvent?: boolean;
readonly invalidBinaryWebSocketEvent?: boolean;