fix(browser): preserve authorized redirect hops (#986)
* fix(browser): preserve authorized redirect hops * fix(browser): compare redirect semantics without event times * fix(browser): classify missing redirect predecessors * fix(browser): distinguish malformed redirect URLs * fix(browser): preserve evidence on malformed redirect envelopes * fix(browser): freeze evidence after malformed redirect * docs: regenerate catalogs after main integration * fix(browser): account for skipped redirect body shapes * fix(browser): mark skipped redirect response shapes unavailable When a malformed redirect keeps a JSON predecessor, record that its response body shape was not captured. --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com>
This commit is contained in:
@@ -133,7 +133,7 @@ REA retains local observation data and removes only URL userinfo credentials, ex
|
||||
- URL observations preserve the complete local URL, including query values, fragments, duplicate parameters, and their order. Only username/password userinfo is removed; malformed original URL strings remain available with `origin: null`.
|
||||
- DOM snapshots retain node types, node names, value lengths, and attribute names, but not text or attribute values.
|
||||
- Accessibility structure and roles are retained by default, while names and descriptions require `include_accessibility_text: true`.
|
||||
- Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained.
|
||||
- Network observations retain method, status, MIME type, size, type, initiator stack location, and complete URLs with userinfo credentials removed. Redirects reported by CDP are attached as ordered `redirects` on the one request ID, with the prior request URL, CDP response URL, method/type, response status/MIME/encoded length, request timestamp, and the next `requestWillBeSent` event timestamp. CDP delivers `redirectResponse` with that next request event, so the latter is an event boundary rather than an exact response timestamp. Redirect response headers and bodies are discarded; hops outside the selected origins are discarded with the request chain. Headers are discarded after an allowlisted projection of length/encoding, structured CSP/Link/policy fields, and untrusted agent hints. Cookies and authorization headers are never retained.
|
||||
- Request/response bodies are not requested or parsed by default. When selected, only allowed-origin JSON media types are read, converted immediately to complete value-free property paths/types, and discarded. `Network.getResponseBody` is never sent unless JSON body shape capture is selected.
|
||||
- Console observations with a stack source retain call type, argument types, timestamp, and source location. When console text capture is selected, already-delivered primitive values are retained verbatim; objects, getters, and remote properties are never expanded.
|
||||
- WebSocket observations retain direction, opcode, and payload byte length. When shape capture is selected, text frames are classified as text or complete value-free JSON shape; binary bytes, hashes, prefixes, and raw frames are never retained.
|
||||
|
||||
@@ -675,9 +675,9 @@
|
||||
"mcp_prompts": 6
|
||||
},
|
||||
"digests": {
|
||||
"tools_sha256": "c77a3b4740fc2e78172e1d6757b1915d2b638185d8e87d2f1885bb41ce538813",
|
||||
"tools_sha256": "4baede40da9b3bb792e6c35b9b461745ac302973931281be26c8ad7a99f72872",
|
||||
"prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d",
|
||||
"combined_sha256": "e652c2ec4cde078a23735789c1d01cb1b9d316c9af7708e86f2c5c31b37b0e51",
|
||||
"combined_sha256": "ed0b788dd5a6bf9446f840e388822db94b54589a5e294816a3e13bf2bec9f48c",
|
||||
"providers_sha256": "aea15f517c822d80918ccaba57a4fa06557f484cf27b8acbaed4e1a660fac896"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"manifest_id": "ecm_aa4a1c78be657e901c87cc215a703938cfe54d7a54411cbfd64872de3ce78b93",
|
||||
"manifest_id": "ecm_4d1cf621854fe1f40698ef60f340fb99263edb0e99218c865b09cb26112d2547",
|
||||
"verifier": {
|
||||
"id": "rea-managed-conformance",
|
||||
"version": "1"
|
||||
@@ -13,7 +13,7 @@
|
||||
"skill_digests": [
|
||||
{
|
||||
"skill_id": "reverse-engineer-anything",
|
||||
"sha256": "57f28ace603b9eb5d6d9ea64d68f563ea8a4833e6a53dd7b4b40ce5fb3ac56e1"
|
||||
"sha256": "1d816388b86e50fff47db599200968788a4bf2272f40d5bdd3bb51aae7f5a9b4"
|
||||
}
|
||||
],
|
||||
"claims": [
|
||||
|
||||
@@ -4,7 +4,7 @@ description: Reverse engineer native, managed, Electron/JavaScript, packaged, fi
|
||||
metadata:
|
||||
version: "27"
|
||||
tool_count: 133
|
||||
catalog_digest: "e652c2ec4cde078a23735789c1d01cb1b9d316c9af7708e86f2c5c31b37b0e51"
|
||||
catalog_digest: "ed0b788dd5a6bf9446f840e388822db94b54589a5e294816a3e13bf2bec9f48c"
|
||||
---
|
||||
|
||||
# REA
|
||||
|
||||
@@ -10,7 +10,10 @@ import {
|
||||
cdpStringValue,
|
||||
type UnknownRecord,
|
||||
} from "./CdpCaptureValues.js";
|
||||
import { requestBodyShape } from "./CdpCaptureEventBodyShapes.js";
|
||||
import {
|
||||
requestBodyShape,
|
||||
updateResponseBodyShape,
|
||||
} from "./CdpCaptureEventBodyShapes.js";
|
||||
import {
|
||||
consolePrimitive,
|
||||
decodeBase64,
|
||||
@@ -19,10 +22,11 @@ import {
|
||||
firstCallFrame,
|
||||
initiatorLocation,
|
||||
integerOrNull,
|
||||
isJsonMediaType,
|
||||
isMainFrameNavigation,
|
||||
} from "./CdpCaptureEventHelpers.js";
|
||||
import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js";
|
||||
import type { CapturedScript } from "./CdpCaptureEventTypes.js";
|
||||
import type { CapturedScript, NetworkState } from "./CdpCaptureEventTypes.js";
|
||||
|
||||
export const handleExecutionContextCreated = (
|
||||
state: CdpCaptureEventsState,
|
||||
@@ -120,9 +124,11 @@ export const handleRequestWillBeSent = (
|
||||
state.completeness.exclude("network_requests", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
if (state.malformedRedirectRequestIds.has(requestId)) return;
|
||||
if (request === undefined) {
|
||||
state.completeness.exclude("network_requests", "invalid_protocol_value");
|
||||
state.network.delete(requestId);
|
||||
state.networkRequestTimestamps.delete(requestId);
|
||||
return;
|
||||
}
|
||||
const sanitized = allowedSanitizedUrl(request.url, state.allowedOrigins);
|
||||
@@ -132,8 +138,60 @@ export const handleRequestWillBeSent = (
|
||||
exclusionReasonForUrl(cdpStringValue(request.url)),
|
||||
);
|
||||
state.network.delete(requestId);
|
||||
state.networkRequestTimestamps.delete(requestId);
|
||||
return;
|
||||
}
|
||||
const previous = state.network.get(requestId);
|
||||
const redirectResponse = recordValue(params.redirectResponse);
|
||||
if (
|
||||
Object.hasOwn(params, "redirectResponse") &&
|
||||
redirectResponse === undefined &&
|
||||
preserveMalformedRedirectEvidence(state, requestId, previous)
|
||||
)
|
||||
return;
|
||||
const redirects = [...(previous?.redirects ?? [])];
|
||||
if (redirectResponse !== undefined) {
|
||||
const rawResponseUrl = cdpStringValue(redirectResponse.url);
|
||||
const responseUrl =
|
||||
rawResponseUrl === undefined || rawResponseUrl === ""
|
||||
? undefined
|
||||
: allowedSanitizedUrl(rawResponseUrl, state.allowedOrigins);
|
||||
const responseUrlReason = redirectResponseUrlReason(rawResponseUrl);
|
||||
if (previous === undefined) {
|
||||
if (responseUrl === undefined)
|
||||
state.completeness.exclude("network_requests", responseUrlReason);
|
||||
else
|
||||
// CDP may deliver the continuation after its predecessor was excluded
|
||||
// by origin policy or before capture began. Keep the final request but
|
||||
// report missing prior coverage without calling it malformed protocol.
|
||||
state.completeness.attachLimited("network_requests");
|
||||
} else if (responseUrl === undefined) {
|
||||
// redirectResponse belongs to the same request ID and can contain an
|
||||
// excluded origin. Drop the entire chain before retaining any details.
|
||||
state.completeness.exclude("network_requests", responseUrlReason);
|
||||
state.network.delete(requestId);
|
||||
state.networkRequestTimestamps.delete(requestId);
|
||||
return;
|
||||
} else {
|
||||
redirects.push({
|
||||
url: previous.url,
|
||||
response_url: responseUrl.url,
|
||||
method: previous.method,
|
||||
resource_type: previous.resource_type,
|
||||
status: numberValue(redirectResponse.status) ?? null,
|
||||
mime_type: cdpStringValue(redirectResponse.mimeType) ?? null,
|
||||
encoded_data_length: (() => {
|
||||
const length = numberValue(redirectResponse.encodedDataLength);
|
||||
return length === undefined ? null : Math.max(0, length);
|
||||
})(),
|
||||
request_timestamp:
|
||||
state.networkRequestTimestamps.get(requestId) ?? null,
|
||||
// CDP reports redirectResponse on the next requestWillBeSent event.
|
||||
// This is the event boundary, not a separately observed response time.
|
||||
redirect_event_timestamp: numberValue(params.timestamp) ?? null,
|
||||
});
|
||||
}
|
||||
}
|
||||
const initiator = recordValue(params.initiator);
|
||||
const initiatorFrame = initiatorLocation(initiator);
|
||||
const rawInitiatorUrl = cdpStringValue(initiatorFrame?.url);
|
||||
@@ -155,6 +213,7 @@ export const handleRequestWillBeSent = (
|
||||
status: null,
|
||||
mime_type: null,
|
||||
encoded_data_length: null,
|
||||
redirects,
|
||||
initiator: {
|
||||
type: cdpStringValue(initiator?.type) ?? "other",
|
||||
url: initiatorUrl?.url ?? null,
|
||||
@@ -163,6 +222,44 @@ export const handleRequestWillBeSent = (
|
||||
},
|
||||
body_shapes: requestBodyShape(state, request),
|
||||
});
|
||||
const timestamp = numberValue(params.timestamp);
|
||||
if (timestamp === undefined) state.networkRequestTimestamps.delete(requestId);
|
||||
else state.networkRequestTimestamps.set(requestId, timestamp);
|
||||
};
|
||||
|
||||
const preserveMalformedRedirectEvidence = (
|
||||
state: CdpCaptureEventsState,
|
||||
requestId: string,
|
||||
previous: NetworkState | undefined,
|
||||
): boolean => {
|
||||
state.completeness.exclude("network_requests", "invalid_protocol_value");
|
||||
if (previous === undefined) return false;
|
||||
state.malformedRedirectRequestIds.add(requestId);
|
||||
if (
|
||||
state.input.include_json_body_shapes &&
|
||||
isJsonMediaType(previous.mime_type) &&
|
||||
previous.body_shapes.response === null
|
||||
) {
|
||||
updateResponseBodyShape(state, requestId, null);
|
||||
state.completeness.exclude("json_body_shapes", "invalid_protocol_value");
|
||||
}
|
||||
return true;
|
||||
};
|
||||
|
||||
const redirectResponseUrlReason = (
|
||||
value: string | undefined,
|
||||
):
|
||||
| "invalid_protocol_value"
|
||||
| "disallowed_origin"
|
||||
| "unsupported_url"
|
||||
| "unattributed_origin" => {
|
||||
if (value === undefined || value === "") return "invalid_protocol_value";
|
||||
try {
|
||||
new URL(value);
|
||||
} catch {
|
||||
return "invalid_protocol_value";
|
||||
}
|
||||
return exclusionReasonForUrl(value);
|
||||
};
|
||||
|
||||
export const handleResponseReceived = (
|
||||
@@ -174,6 +271,7 @@ export const handleResponseReceived = (
|
||||
state.completeness.exclude("network_requests", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
if (state.malformedRedirectRequestIds.has(requestId)) return;
|
||||
const current = state.network.get(requestId);
|
||||
const response = recordValue(params.response);
|
||||
if (current === undefined) return;
|
||||
@@ -186,6 +284,7 @@ export const handleResponseReceived = (
|
||||
: exclusionReasonForUrl(cdpStringValue(response.url)),
|
||||
);
|
||||
state.network.delete(requestId);
|
||||
state.networkRequestTimestamps.delete(requestId);
|
||||
return;
|
||||
}
|
||||
state.network.set(requestId, {
|
||||
@@ -209,6 +308,7 @@ export const handleLoadingFinished = (
|
||||
): void => {
|
||||
const requestId = cdpStringValue(params.requestId);
|
||||
if (requestId === undefined) return;
|
||||
if (state.malformedRedirectRequestIds.has(requestId)) return;
|
||||
const current = state.network.get(requestId);
|
||||
if (current === undefined) return;
|
||||
state.network.set(requestId, {
|
||||
|
||||
@@ -11,6 +11,8 @@ export interface CdpCaptureEventsState {
|
||||
readonly scripts: Map<string, CapturedScript>;
|
||||
readonly executionContextFrames: Map<string, string>;
|
||||
readonly network: Map<string, NetworkState>;
|
||||
readonly networkRequestTimestamps: Map<string, number>;
|
||||
readonly malformedRedirectRequestIds: Set<string>;
|
||||
readonly allowedWebSockets: Set<string>;
|
||||
console: WebPageInspection["console"]["events"];
|
||||
websockets: WebPageInspection["network"]["websocket_events"];
|
||||
|
||||
@@ -18,6 +18,8 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
readonly scripts = new Map<string, CapturedScript>();
|
||||
readonly executionContextFrames = new Map<string, string>();
|
||||
readonly network = new Map<string, NetworkState>();
|
||||
readonly networkRequestTimestamps = new Map<string, number>();
|
||||
readonly malformedRedirectRequestIds = new Set<string>();
|
||||
readonly allowedWebSockets = new Set<string>();
|
||||
console: WebPageInspection["console"]["events"] = [];
|
||||
websockets: WebPageInspection["network"]["websocket_events"] = [];
|
||||
@@ -51,6 +53,8 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
this.scripts.clear();
|
||||
this.executionContextFrames.clear();
|
||||
this.network.clear();
|
||||
this.networkRequestTimestamps.clear();
|
||||
this.malformedRedirectRequestIds.clear();
|
||||
this.allowedWebSockets.clear();
|
||||
this.console.length = 0;
|
||||
this.websockets.length = 0;
|
||||
@@ -135,16 +139,21 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
responseBodyRequestIds(): readonly string[] {
|
||||
if (!this.input.include_json_body_shapes) return [];
|
||||
return [...this.network.values()]
|
||||
.filter(
|
||||
(request) => !this.malformedRedirectRequestIds.has(request.request_id),
|
||||
)
|
||||
.filter((request) => isJsonMediaType(request.mime_type))
|
||||
.map((request) => request.request_id);
|
||||
}
|
||||
|
||||
responseBodyUnavailable(requestId: string): void {
|
||||
if (this.malformedRedirectRequestIds.has(requestId)) return;
|
||||
bodyShapes.updateResponseBodyShape(this, requestId, null);
|
||||
this.completeness.unavailable("json_body_shapes");
|
||||
}
|
||||
|
||||
ingestResponseBody(requestId: string, value: unknown): void {
|
||||
if (this.malformedRedirectRequestIds.has(requestId)) return;
|
||||
bodyShapes.ingestResponseBodyShape(this, requestId, value);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,6 +114,21 @@ const browserNetworkRequestSchema = z.object({
|
||||
status: z.number().nullable(),
|
||||
mime_type: z.string().nullable(),
|
||||
encoded_data_length: z.number().min(0).nullable(),
|
||||
redirects: z
|
||||
.array(
|
||||
z.object({
|
||||
url: z.string(),
|
||||
response_url: z.string(),
|
||||
method: z.string(),
|
||||
resource_type: z.string().nullable(),
|
||||
status: z.number().nullable(),
|
||||
mime_type: z.string().nullable(),
|
||||
encoded_data_length: z.number().min(0).nullable(),
|
||||
request_timestamp: z.number().nullable(),
|
||||
redirect_event_timestamp: z.number().nullable(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
initiator: z.object({
|
||||
type: z.string(),
|
||||
url: z.string().nullable(),
|
||||
|
||||
@@ -249,6 +249,13 @@ const networkMap = (
|
||||
status: request.status,
|
||||
mime_type: request.mime_type,
|
||||
encoded_data_length: request.encoded_data_length,
|
||||
redirects: (request.redirects ?? []).map(
|
||||
({
|
||||
request_timestamp: _requestTimestamp,
|
||||
redirect_event_timestamp: _redirectEventTimestamp,
|
||||
...redirect
|
||||
}) => redirect,
|
||||
),
|
||||
initiator: request.initiator,
|
||||
body_shapes: request.body_shapes,
|
||||
});
|
||||
|
||||
@@ -6,6 +6,59 @@ import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
|
||||
import { describeBrowser, trackBrowser } from "./cdpBrowserProvider.support.js";
|
||||
|
||||
describeBrowser("CdpBrowserProvider: network 1", () => {
|
||||
it("retains same-origin redirect hops on the one final request", async () => {
|
||||
const browser = await startFakeCdpBrowser({ redirectWithinOrigin: true });
|
||||
trackBrowser(browser);
|
||||
const result = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.network.requests).toHaveLength(1);
|
||||
expect(result.value.network.requests[0]).toMatchObject({
|
||||
request_id: "request-1",
|
||||
url: `${browser.allowedOrigin}/redirected?token=final-secret`,
|
||||
status: 200,
|
||||
redirects: [
|
||||
{
|
||||
url: `${browser.allowedOrigin}/api?token=network-secret`,
|
||||
response_url: `${browser.allowedOrigin}/api?token=network-secret`,
|
||||
method: "POST",
|
||||
resource_type: "Fetch",
|
||||
status: 301,
|
||||
mime_type: "text/plain",
|
||||
encoded_data_length: 23,
|
||||
request_timestamp: 7,
|
||||
redirect_event_timestamp: 8,
|
||||
},
|
||||
{
|
||||
url: `${browser.allowedOrigin}/intermediate`,
|
||||
response_url: `${browser.allowedOrigin}/intermediate`,
|
||||
method: "GET",
|
||||
resource_type: "Fetch",
|
||||
status: 302,
|
||||
mime_type: "text/plain",
|
||||
encoded_data_length: 27,
|
||||
request_timestamp: 8,
|
||||
redirect_event_timestamp: 9,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(JSON.stringify(result.value.network.requests)).not.toContain(
|
||||
"redirect-header-secret",
|
||||
);
|
||||
expect(result.value.completeness.excluded).not.toContainEqual({
|
||||
section: "network_requests",
|
||||
reason: "invalid_protocol_value",
|
||||
count: expect.any(Number),
|
||||
});
|
||||
});
|
||||
|
||||
it("drops network evidence when a request redirects outside the approved origin", async () => {
|
||||
const browser = await startFakeCdpBrowser({
|
||||
redirectToDisallowedOrigin: true,
|
||||
@@ -23,4 +76,42 @@ describeBrowser("CdpBrowserProvider: network 1", () => {
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.network.requests).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not retain excluded prior-hop data when a redirect returns in scope", async () => {
|
||||
const browser = await startFakeCdpBrowser({
|
||||
redirectFromDisallowedOrigin: true,
|
||||
});
|
||||
trackBrowser(browser);
|
||||
const result = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
|
||||
if (!result.ok) throw result.error;
|
||||
expect(result.value.network.requests).toHaveLength(1);
|
||||
expect(result.value.network.requests[0]).toMatchObject({
|
||||
url: `${browser.allowedOrigin}/returned`,
|
||||
redirects: [],
|
||||
});
|
||||
const output = JSON.stringify(result.value.network.requests);
|
||||
expect(output).not.toContain("private.example.test");
|
||||
expect(output).not.toContain("redirect-header-secret");
|
||||
expect(output).not.toContain("redirect-body-secret");
|
||||
expect(result.value.completeness.policy_filtered_sections).toContain(
|
||||
"network_requests",
|
||||
);
|
||||
expect(result.value.completeness.status).toBe("policy_filtered");
|
||||
expect(result.value.completeness.unavailable_sections).not.toContain(
|
||||
"network_requests",
|
||||
);
|
||||
expect(result.value.completeness.excluded).not.toContainEqual({
|
||||
section: "network_requests",
|
||||
reason: "invalid_protocol_value",
|
||||
count: expect.any(Number),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { CdpCaptureEvents } from "../../../src/browser/CdpCaptureEvents.js";
|
||||
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
|
||||
|
||||
it("preserves an already captured response shape without marking it unavailable", () => {
|
||||
const origin = "http://127.0.0.1:43127";
|
||||
const events = new CdpCaptureEvents(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: origin,
|
||||
allowed_origins: [origin],
|
||||
target_id: "allowed-page",
|
||||
include_json_body_shapes: true,
|
||||
}),
|
||||
new Set([origin]),
|
||||
);
|
||||
events.ingest({
|
||||
method: "Network.requestWillBeSent",
|
||||
params: {
|
||||
requestId: "request-1",
|
||||
request: {
|
||||
url: `${origin}/prior`,
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
postData: '{"request":true}',
|
||||
},
|
||||
timestamp: 1,
|
||||
},
|
||||
});
|
||||
events.ingest({
|
||||
method: "Network.responseReceived",
|
||||
params: {
|
||||
requestId: "request-1",
|
||||
response: {
|
||||
url: `${origin}/prior`,
|
||||
status: 200,
|
||||
mimeType: "application/json",
|
||||
},
|
||||
},
|
||||
});
|
||||
events.ingestResponseBody("request-1", {
|
||||
body: '{"response":true}',
|
||||
base64Encoded: false,
|
||||
});
|
||||
const capturedShape = events.network.get("request-1")?.body_shapes.response;
|
||||
expect(capturedShape).not.toBeNull();
|
||||
|
||||
events.ingest({
|
||||
method: "Network.requestWillBeSent",
|
||||
params: {
|
||||
requestId: "request-1",
|
||||
request: { url: `${origin}/continuation`, method: "GET" },
|
||||
redirectResponse: null,
|
||||
timestamp: 2,
|
||||
},
|
||||
});
|
||||
|
||||
expect(events.network.get("request-1")?.body_shapes.response).toEqual(
|
||||
capturedShape,
|
||||
);
|
||||
expect(events.network.get("request-1")?.body_shapes.status).toBe("included");
|
||||
expect(events.responseBodyRequestIds()).toEqual([]);
|
||||
expect(events.completeness.snapshot().unavailable_sections).not.toContain(
|
||||
"json_body_shapes",
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,150 @@
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js";
|
||||
import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js";
|
||||
import { startFakeCdpBrowser } from "../../fixtures/fakeCdpBrowser.js";
|
||||
import type { FakeOptions } from "../../fixtures/fakeCdpBrowserTypes.js";
|
||||
import { describeBrowser, trackBrowser } from "./cdpBrowserProvider.support.js";
|
||||
|
||||
const inspectNetwork = async (options: FakeOptions) => {
|
||||
const browser = await startFakeCdpBrowser(options);
|
||||
trackBrowser(browser);
|
||||
const result = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
include_json_body_shapes: true,
|
||||
}),
|
||||
);
|
||||
if (!result.ok) throw result.error;
|
||||
return { browser, inspection: result.value };
|
||||
};
|
||||
|
||||
describeBrowser("CdpBrowserProvider: redirect errors", () => {
|
||||
it.each([
|
||||
{ label: "missing", url: undefined, reason: "invalid_protocol_value" },
|
||||
{ label: "malformed", url: "http://%", reason: "invalid_protocol_value" },
|
||||
{
|
||||
label: "unsupported scheme",
|
||||
url: "file:///private/redirect",
|
||||
reason: "unsupported_url",
|
||||
},
|
||||
])("classifies $label redirect response URLs", async ({ url, reason }) => {
|
||||
const { inspection } = await inspectNetwork({
|
||||
malformedRedirectResponse: true,
|
||||
...(url === undefined ? {} : { redirectResponseUrl: url }),
|
||||
});
|
||||
|
||||
expect(inspection.network.requests).toEqual([]);
|
||||
expect(inspection.completeness.excluded).toContainEqual({
|
||||
section: "network_requests",
|
||||
reason,
|
||||
count: expect.any(Number),
|
||||
});
|
||||
const section =
|
||||
reason === "unsupported_url"
|
||||
? inspection.completeness.policy_filtered_sections
|
||||
: inspection.completeness.unavailable_sections;
|
||||
expect(section).toContain("network_requests");
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ label: "null", envelope: null },
|
||||
{ label: "array", envelope: [] },
|
||||
{ label: "scalar", envelope: "malformed redirect" },
|
||||
])(
|
||||
"preserves prior request for a $label redirect envelope",
|
||||
async ({ envelope }) => {
|
||||
const { browser, inspection } = await inspectNetwork({
|
||||
malformedRedirectResponse: true,
|
||||
redirectResponseEnvelope: envelope,
|
||||
});
|
||||
|
||||
expect(inspection.network.requests).toHaveLength(1);
|
||||
expect(inspection.network.requests[0]).toMatchObject({
|
||||
request_id: "request-1",
|
||||
url: `${browser.allowedOrigin}/malformed-redirect-prior`,
|
||||
status: 201,
|
||||
mime_type: "application/json",
|
||||
encoded_data_length: null,
|
||||
redirects: [
|
||||
{
|
||||
url: `${browser.allowedOrigin}/api?token=network-secret`,
|
||||
response_url: `${browser.allowedOrigin}/api?token=network-secret`,
|
||||
request_timestamp: 7,
|
||||
redirect_event_timestamp: 8,
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(
|
||||
inspection.network.requests[0]?.body_shapes.request,
|
||||
).not.toBeNull();
|
||||
expect(inspection.network.requests[0]?.body_shapes.response).toBeNull();
|
||||
expect(inspection.network.requests[0]?.body_shapes.status).toBe(
|
||||
"partial",
|
||||
);
|
||||
expect(inspection.metadata.responses).toHaveLength(1);
|
||||
expect(inspection.metadata.responses[0]).toMatchObject({
|
||||
url: `${browser.allowedOrigin}/malformed-redirect-prior`,
|
||||
mime_type: "application/json",
|
||||
content_length: 123,
|
||||
content_encoding: "gzip",
|
||||
});
|
||||
expect(inspection.completeness.unavailable_sections).toContain(
|
||||
"network_requests",
|
||||
);
|
||||
expect(inspection.completeness.unavailable_sections).toContain(
|
||||
"json_body_shapes",
|
||||
);
|
||||
expect(inspection.completeness.excluded).toContainEqual({
|
||||
section: "network_requests",
|
||||
reason: "invalid_protocol_value",
|
||||
count: expect.any(Number),
|
||||
});
|
||||
expect(inspection.completeness.excluded).toContainEqual({
|
||||
section: "json_body_shapes",
|
||||
reason: "invalid_protocol_value",
|
||||
count: expect.any(Number),
|
||||
});
|
||||
expect(JSON.stringify(inspection.network.requests)).not.toContain(
|
||||
"malformed-redirect-final",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps authorized prior evidence when the later response URL is disallowed", async () => {
|
||||
const { browser, inspection } = await inspectNetwork({
|
||||
malformedRedirectResponse: true,
|
||||
redirectResponseEnvelope: null,
|
||||
responseAfterMalformedUrl:
|
||||
"https://private.example.test/late?token=unauthorized-secret",
|
||||
});
|
||||
|
||||
expect(inspection.network.requests).toHaveLength(1);
|
||||
expect(inspection.network.requests[0]).toMatchObject({
|
||||
url: `${browser.allowedOrigin}/malformed-redirect-prior`,
|
||||
status: 201,
|
||||
mime_type: "application/json",
|
||||
encoded_data_length: null,
|
||||
body_shapes: { status: "partial", response: null },
|
||||
});
|
||||
expect(inspection.metadata.responses).toHaveLength(1);
|
||||
expect(inspection.metadata.responses[0]?.url).toBe(
|
||||
`${browser.allowedOrigin}/malformed-redirect-prior`,
|
||||
);
|
||||
expect(inspection.completeness.unavailable_sections).toContain(
|
||||
"json_body_shapes",
|
||||
);
|
||||
expect(inspection.completeness.excluded).toContainEqual({
|
||||
section: "network_requests",
|
||||
reason: "invalid_protocol_value",
|
||||
count: expect.any(Number),
|
||||
});
|
||||
const output = JSON.stringify(inspection);
|
||||
expect(output).not.toContain("private.example.test");
|
||||
expect(output).not.toContain("unauthorized-secret");
|
||||
expect(output).not.toContain("header-secret");
|
||||
});
|
||||
});
|
||||
@@ -48,7 +48,22 @@ describe("web capture diff", () => {
|
||||
const after = structuredClone(captured.value);
|
||||
after.scripts.items = [];
|
||||
const request = after.network.requests[0];
|
||||
if (request !== undefined) request.status = 204;
|
||||
if (request !== undefined) {
|
||||
request.status = 204;
|
||||
request.redirects = [
|
||||
{
|
||||
url: `${browser.allowedOrigin}/prior`,
|
||||
response_url: `${browser.allowedOrigin}/prior`,
|
||||
method: "GET",
|
||||
resource_type: "Fetch",
|
||||
status: 302,
|
||||
mime_type: "text/plain",
|
||||
encoded_data_length: 12,
|
||||
request_timestamp: 1,
|
||||
redirect_event_timestamp: 2,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
const result = compareWebCaptures(
|
||||
compareWebCapturesInputSchema.parse({
|
||||
@@ -163,6 +178,68 @@ describe("web capture diff", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("web capture redirect comparison", () => {
|
||||
it("ignores redirect event times but compares redirect semantics", async () => {
|
||||
const browser = await startFakeCdpBrowser();
|
||||
browsers.push(browser);
|
||||
const captured = await new CdpBrowserProvider().inspectPage(
|
||||
inspectWebPageInputSchema.parse({
|
||||
cdp_endpoint: browser.endpoint,
|
||||
allowed_origins: [browser.allowedOrigin],
|
||||
target_id: "allowed-page",
|
||||
observation_ms: 0,
|
||||
}),
|
||||
);
|
||||
if (!captured.ok) throw captured.error;
|
||||
const before = structuredClone(captured.value);
|
||||
const beforeRequest = before.network.requests[0];
|
||||
if (beforeRequest === undefined) throw new Error("Missing network request");
|
||||
beforeRequest.redirects = [
|
||||
{
|
||||
url: `${browser.allowedOrigin}/prior`,
|
||||
response_url: `${browser.allowedOrigin}/prior`,
|
||||
method: "GET",
|
||||
resource_type: "Fetch",
|
||||
status: 302,
|
||||
mime_type: "text/plain",
|
||||
encoded_data_length: 12,
|
||||
request_timestamp: 1,
|
||||
redirect_event_timestamp: 2,
|
||||
},
|
||||
];
|
||||
markSectionsComplete(before, ["network_requests"]);
|
||||
|
||||
const after = structuredClone(before);
|
||||
const afterRequest = after.network.requests[0];
|
||||
if (afterRequest === undefined) throw new Error("Missing network request");
|
||||
const hop = afterRequest.redirects?.[0];
|
||||
if (hop === undefined) throw new Error("Missing redirect hop");
|
||||
hop.request_timestamp = 10;
|
||||
hop.redirect_event_timestamp = 20;
|
||||
|
||||
const compare = (inspection: typeof before) =>
|
||||
compareWebCaptures(
|
||||
compareWebCapturesInputSchema.parse({
|
||||
before: { inspection: before },
|
||||
after: { inspection },
|
||||
}),
|
||||
).dimensions.network;
|
||||
expect(compare(after)).toMatchObject({
|
||||
status: "unchanged",
|
||||
total_changes: 0,
|
||||
});
|
||||
|
||||
const changed = structuredClone(after);
|
||||
const changedHop = changed.network.requests[0]?.redirects?.[0];
|
||||
if (changedHop === undefined) throw new Error("Missing redirect hop");
|
||||
changedHop.status = 307;
|
||||
expect(compare(changed)).toMatchObject({
|
||||
status: "changed",
|
||||
changes: [expect.objectContaining({ change: "modified" })],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("web capture diff incomplete inventories", () => {
|
||||
it.each(["truncated_sections", "unavailable_sections"] as const)(
|
||||
"does not infer script additions or removals from %s",
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { afterEach, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
startFakeCdpBrowser,
|
||||
type FakeCdpBrowser,
|
||||
} from "../../fixtures/fakeCdpBrowser.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const browsers: FakeCdpBrowser[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(browsers.splice(0).map(async (browser) => browser.close()));
|
||||
});
|
||||
|
||||
it("reports missing JSON shapes only when body shapes are selected", async () => {
|
||||
const browser = await startFakeCdpBrowser({
|
||||
malformedRedirectResponse: true,
|
||||
redirectResponseEnvelope: null,
|
||||
});
|
||||
browsers.push(browser);
|
||||
const commonArgs = [
|
||||
"inspect-web-page",
|
||||
browser.endpoint,
|
||||
"allowed-page",
|
||||
"--allowed-origins",
|
||||
browser.allowedOrigin,
|
||||
"--observation-ms",
|
||||
"0",
|
||||
];
|
||||
const selected = normalizedResult(
|
||||
await runCli([...commonArgs, "--include-json-body-shapes", "--json"]),
|
||||
);
|
||||
expect(selected).toMatchObject({
|
||||
network: {
|
||||
requests: [
|
||||
{
|
||||
body_shapes: {
|
||||
status: "partial",
|
||||
request: expect.any(Object),
|
||||
response: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
completeness: {
|
||||
unavailable_sections: expect.arrayContaining([
|
||||
"network_requests",
|
||||
"json_body_shapes",
|
||||
]),
|
||||
excluded: expect.arrayContaining([
|
||||
{
|
||||
section: "json_body_shapes",
|
||||
reason: "invalid_protocol_value",
|
||||
count: expect.any(Number),
|
||||
},
|
||||
]),
|
||||
},
|
||||
});
|
||||
|
||||
const defaulted = normalizedResult(await runCli([...commonArgs, "--json"]));
|
||||
expect(defaulted).toMatchObject({
|
||||
network: {
|
||||
requests: [
|
||||
{
|
||||
body_shapes: {
|
||||
status: "not_approved",
|
||||
request: null,
|
||||
response: null,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
expect(defaulted.completeness).toMatchObject({
|
||||
unavailable_sections: expect.not.arrayContaining(["json_body_shapes"]),
|
||||
});
|
||||
});
|
||||
|
||||
const runCli = async (arguments_: readonly string[]): Promise<unknown> => {
|
||||
try {
|
||||
const { stdout } = await execute(
|
||||
process.execPath,
|
||||
["scripts/rea.mjs", ...arguments_],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
maxBuffer: 16 * 1_024 * 1_024,
|
||||
},
|
||||
);
|
||||
return JSON.parse(stdout);
|
||||
} catch (cause: unknown) {
|
||||
if (
|
||||
typeof cause === "object" &&
|
||||
cause !== null &&
|
||||
"stdout" in cause &&
|
||||
typeof cause.stdout === "string"
|
||||
)
|
||||
return JSON.parse(cause.stdout);
|
||||
throw cause;
|
||||
}
|
||||
};
|
||||
|
||||
const normalizedResult = (value: unknown): Record<string, unknown> => {
|
||||
if (!isRecord(value) || !isRecord(value.normalized_result))
|
||||
throw new TypeError("Missing CLI normalized result");
|
||||
return value.normalized_result;
|
||||
};
|
||||
|
||||
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
+153
-3
@@ -282,6 +282,133 @@ const emitSessionTimeline = (
|
||||
}
|
||||
};
|
||||
|
||||
const emitSameOriginRedirectEvents = (
|
||||
socket: WebSocket,
|
||||
command: FakeCdpCommand,
|
||||
port: number,
|
||||
url: string,
|
||||
): string => {
|
||||
const intermediate = `http://127.0.0.1:${String(port)}/intermediate`;
|
||||
const finalUrl = `http://127.0.0.1:${String(port)}/redirected?token=final-secret`;
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
type: "Fetch",
|
||||
request: { url: intermediate, method: "GET" },
|
||||
redirectResponse: {
|
||||
url,
|
||||
status: 301,
|
||||
mimeType: "text/plain",
|
||||
encodedDataLength: 23,
|
||||
headers: { "X-Private": "redirect-header-secret" },
|
||||
},
|
||||
timestamp: 8,
|
||||
});
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
type: "Fetch",
|
||||
request: { url: finalUrl, method: "GET" },
|
||||
redirectResponse: {
|
||||
url: intermediate,
|
||||
status: 302,
|
||||
mimeType: "text/plain",
|
||||
encodedDataLength: 27,
|
||||
},
|
||||
timestamp: 9,
|
||||
});
|
||||
return finalUrl;
|
||||
};
|
||||
|
||||
const emitReturnFromDisallowedRedirect = (
|
||||
socket: WebSocket,
|
||||
command: FakeCdpCommand,
|
||||
port: number,
|
||||
): string => {
|
||||
const finalUrl = `http://127.0.0.1:${String(port)}/returned`;
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
type: "Fetch",
|
||||
request: { url: finalUrl, method: "GET" },
|
||||
redirectResponse: {
|
||||
url: "https://private.example.test/outside?token=redirect-url-secret",
|
||||
status: 302,
|
||||
mimeType: "text/plain",
|
||||
encodedDataLength: 31,
|
||||
headers: { "X-Private": "redirect-header-secret" },
|
||||
postData: "redirect-body-secret",
|
||||
},
|
||||
timestamp: 8,
|
||||
});
|
||||
return finalUrl;
|
||||
};
|
||||
|
||||
const emitMalformedRedirectPriorResponse = (
|
||||
socket: WebSocket,
|
||||
command: FakeCdpCommand,
|
||||
port: number,
|
||||
initialUrl: string,
|
||||
): void => {
|
||||
const priorUrl = `http://127.0.0.1:${String(port)}/malformed-redirect-prior`;
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
type: "Fetch",
|
||||
request: {
|
||||
url: priorUrl,
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json; charset=utf-8" },
|
||||
postData: JSON.stringify({
|
||||
operation: "preserve-prior",
|
||||
token: "prior-body",
|
||||
}),
|
||||
},
|
||||
redirectResponse: {
|
||||
url: initialUrl,
|
||||
status: 307,
|
||||
mimeType: "text/plain",
|
||||
encodedDataLength: 23,
|
||||
},
|
||||
timestamp: 8,
|
||||
});
|
||||
event(socket, "Network.responseReceived", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
response: {
|
||||
url: priorUrl,
|
||||
status: 201,
|
||||
mimeType: "application/json",
|
||||
headers: {
|
||||
"Content-Length": "123",
|
||||
"Content-Encoding": "gzip",
|
||||
"Content-Security-Policy": "default-src 'self'",
|
||||
"X-Model-Context": "prior-context-hint",
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const emitMalformedRedirect = (
|
||||
socket: WebSocket,
|
||||
command: FakeCdpCommand,
|
||||
port: number,
|
||||
options: FakeOptions,
|
||||
): string => {
|
||||
const finalUrl = `http://127.0.0.1:${String(port)}/malformed-redirect-final`;
|
||||
const redirectResponse = Object.hasOwn(options, "redirectResponseEnvelope")
|
||||
? options.redirectResponseEnvelope
|
||||
: {
|
||||
...(options.redirectResponseUrl === undefined
|
||||
? {}
|
||||
: { url: options.redirectResponseUrl }),
|
||||
status: 302,
|
||||
};
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
type: "Fetch",
|
||||
request: { url: finalUrl, method: "GET" },
|
||||
redirectResponse,
|
||||
timestamp: 9,
|
||||
});
|
||||
return finalUrl;
|
||||
};
|
||||
|
||||
const emitNetworkEvents = (
|
||||
socket: WebSocket,
|
||||
command: FakeCdpCommand,
|
||||
@@ -297,11 +424,15 @@ const emitNetworkEvents = (
|
||||
},
|
||||
});
|
||||
const url = `http://127.0.0.1:${String(port)}/api?token=network-secret`;
|
||||
const requestUrl =
|
||||
options.redirectFromDisallowedOrigin === true
|
||||
? "https://private.example.test/start?token=initial-url-secret"
|
||||
: url;
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
type: "Fetch",
|
||||
request: {
|
||||
url,
|
||||
url: requestUrl,
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: "Bearer request-secret",
|
||||
@@ -313,6 +444,7 @@ const emitNetworkEvents = (
|
||||
filters: { active: true },
|
||||
}),
|
||||
},
|
||||
timestamp: 7,
|
||||
initiator: {
|
||||
type: "script",
|
||||
stack: {
|
||||
@@ -326,6 +458,16 @@ const emitNetworkEvents = (
|
||||
},
|
||||
},
|
||||
});
|
||||
if (options.malformedRedirectResponse === true)
|
||||
emitMalformedRedirectPriorResponse(socket, command, port, url);
|
||||
const responseUrl =
|
||||
options.redirectFromDisallowedOrigin === true
|
||||
? emitReturnFromDisallowedRedirect(socket, command, port)
|
||||
: options.malformedRedirectResponse === true
|
||||
? emitMalformedRedirect(socket, command, port, options)
|
||||
: options.redirectWithinOrigin === true
|
||||
? emitSameOriginRedirectEvents(socket, command, port, url)
|
||||
: url;
|
||||
if (options.redirectToDisallowedOrigin === true)
|
||||
event(socket, "Network.requestWillBeSent", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
@@ -334,14 +476,22 @@ const emitNetworkEvents = (
|
||||
url: "https://private.example.test/redirected",
|
||||
method: "GET",
|
||||
},
|
||||
redirectResponse: {
|
||||
url,
|
||||
status: 302,
|
||||
mimeType: "text/plain",
|
||||
encodedDataLength: 23,
|
||||
headers: { "X-Private": "redirect-header-secret" },
|
||||
},
|
||||
});
|
||||
event(socket, "Network.responseReceived", command.sessionId, {
|
||||
requestId: "request-1",
|
||||
response: {
|
||||
url:
|
||||
options.redirectToDisallowedOrigin === true
|
||||
options.responseAfterMalformedUrl ??
|
||||
(options.redirectToDisallowedOrigin === true
|
||||
? "https://private.example.test/redirected"
|
||||
: url,
|
||||
: responseUrl),
|
||||
status: 200,
|
||||
mimeType: "application/json",
|
||||
headers: {
|
||||
|
||||
+6
@@ -75,6 +75,12 @@ export interface FakeOptions {
|
||||
readonly indexedDbDateKeys?: boolean;
|
||||
readonly foreignSessionEvents?: boolean;
|
||||
readonly redirectToDisallowedOrigin?: boolean;
|
||||
readonly redirectFromDisallowedOrigin?: boolean;
|
||||
readonly redirectWithinOrigin?: boolean;
|
||||
readonly malformedRedirectResponse?: boolean;
|
||||
readonly redirectResponseUrl?: string;
|
||||
readonly redirectResponseEnvelope?: unknown;
|
||||
readonly responseAfterMalformedUrl?: string;
|
||||
readonly unrelatedWorker?: boolean;
|
||||
readonly binaryWebSocketEvent?: boolean;
|
||||
readonly invalidBinaryWebSocketEvent?: boolean;
|
||||
|
||||
Reference in New Issue
Block a user