import type { WebPageInspection } from "../domain/browserObservationSchemas.js"; import { sanitizeBrowserUrl } from "../domain/browserObservation.js"; import { inferJsonShape } from "../domain/jsonShape.js"; import { safeResponseMetadata } from "./CdpSafeMetadata.js"; import { allowedSanitizedUrl, isHttpUrl, numberValue, recordValue, recordsValue, scriptMetadataValues, cdpStringValue, type UnknownRecord, } from "./CdpCaptureValues.js"; import { requestBodyShape, updateResponseBodyShape, } from "./CdpCaptureEventBodyShapes.js"; import { consolePrimitive, decodeBase64, exclusionReasonForUrl, executionContextKey, firstCallFrame, initiatorLocation, integerOrNull, isJsonMediaType, isMainFrameNavigation, } from "./CdpCaptureEventHelpers.js"; import type { CdpCaptureEventsState } from "./CdpCaptureEventState.js"; import { CDP_CAPTURE_SCRIPT_METADATA_LIMITS, type CapturedScript, type NetworkState, } from "./CdpCaptureEventTypes.js"; export const handleExecutionContextCreated = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const context = recordValue(params.context); const identifier = numberValue(context?.id); const frameId = cdpStringValue(recordValue(context?.auxData)?.frameId); if ( identifier === undefined || !Number.isSafeInteger(identifier) || frameId === undefined ) return; const key = String(identifier); // New contexts may reuse an id. Scripts already parsed keep the frame stored on them. state.executionContextFrames.set(key, frameId); }; export const handleExecutionContextDestroyed = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const key = executionContextKey(params.executionContextId); if (key !== null) state.executionContextFrames.delete(key); }; export const handleScriptParsed = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const scriptId = cdpStringValue(params.scriptId); const rawUrl = cdpStringValue(params.url) ?? ""; const sanitized = allowedSanitizedUrl(rawUrl, state.allowedOrigins); if (scriptId === undefined) { state.completeness.exclude("scripts", "invalid_protocol_value"); return; } const replacedBytes = state.scriptMetadataBytesById.get(scriptId) ?? 0; state.scriptMetadataBudgetOmissionsById.delete(scriptId); if (replacedBytes > 0) { state.scripts.delete(scriptId); state.scriptMetadataBytesById.delete(scriptId); state.retainedScriptMetadataBytes -= replacedBytes; } if (sanitized === undefined) { state.completeness.exclude("scripts", exclusionReasonForUrl(rawUrl)); return; } const executionContext = executionContextKey(params.executionContextId); const baseScript = { scriptId, rawUrl, url: sanitized.url, origin: sanitized.origin, ...scriptMetadataValues(params), language: cdpStringValue(params.scriptLanguage) ?? null, executionContextKey: executionContext, frameId: executionContext === null ? null : (state.executionContextFrames.get(executionContext) ?? null), }; const baseBytes = scriptMetadataStringBytes(baseScript); const sourceMapDeclared = typeof params.sourceMapURL === "string" && params.sourceMapURL.length > 0; if ( state.scripts.size >= CDP_CAPTURE_SCRIPT_METADATA_LIMITS.scripts || baseBytes > CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes - state.retainedScriptMetadataBytes ) { state.rejectedScriptMetadataBudgetCount += 1; if (sourceMapDeclared) state.rejectedSourceMapMetadataBudgetCount += 1; return; } const sourceMap = sourceMapForScript( params.sourceMapURL, rawUrl, state, CDP_CAPTURE_SCRIPT_METADATA_LIMITS.retainedBytes - state.retainedScriptMetadataBytes - baseBytes, ); const { budgetExhausted, ...sourceMapFields } = sourceMap; const script: CapturedScript = { ...baseScript, ...sourceMapFields }; if (budgetExhausted) state.scriptMetadataBudgetOmissionsById.set(scriptId, true); const retainedBytes = baseBytes + stringByteLength(sourceMap.sourceMapUrl) + stringByteLength(sourceMap.sourceMapRawUrl); state.retainedScriptMetadataBytes += retainedBytes; state.scriptMetadataBytesById.set(scriptId, retainedBytes); state.scripts.set(scriptId, script); }; const scriptMetadataStringBytes = (script: { readonly scriptId: string; readonly rawUrl: string; readonly url: string; readonly origin: string | null; readonly hash: string | null; readonly language: string | null; readonly executionContextKey: string | null; readonly frameId: string | null; }): number => stringByteLength(script.scriptId) + stringByteLength(script.rawUrl) + stringByteLength(script.url) + stringByteLength(script.origin) + stringByteLength(script.hash) + stringByteLength(script.language) + stringByteLength(script.executionContextKey) + stringByteLength(script.frameId); const stringByteLength = (value: string | null): number => value === null ? 0 : Buffer.byteLength(value, "utf8"); const sourceMapForScript = ( value: unknown, scriptUrl: string, state: CdpCaptureEventsState, remainingBytes: number, ): { readonly sourceMapUrl: string | null; readonly sourceMapRawUrl: string | null; readonly budgetExhausted: boolean; } => { const declaredUrl = cdpStringValue(value); if (declaredUrl === undefined || declaredUrl === "") return { sourceMapUrl: null, sourceMapRawUrl: null, budgetExhausted: false, }; let rawUrl: string; try { rawUrl = new URL(declaredUrl, scriptUrl).href; } catch (cause: unknown) { // Unparseable URLs are recorded as unsupported, not as failures. void cause; state.completeness.exclude("source_maps", "unsupported_url"); return { sourceMapUrl: null, sourceMapRawUrl: null, budgetExhausted: false, }; } const sanitized = allowedSanitizedUrl(rawUrl, state.allowedOrigins); if (sanitized !== undefined) { if ( stringByteLength(rawUrl) + stringByteLength(sanitized.url) > remainingBytes ) { return { sourceMapUrl: null, sourceMapRawUrl: null, budgetExhausted: true, }; } return { sourceMapUrl: sanitized.url, sourceMapRawUrl: rawUrl, budgetExhausted: false, }; } state.completeness.exclude("source_maps", exclusionReasonForUrl(rawUrl)); return { sourceMapUrl: null, sourceMapRawUrl: null, budgetExhausted: false, }; }; export const handleRequestWillBeSent = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const requestId = cdpStringValue(params.requestId); const request = recordValue(params.request); if (requestId === undefined) { state.completeness.exclude("network_requests", "invalid_protocol_value"); return; } if (state.malformedRedirectRequestIds.has(requestId)) return; const previous = state.network.get(requestId); if ( !Object.hasOwn(params, "redirectResponse") && previous !== undefined && preserveMalformedRedirectEvidence(state, requestId, previous) ) return; const method = cdpStringValue(request?.method); const initiator = recordValue(params.initiator); const initiatorType = cdpStringValue(initiator?.type); if ( request === undefined || method === undefined || method.length === 0 || initiatorType === undefined || initiatorType.length === 0 ) { preserveMalformedRedirectEvidence(state, requestId, previous); return; } const sanitized = allowedSanitizedUrl(request.url, state.allowedOrigins); if (sanitized === undefined) { state.completeness.exclude( "network_requests", exclusionReasonForUrl(cdpStringValue(request.url)), ); state.network.delete(requestId); state.networkRequestTimestamps.delete(requestId); return; } const redirectResponse = recordValue(params.redirectResponse); if ( Object.hasOwn(params, "redirectResponse") && redirectResponse === undefined && preserveMalformedRedirectEvidence(state, requestId, previous) ) return; const redirects = [...(previous?.redirects ?? [])]; if (redirectResponse !== undefined) { const rawResponseUrl = cdpStringValue(redirectResponse.url); const responseUrl = rawResponseUrl === undefined || rawResponseUrl === "" ? undefined : allowedSanitizedUrl(rawResponseUrl, state.allowedOrigins); const responseUrlReason = redirectResponseUrlReason(rawResponseUrl); if (previous === undefined) { if (responseUrl === undefined) state.completeness.exclude("network_requests", responseUrlReason); else // CDP may deliver the continuation after its predecessor was excluded // by origin policy or before capture began. Keep the final request but // report missing prior coverage without calling it malformed protocol. state.completeness.attachLimited("network_requests"); } else if (responseUrl === undefined) { // redirectResponse belongs to the same request ID and can contain an // excluded origin. Drop the entire chain before retaining any details. state.completeness.exclude("network_requests", responseUrlReason); state.network.delete(requestId); state.networkRequestTimestamps.delete(requestId); return; } else { const encodedDataLength = numberValue(redirectResponse.encodedDataLength); if (encodedDataLength === undefined || encodedDataLength < 0) state.completeness.exclude( "network_requests", "invalid_protocol_value", ); redirects.push({ url: previous.url, response_url: responseUrl.url, method: previous.method, resource_type: previous.resource_type, status: numberValue(redirectResponse.status) ?? null, mime_type: cdpStringValue(redirectResponse.mimeType) ?? null, encoded_data_length: encodedDataLength === undefined || encodedDataLength < 0 ? null : encodedDataLength, request_timestamp: state.networkRequestTimestamps.get(requestId) ?? null, // CDP reports redirectResponse on the next requestWillBeSent event. // This is the event boundary, not a separately observed response time. redirect_event_timestamp: numberValue(params.timestamp) ?? null, }); } } const initiatorFrame = initiatorLocation(initiator); const rawInitiatorUrl = cdpStringValue(initiatorFrame?.url); const initiatorUrl = allowedSanitizedUrl( rawInitiatorUrl, state.allowedOrigins, ); if (rawInitiatorUrl !== undefined && initiatorUrl === undefined) state.completeness.exclude( "network_initiators", exclusionReasonForUrl(rawInitiatorUrl), ); state.network.set(requestId, { request_id: requestId, url: sanitized.url, origin: sanitized.origin ?? "", method, resource_type: cdpStringValue(params.type) ?? null, status: null, mime_type: null, encoded_data_length: null, redirects, initiator: { type: initiatorType, url: initiatorUrl?.url ?? null, line: integerOrNull(initiatorFrame?.lineNumber), column: integerOrNull(initiatorFrame?.columnNumber), }, body_shapes: requestBodyShape(state, request), }); const timestamp = numberValue(params.timestamp); if (timestamp === undefined) state.networkRequestTimestamps.delete(requestId); else state.networkRequestTimestamps.set(requestId, timestamp); }; const preserveMalformedRedirectEvidence = ( state: CdpCaptureEventsState, requestId: string, previous: NetworkState | undefined, ): boolean => { state.completeness.exclude("network_requests", "invalid_protocol_value"); if (previous === undefined) return false; state.malformedRedirectRequestIds.add(requestId); if ( state.input.include_json_body_shapes && isJsonMediaType(previous.mime_type) && previous.body_shapes.response === null ) { updateResponseBodyShape(state, requestId, null); state.completeness.exclude("json_body_shapes", "invalid_protocol_value"); } return true; }; const redirectResponseUrlReason = ( value: string | undefined, ): | "invalid_protocol_value" | "disallowed_origin" | "unsupported_url" | "unattributed_origin" => { if (value === undefined || value === "") return "invalid_protocol_value"; try { new URL(value); } catch { return "invalid_protocol_value"; } return exclusionReasonForUrl(value); }; export const handleResponseReceived = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const requestId = cdpStringValue(params.requestId); if (requestId === undefined) { state.completeness.exclude("network_requests", "invalid_protocol_value"); return; } if (state.malformedRedirectRequestIds.has(requestId)) return; const current = state.network.get(requestId); const response = recordValue(params.response); if (current === undefined) return; const sanitized = allowedSanitizedUrl(response?.url, state.allowedOrigins); if (response === undefined || sanitized === undefined) { state.completeness.exclude( "network_requests", response === undefined ? "invalid_protocol_value" : exclusionReasonForUrl(cdpStringValue(response.url)), ); state.network.delete(requestId); state.networkRequestTimestamps.delete(requestId); return; } state.network.set(requestId, { ...current, status: numberValue(response.status) ?? null, mime_type: cdpStringValue(response.mimeType) ?? null, }); const metadata = safeResponseMetadata( requestId, sanitized.url, response, state.allowedOrigins, ); state.responseMetadata.push(metadata.response); state.agentHints.push(...metadata.agentHints); }; export const handleLoadingFinished = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const requestId = cdpStringValue(params.requestId); if (requestId === undefined) return; if (state.malformedRedirectRequestIds.has(requestId)) return; const current = state.network.get(requestId); if (current === undefined) return; const encodedDataLength = numberValue(params.encodedDataLength); if (encodedDataLength === undefined || encodedDataLength < 0) { state.completeness.exclude("network_requests", "invalid_protocol_value"); return; } state.network.set(requestId, { ...current, encoded_data_length: encodedDataLength, }); }; export const handleConsoleAPICalled = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const frame = firstCallFrame(recordValue(params.stackTrace)); if (frame === undefined) { state.completeness.exclude("console_events", "unattributed_origin"); return; } const source = allowedSanitizedUrl(frame.url, state.allowedOrigins); if (source === undefined) { state.completeness.exclude( "console_events", exclusionReasonForUrl(cdpStringValue(frame.url)), ); return; } const arguments_ = recordsValue(params.args); state.console.push({ type: cdpStringValue(params.type) ?? "unknown", timestamp: numberValue(params.timestamp) ?? 0, argument_types: arguments_.map( (argument) => cdpStringValue(argument.type) ?? "unknown", ), url: source.url, line: integerOrNull(frame.lineNumber), column: integerOrNull(frame.columnNumber), text_capture: captureConsoleText(state, arguments_), }); }; const captureConsoleText = ( state: CdpCaptureEventsState, arguments_: readonly UnknownRecord[], ): WebPageInspection["console"]["events"][number]["text_capture"] => { if (!state.input.include_console_text) return { status: "not_approved", values: [], retained_bytes: 0, }; const values: WebPageInspection["console"]["events"][number]["text_capture"]["values"] = []; let retainedBytes = 0; for (const [argumentIndex, argument] of arguments_.entries()) { const primitive = consolePrimitive(argument); if (primitive === undefined) continue; const text = primitive.text; values.push({ argument_index: argumentIndex, type: primitive.type, text, }); retainedBytes += Buffer.byteLength(text); } return { status: "included", values, retained_bytes: retainedBytes, }; }; export const handleWebSocketFrame = ( state: CdpCaptureEventsState, params: UnknownRecord, direction: "sent" | "received", ): void => { const requestId = cdpStringValue(params.requestId); if (requestId === undefined) return; const connection = state.webSocketConnectionsById.get(requestId); if (connection === undefined) return; const response = recordValue(params.response); const payload = cdpStringValue(response?.payloadData) ?? ""; const opcode = Math.max(0, Math.trunc(numberValue(response?.opcode) ?? 0)); const decoded = opcode === 1 ? undefined : decodeBase64(payload); if (opcode !== 1 && decoded === undefined) state.completeness.exclude("websocket_frames", "invalid_protocol_value"); connection.events.push({ direction, opcode, payload_bytes: opcode === 1 ? Buffer.byteLength(payload) : (decoded?.byteLength ?? null), payload_shape: captureWebSocketShape(state, payload, opcode), }); }; const captureWebSocketShape = ( state: CdpCaptureEventsState, payload: string, opcode: number, ): WebPageInspection["network"]["websocket_connections"][number]["events"][number]["payload_shape"] => { if (!state.input.include_websocket_shapes) return null; if (opcode !== 1) return { format: "binary", json_shape: null }; const shape = inferJsonShape(payload); return { format: shape === null ? "text" : "json", json_shape: shape, }; }; export const handleWebSocketCreated = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const requestId = cdpStringValue(params.requestId); const rawUrl = cdpStringValue(params.url); if (requestId === undefined) { state.completeness.exclude( "websocket_connections", "invalid_protocol_value", ); return; } if (rawUrl === undefined) { state.completeness.exclude( "websocket_connections", "invalid_protocol_value", ); return; } const sourceUrl = sanitizeBrowserUrl(rawUrl).url; let parsed: URL; try { parsed = new URL(sourceUrl); } catch (cause: unknown) { // Unparseable URLs are recorded as unsupported, not as failures. void cause; state.completeness.exclude("websocket_connections", "unsupported_url"); return; } if (parsed.protocol === "ws:") parsed.protocol = "http:"; else if (parsed.protocol === "wss:") parsed.protocol = "https:"; else { state.completeness.exclude("websocket_connections", "unsupported_url"); return; } if (!state.allowedOrigins.has(parsed.origin)) { state.completeness.exclude("websocket_connections", "disallowed_origin"); return; } if (state.webSocketConnectionsById.has(requestId)) { state.completeness.exclude( "websocket_connections", "invalid_protocol_value", ); return; } state.webSocketConnectionsById.set(requestId, { request_id: requestId, url: sourceUrl, events: [], }); }; export const handleFrameNavigated = ( state: CdpCaptureEventsState, params: UnknownRecord, ): void => { const frame = recordValue(params.frame); if (state.mainFrameId === undefined) return; if ( !isMainFrameNavigation( { method: "Page.frameNavigated", params }, state.mainFrameId, ) ) return; const rawUrl = cdpStringValue(frame?.url); const loaderId = cdpStringValue(frame?.loaderId); // `frameNavigated` re-fires for a document already reported (same URL and // loader). Treating that as a navigation aborted a valid capture with // `target_changed`, which is a false claim that the target moved. const document = `${String(loaderId ?? "")}\0${String(rawUrl ?? "")}`; if (state.committedDocument === document) return; state.committedDocument = document; state.navigationDuringCapture = true; if ( isHttpUrl(rawUrl) && allowedSanitizedUrl(rawUrl, state.allowedOrigins) === undefined ) state.originViolation = true; };