refactor(browser): return complete observation evidence
This commit is contained in:
@@ -143,7 +143,7 @@ export class CdpBrowserProvider implements BrowserObservationPort {
|
||||
: undefined;
|
||||
return ok(
|
||||
webBundleAnalysisSchema.parse(
|
||||
analyzeCapturedWebBundle(captured.inspection, input, sourceMaps),
|
||||
analyzeCapturedWebBundle(captured.inspection, sourceMaps),
|
||||
),
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
@@ -368,7 +368,7 @@ const selectTargets = (
|
||||
allowed.push({
|
||||
target_id: target.id,
|
||||
type: target.type,
|
||||
title: target.title.slice(0, 16_384),
|
||||
title: target.title,
|
||||
url: url.url,
|
||||
origin: url.origin,
|
||||
attached: target.attached,
|
||||
|
||||
@@ -70,7 +70,7 @@ describe("CDP document capture", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("bounds approved accessibility text by UTF-8 bytes without splitting characters", () => {
|
||||
it("retains full approved accessibility text", () => {
|
||||
const capture = captureAccessibility(
|
||||
[
|
||||
{
|
||||
@@ -91,21 +91,19 @@ describe("CDP document capture", () => {
|
||||
],
|
||||
{
|
||||
includeText: true,
|
||||
maximumFieldBytes: 4,
|
||||
},
|
||||
);
|
||||
|
||||
expect(capture.nodes[0]).toMatchObject({
|
||||
name: "😀",
|
||||
name: "😀a",
|
||||
description: "éé",
|
||||
states: [{ name: "disabled", value: true }],
|
||||
});
|
||||
expect(capture.treeIncomplete).toBe(true);
|
||||
expect(capture.textCapture).toEqual({
|
||||
status: "truncated",
|
||||
retained_bytes: 8,
|
||||
status: "included",
|
||||
retained_bytes: 9,
|
||||
excluded_fields: 0,
|
||||
truncated_fields: 1,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -124,7 +122,6 @@ describe("CDP document capture", () => {
|
||||
],
|
||||
{
|
||||
includeText: false,
|
||||
maximumFieldBytes: 1_024,
|
||||
},
|
||||
);
|
||||
|
||||
@@ -133,7 +130,6 @@ describe("CDP document capture", () => {
|
||||
status: "not_approved",
|
||||
retained_bytes: 0,
|
||||
excluded_fields: 2,
|
||||
truncated_fields: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,7 +4,6 @@ import type {
|
||||
} from "../domain/browserObservation.js";
|
||||
import {
|
||||
allowedSanitizedUrl,
|
||||
boundedText,
|
||||
numberValue,
|
||||
recordValue,
|
||||
recordsValue,
|
||||
@@ -36,7 +35,7 @@ export const captureFrames = (
|
||||
const frame = recordValue(tree.frame);
|
||||
const frameId = stringValue(frame?.id);
|
||||
const sanitized = allowedSanitizedUrl(frame?.url, allowedOrigins);
|
||||
if (frame === undefined || frameId === undefined || frameId.length > 256) {
|
||||
if (frame === undefined || frameId === undefined) {
|
||||
completeness?.exclude("frames", "invalid_protocol_value");
|
||||
continue;
|
||||
}
|
||||
@@ -50,8 +49,7 @@ export const captureFrames = (
|
||||
if (maximum !== undefined && items.length >= maximum) continue;
|
||||
items.push({
|
||||
frame_id: frameId,
|
||||
parent_frame_id:
|
||||
(stringValue(frame.parentId) ?? "").slice(0, 256) || null,
|
||||
parent_frame_id: stringValue(frame.parentId) ?? null,
|
||||
url: sanitized.url,
|
||||
origin: sanitized.origin,
|
||||
});
|
||||
@@ -91,8 +89,8 @@ export const captureResources = (
|
||||
rawUrl: stringValue(resource.url) ?? "",
|
||||
url: url.url,
|
||||
origin: url.origin,
|
||||
type: (stringValue(resource.type) ?? "Other").slice(0, 100),
|
||||
mime_type: (stringValue(resource.mimeType) ?? "").slice(0, 256),
|
||||
type: stringValue(resource.type) ?? "Other",
|
||||
mime_type: stringValue(resource.mimeType) ?? "",
|
||||
content_size:
|
||||
contentSize === undefined ? null : Math.max(0, contentSize),
|
||||
});
|
||||
@@ -159,7 +157,7 @@ export const captureDom = (
|
||||
const attributeIndexes = numberArray(attributes[index]);
|
||||
const parent = Math.trunc(parents[index] ?? -1);
|
||||
const nodeIndex = nodes.length;
|
||||
const nodeName = indexedString(strings, nodeNames[index]).slice(0, 256);
|
||||
const nodeName = indexedString(strings, nodeNames[index]);
|
||||
nodes.push({
|
||||
index: nodeIndex,
|
||||
parent_index: parent < 0 ? -1 : baseIndex + parent,
|
||||
@@ -168,7 +166,7 @@ export const captureDom = (
|
||||
node_value_length: indexedString(strings, nodeValues[index]).length,
|
||||
attribute_names: attributeIndexes
|
||||
.filter((_value, attributeIndex) => attributeIndex % 2 === 0)
|
||||
.map((value) => indexedString(strings, value).slice(0, 256)),
|
||||
.map((value) => indexedString(strings, value)),
|
||||
});
|
||||
const metadata = domMetadata({
|
||||
strings,
|
||||
@@ -206,7 +204,6 @@ export const captureAccessibility = (
|
||||
results: readonly unknown[],
|
||||
options: {
|
||||
readonly includeText: boolean;
|
||||
readonly maximumFieldBytes: number;
|
||||
readonly unavailable?: boolean;
|
||||
},
|
||||
): {
|
||||
@@ -225,7 +222,6 @@ export const captureAccessibility = (
|
||||
}),
|
||||
);
|
||||
let excludedFields = 0;
|
||||
let truncatedFields = 0;
|
||||
const nodes = all.map((node) => {
|
||||
const captureText = (value: unknown): string | null => {
|
||||
const raw = stringValue(recordValue(value)?.value);
|
||||
@@ -234,19 +230,11 @@ export const captureAccessibility = (
|
||||
excludedFields += 1;
|
||||
return null;
|
||||
}
|
||||
const captured = boundedUtf8(raw, options.maximumFieldBytes);
|
||||
if (captured.truncated) {
|
||||
truncatedFields += 1;
|
||||
if (captured.text === "" && raw !== "") {
|
||||
excludedFields += 1;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return captured.text;
|
||||
return raw;
|
||||
};
|
||||
return {
|
||||
node_id: (stringValue(node.nodeId) ?? "").slice(0, 256),
|
||||
parent_id: (stringValue(node.parentId) ?? "").slice(0, 256) || null,
|
||||
node_id: stringValue(node.nodeId) ?? "",
|
||||
parent_id: stringValue(node.parentId) ?? null,
|
||||
role: axText(node.role),
|
||||
name: captureText(node.name),
|
||||
description: captureText(node.description),
|
||||
@@ -270,9 +258,7 @@ export const captureAccessibility = (
|
||||
? "unavailable"
|
||||
: !options.includeText
|
||||
? "not_approved"
|
||||
: truncatedFields > 0
|
||||
? "truncated"
|
||||
: "included",
|
||||
: "included",
|
||||
retained_bytes: nodes.reduce(
|
||||
(total, node) =>
|
||||
total +
|
||||
@@ -281,7 +267,6 @@ export const captureAccessibility = (
|
||||
0,
|
||||
),
|
||||
excluded_fields: excludedFields,
|
||||
truncated_fields: truncatedFields,
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -327,7 +312,7 @@ const accessibilityStates = (
|
||||
: typeof raw === "number"
|
||||
? numberValue(raw)
|
||||
: typeof raw === "string"
|
||||
? raw.slice(0, 256)
|
||||
? raw
|
||||
: undefined;
|
||||
if (state !== undefined) states.set(name, state);
|
||||
}
|
||||
@@ -337,7 +322,7 @@ const accessibilityStates = (
|
||||
};
|
||||
|
||||
const axText = (value: unknown): string | null =>
|
||||
boundedText(recordValue(value)?.value, 1_024);
|
||||
stringValue(recordValue(value)?.value) ?? null;
|
||||
|
||||
const indexedString = (strings: readonly string[], index: unknown): string => {
|
||||
const integer = numberValue(index);
|
||||
@@ -433,28 +418,6 @@ const domDestination = (
|
||||
}
|
||||
};
|
||||
|
||||
const boundedUtf8 = (
|
||||
value: string,
|
||||
maximumBytes: number,
|
||||
): {
|
||||
readonly text: string;
|
||||
readonly bytes: number;
|
||||
readonly truncated: boolean;
|
||||
} => {
|
||||
const total = Buffer.byteLength(value);
|
||||
if (total <= maximumBytes)
|
||||
return { text: value, bytes: total, truncated: false };
|
||||
let text = "";
|
||||
let bytes = 0;
|
||||
for (const character of value) {
|
||||
const characterBytes = Buffer.byteLength(character);
|
||||
if (bytes + characterBytes > maximumBytes) break;
|
||||
text += character;
|
||||
bytes += characterBytes;
|
||||
}
|
||||
return { text, bytes, truncated: true };
|
||||
};
|
||||
|
||||
const domUrlAttributes = [
|
||||
"href",
|
||||
"src",
|
||||
|
||||
@@ -27,8 +27,8 @@ export const ingestResponseBodyShape = (
|
||||
invalidResponseBodyShape(state, requestId);
|
||||
return;
|
||||
}
|
||||
const inferred = inferBodyShape(state, decoded);
|
||||
updateResponseBodyShape(state, requestId, inferred.shape, inferred.truncated);
|
||||
const inferred = inferBodyShape(decoded);
|
||||
updateResponseBodyShape(state, requestId, inferred.shape);
|
||||
};
|
||||
|
||||
export const requestBodyShape = (
|
||||
@@ -40,13 +40,9 @@ export const requestBodyShape = (
|
||||
const body = stringValue(request.postData);
|
||||
if (!isJsonContentType(recordValue(request.headers)) || body === undefined)
|
||||
return { status: "unavailable", request: null, response: null };
|
||||
const inferred = inferBodyShape(state, body);
|
||||
const inferred = inferBodyShape(body);
|
||||
return {
|
||||
status: inferred.truncated
|
||||
? "truncated"
|
||||
: inferred.shape === null
|
||||
? "unavailable"
|
||||
: "included",
|
||||
status: inferred.shape === null ? "unavailable" : "included",
|
||||
request: inferred.shape,
|
||||
response: null,
|
||||
};
|
||||
@@ -56,48 +52,30 @@ export const updateResponseBodyShape = (
|
||||
state: CdpCaptureEventsState,
|
||||
requestId: string,
|
||||
response: JsonShape | null,
|
||||
truncated: boolean,
|
||||
): void => {
|
||||
const current = state.network.get(requestId);
|
||||
if (current === undefined) return;
|
||||
const request = current.body_shapes.request;
|
||||
const status =
|
||||
truncated || current.body_shapes.status === "truncated"
|
||||
? "truncated"
|
||||
: response !== null
|
||||
? "included"
|
||||
: request !== null
|
||||
? "partial"
|
||||
: "unavailable";
|
||||
response !== null
|
||||
? "included"
|
||||
: request !== null
|
||||
? "partial"
|
||||
: "unavailable";
|
||||
state.network.set(requestId, {
|
||||
...current,
|
||||
body_shapes: { status, request, response },
|
||||
});
|
||||
};
|
||||
|
||||
const inferBodyShape = (
|
||||
state: CdpCaptureEventsState,
|
||||
text: string,
|
||||
): { readonly shape: JsonShape | null; readonly truncated: boolean } => {
|
||||
const bytes = Buffer.byteLength(text);
|
||||
if (bytes > state.input.limits.max_json_body_bytes) {
|
||||
state.completeness.truncate("json_body_shapes");
|
||||
return { shape: null, truncated: true };
|
||||
}
|
||||
const shape = inferJsonShape(text, {
|
||||
maximumBytes: state.input.limits.max_json_body_bytes,
|
||||
maximumNodes: state.input.limits.max_json_shape_nodes,
|
||||
maximumDepth: state.input.limits.max_json_shape_depth,
|
||||
});
|
||||
if (shape?.truncated === true)
|
||||
state.completeness.truncate("json_body_shapes");
|
||||
return { shape, truncated: shape?.truncated === true };
|
||||
const inferBodyShape = (text: string): { readonly shape: JsonShape | null } => {
|
||||
return { shape: inferJsonShape(text) };
|
||||
};
|
||||
|
||||
const invalidResponseBodyShape = (
|
||||
state: CdpCaptureEventsState,
|
||||
requestId: string,
|
||||
): void => {
|
||||
updateResponseBodyShape(state, requestId, null, false);
|
||||
updateResponseBodyShape(state, requestId, null);
|
||||
state.completeness.exclude("json_body_shapes", "invalid_protocol_value");
|
||||
};
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
import type { WebPageInspection } from "../domain/browserObservation.js";
|
||||
import { inferJsonShape } from "../domain/jsonShape.js";
|
||||
import { safeResponseMetadata } from "./CdpSafeMetadata.js";
|
||||
import { boundedSensitiveText } from "./SensitiveTextCapture.js";
|
||||
import { redactSensitiveText } from "./SensitiveTextCapture.js";
|
||||
import {
|
||||
allowedSanitizedUrl,
|
||||
boundedText,
|
||||
isHttpUrl,
|
||||
numberValue,
|
||||
recordValue,
|
||||
@@ -35,8 +34,7 @@ export const handleExecutionContextCreated = (
|
||||
if (
|
||||
identifier === undefined ||
|
||||
!Number.isSafeInteger(identifier) ||
|
||||
frameId === undefined ||
|
||||
frameId.length > 256
|
||||
frameId === undefined
|
||||
)
|
||||
return;
|
||||
const key = String(identifier);
|
||||
@@ -58,7 +56,7 @@ export const handleScriptParsed = (
|
||||
const scriptId = stringValue(params.scriptId);
|
||||
const rawUrl = stringValue(params.url) ?? "";
|
||||
const sanitized = allowedSanitizedUrl(rawUrl, state.allowedOrigins);
|
||||
if (scriptId === undefined || scriptId.length > 256) {
|
||||
if (scriptId === undefined) {
|
||||
state.completeness.exclude("scripts", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
@@ -72,7 +70,7 @@ export const handleScriptParsed = (
|
||||
rawUrl,
|
||||
url: sanitized.url,
|
||||
origin: sanitized.origin,
|
||||
hash: (stringValue(params.hash) ?? "").slice(0, 512),
|
||||
hash: stringValue(params.hash) ?? "",
|
||||
length: Math.max(
|
||||
0,
|
||||
Math.min(
|
||||
@@ -81,7 +79,7 @@ export const handleScriptParsed = (
|
||||
),
|
||||
),
|
||||
isModule: params.isModule === true,
|
||||
language: boundedText(params.scriptLanguage, 100),
|
||||
language: stringValue(params.scriptLanguage) ?? null,
|
||||
sourceMapUrl: sourceMap.sanitized,
|
||||
sourceMapRawUrl: sourceMap.raw,
|
||||
executionContextKey: executionContextKey(params.executionContextId),
|
||||
@@ -116,7 +114,7 @@ export const handleRequestWillBeSent = (
|
||||
): void => {
|
||||
const requestId = stringValue(params.requestId);
|
||||
const request = recordValue(params.request);
|
||||
if (requestId === undefined || requestId.length > 256) {
|
||||
if (requestId === undefined) {
|
||||
state.completeness.exclude("network_requests", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
@@ -150,13 +148,13 @@ export const handleRequestWillBeSent = (
|
||||
request_id: requestId,
|
||||
url: sanitized.url,
|
||||
origin: sanitized.origin ?? "",
|
||||
method: (stringValue(request.method) ?? "GET").slice(0, 32),
|
||||
resource_type: boundedText(params.type, 100),
|
||||
method: stringValue(request.method) ?? "GET",
|
||||
resource_type: stringValue(params.type) ?? null,
|
||||
status: null,
|
||||
mime_type: null,
|
||||
encoded_data_length: null,
|
||||
initiator: {
|
||||
type: (stringValue(initiator?.type) ?? "other").slice(0, 100),
|
||||
type: stringValue(initiator?.type) ?? "other",
|
||||
url: initiatorUrl?.url ?? null,
|
||||
line: integerOrNull(initiatorFrame?.lineNumber),
|
||||
column: integerOrNull(initiatorFrame?.columnNumber),
|
||||
@@ -170,7 +168,7 @@ export const handleResponseReceived = (
|
||||
params: UnknownRecord,
|
||||
): void => {
|
||||
const requestId = stringValue(params.requestId);
|
||||
if (requestId === undefined || requestId.length > 256) {
|
||||
if (requestId === undefined) {
|
||||
state.completeness.exclude("network_requests", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
@@ -191,7 +189,7 @@ export const handleResponseReceived = (
|
||||
state.network.set(requestId, {
|
||||
...current,
|
||||
status: numberValue(response.status) ?? null,
|
||||
mime_type: boundedText(response.mimeType, 256),
|
||||
mime_type: stringValue(response.mimeType) ?? null,
|
||||
});
|
||||
const metadata = safeResponseMetadata(
|
||||
requestId,
|
||||
@@ -208,7 +206,7 @@ export const handleLoadingFinished = (
|
||||
params: UnknownRecord,
|
||||
): void => {
|
||||
const requestId = stringValue(params.requestId);
|
||||
if (requestId === undefined || requestId.length > 256) return;
|
||||
if (requestId === undefined) return;
|
||||
const current = state.network.get(requestId);
|
||||
if (current === undefined) return;
|
||||
state.network.set(requestId, {
|
||||
@@ -239,10 +237,10 @@ export const handleConsoleAPICalled = (
|
||||
}
|
||||
const arguments_ = recordsValue(params.args);
|
||||
state.console.push({
|
||||
type: (stringValue(params.type) ?? "unknown").slice(0, 100),
|
||||
type: stringValue(params.type) ?? "unknown",
|
||||
timestamp: numberValue(params.timestamp) ?? 0,
|
||||
argument_types: arguments_.map((argument) =>
|
||||
(stringValue(argument.type) ?? "unknown").slice(0, 100),
|
||||
argument_types: arguments_.map(
|
||||
(argument) => stringValue(argument.type) ?? "unknown",
|
||||
),
|
||||
url: source.url,
|
||||
line: integerOrNull(frame.lineNumber),
|
||||
@@ -260,33 +258,25 @@ const captureConsoleText = (
|
||||
status: "not_approved",
|
||||
values: [],
|
||||
retained_bytes: 0,
|
||||
truncated_values: 0,
|
||||
};
|
||||
const values: WebPageInspection["console"]["events"][number]["text_capture"]["values"] =
|
||||
[];
|
||||
let retainedBytes = 0;
|
||||
let truncatedValues = 0;
|
||||
for (const [argumentIndex, argument] of arguments_.entries()) {
|
||||
const primitive = consolePrimitive(argument);
|
||||
if (primitive === undefined) continue;
|
||||
const bounded = boundedSensitiveText(
|
||||
primitive.text,
|
||||
state.input.limits.max_console_text_field_bytes,
|
||||
);
|
||||
const text = redactSensitiveText(primitive.text);
|
||||
values.push({
|
||||
argument_index: argumentIndex,
|
||||
type: primitive.type,
|
||||
text: bounded.text,
|
||||
text,
|
||||
});
|
||||
retainedBytes += bounded.bytes;
|
||||
if (bounded.truncated) truncatedValues += 1;
|
||||
retainedBytes += Buffer.byteLength(text);
|
||||
}
|
||||
if (truncatedValues > 0) state.completeness.truncate("console_text");
|
||||
return {
|
||||
status: truncatedValues > 0 ? "truncated" : "included",
|
||||
status: "included",
|
||||
values,
|
||||
retained_bytes: retainedBytes,
|
||||
truncated_values: truncatedValues,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -298,7 +288,6 @@ export const handleWebSocketFrame = (
|
||||
const requestId = stringValue(params.requestId);
|
||||
if (
|
||||
requestId === undefined ||
|
||||
requestId.length > 256 ||
|
||||
(!state.network.has(requestId) && !state.allowedWebSockets.has(requestId))
|
||||
)
|
||||
return;
|
||||
@@ -324,24 +313,11 @@ const captureWebSocketShape = (
|
||||
opcode: number,
|
||||
): WebPageInspection["network"]["websocket_events"][number]["payload_shape"] => {
|
||||
if (!state.input.include_websocket_shapes) return null;
|
||||
if (opcode !== 1)
|
||||
return { format: "binary", json_shape: null, truncated: false };
|
||||
const bytes = Buffer.byteLength(payload);
|
||||
if (bytes > state.input.limits.max_websocket_shape_bytes) {
|
||||
state.completeness.truncate("websocket_shapes");
|
||||
return { format: "text", json_shape: null, truncated: true };
|
||||
}
|
||||
const shape = inferJsonShape(payload, {
|
||||
maximumBytes: state.input.limits.max_websocket_shape_bytes,
|
||||
maximumNodes: state.input.limits.max_json_shape_nodes,
|
||||
maximumDepth: state.input.limits.max_json_shape_depth,
|
||||
});
|
||||
if (shape?.truncated === true)
|
||||
state.completeness.truncate("websocket_shapes");
|
||||
if (opcode !== 1) return { format: "binary", json_shape: null };
|
||||
const shape = inferJsonShape(payload);
|
||||
return {
|
||||
format: shape === null ? "text" : "json",
|
||||
json_shape: shape,
|
||||
truncated: shape?.truncated === true,
|
||||
};
|
||||
};
|
||||
|
||||
@@ -351,7 +327,7 @@ export const handleWebSocketCreated = (
|
||||
): void => {
|
||||
const requestId = stringValue(params.requestId);
|
||||
const rawUrl = stringValue(params.url);
|
||||
if (requestId === undefined || requestId.length > 256) {
|
||||
if (requestId === undefined) {
|
||||
state.completeness.exclude(
|
||||
"websocket_connections",
|
||||
"invalid_protocol_value",
|
||||
|
||||
@@ -131,7 +131,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
|
||||
}
|
||||
|
||||
responseBodyUnavailable(requestId: string): void {
|
||||
bodyShapes.updateResponseBodyShape(this, requestId, null, false);
|
||||
bodyShapes.updateResponseBodyShape(this, requestId, null);
|
||||
this.completeness.unavailable("json_body_shapes");
|
||||
}
|
||||
|
||||
|
||||
@@ -85,10 +85,10 @@ export const captureStorage = async (
|
||||
origin,
|
||||
usage_bytes: numberValue(quota?.usage) ?? null,
|
||||
quota_bytes: numberValue(quota?.quota) ?? null,
|
||||
local_storage_keys: local.items.map(({ key }) => key.slice(0, 1_024)),
|
||||
session_storage_keys: session.items.map(({ key }) => key.slice(0, 1_024)),
|
||||
indexed_db_names: indexed.map((name) => name.slice(0, 1_024)),
|
||||
cache_names: caches.map(({ name }) => name.slice(0, 1_024)),
|
||||
local_storage_keys: local.items.map(({ key }) => key),
|
||||
session_storage_keys: session.items.map(({ key }) => key),
|
||||
indexed_db_names: [...indexed],
|
||||
cache_names: caches.map(({ name }) => name),
|
||||
content_fingerprints: [...fingerprints.items],
|
||||
fingerprint_algorithm: "sha256",
|
||||
fingerprints_complete: fingerprints.complete && structuredStorageComplete,
|
||||
|
||||
@@ -10,8 +10,6 @@ import { CdpConnection } from "./CdpConnection.js";
|
||||
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
|
||||
import { recordValue, recordsValue, stringValue } from "./CdpCaptureValues.js";
|
||||
|
||||
const MAX_CACHE_BODY_BYTES = 64 * 1_024;
|
||||
|
||||
type StorageFingerprint =
|
||||
WebPageInspection["storage"]["content_fingerprints"][number];
|
||||
|
||||
@@ -383,8 +381,7 @@ const addCacheEntry = async (
|
||||
);
|
||||
const body = stringValue(recordValue(recordValue(raw)?.response)?.body);
|
||||
const decoded = body === undefined ? null : decodeBase64(body);
|
||||
const complete =
|
||||
decoded !== null && decoded.byteLength <= MAX_CACHE_BODY_BYTES;
|
||||
const complete = decoded !== null;
|
||||
if (!complete) run.state.complete = false;
|
||||
addFingerprint(run.state, {
|
||||
scope: "cache_entry",
|
||||
@@ -426,7 +423,7 @@ const encodedValue = (value: unknown): Uint8Array | string => {
|
||||
return encoded;
|
||||
};
|
||||
|
||||
const stableValue = (value: unknown, depth = 0): unknown => {
|
||||
const stableValue = (value: unknown): unknown => {
|
||||
if (
|
||||
value === null ||
|
||||
typeof value === "string" ||
|
||||
@@ -434,21 +431,18 @@ const stableValue = (value: unknown, depth = 0): unknown => {
|
||||
typeof value === "boolean"
|
||||
)
|
||||
return value;
|
||||
if (depth >= 8) return "<depth-limit>";
|
||||
if (Array.isArray(value))
|
||||
return value.slice(0, 64).map((item) => stableValue(item, depth + 1));
|
||||
if (Array.isArray(value)) return value.map((item) => stableValue(item));
|
||||
const record = recordValue(value);
|
||||
if (record === undefined) return String(value);
|
||||
return Object.fromEntries(
|
||||
Object.entries(record)
|
||||
.filter(([key]) => key !== "objectId" && key !== "customPreview")
|
||||
.sort(([left], [right]) => compareCodePoints(left, right))
|
||||
.slice(0, 64)
|
||||
.map(([key, item]) => [key, stableValue(item, depth + 1)]),
|
||||
.map(([key, item]) => [key, stableValue(item)]),
|
||||
);
|
||||
};
|
||||
|
||||
const stableValueComplete = (value: unknown, depth = 0): boolean => {
|
||||
const stableValueComplete = (value: unknown): boolean => {
|
||||
if (value instanceof Uint8Array) return true;
|
||||
if (
|
||||
value === null ||
|
||||
@@ -458,21 +452,14 @@ const stableValueComplete = (value: unknown, depth = 0): boolean => {
|
||||
typeof value === "boolean"
|
||||
)
|
||||
return true;
|
||||
if (depth >= 8) return false;
|
||||
if (Array.isArray(value))
|
||||
return (
|
||||
value.length <= 64 &&
|
||||
value.every((item) => stableValueComplete(item, depth + 1))
|
||||
);
|
||||
return value.every((item) => stableValueComplete(item));
|
||||
const record = recordValue(value);
|
||||
if (record === undefined) return false;
|
||||
const entries = Object.entries(record).filter(
|
||||
([key]) => key !== "objectId" && key !== "customPreview",
|
||||
);
|
||||
return (
|
||||
entries.length <= 64 &&
|
||||
entries.every(([, item]) => stableValueComplete(item, depth + 1))
|
||||
);
|
||||
return entries.every(([, item]) => stableValueComplete(item));
|
||||
};
|
||||
|
||||
const remoteObjectComplete = (value: unknown): boolean => {
|
||||
|
||||
@@ -23,8 +23,6 @@ interface PendingCommand {
|
||||
|
||||
const CONNECT_TIMEOUT_MS = 5_000;
|
||||
const COMMAND_TIMEOUT_MS = 5_000;
|
||||
const MAX_PAYLOAD_BYTES = 16 * 1_024 * 1_024;
|
||||
const MAX_PENDING_COMMANDS = 128;
|
||||
|
||||
/** Correlated, bounded JSON-RPC transport for one CDP target operation. */
|
||||
export class CdpConnection {
|
||||
@@ -53,7 +51,7 @@ export class CdpConnection {
|
||||
if (signal?.aborted === true) throw new AnalysisCancelledError(operation);
|
||||
const socket = new WebSocket(url, {
|
||||
handshakeTimeout: CONNECT_TIMEOUT_MS,
|
||||
maxPayload: MAX_PAYLOAD_BYTES,
|
||||
maxPayload: 0,
|
||||
perMessageDeflate: false,
|
||||
});
|
||||
await waitForOpen(socket, operation, signal);
|
||||
@@ -87,8 +85,6 @@ export class CdpConnection {
|
||||
throw new BrowserObservationError(this.operation, "protocol_error");
|
||||
if (this.#closed || this.socket.readyState !== WebSocket.OPEN)
|
||||
throw new BrowserObservationError(this.operation, "disconnected");
|
||||
if (this.#pending.size >= MAX_PENDING_COMMANDS)
|
||||
throw new BrowserObservationError(this.operation, "payload_limit");
|
||||
if (signal?.aborted === true)
|
||||
throw new AnalysisCancelledError(this.operation);
|
||||
const id = this.#nextId;
|
||||
|
||||
@@ -117,7 +117,7 @@ const runElectronInspection = async (
|
||||
target: {
|
||||
target_id: context.target.id,
|
||||
type: context.target.type,
|
||||
title: context.target.title.slice(0, 16_384),
|
||||
title: context.target.title,
|
||||
file_path: main.file_path,
|
||||
attached: context.target.attached,
|
||||
},
|
||||
@@ -267,12 +267,12 @@ const captureFrames = async (
|
||||
continue;
|
||||
}
|
||||
frames.push({
|
||||
frame_id: frameId.slice(0, 256),
|
||||
frame_id: frameId,
|
||||
parent_frame_id: current.parent,
|
||||
file_path: path,
|
||||
});
|
||||
for (const child of recordsValue(current.tree.childFrames))
|
||||
pending.push({ tree: child, parent: frameId.slice(0, 256) });
|
||||
pending.push({ tree: child, parent: frameId });
|
||||
}
|
||||
return frames;
|
||||
};
|
||||
@@ -308,8 +308,8 @@ const captureResources = async (
|
||||
}
|
||||
const item = {
|
||||
file_path: path,
|
||||
type: (stringValue(resource.type) ?? "Other").slice(0, 100),
|
||||
mime_type: (stringValue(resource.mimeType) ?? "").slice(0, 256),
|
||||
type: stringValue(resource.type) ?? "Other",
|
||||
mime_type: stringValue(resource.mimeType) ?? "",
|
||||
content_size:
|
||||
numberValue(resource.contentSize) === undefined
|
||||
? null
|
||||
@@ -369,13 +369,13 @@ const captureDom = async (
|
||||
index: nodes.length,
|
||||
parent_index: parent < 0 ? -1 : baseIndex + parent,
|
||||
node_type: integer(nodeTypes[index], 0),
|
||||
node_name: stringAt(strings, nodeNames[index]).slice(0, 1_024),
|
||||
node_name: stringAt(strings, nodeNames[index]),
|
||||
node_value_length: Buffer.byteLength(
|
||||
stringAt(strings, nodeValues[index]),
|
||||
),
|
||||
attribute_names: attributeIndexes
|
||||
.filter((_value, attributeIndex) => attributeIndex % 2 === 0)
|
||||
.map((value) => stringAt(strings, value).slice(0, 1_024)),
|
||||
.map((value) => stringAt(strings, value)),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -82,7 +82,7 @@ export class CdpElectronProvider implements ElectronObservationPort {
|
||||
allowed.push({
|
||||
target_id: target.id,
|
||||
type: target.type,
|
||||
title: target.title.slice(0, 16_384),
|
||||
title: target.title,
|
||||
file_path: path,
|
||||
attached: target.attached,
|
||||
});
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
import type { CdpEvent } from "./CdpConnection.js";
|
||||
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
|
||||
import {
|
||||
boundedText,
|
||||
numberValue,
|
||||
recordValue,
|
||||
stringValue,
|
||||
} from "./CdpCaptureValues.js";
|
||||
import { numberValue, recordValue, stringValue } from "./CdpCaptureValues.js";
|
||||
|
||||
export interface ElectronScriptDraft {
|
||||
readonly scriptId: string;
|
||||
@@ -17,7 +12,7 @@ export interface ElectronScriptDraft {
|
||||
readonly executionContextKey: string | null;
|
||||
}
|
||||
|
||||
/** Retain only bounded script/frame metadata from passive Electron events. */
|
||||
/** Retain script/frame metadata from passive Electron events. */
|
||||
export const ingestElectronScriptEvent = (input: {
|
||||
readonly event: CdpEvent;
|
||||
readonly scripts: ElectronScriptDraft[];
|
||||
@@ -35,18 +30,14 @@ export const ingestElectronScriptEvent = (input: {
|
||||
return;
|
||||
}
|
||||
if (input.event.method === "Runtime.executionContextCreated") {
|
||||
retainExecutionContext(
|
||||
input.event,
|
||||
input.executionContextFrames,
|
||||
input.completeness,
|
||||
);
|
||||
retainExecutionContext(input.event, input.executionContextFrames);
|
||||
return;
|
||||
}
|
||||
if (input.event.method !== "Debugger.scriptParsed") return;
|
||||
const value = recordValue(input.event.params);
|
||||
const scriptId = stringValue(value?.scriptId);
|
||||
const rawUrl = stringValue(value?.url);
|
||||
if (scriptId === undefined || scriptId.length > 256) {
|
||||
if (scriptId === undefined) {
|
||||
input.completeness.exclude("scripts", "invalid_protocol_value");
|
||||
return;
|
||||
}
|
||||
@@ -54,17 +45,13 @@ export const ingestElectronScriptEvent = (input: {
|
||||
input.completeness.exclude("scripts", "unattributed_origin");
|
||||
return;
|
||||
}
|
||||
if (input.scripts.length >= 2_000) {
|
||||
input.completeness.drop("scripts");
|
||||
return;
|
||||
}
|
||||
input.scripts.push({
|
||||
scriptId,
|
||||
rawUrl,
|
||||
hash: (stringValue(value?.hash) ?? "").slice(0, 512),
|
||||
hash: stringValue(value?.hash) ?? "",
|
||||
length: nonnegativeInteger(value?.length),
|
||||
isModule: value?.isModule === true,
|
||||
language: boundedText(value?.scriptLanguage, 100),
|
||||
language: stringValue(value?.scriptLanguage) ?? null,
|
||||
executionContextKey: executionContextKey(value?.executionContextId),
|
||||
});
|
||||
};
|
||||
@@ -72,18 +59,12 @@ export const ingestElectronScriptEvent = (input: {
|
||||
const retainExecutionContext = (
|
||||
event: CdpEvent,
|
||||
frames: Map<string, string>,
|
||||
completeness: CdpCaptureCompleteness,
|
||||
): void => {
|
||||
const parameters = recordValue(event.params);
|
||||
const runtimeContext = recordValue(parameters?.context);
|
||||
const contextKey = executionContextKey(runtimeContext?.id);
|
||||
const frameId = stringValue(recordValue(runtimeContext?.auxData)?.frameId);
|
||||
if (contextKey === null || frameId === undefined || frameId.length > 256)
|
||||
return;
|
||||
if (frames.size >= 2_000 && !frames.has(contextKey)) {
|
||||
completeness.truncate("scripts");
|
||||
return;
|
||||
}
|
||||
if (contextKey === null || frameId === undefined) return;
|
||||
frames.set(contextKey, frameId);
|
||||
};
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ import type {
|
||||
ElectronPageInspection,
|
||||
InspectElectronPageInput,
|
||||
} from "../domain/electronObservation.js";
|
||||
import { MAX_ELECTRON_SCRIPT_SOURCE_BYTES } from "../domain/electronObservation.js";
|
||||
import { createWebTextArtifact } from "../domain/webContentArtifact.js";
|
||||
import type { CdpConnection } from "./CdpConnection.js";
|
||||
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
|
||||
@@ -91,8 +90,6 @@ const captureScriptSource = async (
|
||||
reason: "source capture was not approved",
|
||||
},
|
||||
};
|
||||
if (script.length > MAX_ELECTRON_SCRIPT_SOURCE_BYTES)
|
||||
return sourceLimitResult(input.completeness);
|
||||
const result = requiredRecord(
|
||||
await input.connection.send(
|
||||
"Debugger.getScriptSource",
|
||||
@@ -102,9 +99,6 @@ const captureScriptSource = async (
|
||||
),
|
||||
);
|
||||
const text = stringValue(result.scriptSource) ?? "";
|
||||
const bytes = Buffer.byteLength(text);
|
||||
if (bytes > MAX_ELECTRON_SCRIPT_SOURCE_BYTES)
|
||||
return sourceLimitResult(input.completeness);
|
||||
return {
|
||||
source: {
|
||||
included: true,
|
||||
@@ -113,15 +107,6 @@ const captureScriptSource = async (
|
||||
};
|
||||
};
|
||||
|
||||
const sourceLimitResult = (
|
||||
completeness: CdpCaptureCompleteness,
|
||||
): { readonly source: ElectronScriptSource } => {
|
||||
completeness.truncate("script_sources");
|
||||
return {
|
||||
source: { included: false, reason: "script source byte limit reached" },
|
||||
};
|
||||
};
|
||||
|
||||
const digest = (value: unknown): string => {
|
||||
const encoded = canonicalize(value);
|
||||
if (encoded === undefined) throw new TypeError("Expected canonical JSON");
|
||||
|
||||
@@ -4,12 +4,7 @@ import type {
|
||||
} from "../domain/electronObservation.js";
|
||||
import type { CdpConnection } from "./CdpConnection.js";
|
||||
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
|
||||
import {
|
||||
boundedText,
|
||||
recordValue,
|
||||
recordsValue,
|
||||
stringValue,
|
||||
} from "./CdpCaptureValues.js";
|
||||
import { recordValue, recordsValue, stringValue } from "./CdpCaptureValues.js";
|
||||
import type { CdpEndpointTarget } from "./CdpEndpoint.js";
|
||||
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
|
||||
import { authorizedElectronFile } from "./ElectronFileScope.js";
|
||||
@@ -67,7 +62,7 @@ export const captureElectronWorkers = async (input: {
|
||||
}
|
||||
workers.push({
|
||||
target_id: targetId,
|
||||
type: type.slice(0, 100),
|
||||
type,
|
||||
file_path: path,
|
||||
attached: target.attached === true,
|
||||
opener_target_id: openerTargetId,
|
||||
@@ -80,6 +75,6 @@ export const captureElectronWorkers = async (input: {
|
||||
};
|
||||
|
||||
const boundedTargetField = (value: unknown): string | null => {
|
||||
const field = boundedText(value, 256);
|
||||
return field === null || field === "" ? null : field;
|
||||
const field = stringValue(value);
|
||||
return field === undefined || field === "" ? null : field;
|
||||
};
|
||||
|
||||
+24
-44
@@ -12,25 +12,23 @@ import {
|
||||
import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
|
||||
|
||||
const HTTP_TIMEOUT_MS = 5_000;
|
||||
const MAX_VERSION_BYTES = 64 * 1_024;
|
||||
const MAX_TARGET_LIST_BYTES = 2 * 1_024 * 1_024;
|
||||
|
||||
const endpointVersionSchema = z.object({
|
||||
Browser: z.string().min(1).max(1_024),
|
||||
"Protocol-Version": z.string().min(1).max(100),
|
||||
"User-Agent": z.string().max(16_384),
|
||||
"V8-Version": z.string().max(1_024),
|
||||
"WebKit-Version": z.string().max(1_024),
|
||||
webSocketDebuggerUrl: z.string().min(1).max(2_048),
|
||||
Browser: z.string().min(1),
|
||||
"Protocol-Version": z.string().min(1),
|
||||
"User-Agent": z.string(),
|
||||
"V8-Version": z.string(),
|
||||
"WebKit-Version": z.string(),
|
||||
webSocketDebuggerUrl: z.string().min(1),
|
||||
});
|
||||
|
||||
const endpointTargetSchema = z.object({
|
||||
id: z.string().min(1).max(256),
|
||||
type: z.string().min(1).max(100),
|
||||
title: z.string().max(16_384).default(""),
|
||||
url: z.string().max(65_536),
|
||||
id: z.string().min(1),
|
||||
type: z.string().min(1),
|
||||
title: z.string().default(""),
|
||||
url: z.string(),
|
||||
attached: z.boolean().default(false),
|
||||
webSocketDebuggerUrl: z.string().min(1).max(2_048).optional(),
|
||||
webSocketDebuggerUrl: z.string().min(1).optional(),
|
||||
});
|
||||
const endpointTargetsSchema = z.array(endpointTargetSchema);
|
||||
|
||||
@@ -68,21 +66,19 @@ export interface CdpEndpointDiscovery {
|
||||
readonly targets: readonly CdpEndpointTarget[];
|
||||
}
|
||||
|
||||
/** Read bounded CDP discovery endpoints without following redirects. */
|
||||
/** Read CDP discovery endpoints without following redirects. */
|
||||
export const discoverCdpEndpoint = async (
|
||||
endpoint: string,
|
||||
operation: BrowserObservationOperation,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CdpEndpointDiscovery> => {
|
||||
const versionInput = await readBoundedCdpJson(
|
||||
const versionInput = await readCdpJson(
|
||||
new URL("/json/version", endpoint),
|
||||
MAX_VERSION_BYTES,
|
||||
operation,
|
||||
signal,
|
||||
);
|
||||
const targetsInput = await readBoundedCdpJson(
|
||||
const targetsInput = await readCdpJson(
|
||||
new URL("/json/list", endpoint),
|
||||
MAX_TARGET_LIST_BYTES,
|
||||
operation,
|
||||
signal,
|
||||
);
|
||||
@@ -128,7 +124,7 @@ const sanitizeTargetTitle = (value: string, targetUrl: string): string => {
|
||||
return sanitizeTargetUrlAlias(value, targetUrl);
|
||||
}
|
||||
return url.protocol === "http:" || url.protocol === "https:"
|
||||
? sanitizeBrowserUrl(value).url.slice(0, 16_384)
|
||||
? sanitizeBrowserUrl(value).url
|
||||
: sanitizeTargetUrlAlias(value, targetUrl);
|
||||
};
|
||||
|
||||
@@ -145,11 +141,9 @@ const sanitizeTargetUrlAlias = (value: string, targetUrl: string): string => {
|
||||
`(?:${markers.map(escapeRegExp).join("|")})[^\\s<>"']*`,
|
||||
"gu",
|
||||
);
|
||||
return value
|
||||
.replace(candidatePattern, (candidate) =>
|
||||
sanitizeSameOriginTitleUrl(candidate, parsed),
|
||||
)
|
||||
.slice(0, 16_384);
|
||||
return value.replace(candidatePattern, (candidate) =>
|
||||
sanitizeSameOriginTitleUrl(candidate, parsed),
|
||||
);
|
||||
};
|
||||
|
||||
const sanitizeSameOriginTitleUrl = (candidate: string, target: URL): string => {
|
||||
@@ -250,22 +244,19 @@ const cdpWebSocketPath = (
|
||||
| { readonly scope: "browser" }
|
||||
| { readonly scope: "page"; targetId: string }
|
||||
| undefined => {
|
||||
const browserId = pathIdentifier(pathname, "/devtools/browser/", 1_024);
|
||||
const browserId = pathIdentifier(pathname, "/devtools/browser/");
|
||||
if (browserId !== undefined) return { scope: "browser" };
|
||||
const targetId = pathIdentifier(pathname, "/devtools/page/", 256);
|
||||
const targetId = pathIdentifier(pathname, "/devtools/page/");
|
||||
return targetId === undefined ? undefined : { scope: "page", targetId };
|
||||
};
|
||||
|
||||
const pathIdentifier = (
|
||||
pathname: string,
|
||||
prefix: string,
|
||||
maximumLength: number,
|
||||
): string | undefined => {
|
||||
if (!pathname.startsWith(prefix)) return undefined;
|
||||
const identifier = pathname.slice(prefix.length);
|
||||
return identifier.length > 0 &&
|
||||
identifier.length <= maximumLength &&
|
||||
!identifier.includes("/")
|
||||
return identifier.length > 0 && !identifier.includes("/")
|
||||
? identifier
|
||||
: undefined;
|
||||
};
|
||||
@@ -290,10 +281,9 @@ const targetWebSocket = (
|
||||
}
|
||||
};
|
||||
|
||||
/** Read one bounded loopback CDP discovery document without redirects. */
|
||||
export const readBoundedCdpJson = async (
|
||||
/** Read one loopback CDP discovery document without redirects. */
|
||||
export const readCdpJson = async (
|
||||
url: URL,
|
||||
maximumBytes: number,
|
||||
operation: BrowserObservationOperation,
|
||||
signal?: AbortSignal,
|
||||
): Promise<unknown> =>
|
||||
@@ -314,17 +304,7 @@ export const readBoundedCdpJson = async (
|
||||
return;
|
||||
}
|
||||
const chunks: Buffer[] = [];
|
||||
let bytes = 0;
|
||||
response.on("data", (chunk: Buffer) => {
|
||||
bytes += chunk.length;
|
||||
if (bytes > maximumBytes) {
|
||||
response.destroy(
|
||||
new BrowserObservationError(operation, "payload_limit"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
response.on("data", (chunk: Buffer) => chunks.push(chunk));
|
||||
response.on("end", () => {
|
||||
try {
|
||||
resolve(JSON.parse(Buffer.concat(chunks).toString("utf8")));
|
||||
|
||||
@@ -52,12 +52,7 @@ export const observeCdpSession = async (
|
||||
const mainFrameId = frameId(initial);
|
||||
if (mainFrameId === undefined)
|
||||
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
||||
const capture = new TimelineCapture(
|
||||
context.input.max_timeline_events,
|
||||
allowedOrigins,
|
||||
mainFrameId,
|
||||
initialUrl,
|
||||
);
|
||||
const capture = new TimelineCapture(allowedOrigins, mainFrameId, initialUrl);
|
||||
const removeListener = context.connection.onEvent((event) => {
|
||||
if (observationEventMatches(event, context.sessionId))
|
||||
capture.ingest(event);
|
||||
@@ -74,7 +69,7 @@ export const observeCdpSession = async (
|
||||
);
|
||||
await context.connection.send(
|
||||
"Network.enable",
|
||||
{ maxTotalBufferSize: 1_024 * 1_024 },
|
||||
{},
|
||||
context.sessionId,
|
||||
context.signal,
|
||||
);
|
||||
@@ -143,7 +138,6 @@ class TimelineCapture {
|
||||
readonly #listeners = new Set<(reason: EndReason) => void>();
|
||||
|
||||
constructor(
|
||||
private readonly maximum: number,
|
||||
private readonly allowedOrigins: ReadonlySet<string>,
|
||||
private readonly mainFrameId: string,
|
||||
initialUrl: string,
|
||||
@@ -320,10 +314,6 @@ class TimelineCapture {
|
||||
} = options;
|
||||
const destination =
|
||||
rawUrl == null ? null : scopedUrl(rawUrl, this.allowedOrigins);
|
||||
if (this.timeline.length >= this.maximum) {
|
||||
this.completeness.drop("timeline_events");
|
||||
return;
|
||||
}
|
||||
this.#sequence += 1;
|
||||
this.timeline.push({
|
||||
sequence: this.#sequence,
|
||||
@@ -331,7 +321,7 @@ class TimelineCapture {
|
||||
timestamp: Math.max(0, numberValue(params.timestamp) ?? 0),
|
||||
frame_id: boundedId(params.frameId),
|
||||
loader_id: boundedId(params.loaderId),
|
||||
request_id: requestId?.slice(0, 256) ?? null,
|
||||
request_id: requestId ?? null,
|
||||
url: destination?.scope === "approved" ? destination.url : null,
|
||||
destination_scope: destination?.scope ?? null,
|
||||
detail,
|
||||
@@ -449,8 +439,7 @@ const scopedUrl = (
|
||||
: { url: null, scope: "unsupported" };
|
||||
};
|
||||
|
||||
const boundedId = (value: unknown): string | null =>
|
||||
stringValue(value)?.slice(0, 256) ?? null;
|
||||
const boundedId = (value: unknown): string | null => stringValue(value) ?? null;
|
||||
|
||||
const safeDetail = (value: unknown): string | null => {
|
||||
const detail = stringValue(value)?.toLowerCase();
|
||||
|
||||
@@ -319,21 +319,8 @@ const enableObservationDomains = async (
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> => {
|
||||
await connection.send("Runtime.enable", {}, sessionId, signal);
|
||||
await connection.send(
|
||||
"Debugger.enable",
|
||||
{ maxScriptsCacheSize: 4 * 1_024 * 1_024 },
|
||||
sessionId,
|
||||
signal,
|
||||
);
|
||||
await connection.send(
|
||||
"Network.enable",
|
||||
{
|
||||
maxTotalBufferSize: 4 * 1_024 * 1_024,
|
||||
maxResourceBufferSize: 1_024 * 1_024,
|
||||
},
|
||||
sessionId,
|
||||
signal,
|
||||
);
|
||||
await connection.send("Debugger.enable", {}, sessionId, signal);
|
||||
await connection.send("Network.enable", {}, sessionId, signal);
|
||||
};
|
||||
|
||||
const accessibilityForFrames = async (
|
||||
@@ -347,7 +334,7 @@ const accessibilityForFrames = async (
|
||||
const result = await optionalCdpCommand(
|
||||
context,
|
||||
"Accessibility.getFullAXTree",
|
||||
{ depth: 32, frameId: frame.frame_id },
|
||||
{ frameId: frame.frame_id },
|
||||
limitations,
|
||||
);
|
||||
if (result === undefined) unavailable = true;
|
||||
@@ -356,7 +343,6 @@ const accessibilityForFrames = async (
|
||||
if (unavailable) events.completeness.unavailable("accessibility");
|
||||
const capture = captureAccessibility(results, {
|
||||
includeText: context.input.include_accessibility_text,
|
||||
maximumFieldBytes: context.input.limits.max_ax_text_field_bytes,
|
||||
...(results.length === 0 && unavailable ? { unavailable: true } : {}),
|
||||
});
|
||||
if (!context.input.include_accessibility_text)
|
||||
@@ -365,8 +351,6 @@ const accessibilityForFrames = async (
|
||||
"not_approved",
|
||||
capture.textCapture.excluded_fields,
|
||||
);
|
||||
if (capture.textCapture.status === "truncated")
|
||||
events.completeness.truncate("accessibility");
|
||||
if (capture.treeIncomplete) events.completeness.truncate("accessibility");
|
||||
return capture;
|
||||
};
|
||||
@@ -395,7 +379,7 @@ const normalizedTarget = (
|
||||
return {
|
||||
target_id: target.id,
|
||||
type: target.type,
|
||||
title: target.title.slice(0, 16_384),
|
||||
title: target.title,
|
||||
url: url?.url ?? "[unsupported-url]",
|
||||
origin: url?.origin ?? "",
|
||||
attached: target.attached,
|
||||
|
||||
@@ -106,11 +106,7 @@ const captureScriptSource = async (
|
||||
script: CapturedScript,
|
||||
state: ScriptDraftState,
|
||||
): Promise<WebPageInspection["scripts"]["items"][number]["source"]> => {
|
||||
const { context, events } = state;
|
||||
if (script.length > context.input.limits.max_script_source_bytes) {
|
||||
events.completeness.truncate("script_sources");
|
||||
return sourceExcluded("declared script length exceeds per-script limit");
|
||||
}
|
||||
const { context } = state;
|
||||
const result = requiredRecord(
|
||||
await context.connection.send(
|
||||
"Debugger.getScriptSource",
|
||||
@@ -120,11 +116,6 @@ const captureScriptSource = async (
|
||||
),
|
||||
);
|
||||
const content = stringValue(result.scriptSource) ?? "";
|
||||
const bytes = Buffer.byteLength(content);
|
||||
if (bytes > context.input.limits.max_script_source_bytes) {
|
||||
events.completeness.truncate("script_sources");
|
||||
return sourceExcluded("actual source exceeds the per-script byte limit");
|
||||
}
|
||||
return sourceResult(content);
|
||||
};
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@ import type { WebPageInspection } from "../domain/browserObservation.js";
|
||||
import type { CdpCaptureEvents } from "./CdpCaptureEvents.js";
|
||||
import {
|
||||
allowedSanitizedUrl,
|
||||
boundedText,
|
||||
isHttpUrl,
|
||||
recordValue,
|
||||
recordsValue,
|
||||
@@ -57,13 +56,13 @@ export const captureWorkers = async (
|
||||
continue;
|
||||
}
|
||||
items.push({
|
||||
target_id: (stringValue(target.targetId) ?? "").slice(0, 256),
|
||||
type: type.slice(0, 100),
|
||||
target_id: stringValue(target.targetId) ?? "",
|
||||
type,
|
||||
url: url.url,
|
||||
origin: url.origin,
|
||||
attached: target.attached === true,
|
||||
opener_target_id: boundedText(target.openerId, 256) ?? null,
|
||||
parent_frame_id: boundedText(target.parentFrameId, 256) ?? null,
|
||||
opener_target_id: stringValue(target.openerId) ?? null,
|
||||
parent_frame_id: stringValue(target.parentFrameId) ?? null,
|
||||
});
|
||||
}
|
||||
return items;
|
||||
|
||||
@@ -68,15 +68,15 @@ const normalizedHeaders = (value: unknown): ReadonlyMap<string, string> => {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
return new Map();
|
||||
const headers = new Map<string, string>();
|
||||
for (const [name, raw] of Object.entries(value).slice(0, 500)) {
|
||||
const normalized = name.toLowerCase().slice(0, 256);
|
||||
for (const [name, raw] of Object.entries(value)) {
|
||||
const normalized = name.toLowerCase();
|
||||
const header =
|
||||
typeof raw === "string"
|
||||
? raw
|
||||
: typeof raw === "number" && Number.isFinite(raw)
|
||||
? String(raw)
|
||||
: undefined;
|
||||
if (header !== undefined) headers.set(normalized, header.slice(0, 65_536));
|
||||
if (header !== undefined) headers.set(normalized, header);
|
||||
}
|
||||
return headers;
|
||||
};
|
||||
@@ -89,13 +89,13 @@ const parseCsp = (
|
||||
const directives: ResponseMetadata["csp"]["directives"] = [];
|
||||
let nonceCount = 0;
|
||||
let hashCount = 0;
|
||||
for (const rawDirective of (value ?? "").split(";").slice(0, 100)) {
|
||||
for (const rawDirective of (value ?? "").split(";")) {
|
||||
const [rawName, ...tokens] = rawDirective.trim().split(/\s+/u);
|
||||
const name = (rawName ?? "").toLowerCase();
|
||||
if (!/^[a-z][a-z0-9-]{0,99}$/u.test(name)) continue;
|
||||
if (!/^[a-z][a-z0-9-]*$/u.test(name)) continue;
|
||||
const sources: ResponseMetadata["csp"]["directives"][number]["sources"] =
|
||||
[];
|
||||
for (const token of tokens.slice(0, 100)) {
|
||||
for (const token of tokens) {
|
||||
const lower = token.toLowerCase();
|
||||
if (lower.startsWith("'nonce-")) {
|
||||
nonceCount += 1;
|
||||
@@ -122,10 +122,9 @@ const cspSource = (
|
||||
allowedOrigins: ReadonlySet<string>,
|
||||
): ResponseMetadata["csp"]["directives"][number]["sources"][number] => {
|
||||
const lower = token.toLowerCase();
|
||||
if (/^'[a-z0-9-]+'$/u.test(lower))
|
||||
return { kind: "keyword", value: lower.slice(0, 100) };
|
||||
if (/^'[a-z0-9-]+'$/u.test(lower)) return { kind: "keyword", value: lower };
|
||||
if (/^[a-z][a-z0-9+.-]*:$/u.test(lower))
|
||||
return { kind: "scheme", value: lower.slice(0, 100) };
|
||||
return { kind: "scheme", value: lower };
|
||||
try {
|
||||
const parsed = new URL(token, baseUrl);
|
||||
return allowedOrigins.has(parsed.origin)
|
||||
@@ -142,8 +141,8 @@ const parseLinks = (
|
||||
allowedOrigins: ReadonlySet<string>,
|
||||
): LinkMetadata[] => {
|
||||
const links: LinkMetadata[] = [];
|
||||
for (const entry of splitLinkHeader(value ?? "").slice(0, 100)) {
|
||||
const match = /^\s*<([^>]{1,4096})>(.*)$/u.exec(entry);
|
||||
for (const entry of splitLinkHeader(value ?? "")) {
|
||||
const match = /^\s*<([^>]*)>(.*)$/u.exec(entry);
|
||||
if (match === null) continue;
|
||||
const parameters = linkParameters(match[2] ?? "");
|
||||
const destination = safeDestination(
|
||||
@@ -157,8 +156,7 @@ const parseLinks = (
|
||||
rel: (parameters.get("rel") ?? "")
|
||||
.toLowerCase()
|
||||
.split(/\s+/u)
|
||||
.filter(Boolean)
|
||||
.slice(0, 32),
|
||||
.filter(Boolean),
|
||||
as: boundedHeader(parameters.get("as")),
|
||||
type: boundedHeader(parameters.get("type")),
|
||||
crossorigin: boundedHeader(parameters.get("crossorigin")),
|
||||
@@ -184,14 +182,14 @@ const splitLinkHeader = (value: string): string[] => {
|
||||
|
||||
const linkParameters = (value: string): ReadonlyMap<string, string> => {
|
||||
const parameters = new Map<string, string>();
|
||||
for (const raw of value.split(";").slice(1, 50)) {
|
||||
for (const raw of value.split(";").slice(1)) {
|
||||
const separator = raw.indexOf("=");
|
||||
const name = (separator < 0 ? raw : raw.slice(0, separator))
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!/^[a-z][a-z0-9-]{0,99}$/u.test(name)) continue;
|
||||
if (!/^[a-z][a-z0-9-]*$/u.test(name)) continue;
|
||||
const parameter = separator < 0 ? "" : raw.slice(separator + 1).trim();
|
||||
parameters.set(name, unquote(parameter).slice(0, 1_024));
|
||||
parameters.set(name, unquote(parameter));
|
||||
}
|
||||
return parameters;
|
||||
};
|
||||
@@ -222,11 +220,9 @@ const permissionFeatures = (value: string | undefined): string[] =>
|
||||
(value ?? "")
|
||||
.split(",")
|
||||
.map((entry) => entry.split("=", 1)[0]?.trim().toLowerCase() ?? "")
|
||||
.filter((feature) => /^[a-z][a-z0-9-]{0,99}$/u.test(feature)),
|
||||
.filter((feature) => /^[a-z][a-z0-9-]*$/u.test(feature)),
|
||||
),
|
||||
]
|
||||
.sort()
|
||||
.slice(0, 200);
|
||||
].sort();
|
||||
|
||||
const wellKnownHint = (url: string): AgentHint[] => {
|
||||
try {
|
||||
@@ -245,7 +241,7 @@ const agentHint = (
|
||||
url: string | null,
|
||||
): AgentHint => ({
|
||||
mechanism,
|
||||
declaration: declaration.slice(0, 256),
|
||||
declaration,
|
||||
url,
|
||||
trust: "page-declared-untrusted",
|
||||
});
|
||||
@@ -255,7 +251,7 @@ const isAgentRel = (value: string): boolean =>
|
||||
|
||||
const policyToken = (value: string | undefined): string | null => {
|
||||
const token = (value ?? "").trim().toLowerCase();
|
||||
return /^[a-z][a-z0-9_.-]{0,99}$/u.test(token) ? token : null;
|
||||
return /^[a-z][a-z0-9_.-]*$/u.test(token) ? token : null;
|
||||
};
|
||||
|
||||
const nonnegativeInteger = (value: string | undefined): number | null => {
|
||||
@@ -264,7 +260,7 @@ const nonnegativeInteger = (value: string | undefined): number | null => {
|
||||
};
|
||||
|
||||
const boundedHeader = (value: string | undefined): string | null =>
|
||||
value === undefined ? null : value.trim().toLowerCase().slice(0, 256);
|
||||
value === undefined ? null : value.trim().toLowerCase();
|
||||
|
||||
const unquote = (value: string): string =>
|
||||
value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value;
|
||||
|
||||
@@ -83,10 +83,7 @@ export const captureCdpScreenshot = async (
|
||||
const encoded = stringValue(result?.data);
|
||||
const bytes =
|
||||
encoded === undefined ? undefined : decodeCanonicalBase64(encoded);
|
||||
if (
|
||||
bytes === undefined ||
|
||||
bytes.byteLength > context.input.maximum_image_bytes
|
||||
)
|
||||
if (bytes === undefined)
|
||||
throw new BrowserObservationError("inspect_web_page", "payload_limit");
|
||||
const dimensions = pngDimensions(bytes);
|
||||
if (navigation.leftScope)
|
||||
@@ -155,8 +152,8 @@ const pngDimensions = (
|
||||
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
||||
const width = bytes.readUInt32BE(16);
|
||||
const height = bytes.readUInt32BE(20);
|
||||
if (width === 0 || height === 0 || width * height > 32_000_000)
|
||||
throw new BrowserObservationError("inspect_web_page", "payload_limit");
|
||||
if (width === 0 || height === 0)
|
||||
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
||||
return { width, height };
|
||||
};
|
||||
|
||||
|
||||
@@ -73,8 +73,7 @@ const attachedSessionId = (
|
||||
value === null ||
|
||||
!("sessionId" in value) ||
|
||||
typeof value.sessionId !== "string" ||
|
||||
value.sessionId.length === 0 ||
|
||||
value.sessionId.length > 256
|
||||
value.sessionId.length === 0
|
||||
)
|
||||
throw new BrowserObservationError(operation, "protocol_error");
|
||||
return value.sessionId;
|
||||
|
||||
@@ -21,7 +21,7 @@ import {
|
||||
} from "./CdpCaptureValues.js";
|
||||
import { captureFrames, mainFrameUrl } from "./CdpCaptureDocuments.js";
|
||||
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
|
||||
import { boundedSensitiveText } from "./SensitiveTextCapture.js";
|
||||
import { redactSensitiveText } from "./SensitiveTextCapture.js";
|
||||
|
||||
interface DiscoveryContext {
|
||||
readonly connection: CdpConnection;
|
||||
@@ -106,7 +106,7 @@ export const discoverWebMcp = async (
|
||||
} else
|
||||
// CDP sends the enable response before the required toolsAdded replay.
|
||||
await delayWithCancellation(
|
||||
Math.max(context.input.observation_ms, 25),
|
||||
context.input.observation_ms,
|
||||
"discover_webmcp_tools",
|
||||
context.signal,
|
||||
);
|
||||
@@ -167,7 +167,7 @@ const handleFrameDetached = (
|
||||
state: FrameScopeState,
|
||||
): boolean => {
|
||||
const frameId = stringValue(params.frameId);
|
||||
if (frameId === undefined || frameId.length > 256) {
|
||||
if (frameId === undefined) {
|
||||
state.completeness.exclude("webmcp_tools", "invalid_protocol_value");
|
||||
return true;
|
||||
}
|
||||
@@ -180,7 +180,7 @@ const handleFrameUpdate = (
|
||||
state: FrameScopeState,
|
||||
): boolean => {
|
||||
const frameId = stringValue(frame?.id) ?? stringValue(frame?.frameId);
|
||||
if (frameId === undefined || frameId.length > 256) {
|
||||
if (frameId === undefined) {
|
||||
state.completeness.exclude("webmcp_tools", "invalid_protocol_value");
|
||||
return true;
|
||||
}
|
||||
@@ -293,16 +293,13 @@ const normalizeTool = (
|
||||
completeness.exclude("webmcp_tools", "out_of_target_scope");
|
||||
return undefined;
|
||||
}
|
||||
const description = boundedSensitiveText(
|
||||
stringValue(value.description) ?? "",
|
||||
4_096,
|
||||
).text;
|
||||
const description = redactSensitiveText(stringValue(value.description) ?? "");
|
||||
const annotations = recordValue(value.annotations);
|
||||
return {
|
||||
tool_key: toolKey(frame.url, name),
|
||||
name: boundedSensitiveText(name, 512).text,
|
||||
name: redactSensitiveText(name),
|
||||
description,
|
||||
frame_id: frameId.slice(0, 256),
|
||||
frame_id: frameId,
|
||||
frame_url: frame.url,
|
||||
owner_origin: frame.origin,
|
||||
declaration_kind:
|
||||
@@ -326,15 +323,9 @@ const normalizeTool = (
|
||||
const schemaShape = (value: unknown) => {
|
||||
if (recordValue(value) === undefined) return null;
|
||||
const encoded = JSON.stringify(value);
|
||||
const shape = inferJsonShape(encoded, {
|
||||
maximumBytes: 256 * 1_024,
|
||||
maximumNodes: 5_000,
|
||||
maximumDepth: 20,
|
||||
});
|
||||
const shape = inferJsonShape(encoded);
|
||||
if (shape === null)
|
||||
throw new BrowserObservationError("inspect_web_page", "protocol_error");
|
||||
if (shape.truncated)
|
||||
throw new BrowserObservationError("inspect_web_page", "payload_limit");
|
||||
return shape;
|
||||
};
|
||||
|
||||
|
||||
@@ -49,17 +49,16 @@ describe("PNG visual diff", () => {
|
||||
).toMatchObject({ status: "dimension_mismatch", compared_pixels: 0 });
|
||||
});
|
||||
|
||||
it("rejects images above the caller's decoded pixel limit", () => {
|
||||
const image = artifact(2, 1, [0, 0, 0, 255, 0, 0, 0, 255]);
|
||||
it("rejects malformed PNG dimensions after validating image data", () => {
|
||||
const image = artifact(32_000_001, 1, [0, 0, 0, 255]);
|
||||
expect(() =>
|
||||
comparePngScreenshots(
|
||||
compareWebScreenshotsInputSchema.parse({
|
||||
before: image,
|
||||
after: image,
|
||||
maximum_pixels: 1,
|
||||
}),
|
||||
),
|
||||
).toThrow("oversized PNG");
|
||||
).toThrow("Unexpected PNG size");
|
||||
});
|
||||
|
||||
it("rejects metrics that cannot represent their comparison status", () => {
|
||||
|
||||
@@ -13,12 +13,12 @@ interface DecodedPng {
|
||||
readonly rgba: Buffer;
|
||||
}
|
||||
|
||||
/** Compare two validated screenshots using bounded, deterministic RGBA metrics. */
|
||||
/** Compare two validated screenshots using deterministic RGBA metrics. */
|
||||
export const comparePngScreenshots = (
|
||||
input: CompareWebScreenshotsInput,
|
||||
): WebScreenshotDiff => {
|
||||
const before = decodePng(input.before, input.maximum_pixels);
|
||||
const after = decodePng(input.after, input.maximum_pixels);
|
||||
const before = decodePng(input.before);
|
||||
const after = decodePng(input.after);
|
||||
if (before.width !== after.width || before.height !== after.height)
|
||||
return {
|
||||
status: "dimension_mismatch",
|
||||
@@ -73,10 +73,7 @@ export const comparePngScreenshots = (
|
||||
};
|
||||
};
|
||||
|
||||
const decodePng = (
|
||||
artifact: WebScreenshotArtifact,
|
||||
maximumPixels: number,
|
||||
): DecodedPng => {
|
||||
const decodePng = (artifact: WebScreenshotArtifact): DecodedPng => {
|
||||
const bytes = decodeCanonicalBase64(artifact.data_base64);
|
||||
if (bytes === undefined || !bytes.subarray(0, 8).equals(PNG_SIGNATURE))
|
||||
throw new TypeError("Invalid PNG signature");
|
||||
@@ -92,7 +89,7 @@ const decodePng = (
|
||||
const dataEnd = dataStart + length;
|
||||
if (dataEnd + 4 > bytes.length) throw new TypeError("Truncated PNG data");
|
||||
const data = bytes.subarray(dataStart, dataEnd);
|
||||
if (type === "IHDR") header = parseHeader(data, maximumPixels);
|
||||
if (type === "IHDR") header = parseHeader(data);
|
||||
else if (type === "IDAT") compressed.push(data);
|
||||
else if (type === "IEND") {
|
||||
sawEnd = true;
|
||||
@@ -115,7 +112,7 @@ const decodePng = (
|
||||
};
|
||||
};
|
||||
|
||||
const parseHeader = (data: Buffer, maximumPixels: number) => {
|
||||
const parseHeader = (data: Buffer) => {
|
||||
if (data.byteLength !== 13) throw new TypeError("Invalid PNG header");
|
||||
const width = data.readUInt32BE(0);
|
||||
const height = data.readUInt32BE(4);
|
||||
@@ -124,14 +121,14 @@ const parseHeader = (data: Buffer, maximumPixels: number) => {
|
||||
if (
|
||||
width === 0 ||
|
||||
height === 0 ||
|
||||
width * height > maximumPixels ||
|
||||
!Number.isSafeInteger(width * height) ||
|
||||
bitDepth !== 8 ||
|
||||
(colorType !== 2 && colorType !== 6) ||
|
||||
data[10] !== 0 ||
|
||||
data[11] !== 0 ||
|
||||
data[12] !== 0
|
||||
)
|
||||
throw new TypeError("Unsupported or oversized PNG");
|
||||
throw new TypeError("Unsupported PNG");
|
||||
return { width, height, channels: colorType === 6 ? 4 : 3 };
|
||||
};
|
||||
|
||||
|
||||
@@ -1,9 +1,6 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
boundedSensitiveText,
|
||||
redactSensitiveText,
|
||||
} from "./SensitiveTextCapture.js";
|
||||
import { redactSensitiveText } from "./SensitiveTextCapture.js";
|
||||
|
||||
describe("sensitive console text", () => {
|
||||
it("redacts assignments, bearer credentials, and JWT-shaped values", () => {
|
||||
@@ -16,12 +13,4 @@ describe("sensitive console text", () => {
|
||||
"authorization=[REDACTED] password=[REDACTED] api_key=[REDACTED] [REDACTED_JWT]",
|
||||
);
|
||||
});
|
||||
|
||||
it("bounds UTF-8 after redaction without splitting a code point", () => {
|
||||
expect(boundedSensitiveText("password=secret 你好", 22)).toEqual({
|
||||
text: "password=[REDACTED] ",
|
||||
bytes: 20,
|
||||
truncated: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -9,27 +9,3 @@ export const redactSensitiveText = (value: string): string =>
|
||||
.replace(SECRET_ASSIGNMENT, (_match, name: string) => `${name}=[REDACTED]`)
|
||||
.replace(BEARER, "Bearer [REDACTED]")
|
||||
.replace(JWT, "[REDACTED_JWT]");
|
||||
|
||||
/** Return a UTF-8-bounded prefix without splitting a code point. */
|
||||
export const boundedSensitiveText = (
|
||||
value: string,
|
||||
maximumBytes: number,
|
||||
): {
|
||||
readonly text: string;
|
||||
readonly bytes: number;
|
||||
readonly truncated: boolean;
|
||||
} => {
|
||||
const redacted = redactSensitiveText(value);
|
||||
const total = Buffer.byteLength(redacted);
|
||||
if (total <= maximumBytes)
|
||||
return { text: redacted, bytes: total, truncated: false };
|
||||
let text = "";
|
||||
let bytes = 0;
|
||||
for (const character of redacted) {
|
||||
const size = Buffer.byteLength(character);
|
||||
if (bytes + size > maximumBytes) break;
|
||||
text += character;
|
||||
bytes += size;
|
||||
}
|
||||
return { text, bytes, truncated: true };
|
||||
};
|
||||
|
||||
@@ -8,26 +8,23 @@ import {
|
||||
BrowserObservationError,
|
||||
type BrowserObservationOperation,
|
||||
} from "../domain/errors.js";
|
||||
import { parseCdpEndpointValue, readBoundedCdpJson } from "./CdpEndpoint.js";
|
||||
|
||||
const MAX_VERSION_BYTES = 64 * 1_024;
|
||||
const MAX_TARGET_LIST_BYTES = 2 * 1_024 * 1_024;
|
||||
import { parseCdpEndpointValue, readCdpJson } from "./CdpEndpoint.js";
|
||||
|
||||
const versionSchema = z
|
||||
.object({
|
||||
Browser: z.string().min(1).max(1_024),
|
||||
"Protocol-Version": z.string().min(1).max(100),
|
||||
"V8-Version": z.string().max(1_024).optional(),
|
||||
Browser: z.string().min(1),
|
||||
"Protocol-Version": z.string().min(1),
|
||||
"V8-Version": z.string().optional(),
|
||||
})
|
||||
.passthrough();
|
||||
|
||||
const targetSchema = z
|
||||
.object({
|
||||
id: z.string().min(1).max(256),
|
||||
type: z.string().min(1).max(100),
|
||||
url: z.string().max(65_536),
|
||||
id: z.string().min(1),
|
||||
type: z.string().min(1),
|
||||
url: z.string(),
|
||||
attached: z.boolean().default(false),
|
||||
webSocketDebuggerUrl: z.string().min(1).max(2_048),
|
||||
webSocketDebuggerUrl: z.string().min(1),
|
||||
})
|
||||
.passthrough();
|
||||
const targetsSchema = z.array(targetSchema);
|
||||
@@ -45,25 +42,15 @@ export interface V8InspectorDiscovery {
|
||||
readonly targets: readonly V8InspectorTarget[];
|
||||
}
|
||||
|
||||
/** Discover bounded Node/Electron Inspector targets from one loopback endpoint. */
|
||||
/** Discover Node/Electron Inspector targets from one loopback endpoint. */
|
||||
export const discoverV8Inspector = async (
|
||||
endpoint: string,
|
||||
operation: BrowserObservationOperation,
|
||||
signal?: AbortSignal,
|
||||
): Promise<V8InspectorDiscovery> => {
|
||||
const [versionInput, targetsInput] = await Promise.all([
|
||||
readBoundedCdpJson(
|
||||
new URL("/json/version", endpoint),
|
||||
MAX_VERSION_BYTES,
|
||||
operation,
|
||||
signal,
|
||||
),
|
||||
readBoundedCdpJson(
|
||||
new URL("/json/list", endpoint),
|
||||
MAX_TARGET_LIST_BYTES,
|
||||
operation,
|
||||
signal,
|
||||
),
|
||||
readCdpJson(new URL("/json/version", endpoint), operation, signal),
|
||||
readCdpJson(new URL("/json/list", endpoint), operation, signal),
|
||||
]);
|
||||
const version = parseCdpEndpointValue(versionSchema, versionInput, operation);
|
||||
const targets = parseCdpEndpointValue(targetsSchema, targetsInput, operation);
|
||||
|
||||
@@ -31,33 +31,14 @@ export const fetchWebSourceMaps = async (
|
||||
host: SourceMapFetchHost = { fetch: globalThis.fetch },
|
||||
): Promise<SourceMaps> => {
|
||||
const items: SourceMapItem[] = [];
|
||||
let remainingBytes = input.analysis_limits.max_total_source_map_bytes;
|
||||
for (const request of requests.slice(
|
||||
0,
|
||||
input.analysis_limits.max_source_maps,
|
||||
)) {
|
||||
for (const request of requests) {
|
||||
if (signal?.aborted === true) throw signal.reason;
|
||||
if (remainingBytes === 0) break;
|
||||
const fetched = await fetchOne(request, {
|
||||
input,
|
||||
maximumBytes: Math.min(
|
||||
input.analysis_limits.max_source_map_bytes,
|
||||
remainingBytes,
|
||||
),
|
||||
signal,
|
||||
host,
|
||||
});
|
||||
items.push(fetched.item);
|
||||
remainingBytes = Math.max(0, remainingBytes - fetched.bytesRead);
|
||||
items.push(await fetchOne(request, input, signal, host));
|
||||
}
|
||||
const included = items.filter(({ status }) => status === "included").length;
|
||||
const dropped = requests.length - items.length;
|
||||
const truncated =
|
||||
dropped > 0 || items.some(({ status }) => status === "truncated");
|
||||
return webSourceMapsSchema.parse({
|
||||
status: truncated
|
||||
? "truncated"
|
||||
: items.length === 0
|
||||
status:
|
||||
items.length === 0
|
||||
? "unavailable"
|
||||
: included === items.length
|
||||
? "included"
|
||||
@@ -66,40 +47,22 @@ export const fetchWebSourceMaps = async (
|
||||
: "unavailable",
|
||||
requested: requests.length,
|
||||
processed: items.length,
|
||||
dropped,
|
||||
dropped_script_keys: requests
|
||||
.slice(items.length)
|
||||
.map(({ scriptKey }) => scriptKey),
|
||||
items,
|
||||
});
|
||||
};
|
||||
|
||||
interface SourceMapFetchResult {
|
||||
readonly item: SourceMapItem;
|
||||
readonly bytesRead: number;
|
||||
}
|
||||
|
||||
interface SourceMapFetchContext {
|
||||
readonly input: AnalyzeWebBundleInput;
|
||||
readonly maximumBytes: number;
|
||||
readonly signal: AbortSignal | undefined;
|
||||
readonly host: SourceMapFetchHost;
|
||||
}
|
||||
|
||||
const fetchOne = async (
|
||||
request: WebSourceMapRequest,
|
||||
context: SourceMapFetchContext,
|
||||
): Promise<SourceMapFetchResult> => {
|
||||
const { input, maximumBytes, signal, host } = context;
|
||||
input: AnalyzeWebBundleInput,
|
||||
signal: AbortSignal | undefined,
|
||||
host: SourceMapFetchHost,
|
||||
): Promise<SourceMapItem> => {
|
||||
if (!approvedUrl(request.fetchUrl, input.allowed_origins))
|
||||
return {
|
||||
item: emptySourceMapItem(
|
||||
request,
|
||||
"policy_filtered",
|
||||
"Declared source-map URL is outside the approved exact origins.",
|
||||
),
|
||||
bytesRead: 0,
|
||||
};
|
||||
return emptySourceMapItem(
|
||||
request,
|
||||
"policy_filtered",
|
||||
"Declared source-map URL is outside the approved exact origins.",
|
||||
);
|
||||
try {
|
||||
const response = await fetchFollowingApprovedRedirects(
|
||||
request.fetchUrl,
|
||||
@@ -108,40 +71,25 @@ const fetchOne = async (
|
||||
host,
|
||||
);
|
||||
if (response === undefined)
|
||||
return {
|
||||
item: emptySourceMapItem(
|
||||
request,
|
||||
"policy_filtered",
|
||||
"A source-map redirect left the approved exact origins.",
|
||||
),
|
||||
bytesRead: 0,
|
||||
};
|
||||
return emptySourceMapItem(
|
||||
request,
|
||||
"policy_filtered",
|
||||
"A source-map redirect left the approved exact origins.",
|
||||
);
|
||||
if (!response.ok)
|
||||
return {
|
||||
item: emptySourceMapItem(
|
||||
request,
|
||||
"fetch_failed",
|
||||
`Source-map server returned HTTP ${String(response.status)}.`,
|
||||
),
|
||||
bytesRead: 0,
|
||||
};
|
||||
const body = await boundedResponseText(response, maximumBytes);
|
||||
return {
|
||||
item: normalizeSourceMap(request, body.text, input),
|
||||
bytesRead: body.bytes,
|
||||
};
|
||||
return emptySourceMapItem(
|
||||
request,
|
||||
"fetch_failed",
|
||||
`Source-map server returned HTTP ${String(response.status)}.`,
|
||||
);
|
||||
return normalizeSourceMap(request, await response.text());
|
||||
} catch (cause: unknown) {
|
||||
if (signal?.aborted === true) throw cause;
|
||||
return {
|
||||
item: emptySourceMapItem(
|
||||
request,
|
||||
isLimitError(cause) ? "truncated" : "fetch_failed",
|
||||
isLimitError(cause)
|
||||
? "Source-map response exceeded the remaining approved byte budget."
|
||||
: "Source-map fetch or validation failed.",
|
||||
),
|
||||
bytesRead: cause instanceof SourceMapByteLimitError ? cause.bytesRead : 0,
|
||||
};
|
||||
return emptySourceMapItem(
|
||||
request,
|
||||
"fetch_failed",
|
||||
"Source-map fetch or validation failed.",
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -152,75 +100,37 @@ const fetchFollowingApprovedRedirects = async (
|
||||
host: SourceMapFetchHost,
|
||||
): Promise<Response | undefined> => {
|
||||
let current = initialUrl;
|
||||
for (let redirect = 0; redirect <= 5; redirect += 1) {
|
||||
const visited = new Set<string>();
|
||||
for (;;) {
|
||||
if (!approvedUrl(current, allowedOrigins)) return undefined;
|
||||
const response = await timedFetch(current, signal, host);
|
||||
if (visited.has(current)) throw new Error("source_map_redirect_loop");
|
||||
visited.add(current);
|
||||
const response = await host.fetch(current, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Accept: "application/json, application/source-map+json;q=0.9",
|
||||
},
|
||||
redirect: "manual",
|
||||
credentials: "omit",
|
||||
referrerPolicy: "no-referrer",
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
});
|
||||
if (response.status < 300 || response.status >= 400) return response;
|
||||
const location = response.headers.get("location");
|
||||
if (location === null) return response;
|
||||
current = new URL(location, current).href;
|
||||
}
|
||||
throw new Error("source_map_redirect_limit");
|
||||
};
|
||||
|
||||
const timedFetch = async (
|
||||
url: string,
|
||||
signal: AbortSignal | undefined,
|
||||
host: SourceMapFetchHost,
|
||||
): Promise<Response> => {
|
||||
const timeout = AbortSignal.timeout(5_000);
|
||||
return await host.fetch(url, {
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json, application/source-map+json;q=0.9" },
|
||||
redirect: "manual",
|
||||
credentials: "omit",
|
||||
referrerPolicy: "no-referrer",
|
||||
signal: signal === undefined ? timeout : AbortSignal.any([signal, timeout]),
|
||||
});
|
||||
};
|
||||
|
||||
const boundedResponseText = async (
|
||||
response: Response,
|
||||
maximumBytes: number,
|
||||
): Promise<{ readonly text: string; readonly bytes: number }> => {
|
||||
const declared = Number(response.headers.get("content-length"));
|
||||
if (Number.isFinite(declared) && declared > maximumBytes)
|
||||
throw new SourceMapByteLimitError(0);
|
||||
if (response.body === null) return { text: "", bytes: 0 };
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true });
|
||||
let bytes = 0;
|
||||
let text = "";
|
||||
for (;;) {
|
||||
const chunk = await reader.read();
|
||||
if (chunk.done) break;
|
||||
bytes += chunk.value.byteLength;
|
||||
if (bytes > maximumBytes) {
|
||||
await reader.cancel();
|
||||
throw new SourceMapByteLimitError(bytes);
|
||||
}
|
||||
text += decoder.decode(chunk.value, { stream: true });
|
||||
}
|
||||
text += decoder.decode();
|
||||
return { text, bytes };
|
||||
};
|
||||
|
||||
class SourceMapByteLimitError extends RangeError {
|
||||
constructor(readonly bytesRead: number) {
|
||||
super("source_map_bytes");
|
||||
}
|
||||
}
|
||||
|
||||
const normalizeSourceMap = (
|
||||
request: WebSourceMapRequest,
|
||||
text: string,
|
||||
input: AnalyzeWebBundleInput,
|
||||
): SourceMapItem => {
|
||||
if (!validSourceMapEnvelope(text))
|
||||
return emptySourceMapItem(
|
||||
request,
|
||||
"invalid",
|
||||
"Source-map JSON is not a bounded version 3 map.",
|
||||
"Source-map JSON is not a version 3 map.",
|
||||
);
|
||||
try {
|
||||
const map = new AnyMap(text, request.fetchUrl);
|
||||
@@ -239,7 +149,6 @@ const normalizeSourceMap = (
|
||||
};
|
||||
});
|
||||
const mappings: ParsedSourceMapItem["mappings"] = [];
|
||||
let totalMappings = 0;
|
||||
eachMapping(map, (mapping) => {
|
||||
if (
|
||||
mapping.source === null ||
|
||||
@@ -247,9 +156,6 @@ const normalizeSourceMap = (
|
||||
mapping.originalColumn === null
|
||||
)
|
||||
return;
|
||||
totalMappings += 1;
|
||||
if (mappings.length >= input.analysis_limits.max_source_map_mappings)
|
||||
return;
|
||||
mappings.push({
|
||||
generated_line: mapping.generatedLine,
|
||||
generated_column: mapping.generatedColumn,
|
||||
@@ -258,16 +164,10 @@ const normalizeSourceMap = (
|
||||
),
|
||||
original_line: mapping.originalLine,
|
||||
original_column: mapping.originalColumn,
|
||||
name: mapping.name?.slice(0, 1_024) ?? null,
|
||||
name: mapping.name ?? null,
|
||||
});
|
||||
});
|
||||
const modules = originalModuleEdges(originalSources);
|
||||
const mappingTruncated = totalMappings > mappings.length;
|
||||
const limitations = mappingTruncated
|
||||
? [
|
||||
`Mappings were truncated from ${String(totalMappings)} to ${String(mappings.length)}.`,
|
||||
]
|
||||
: [];
|
||||
const parsed = {
|
||||
...sourceMapContext(request),
|
||||
artifact: createWebTextArtifact(text, "application/source-map+json"),
|
||||
@@ -275,13 +175,7 @@ const normalizeSourceMap = (
|
||||
original_module_edges: modules,
|
||||
mappings,
|
||||
};
|
||||
return limitations.length === 0
|
||||
? { ...parsed, status: "included", limitation: null }
|
||||
: {
|
||||
...parsed,
|
||||
status: "truncated",
|
||||
limitation: limitations.join(" "),
|
||||
};
|
||||
return { ...parsed, status: "included", limitation: null };
|
||||
} catch {
|
||||
return emptySourceMapItem(
|
||||
request,
|
||||
@@ -300,7 +194,7 @@ const originalModuleEdges = (
|
||||
if (source.artifact === null) continue;
|
||||
for (const detector of originalImportDetectors) {
|
||||
for (const match of source.artifact.text.matchAll(detector.pattern)) {
|
||||
const specifier = match[1]?.slice(0, 4_096);
|
||||
const specifier = match[1];
|
||||
if (specifier === undefined) continue;
|
||||
const key = `${source.source}\0${detector.kind}\0${specifier}`;
|
||||
if (seen.has(key)) continue;
|
||||
@@ -327,19 +221,16 @@ const validSourceMapEnvelope = (text: string): boolean => {
|
||||
if (!isRecord(parsed) || parsed.version !== 3) return false;
|
||||
if (typeof parsed.mappings === "string")
|
||||
return Array.isArray(parsed.sources) && Array.isArray(parsed.names);
|
||||
if (!Array.isArray(parsed.sections) || parsed.sections.length > 10_000)
|
||||
return false;
|
||||
if (!Array.isArray(parsed.sections)) return false;
|
||||
const pending: unknown[] = [...parsed.sections];
|
||||
let visited = 0;
|
||||
while (pending.length > 0) {
|
||||
const section = pending.pop();
|
||||
if (!isRecord(section) || !isRecord(section.offset) || !("map" in section))
|
||||
return false;
|
||||
visited += 1;
|
||||
if (visited > 10_000) return false;
|
||||
const map = section.map;
|
||||
if (!isRecord(map) || map.version !== 3) return false;
|
||||
if (Array.isArray(map.sections)) pending.push(...map.sections);
|
||||
if (Array.isArray(map.sections))
|
||||
for (const child of map.sections) pending.push(child);
|
||||
else if (
|
||||
typeof map.mappings !== "string" ||
|
||||
!Array.isArray(map.sources) ||
|
||||
@@ -387,14 +278,13 @@ const emptySourceMapItem = (
|
||||
});
|
||||
|
||||
const sanitizeSource = (value: string): string => {
|
||||
const bounded = value.slice(0, 4_096);
|
||||
try {
|
||||
const parsed = new URL(bounded);
|
||||
const parsed = new URL(value);
|
||||
return parsed.protocol === "http:" || parsed.protocol === "https:"
|
||||
? sanitizeBrowserUrl(parsed.href).url
|
||||
: `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
|
||||
} catch {
|
||||
return bounded.split("#", 1)[0]?.split("?", 1)[0] ?? "";
|
||||
return value.split("#", 1)[0]?.split("?", 1)[0] ?? "";
|
||||
}
|
||||
};
|
||||
|
||||
@@ -417,9 +307,6 @@ const sourceMediaType = (source: string | null): string =>
|
||||
const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
|
||||
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
|
||||
const isLimitError = (value: unknown): value is RangeError =>
|
||||
value instanceof RangeError;
|
||||
|
||||
const originalImportDetectors = [
|
||||
{
|
||||
kind: "static_import" as const,
|
||||
|
||||
Reference in New Issue
Block a user