refactor(browser): return complete observation evidence

This commit is contained in:
morluto
2026-09-29 13:56:49 +08:00
parent 9d1f88cbe8
commit 0e2738ffb6
69 changed files with 747 additions and 1206 deletions
+2 -2
View File
@@ -143,7 +143,7 @@ export class CdpBrowserProvider implements BrowserObservationPort {
: undefined;
return ok(
webBundleAnalysisSchema.parse(
analyzeCapturedWebBundle(captured.inspection, input, sourceMaps),
analyzeCapturedWebBundle(captured.inspection, sourceMaps),
),
);
} catch (cause: unknown) {
@@ -368,7 +368,7 @@ const selectTargets = (
allowed.push({
target_id: target.id,
type: target.type,
title: target.title.slice(0, 16_384),
title: target.title,
url: url.url,
origin: url.origin,
attached: target.attached,
+4 -8
View File
@@ -70,7 +70,7 @@ describe("CDP document capture", () => {
]);
});
it("bounds approved accessibility text by UTF-8 bytes without splitting characters", () => {
it("retains full approved accessibility text", () => {
const capture = captureAccessibility(
[
{
@@ -91,21 +91,19 @@ describe("CDP document capture", () => {
],
{
includeText: true,
maximumFieldBytes: 4,
},
);
expect(capture.nodes[0]).toMatchObject({
name: "😀",
name: "😀a",
description: "éé",
states: [{ name: "disabled", value: true }],
});
expect(capture.treeIncomplete).toBe(true);
expect(capture.textCapture).toEqual({
status: "truncated",
retained_bytes: 8,
status: "included",
retained_bytes: 9,
excluded_fields: 0,
truncated_fields: 1,
});
});
@@ -124,7 +122,6 @@ describe("CDP document capture", () => {
],
{
includeText: false,
maximumFieldBytes: 1_024,
},
);
@@ -133,7 +130,6 @@ describe("CDP document capture", () => {
status: "not_approved",
retained_bytes: 0,
excluded_fields: 2,
truncated_fields: 0,
});
});
});
+12 -49
View File
@@ -4,7 +4,6 @@ import type {
} from "../domain/browserObservation.js";
import {
allowedSanitizedUrl,
boundedText,
numberValue,
recordValue,
recordsValue,
@@ -36,7 +35,7 @@ export const captureFrames = (
const frame = recordValue(tree.frame);
const frameId = stringValue(frame?.id);
const sanitized = allowedSanitizedUrl(frame?.url, allowedOrigins);
if (frame === undefined || frameId === undefined || frameId.length > 256) {
if (frame === undefined || frameId === undefined) {
completeness?.exclude("frames", "invalid_protocol_value");
continue;
}
@@ -50,8 +49,7 @@ export const captureFrames = (
if (maximum !== undefined && items.length >= maximum) continue;
items.push({
frame_id: frameId,
parent_frame_id:
(stringValue(frame.parentId) ?? "").slice(0, 256) || null,
parent_frame_id: stringValue(frame.parentId) ?? null,
url: sanitized.url,
origin: sanitized.origin,
});
@@ -91,8 +89,8 @@ export const captureResources = (
rawUrl: stringValue(resource.url) ?? "",
url: url.url,
origin: url.origin,
type: (stringValue(resource.type) ?? "Other").slice(0, 100),
mime_type: (stringValue(resource.mimeType) ?? "").slice(0, 256),
type: stringValue(resource.type) ?? "Other",
mime_type: stringValue(resource.mimeType) ?? "",
content_size:
contentSize === undefined ? null : Math.max(0, contentSize),
});
@@ -159,7 +157,7 @@ export const captureDom = (
const attributeIndexes = numberArray(attributes[index]);
const parent = Math.trunc(parents[index] ?? -1);
const nodeIndex = nodes.length;
const nodeName = indexedString(strings, nodeNames[index]).slice(0, 256);
const nodeName = indexedString(strings, nodeNames[index]);
nodes.push({
index: nodeIndex,
parent_index: parent < 0 ? -1 : baseIndex + parent,
@@ -168,7 +166,7 @@ export const captureDom = (
node_value_length: indexedString(strings, nodeValues[index]).length,
attribute_names: attributeIndexes
.filter((_value, attributeIndex) => attributeIndex % 2 === 0)
.map((value) => indexedString(strings, value).slice(0, 256)),
.map((value) => indexedString(strings, value)),
});
const metadata = domMetadata({
strings,
@@ -206,7 +204,6 @@ export const captureAccessibility = (
results: readonly unknown[],
options: {
readonly includeText: boolean;
readonly maximumFieldBytes: number;
readonly unavailable?: boolean;
},
): {
@@ -225,7 +222,6 @@ export const captureAccessibility = (
}),
);
let excludedFields = 0;
let truncatedFields = 0;
const nodes = all.map((node) => {
const captureText = (value: unknown): string | null => {
const raw = stringValue(recordValue(value)?.value);
@@ -234,19 +230,11 @@ export const captureAccessibility = (
excludedFields += 1;
return null;
}
const captured = boundedUtf8(raw, options.maximumFieldBytes);
if (captured.truncated) {
truncatedFields += 1;
if (captured.text === "" && raw !== "") {
excludedFields += 1;
return null;
}
}
return captured.text;
return raw;
};
return {
node_id: (stringValue(node.nodeId) ?? "").slice(0, 256),
parent_id: (stringValue(node.parentId) ?? "").slice(0, 256) || null,
node_id: stringValue(node.nodeId) ?? "",
parent_id: stringValue(node.parentId) ?? null,
role: axText(node.role),
name: captureText(node.name),
description: captureText(node.description),
@@ -270,9 +258,7 @@ export const captureAccessibility = (
? "unavailable"
: !options.includeText
? "not_approved"
: truncatedFields > 0
? "truncated"
: "included",
: "included",
retained_bytes: nodes.reduce(
(total, node) =>
total +
@@ -281,7 +267,6 @@ export const captureAccessibility = (
0,
),
excluded_fields: excludedFields,
truncated_fields: truncatedFields,
},
};
};
@@ -327,7 +312,7 @@ const accessibilityStates = (
: typeof raw === "number"
? numberValue(raw)
: typeof raw === "string"
? raw.slice(0, 256)
? raw
: undefined;
if (state !== undefined) states.set(name, state);
}
@@ -337,7 +322,7 @@ const accessibilityStates = (
};
const axText = (value: unknown): string | null =>
boundedText(recordValue(value)?.value, 1_024);
stringValue(recordValue(value)?.value) ?? null;
const indexedString = (strings: readonly string[], index: unknown): string => {
const integer = numberValue(index);
@@ -433,28 +418,6 @@ const domDestination = (
}
};
const boundedUtf8 = (
value: string,
maximumBytes: number,
): {
readonly text: string;
readonly bytes: number;
readonly truncated: boolean;
} => {
const total = Buffer.byteLength(value);
if (total <= maximumBytes)
return { text: value, bytes: total, truncated: false };
let text = "";
let bytes = 0;
for (const character of value) {
const characterBytes = Buffer.byteLength(character);
if (bytes + characterBytes > maximumBytes) break;
text += character;
bytes += characterBytes;
}
return { text, bytes, truncated: true };
};
const domUrlAttributes = [
"href",
"src",
+12 -34
View File
@@ -27,8 +27,8 @@ export const ingestResponseBodyShape = (
invalidResponseBodyShape(state, requestId);
return;
}
const inferred = inferBodyShape(state, decoded);
updateResponseBodyShape(state, requestId, inferred.shape, inferred.truncated);
const inferred = inferBodyShape(decoded);
updateResponseBodyShape(state, requestId, inferred.shape);
};
export const requestBodyShape = (
@@ -40,13 +40,9 @@ export const requestBodyShape = (
const body = stringValue(request.postData);
if (!isJsonContentType(recordValue(request.headers)) || body === undefined)
return { status: "unavailable", request: null, response: null };
const inferred = inferBodyShape(state, body);
const inferred = inferBodyShape(body);
return {
status: inferred.truncated
? "truncated"
: inferred.shape === null
? "unavailable"
: "included",
status: inferred.shape === null ? "unavailable" : "included",
request: inferred.shape,
response: null,
};
@@ -56,48 +52,30 @@ export const updateResponseBodyShape = (
state: CdpCaptureEventsState,
requestId: string,
response: JsonShape | null,
truncated: boolean,
): void => {
const current = state.network.get(requestId);
if (current === undefined) return;
const request = current.body_shapes.request;
const status =
truncated || current.body_shapes.status === "truncated"
? "truncated"
: response !== null
? "included"
: request !== null
? "partial"
: "unavailable";
response !== null
? "included"
: request !== null
? "partial"
: "unavailable";
state.network.set(requestId, {
...current,
body_shapes: { status, request, response },
});
};
const inferBodyShape = (
state: CdpCaptureEventsState,
text: string,
): { readonly shape: JsonShape | null; readonly truncated: boolean } => {
const bytes = Buffer.byteLength(text);
if (bytes > state.input.limits.max_json_body_bytes) {
state.completeness.truncate("json_body_shapes");
return { shape: null, truncated: true };
}
const shape = inferJsonShape(text, {
maximumBytes: state.input.limits.max_json_body_bytes,
maximumNodes: state.input.limits.max_json_shape_nodes,
maximumDepth: state.input.limits.max_json_shape_depth,
});
if (shape?.truncated === true)
state.completeness.truncate("json_body_shapes");
return { shape, truncated: shape?.truncated === true };
const inferBodyShape = (text: string): { readonly shape: JsonShape | null } => {
return { shape: inferJsonShape(text) };
};
const invalidResponseBodyShape = (
state: CdpCaptureEventsState,
requestId: string,
): void => {
updateResponseBodyShape(state, requestId, null, false);
updateResponseBodyShape(state, requestId, null);
state.completeness.exclude("json_body_shapes", "invalid_protocol_value");
};
+22 -46
View File
@@ -1,10 +1,9 @@
import type { WebPageInspection } from "../domain/browserObservation.js";
import { inferJsonShape } from "../domain/jsonShape.js";
import { safeResponseMetadata } from "./CdpSafeMetadata.js";
import { boundedSensitiveText } from "./SensitiveTextCapture.js";
import { redactSensitiveText } from "./SensitiveTextCapture.js";
import {
allowedSanitizedUrl,
boundedText,
isHttpUrl,
numberValue,
recordValue,
@@ -35,8 +34,7 @@ export const handleExecutionContextCreated = (
if (
identifier === undefined ||
!Number.isSafeInteger(identifier) ||
frameId === undefined ||
frameId.length > 256
frameId === undefined
)
return;
const key = String(identifier);
@@ -58,7 +56,7 @@ export const handleScriptParsed = (
const scriptId = stringValue(params.scriptId);
const rawUrl = stringValue(params.url) ?? "";
const sanitized = allowedSanitizedUrl(rawUrl, state.allowedOrigins);
if (scriptId === undefined || scriptId.length > 256) {
if (scriptId === undefined) {
state.completeness.exclude("scripts", "invalid_protocol_value");
return;
}
@@ -72,7 +70,7 @@ export const handleScriptParsed = (
rawUrl,
url: sanitized.url,
origin: sanitized.origin,
hash: (stringValue(params.hash) ?? "").slice(0, 512),
hash: stringValue(params.hash) ?? "",
length: Math.max(
0,
Math.min(
@@ -81,7 +79,7 @@ export const handleScriptParsed = (
),
),
isModule: params.isModule === true,
language: boundedText(params.scriptLanguage, 100),
language: stringValue(params.scriptLanguage) ?? null,
sourceMapUrl: sourceMap.sanitized,
sourceMapRawUrl: sourceMap.raw,
executionContextKey: executionContextKey(params.executionContextId),
@@ -116,7 +114,7 @@ export const handleRequestWillBeSent = (
): void => {
const requestId = stringValue(params.requestId);
const request = recordValue(params.request);
if (requestId === undefined || requestId.length > 256) {
if (requestId === undefined) {
state.completeness.exclude("network_requests", "invalid_protocol_value");
return;
}
@@ -150,13 +148,13 @@ export const handleRequestWillBeSent = (
request_id: requestId,
url: sanitized.url,
origin: sanitized.origin ?? "",
method: (stringValue(request.method) ?? "GET").slice(0, 32),
resource_type: boundedText(params.type, 100),
method: stringValue(request.method) ?? "GET",
resource_type: stringValue(params.type) ?? null,
status: null,
mime_type: null,
encoded_data_length: null,
initiator: {
type: (stringValue(initiator?.type) ?? "other").slice(0, 100),
type: stringValue(initiator?.type) ?? "other",
url: initiatorUrl?.url ?? null,
line: integerOrNull(initiatorFrame?.lineNumber),
column: integerOrNull(initiatorFrame?.columnNumber),
@@ -170,7 +168,7 @@ export const handleResponseReceived = (
params: UnknownRecord,
): void => {
const requestId = stringValue(params.requestId);
if (requestId === undefined || requestId.length > 256) {
if (requestId === undefined) {
state.completeness.exclude("network_requests", "invalid_protocol_value");
return;
}
@@ -191,7 +189,7 @@ export const handleResponseReceived = (
state.network.set(requestId, {
...current,
status: numberValue(response.status) ?? null,
mime_type: boundedText(response.mimeType, 256),
mime_type: stringValue(response.mimeType) ?? null,
});
const metadata = safeResponseMetadata(
requestId,
@@ -208,7 +206,7 @@ export const handleLoadingFinished = (
params: UnknownRecord,
): void => {
const requestId = stringValue(params.requestId);
if (requestId === undefined || requestId.length > 256) return;
if (requestId === undefined) return;
const current = state.network.get(requestId);
if (current === undefined) return;
state.network.set(requestId, {
@@ -239,10 +237,10 @@ export const handleConsoleAPICalled = (
}
const arguments_ = recordsValue(params.args);
state.console.push({
type: (stringValue(params.type) ?? "unknown").slice(0, 100),
type: stringValue(params.type) ?? "unknown",
timestamp: numberValue(params.timestamp) ?? 0,
argument_types: arguments_.map((argument) =>
(stringValue(argument.type) ?? "unknown").slice(0, 100),
argument_types: arguments_.map(
(argument) => stringValue(argument.type) ?? "unknown",
),
url: source.url,
line: integerOrNull(frame.lineNumber),
@@ -260,33 +258,25 @@ const captureConsoleText = (
status: "not_approved",
values: [],
retained_bytes: 0,
truncated_values: 0,
};
const values: WebPageInspection["console"]["events"][number]["text_capture"]["values"] =
[];
let retainedBytes = 0;
let truncatedValues = 0;
for (const [argumentIndex, argument] of arguments_.entries()) {
const primitive = consolePrimitive(argument);
if (primitive === undefined) continue;
const bounded = boundedSensitiveText(
primitive.text,
state.input.limits.max_console_text_field_bytes,
);
const text = redactSensitiveText(primitive.text);
values.push({
argument_index: argumentIndex,
type: primitive.type,
text: bounded.text,
text,
});
retainedBytes += bounded.bytes;
if (bounded.truncated) truncatedValues += 1;
retainedBytes += Buffer.byteLength(text);
}
if (truncatedValues > 0) state.completeness.truncate("console_text");
return {
status: truncatedValues > 0 ? "truncated" : "included",
status: "included",
values,
retained_bytes: retainedBytes,
truncated_values: truncatedValues,
};
};
@@ -298,7 +288,6 @@ export const handleWebSocketFrame = (
const requestId = stringValue(params.requestId);
if (
requestId === undefined ||
requestId.length > 256 ||
(!state.network.has(requestId) && !state.allowedWebSockets.has(requestId))
)
return;
@@ -324,24 +313,11 @@ const captureWebSocketShape = (
opcode: number,
): WebPageInspection["network"]["websocket_events"][number]["payload_shape"] => {
if (!state.input.include_websocket_shapes) return null;
if (opcode !== 1)
return { format: "binary", json_shape: null, truncated: false };
const bytes = Buffer.byteLength(payload);
if (bytes > state.input.limits.max_websocket_shape_bytes) {
state.completeness.truncate("websocket_shapes");
return { format: "text", json_shape: null, truncated: true };
}
const shape = inferJsonShape(payload, {
maximumBytes: state.input.limits.max_websocket_shape_bytes,
maximumNodes: state.input.limits.max_json_shape_nodes,
maximumDepth: state.input.limits.max_json_shape_depth,
});
if (shape?.truncated === true)
state.completeness.truncate("websocket_shapes");
if (opcode !== 1) return { format: "binary", json_shape: null };
const shape = inferJsonShape(payload);
return {
format: shape === null ? "text" : "json",
json_shape: shape,
truncated: shape?.truncated === true,
};
};
@@ -351,7 +327,7 @@ export const handleWebSocketCreated = (
): void => {
const requestId = stringValue(params.requestId);
const rawUrl = stringValue(params.url);
if (requestId === undefined || requestId.length > 256) {
if (requestId === undefined) {
state.completeness.exclude(
"websocket_connections",
"invalid_protocol_value",
+1 -1
View File
@@ -131,7 +131,7 @@ export class CdpCaptureEvents implements CdpCaptureEventsState {
}
responseBodyUnavailable(requestId: string): void {
bodyShapes.updateResponseBodyShape(this, requestId, null, false);
bodyShapes.updateResponseBodyShape(this, requestId, null);
this.completeness.unavailable("json_body_shapes");
}
+4 -4
View File
@@ -85,10 +85,10 @@ export const captureStorage = async (
origin,
usage_bytes: numberValue(quota?.usage) ?? null,
quota_bytes: numberValue(quota?.quota) ?? null,
local_storage_keys: local.items.map(({ key }) => key.slice(0, 1_024)),
session_storage_keys: session.items.map(({ key }) => key.slice(0, 1_024)),
indexed_db_names: indexed.map((name) => name.slice(0, 1_024)),
cache_names: caches.map(({ name }) => name.slice(0, 1_024)),
local_storage_keys: local.items.map(({ key }) => key),
session_storage_keys: session.items.map(({ key }) => key),
indexed_db_names: [...indexed],
cache_names: caches.map(({ name }) => name),
content_fingerprints: [...fingerprints.items],
fingerprint_algorithm: "sha256",
fingerprints_complete: fingerprints.complete && structuredStorageComplete,
+7 -20
View File
@@ -10,8 +10,6 @@ import { CdpConnection } from "./CdpConnection.js";
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
import { recordValue, recordsValue, stringValue } from "./CdpCaptureValues.js";
const MAX_CACHE_BODY_BYTES = 64 * 1_024;
type StorageFingerprint =
WebPageInspection["storage"]["content_fingerprints"][number];
@@ -383,8 +381,7 @@ const addCacheEntry = async (
);
const body = stringValue(recordValue(recordValue(raw)?.response)?.body);
const decoded = body === undefined ? null : decodeBase64(body);
const complete =
decoded !== null && decoded.byteLength <= MAX_CACHE_BODY_BYTES;
const complete = decoded !== null;
if (!complete) run.state.complete = false;
addFingerprint(run.state, {
scope: "cache_entry",
@@ -426,7 +423,7 @@ const encodedValue = (value: unknown): Uint8Array | string => {
return encoded;
};
const stableValue = (value: unknown, depth = 0): unknown => {
const stableValue = (value: unknown): unknown => {
if (
value === null ||
typeof value === "string" ||
@@ -434,21 +431,18 @@ const stableValue = (value: unknown, depth = 0): unknown => {
typeof value === "boolean"
)
return value;
if (depth >= 8) return "<depth-limit>";
if (Array.isArray(value))
return value.slice(0, 64).map((item) => stableValue(item, depth + 1));
if (Array.isArray(value)) return value.map((item) => stableValue(item));
const record = recordValue(value);
if (record === undefined) return String(value);
return Object.fromEntries(
Object.entries(record)
.filter(([key]) => key !== "objectId" && key !== "customPreview")
.sort(([left], [right]) => compareCodePoints(left, right))
.slice(0, 64)
.map(([key, item]) => [key, stableValue(item, depth + 1)]),
.map(([key, item]) => [key, stableValue(item)]),
);
};
const stableValueComplete = (value: unknown, depth = 0): boolean => {
const stableValueComplete = (value: unknown): boolean => {
if (value instanceof Uint8Array) return true;
if (
value === null ||
@@ -458,21 +452,14 @@ const stableValueComplete = (value: unknown, depth = 0): boolean => {
typeof value === "boolean"
)
return true;
if (depth >= 8) return false;
if (Array.isArray(value))
return (
value.length <= 64 &&
value.every((item) => stableValueComplete(item, depth + 1))
);
return value.every((item) => stableValueComplete(item));
const record = recordValue(value);
if (record === undefined) return false;
const entries = Object.entries(record).filter(
([key]) => key !== "objectId" && key !== "customPreview",
);
return (
entries.length <= 64 &&
entries.every(([, item]) => stableValueComplete(item, depth + 1))
);
return entries.every(([, item]) => stableValueComplete(item));
};
const remoteObjectComplete = (value: unknown): boolean => {
+1 -5
View File
@@ -23,8 +23,6 @@ interface PendingCommand {
const CONNECT_TIMEOUT_MS = 5_000;
const COMMAND_TIMEOUT_MS = 5_000;
const MAX_PAYLOAD_BYTES = 16 * 1_024 * 1_024;
const MAX_PENDING_COMMANDS = 128;
/** Correlated, bounded JSON-RPC transport for one CDP target operation. */
export class CdpConnection {
@@ -53,7 +51,7 @@ export class CdpConnection {
if (signal?.aborted === true) throw new AnalysisCancelledError(operation);
const socket = new WebSocket(url, {
handshakeTimeout: CONNECT_TIMEOUT_MS,
maxPayload: MAX_PAYLOAD_BYTES,
maxPayload: 0,
perMessageDeflate: false,
});
await waitForOpen(socket, operation, signal);
@@ -87,8 +85,6 @@ export class CdpConnection {
throw new BrowserObservationError(this.operation, "protocol_error");
if (this.#closed || this.socket.readyState !== WebSocket.OPEN)
throw new BrowserObservationError(this.operation, "disconnected");
if (this.#pending.size >= MAX_PENDING_COMMANDS)
throw new BrowserObservationError(this.operation, "payload_limit");
if (signal?.aborted === true)
throw new AnalysisCancelledError(this.operation);
const id = this.#nextId;
+7 -7
View File
@@ -117,7 +117,7 @@ const runElectronInspection = async (
target: {
target_id: context.target.id,
type: context.target.type,
title: context.target.title.slice(0, 16_384),
title: context.target.title,
file_path: main.file_path,
attached: context.target.attached,
},
@@ -267,12 +267,12 @@ const captureFrames = async (
continue;
}
frames.push({
frame_id: frameId.slice(0, 256),
frame_id: frameId,
parent_frame_id: current.parent,
file_path: path,
});
for (const child of recordsValue(current.tree.childFrames))
pending.push({ tree: child, parent: frameId.slice(0, 256) });
pending.push({ tree: child, parent: frameId });
}
return frames;
};
@@ -308,8 +308,8 @@ const captureResources = async (
}
const item = {
file_path: path,
type: (stringValue(resource.type) ?? "Other").slice(0, 100),
mime_type: (stringValue(resource.mimeType) ?? "").slice(0, 256),
type: stringValue(resource.type) ?? "Other",
mime_type: stringValue(resource.mimeType) ?? "",
content_size:
numberValue(resource.contentSize) === undefined
? null
@@ -369,13 +369,13 @@ const captureDom = async (
index: nodes.length,
parent_index: parent < 0 ? -1 : baseIndex + parent,
node_type: integer(nodeTypes[index], 0),
node_name: stringAt(strings, nodeNames[index]).slice(0, 1_024),
node_name: stringAt(strings, nodeNames[index]),
node_value_length: Buffer.byteLength(
stringAt(strings, nodeValues[index]),
),
attribute_names: attributeIndexes
.filter((_value, attributeIndex) => attributeIndex % 2 === 0)
.map((value) => stringAt(strings, value).slice(0, 1_024)),
.map((value) => stringAt(strings, value)),
});
}
}
+1 -1
View File
@@ -82,7 +82,7 @@ export class CdpElectronProvider implements ElectronObservationPort {
allowed.push({
target_id: target.id,
type: target.type,
title: target.title.slice(0, 16_384),
title: target.title,
file_path: path,
attached: target.attached,
});
+7 -26
View File
@@ -1,11 +1,6 @@
import type { CdpEvent } from "./CdpConnection.js";
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
import {
boundedText,
numberValue,
recordValue,
stringValue,
} from "./CdpCaptureValues.js";
import { numberValue, recordValue, stringValue } from "./CdpCaptureValues.js";
export interface ElectronScriptDraft {
readonly scriptId: string;
@@ -17,7 +12,7 @@ export interface ElectronScriptDraft {
readonly executionContextKey: string | null;
}
/** Retain only bounded script/frame metadata from passive Electron events. */
/** Retain script/frame metadata from passive Electron events. */
export const ingestElectronScriptEvent = (input: {
readonly event: CdpEvent;
readonly scripts: ElectronScriptDraft[];
@@ -35,18 +30,14 @@ export const ingestElectronScriptEvent = (input: {
return;
}
if (input.event.method === "Runtime.executionContextCreated") {
retainExecutionContext(
input.event,
input.executionContextFrames,
input.completeness,
);
retainExecutionContext(input.event, input.executionContextFrames);
return;
}
if (input.event.method !== "Debugger.scriptParsed") return;
const value = recordValue(input.event.params);
const scriptId = stringValue(value?.scriptId);
const rawUrl = stringValue(value?.url);
if (scriptId === undefined || scriptId.length > 256) {
if (scriptId === undefined) {
input.completeness.exclude("scripts", "invalid_protocol_value");
return;
}
@@ -54,17 +45,13 @@ export const ingestElectronScriptEvent = (input: {
input.completeness.exclude("scripts", "unattributed_origin");
return;
}
if (input.scripts.length >= 2_000) {
input.completeness.drop("scripts");
return;
}
input.scripts.push({
scriptId,
rawUrl,
hash: (stringValue(value?.hash) ?? "").slice(0, 512),
hash: stringValue(value?.hash) ?? "",
length: nonnegativeInteger(value?.length),
isModule: value?.isModule === true,
language: boundedText(value?.scriptLanguage, 100),
language: stringValue(value?.scriptLanguage) ?? null,
executionContextKey: executionContextKey(value?.executionContextId),
});
};
@@ -72,18 +59,12 @@ export const ingestElectronScriptEvent = (input: {
const retainExecutionContext = (
event: CdpEvent,
frames: Map<string, string>,
completeness: CdpCaptureCompleteness,
): void => {
const parameters = recordValue(event.params);
const runtimeContext = recordValue(parameters?.context);
const contextKey = executionContextKey(runtimeContext?.id);
const frameId = stringValue(recordValue(runtimeContext?.auxData)?.frameId);
if (contextKey === null || frameId === undefined || frameId.length > 256)
return;
if (frames.size >= 2_000 && !frames.has(contextKey)) {
completeness.truncate("scripts");
return;
}
if (contextKey === null || frameId === undefined) return;
frames.set(contextKey, frameId);
};
-15
View File
@@ -6,7 +6,6 @@ import type {
ElectronPageInspection,
InspectElectronPageInput,
} from "../domain/electronObservation.js";
import { MAX_ELECTRON_SCRIPT_SOURCE_BYTES } from "../domain/electronObservation.js";
import { createWebTextArtifact } from "../domain/webContentArtifact.js";
import type { CdpConnection } from "./CdpConnection.js";
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
@@ -91,8 +90,6 @@ const captureScriptSource = async (
reason: "source capture was not approved",
},
};
if (script.length > MAX_ELECTRON_SCRIPT_SOURCE_BYTES)
return sourceLimitResult(input.completeness);
const result = requiredRecord(
await input.connection.send(
"Debugger.getScriptSource",
@@ -102,9 +99,6 @@ const captureScriptSource = async (
),
);
const text = stringValue(result.scriptSource) ?? "";
const bytes = Buffer.byteLength(text);
if (bytes > MAX_ELECTRON_SCRIPT_SOURCE_BYTES)
return sourceLimitResult(input.completeness);
return {
source: {
included: true,
@@ -113,15 +107,6 @@ const captureScriptSource = async (
};
};
const sourceLimitResult = (
completeness: CdpCaptureCompleteness,
): { readonly source: ElectronScriptSource } => {
completeness.truncate("script_sources");
return {
source: { included: false, reason: "script source byte limit reached" },
};
};
const digest = (value: unknown): string => {
const encoded = canonicalize(value);
if (encoded === undefined) throw new TypeError("Expected canonical JSON");
+4 -9
View File
@@ -4,12 +4,7 @@ import type {
} from "../domain/electronObservation.js";
import type { CdpConnection } from "./CdpConnection.js";
import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js";
import {
boundedText,
recordValue,
recordsValue,
stringValue,
} from "./CdpCaptureValues.js";
import { recordValue, recordsValue, stringValue } from "./CdpCaptureValues.js";
import type { CdpEndpointTarget } from "./CdpEndpoint.js";
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
import { authorizedElectronFile } from "./ElectronFileScope.js";
@@ -67,7 +62,7 @@ export const captureElectronWorkers = async (input: {
}
workers.push({
target_id: targetId,
type: type.slice(0, 100),
type,
file_path: path,
attached: target.attached === true,
opener_target_id: openerTargetId,
@@ -80,6 +75,6 @@ export const captureElectronWorkers = async (input: {
};
const boundedTargetField = (value: unknown): string | null => {
const field = boundedText(value, 256);
return field === null || field === "" ? null : field;
const field = stringValue(value);
return field === undefined || field === "" ? null : field;
};
+24 -44
View File
@@ -12,25 +12,23 @@ import {
import { sanitizeBrowserUrl } from "../domain/browserObservation.js";
const HTTP_TIMEOUT_MS = 5_000;
const MAX_VERSION_BYTES = 64 * 1_024;
const MAX_TARGET_LIST_BYTES = 2 * 1_024 * 1_024;
const endpointVersionSchema = z.object({
Browser: z.string().min(1).max(1_024),
"Protocol-Version": z.string().min(1).max(100),
"User-Agent": z.string().max(16_384),
"V8-Version": z.string().max(1_024),
"WebKit-Version": z.string().max(1_024),
webSocketDebuggerUrl: z.string().min(1).max(2_048),
Browser: z.string().min(1),
"Protocol-Version": z.string().min(1),
"User-Agent": z.string(),
"V8-Version": z.string(),
"WebKit-Version": z.string(),
webSocketDebuggerUrl: z.string().min(1),
});
const endpointTargetSchema = z.object({
id: z.string().min(1).max(256),
type: z.string().min(1).max(100),
title: z.string().max(16_384).default(""),
url: z.string().max(65_536),
id: z.string().min(1),
type: z.string().min(1),
title: z.string().default(""),
url: z.string(),
attached: z.boolean().default(false),
webSocketDebuggerUrl: z.string().min(1).max(2_048).optional(),
webSocketDebuggerUrl: z.string().min(1).optional(),
});
const endpointTargetsSchema = z.array(endpointTargetSchema);
@@ -68,21 +66,19 @@ export interface CdpEndpointDiscovery {
readonly targets: readonly CdpEndpointTarget[];
}
/** Read bounded CDP discovery endpoints without following redirects. */
/** Read CDP discovery endpoints without following redirects. */
export const discoverCdpEndpoint = async (
endpoint: string,
operation: BrowserObservationOperation,
signal?: AbortSignal,
): Promise<CdpEndpointDiscovery> => {
const versionInput = await readBoundedCdpJson(
const versionInput = await readCdpJson(
new URL("/json/version", endpoint),
MAX_VERSION_BYTES,
operation,
signal,
);
const targetsInput = await readBoundedCdpJson(
const targetsInput = await readCdpJson(
new URL("/json/list", endpoint),
MAX_TARGET_LIST_BYTES,
operation,
signal,
);
@@ -128,7 +124,7 @@ const sanitizeTargetTitle = (value: string, targetUrl: string): string => {
return sanitizeTargetUrlAlias(value, targetUrl);
}
return url.protocol === "http:" || url.protocol === "https:"
? sanitizeBrowserUrl(value).url.slice(0, 16_384)
? sanitizeBrowserUrl(value).url
: sanitizeTargetUrlAlias(value, targetUrl);
};
@@ -145,11 +141,9 @@ const sanitizeTargetUrlAlias = (value: string, targetUrl: string): string => {
`(?:${markers.map(escapeRegExp).join("|")})[^\\s<>"']*`,
"gu",
);
return value
.replace(candidatePattern, (candidate) =>
sanitizeSameOriginTitleUrl(candidate, parsed),
)
.slice(0, 16_384);
return value.replace(candidatePattern, (candidate) =>
sanitizeSameOriginTitleUrl(candidate, parsed),
);
};
const sanitizeSameOriginTitleUrl = (candidate: string, target: URL): string => {
@@ -250,22 +244,19 @@ const cdpWebSocketPath = (
| { readonly scope: "browser" }
| { readonly scope: "page"; targetId: string }
| undefined => {
const browserId = pathIdentifier(pathname, "/devtools/browser/", 1_024);
const browserId = pathIdentifier(pathname, "/devtools/browser/");
if (browserId !== undefined) return { scope: "browser" };
const targetId = pathIdentifier(pathname, "/devtools/page/", 256);
const targetId = pathIdentifier(pathname, "/devtools/page/");
return targetId === undefined ? undefined : { scope: "page", targetId };
};
const pathIdentifier = (
pathname: string,
prefix: string,
maximumLength: number,
): string | undefined => {
if (!pathname.startsWith(prefix)) return undefined;
const identifier = pathname.slice(prefix.length);
return identifier.length > 0 &&
identifier.length <= maximumLength &&
!identifier.includes("/")
return identifier.length > 0 && !identifier.includes("/")
? identifier
: undefined;
};
@@ -290,10 +281,9 @@ const targetWebSocket = (
}
};
/** Read one bounded loopback CDP discovery document without redirects. */
export const readBoundedCdpJson = async (
/** Read one loopback CDP discovery document without redirects. */
export const readCdpJson = async (
url: URL,
maximumBytes: number,
operation: BrowserObservationOperation,
signal?: AbortSignal,
): Promise<unknown> =>
@@ -314,17 +304,7 @@ export const readBoundedCdpJson = async (
return;
}
const chunks: Buffer[] = [];
let bytes = 0;
response.on("data", (chunk: Buffer) => {
bytes += chunk.length;
if (bytes > maximumBytes) {
response.destroy(
new BrowserObservationError(operation, "payload_limit"),
);
return;
}
chunks.push(chunk);
});
response.on("data", (chunk: Buffer) => chunks.push(chunk));
response.on("end", () => {
try {
resolve(JSON.parse(Buffer.concat(chunks).toString("utf8")));
+4 -15
View File
@@ -52,12 +52,7 @@ export const observeCdpSession = async (
const mainFrameId = frameId(initial);
if (mainFrameId === undefined)
throw new BrowserObservationError("inspect_web_page", "protocol_error");
const capture = new TimelineCapture(
context.input.max_timeline_events,
allowedOrigins,
mainFrameId,
initialUrl,
);
const capture = new TimelineCapture(allowedOrigins, mainFrameId, initialUrl);
const removeListener = context.connection.onEvent((event) => {
if (observationEventMatches(event, context.sessionId))
capture.ingest(event);
@@ -74,7 +69,7 @@ export const observeCdpSession = async (
);
await context.connection.send(
"Network.enable",
{ maxTotalBufferSize: 1_024 * 1_024 },
{},
context.sessionId,
context.signal,
);
@@ -143,7 +138,6 @@ class TimelineCapture {
readonly #listeners = new Set<(reason: EndReason) => void>();
constructor(
private readonly maximum: number,
private readonly allowedOrigins: ReadonlySet<string>,
private readonly mainFrameId: string,
initialUrl: string,
@@ -320,10 +314,6 @@ class TimelineCapture {
} = options;
const destination =
rawUrl == null ? null : scopedUrl(rawUrl, this.allowedOrigins);
if (this.timeline.length >= this.maximum) {
this.completeness.drop("timeline_events");
return;
}
this.#sequence += 1;
this.timeline.push({
sequence: this.#sequence,
@@ -331,7 +321,7 @@ class TimelineCapture {
timestamp: Math.max(0, numberValue(params.timestamp) ?? 0),
frame_id: boundedId(params.frameId),
loader_id: boundedId(params.loaderId),
request_id: requestId?.slice(0, 256) ?? null,
request_id: requestId ?? null,
url: destination?.scope === "approved" ? destination.url : null,
destination_scope: destination?.scope ?? null,
detail,
@@ -449,8 +439,7 @@ const scopedUrl = (
: { url: null, scope: "unsupported" };
};
const boundedId = (value: unknown): string | null =>
stringValue(value)?.slice(0, 256) ?? null;
const boundedId = (value: unknown): string | null => stringValue(value) ?? null;
const safeDetail = (value: unknown): string | null => {
const detail = stringValue(value)?.toLowerCase();
+4 -20
View File
@@ -319,21 +319,8 @@ const enableObservationDomains = async (
signal?: AbortSignal,
): Promise<void> => {
await connection.send("Runtime.enable", {}, sessionId, signal);
await connection.send(
"Debugger.enable",
{ maxScriptsCacheSize: 4 * 1_024 * 1_024 },
sessionId,
signal,
);
await connection.send(
"Network.enable",
{
maxTotalBufferSize: 4 * 1_024 * 1_024,
maxResourceBufferSize: 1_024 * 1_024,
},
sessionId,
signal,
);
await connection.send("Debugger.enable", {}, sessionId, signal);
await connection.send("Network.enable", {}, sessionId, signal);
};
const accessibilityForFrames = async (
@@ -347,7 +334,7 @@ const accessibilityForFrames = async (
const result = await optionalCdpCommand(
context,
"Accessibility.getFullAXTree",
{ depth: 32, frameId: frame.frame_id },
{ frameId: frame.frame_id },
limitations,
);
if (result === undefined) unavailable = true;
@@ -356,7 +343,6 @@ const accessibilityForFrames = async (
if (unavailable) events.completeness.unavailable("accessibility");
const capture = captureAccessibility(results, {
includeText: context.input.include_accessibility_text,
maximumFieldBytes: context.input.limits.max_ax_text_field_bytes,
...(results.length === 0 && unavailable ? { unavailable: true } : {}),
});
if (!context.input.include_accessibility_text)
@@ -365,8 +351,6 @@ const accessibilityForFrames = async (
"not_approved",
capture.textCapture.excluded_fields,
);
if (capture.textCapture.status === "truncated")
events.completeness.truncate("accessibility");
if (capture.treeIncomplete) events.completeness.truncate("accessibility");
return capture;
};
@@ -395,7 +379,7 @@ const normalizedTarget = (
return {
target_id: target.id,
type: target.type,
title: target.title.slice(0, 16_384),
title: target.title,
url: url?.url ?? "[unsupported-url]",
origin: url?.origin ?? "",
attached: target.attached,
+1 -10
View File
@@ -106,11 +106,7 @@ const captureScriptSource = async (
script: CapturedScript,
state: ScriptDraftState,
): Promise<WebPageInspection["scripts"]["items"][number]["source"]> => {
const { context, events } = state;
if (script.length > context.input.limits.max_script_source_bytes) {
events.completeness.truncate("script_sources");
return sourceExcluded("declared script length exceeds per-script limit");
}
const { context } = state;
const result = requiredRecord(
await context.connection.send(
"Debugger.getScriptSource",
@@ -120,11 +116,6 @@ const captureScriptSource = async (
),
);
const content = stringValue(result.scriptSource) ?? "";
const bytes = Buffer.byteLength(content);
if (bytes > context.input.limits.max_script_source_bytes) {
events.completeness.truncate("script_sources");
return sourceExcluded("actual source exceeds the per-script byte limit");
}
return sourceResult(content);
};
+4 -5
View File
@@ -2,7 +2,6 @@ import type { WebPageInspection } from "../domain/browserObservation.js";
import type { CdpCaptureEvents } from "./CdpCaptureEvents.js";
import {
allowedSanitizedUrl,
boundedText,
isHttpUrl,
recordValue,
recordsValue,
@@ -57,13 +56,13 @@ export const captureWorkers = async (
continue;
}
items.push({
target_id: (stringValue(target.targetId) ?? "").slice(0, 256),
type: type.slice(0, 100),
target_id: stringValue(target.targetId) ?? "",
type,
url: url.url,
origin: url.origin,
attached: target.attached === true,
opener_target_id: boundedText(target.openerId, 256) ?? null,
parent_frame_id: boundedText(target.parentFrameId, 256) ?? null,
opener_target_id: stringValue(target.openerId) ?? null,
parent_frame_id: stringValue(target.parentFrameId) ?? null,
});
}
return items;
+19 -23
View File
@@ -68,15 +68,15 @@ const normalizedHeaders = (value: unknown): ReadonlyMap<string, string> => {
if (typeof value !== "object" || value === null || Array.isArray(value))
return new Map();
const headers = new Map<string, string>();
for (const [name, raw] of Object.entries(value).slice(0, 500)) {
const normalized = name.toLowerCase().slice(0, 256);
for (const [name, raw] of Object.entries(value)) {
const normalized = name.toLowerCase();
const header =
typeof raw === "string"
? raw
: typeof raw === "number" && Number.isFinite(raw)
? String(raw)
: undefined;
if (header !== undefined) headers.set(normalized, header.slice(0, 65_536));
if (header !== undefined) headers.set(normalized, header);
}
return headers;
};
@@ -89,13 +89,13 @@ const parseCsp = (
const directives: ResponseMetadata["csp"]["directives"] = [];
let nonceCount = 0;
let hashCount = 0;
for (const rawDirective of (value ?? "").split(";").slice(0, 100)) {
for (const rawDirective of (value ?? "").split(";")) {
const [rawName, ...tokens] = rawDirective.trim().split(/\s+/u);
const name = (rawName ?? "").toLowerCase();
if (!/^[a-z][a-z0-9-]{0,99}$/u.test(name)) continue;
if (!/^[a-z][a-z0-9-]*$/u.test(name)) continue;
const sources: ResponseMetadata["csp"]["directives"][number]["sources"] =
[];
for (const token of tokens.slice(0, 100)) {
for (const token of tokens) {
const lower = token.toLowerCase();
if (lower.startsWith("'nonce-")) {
nonceCount += 1;
@@ -122,10 +122,9 @@ const cspSource = (
allowedOrigins: ReadonlySet<string>,
): ResponseMetadata["csp"]["directives"][number]["sources"][number] => {
const lower = token.toLowerCase();
if (/^'[a-z0-9-]+'$/u.test(lower))
return { kind: "keyword", value: lower.slice(0, 100) };
if (/^'[a-z0-9-]+'$/u.test(lower)) return { kind: "keyword", value: lower };
if (/^[a-z][a-z0-9+.-]*:$/u.test(lower))
return { kind: "scheme", value: lower.slice(0, 100) };
return { kind: "scheme", value: lower };
try {
const parsed = new URL(token, baseUrl);
return allowedOrigins.has(parsed.origin)
@@ -142,8 +141,8 @@ const parseLinks = (
allowedOrigins: ReadonlySet<string>,
): LinkMetadata[] => {
const links: LinkMetadata[] = [];
for (const entry of splitLinkHeader(value ?? "").slice(0, 100)) {
const match = /^\s*<([^>]{1,4096})>(.*)$/u.exec(entry);
for (const entry of splitLinkHeader(value ?? "")) {
const match = /^\s*<([^>]*)>(.*)$/u.exec(entry);
if (match === null) continue;
const parameters = linkParameters(match[2] ?? "");
const destination = safeDestination(
@@ -157,8 +156,7 @@ const parseLinks = (
rel: (parameters.get("rel") ?? "")
.toLowerCase()
.split(/\s+/u)
.filter(Boolean)
.slice(0, 32),
.filter(Boolean),
as: boundedHeader(parameters.get("as")),
type: boundedHeader(parameters.get("type")),
crossorigin: boundedHeader(parameters.get("crossorigin")),
@@ -184,14 +182,14 @@ const splitLinkHeader = (value: string): string[] => {
const linkParameters = (value: string): ReadonlyMap<string, string> => {
const parameters = new Map<string, string>();
for (const raw of value.split(";").slice(1, 50)) {
for (const raw of value.split(";").slice(1)) {
const separator = raw.indexOf("=");
const name = (separator < 0 ? raw : raw.slice(0, separator))
.trim()
.toLowerCase();
if (!/^[a-z][a-z0-9-]{0,99}$/u.test(name)) continue;
if (!/^[a-z][a-z0-9-]*$/u.test(name)) continue;
const parameter = separator < 0 ? "" : raw.slice(separator + 1).trim();
parameters.set(name, unquote(parameter).slice(0, 1_024));
parameters.set(name, unquote(parameter));
}
return parameters;
};
@@ -222,11 +220,9 @@ const permissionFeatures = (value: string | undefined): string[] =>
(value ?? "")
.split(",")
.map((entry) => entry.split("=", 1)[0]?.trim().toLowerCase() ?? "")
.filter((feature) => /^[a-z][a-z0-9-]{0,99}$/u.test(feature)),
.filter((feature) => /^[a-z][a-z0-9-]*$/u.test(feature)),
),
]
.sort()
.slice(0, 200);
].sort();
const wellKnownHint = (url: string): AgentHint[] => {
try {
@@ -245,7 +241,7 @@ const agentHint = (
url: string | null,
): AgentHint => ({
mechanism,
declaration: declaration.slice(0, 256),
declaration,
url,
trust: "page-declared-untrusted",
});
@@ -255,7 +251,7 @@ const isAgentRel = (value: string): boolean =>
const policyToken = (value: string | undefined): string | null => {
const token = (value ?? "").trim().toLowerCase();
return /^[a-z][a-z0-9_.-]{0,99}$/u.test(token) ? token : null;
return /^[a-z][a-z0-9_.-]*$/u.test(token) ? token : null;
};
const nonnegativeInteger = (value: string | undefined): number | null => {
@@ -264,7 +260,7 @@ const nonnegativeInteger = (value: string | undefined): number | null => {
};
const boundedHeader = (value: string | undefined): string | null =>
value === undefined ? null : value.trim().toLowerCase().slice(0, 256);
value === undefined ? null : value.trim().toLowerCase();
const unquote = (value: string): string =>
value.startsWith('"') && value.endsWith('"') ? value.slice(1, -1) : value;
+3 -6
View File
@@ -83,10 +83,7 @@ export const captureCdpScreenshot = async (
const encoded = stringValue(result?.data);
const bytes =
encoded === undefined ? undefined : decodeCanonicalBase64(encoded);
if (
bytes === undefined ||
bytes.byteLength > context.input.maximum_image_bytes
)
if (bytes === undefined)
throw new BrowserObservationError("inspect_web_page", "payload_limit");
const dimensions = pngDimensions(bytes);
if (navigation.leftScope)
@@ -155,8 +152,8 @@ const pngDimensions = (
throw new BrowserObservationError("inspect_web_page", "protocol_error");
const width = bytes.readUInt32BE(16);
const height = bytes.readUInt32BE(20);
if (width === 0 || height === 0 || width * height > 32_000_000)
throw new BrowserObservationError("inspect_web_page", "payload_limit");
if (width === 0 || height === 0)
throw new BrowserObservationError("inspect_web_page", "protocol_error");
return { width, height };
};
+1 -2
View File
@@ -73,8 +73,7 @@ const attachedSessionId = (
value === null ||
!("sessionId" in value) ||
typeof value.sessionId !== "string" ||
value.sessionId.length === 0 ||
value.sessionId.length > 256
value.sessionId.length === 0
)
throw new BrowserObservationError(operation, "protocol_error");
return value.sessionId;
+8 -17
View File
@@ -21,7 +21,7 @@ import {
} from "./CdpCaptureValues.js";
import { captureFrames, mainFrameUrl } from "./CdpCaptureDocuments.js";
import { optionalCdpCommand } from "./CdpOptionalCommand.js";
import { boundedSensitiveText } from "./SensitiveTextCapture.js";
import { redactSensitiveText } from "./SensitiveTextCapture.js";
interface DiscoveryContext {
readonly connection: CdpConnection;
@@ -106,7 +106,7 @@ export const discoverWebMcp = async (
} else
// CDP sends the enable response before the required toolsAdded replay.
await delayWithCancellation(
Math.max(context.input.observation_ms, 25),
context.input.observation_ms,
"discover_webmcp_tools",
context.signal,
);
@@ -167,7 +167,7 @@ const handleFrameDetached = (
state: FrameScopeState,
): boolean => {
const frameId = stringValue(params.frameId);
if (frameId === undefined || frameId.length > 256) {
if (frameId === undefined) {
state.completeness.exclude("webmcp_tools", "invalid_protocol_value");
return true;
}
@@ -180,7 +180,7 @@ const handleFrameUpdate = (
state: FrameScopeState,
): boolean => {
const frameId = stringValue(frame?.id) ?? stringValue(frame?.frameId);
if (frameId === undefined || frameId.length > 256) {
if (frameId === undefined) {
state.completeness.exclude("webmcp_tools", "invalid_protocol_value");
return true;
}
@@ -293,16 +293,13 @@ const normalizeTool = (
completeness.exclude("webmcp_tools", "out_of_target_scope");
return undefined;
}
const description = boundedSensitiveText(
stringValue(value.description) ?? "",
4_096,
).text;
const description = redactSensitiveText(stringValue(value.description) ?? "");
const annotations = recordValue(value.annotations);
return {
tool_key: toolKey(frame.url, name),
name: boundedSensitiveText(name, 512).text,
name: redactSensitiveText(name),
description,
frame_id: frameId.slice(0, 256),
frame_id: frameId,
frame_url: frame.url,
owner_origin: frame.origin,
declaration_kind:
@@ -326,15 +323,9 @@ const normalizeTool = (
const schemaShape = (value: unknown) => {
if (recordValue(value) === undefined) return null;
const encoded = JSON.stringify(value);
const shape = inferJsonShape(encoded, {
maximumBytes: 256 * 1_024,
maximumNodes: 5_000,
maximumDepth: 20,
});
const shape = inferJsonShape(encoded);
if (shape === null)
throw new BrowserObservationError("inspect_web_page", "protocol_error");
if (shape.truncated)
throw new BrowserObservationError("inspect_web_page", "payload_limit");
return shape;
};
+3 -4
View File
@@ -49,17 +49,16 @@ describe("PNG visual diff", () => {
).toMatchObject({ status: "dimension_mismatch", compared_pixels: 0 });
});
it("rejects images above the caller's decoded pixel limit", () => {
const image = artifact(2, 1, [0, 0, 0, 255, 0, 0, 0, 255]);
it("rejects malformed PNG dimensions after validating image data", () => {
const image = artifact(32_000_001, 1, [0, 0, 0, 255]);
expect(() =>
comparePngScreenshots(
compareWebScreenshotsInputSchema.parse({
before: image,
after: image,
maximum_pixels: 1,
}),
),
).toThrow("oversized PNG");
).toThrow("Unexpected PNG size");
});
it("rejects metrics that cannot represent their comparison status", () => {
+8 -11
View File
@@ -13,12 +13,12 @@ interface DecodedPng {
readonly rgba: Buffer;
}
/** Compare two validated screenshots using bounded, deterministic RGBA metrics. */
/** Compare two validated screenshots using deterministic RGBA metrics. */
export const comparePngScreenshots = (
input: CompareWebScreenshotsInput,
): WebScreenshotDiff => {
const before = decodePng(input.before, input.maximum_pixels);
const after = decodePng(input.after, input.maximum_pixels);
const before = decodePng(input.before);
const after = decodePng(input.after);
if (before.width !== after.width || before.height !== after.height)
return {
status: "dimension_mismatch",
@@ -73,10 +73,7 @@ export const comparePngScreenshots = (
};
};
const decodePng = (
artifact: WebScreenshotArtifact,
maximumPixels: number,
): DecodedPng => {
const decodePng = (artifact: WebScreenshotArtifact): DecodedPng => {
const bytes = decodeCanonicalBase64(artifact.data_base64);
if (bytes === undefined || !bytes.subarray(0, 8).equals(PNG_SIGNATURE))
throw new TypeError("Invalid PNG signature");
@@ -92,7 +89,7 @@ const decodePng = (
const dataEnd = dataStart + length;
if (dataEnd + 4 > bytes.length) throw new TypeError("Truncated PNG data");
const data = bytes.subarray(dataStart, dataEnd);
if (type === "IHDR") header = parseHeader(data, maximumPixels);
if (type === "IHDR") header = parseHeader(data);
else if (type === "IDAT") compressed.push(data);
else if (type === "IEND") {
sawEnd = true;
@@ -115,7 +112,7 @@ const decodePng = (
};
};
const parseHeader = (data: Buffer, maximumPixels: number) => {
const parseHeader = (data: Buffer) => {
if (data.byteLength !== 13) throw new TypeError("Invalid PNG header");
const width = data.readUInt32BE(0);
const height = data.readUInt32BE(4);
@@ -124,14 +121,14 @@ const parseHeader = (data: Buffer, maximumPixels: number) => {
if (
width === 0 ||
height === 0 ||
width * height > maximumPixels ||
!Number.isSafeInteger(width * height) ||
bitDepth !== 8 ||
(colorType !== 2 && colorType !== 6) ||
data[10] !== 0 ||
data[11] !== 0 ||
data[12] !== 0
)
throw new TypeError("Unsupported or oversized PNG");
throw new TypeError("Unsupported PNG");
return { width, height, channels: colorType === 6 ? 4 : 3 };
};
+1 -12
View File
@@ -1,9 +1,6 @@
import { describe, expect, it } from "vitest";
import {
boundedSensitiveText,
redactSensitiveText,
} from "./SensitiveTextCapture.js";
import { redactSensitiveText } from "./SensitiveTextCapture.js";
describe("sensitive console text", () => {
it("redacts assignments, bearer credentials, and JWT-shaped values", () => {
@@ -16,12 +13,4 @@ describe("sensitive console text", () => {
"authorization=[REDACTED] password=[REDACTED] api_key=[REDACTED] [REDACTED_JWT]",
);
});
it("bounds UTF-8 after redaction without splitting a code point", () => {
expect(boundedSensitiveText("password=secret 你好", 22)).toEqual({
text: "password=[REDACTED] ",
bytes: 20,
truncated: true,
});
});
});
-24
View File
@@ -9,27 +9,3 @@ export const redactSensitiveText = (value: string): string =>
.replace(SECRET_ASSIGNMENT, (_match, name: string) => `${name}=[REDACTED]`)
.replace(BEARER, "Bearer [REDACTED]")
.replace(JWT, "[REDACTED_JWT]");
/** Return a UTF-8-bounded prefix without splitting a code point. */
export const boundedSensitiveText = (
value: string,
maximumBytes: number,
): {
readonly text: string;
readonly bytes: number;
readonly truncated: boolean;
} => {
const redacted = redactSensitiveText(value);
const total = Buffer.byteLength(redacted);
if (total <= maximumBytes)
return { text: redacted, bytes: total, truncated: false };
let text = "";
let bytes = 0;
for (const character of redacted) {
const size = Buffer.byteLength(character);
if (bytes + size > maximumBytes) break;
text += character;
bytes += size;
}
return { text, bytes, truncated: true };
};
+11 -24
View File
@@ -8,26 +8,23 @@ import {
BrowserObservationError,
type BrowserObservationOperation,
} from "../domain/errors.js";
import { parseCdpEndpointValue, readBoundedCdpJson } from "./CdpEndpoint.js";
const MAX_VERSION_BYTES = 64 * 1_024;
const MAX_TARGET_LIST_BYTES = 2 * 1_024 * 1_024;
import { parseCdpEndpointValue, readCdpJson } from "./CdpEndpoint.js";
const versionSchema = z
.object({
Browser: z.string().min(1).max(1_024),
"Protocol-Version": z.string().min(1).max(100),
"V8-Version": z.string().max(1_024).optional(),
Browser: z.string().min(1),
"Protocol-Version": z.string().min(1),
"V8-Version": z.string().optional(),
})
.passthrough();
const targetSchema = z
.object({
id: z.string().min(1).max(256),
type: z.string().min(1).max(100),
url: z.string().max(65_536),
id: z.string().min(1),
type: z.string().min(1),
url: z.string(),
attached: z.boolean().default(false),
webSocketDebuggerUrl: z.string().min(1).max(2_048),
webSocketDebuggerUrl: z.string().min(1),
})
.passthrough();
const targetsSchema = z.array(targetSchema);
@@ -45,25 +42,15 @@ export interface V8InspectorDiscovery {
readonly targets: readonly V8InspectorTarget[];
}
/** Discover bounded Node/Electron Inspector targets from one loopback endpoint. */
/** Discover Node/Electron Inspector targets from one loopback endpoint. */
export const discoverV8Inspector = async (
endpoint: string,
operation: BrowserObservationOperation,
signal?: AbortSignal,
): Promise<V8InspectorDiscovery> => {
const [versionInput, targetsInput] = await Promise.all([
readBoundedCdpJson(
new URL("/json/version", endpoint),
MAX_VERSION_BYTES,
operation,
signal,
),
readBoundedCdpJson(
new URL("/json/list", endpoint),
MAX_TARGET_LIST_BYTES,
operation,
signal,
),
readCdpJson(new URL("/json/version", endpoint), operation, signal),
readCdpJson(new URL("/json/list", endpoint), operation, signal),
]);
const version = parseCdpEndpointValue(versionSchema, versionInput, operation);
const targets = parseCdpEndpointValue(targetsSchema, targetsInput, operation);
+52 -165
View File
@@ -31,33 +31,14 @@ export const fetchWebSourceMaps = async (
host: SourceMapFetchHost = { fetch: globalThis.fetch },
): Promise<SourceMaps> => {
const items: SourceMapItem[] = [];
let remainingBytes = input.analysis_limits.max_total_source_map_bytes;
for (const request of requests.slice(
0,
input.analysis_limits.max_source_maps,
)) {
for (const request of requests) {
if (signal?.aborted === true) throw signal.reason;
if (remainingBytes === 0) break;
const fetched = await fetchOne(request, {
input,
maximumBytes: Math.min(
input.analysis_limits.max_source_map_bytes,
remainingBytes,
),
signal,
host,
});
items.push(fetched.item);
remainingBytes = Math.max(0, remainingBytes - fetched.bytesRead);
items.push(await fetchOne(request, input, signal, host));
}
const included = items.filter(({ status }) => status === "included").length;
const dropped = requests.length - items.length;
const truncated =
dropped > 0 || items.some(({ status }) => status === "truncated");
return webSourceMapsSchema.parse({
status: truncated
? "truncated"
: items.length === 0
status:
items.length === 0
? "unavailable"
: included === items.length
? "included"
@@ -66,40 +47,22 @@ export const fetchWebSourceMaps = async (
: "unavailable",
requested: requests.length,
processed: items.length,
dropped,
dropped_script_keys: requests
.slice(items.length)
.map(({ scriptKey }) => scriptKey),
items,
});
};
interface SourceMapFetchResult {
readonly item: SourceMapItem;
readonly bytesRead: number;
}
interface SourceMapFetchContext {
readonly input: AnalyzeWebBundleInput;
readonly maximumBytes: number;
readonly signal: AbortSignal | undefined;
readonly host: SourceMapFetchHost;
}
const fetchOne = async (
request: WebSourceMapRequest,
context: SourceMapFetchContext,
): Promise<SourceMapFetchResult> => {
const { input, maximumBytes, signal, host } = context;
input: AnalyzeWebBundleInput,
signal: AbortSignal | undefined,
host: SourceMapFetchHost,
): Promise<SourceMapItem> => {
if (!approvedUrl(request.fetchUrl, input.allowed_origins))
return {
item: emptySourceMapItem(
request,
"policy_filtered",
"Declared source-map URL is outside the approved exact origins.",
),
bytesRead: 0,
};
return emptySourceMapItem(
request,
"policy_filtered",
"Declared source-map URL is outside the approved exact origins.",
);
try {
const response = await fetchFollowingApprovedRedirects(
request.fetchUrl,
@@ -108,40 +71,25 @@ const fetchOne = async (
host,
);
if (response === undefined)
return {
item: emptySourceMapItem(
request,
"policy_filtered",
"A source-map redirect left the approved exact origins.",
),
bytesRead: 0,
};
return emptySourceMapItem(
request,
"policy_filtered",
"A source-map redirect left the approved exact origins.",
);
if (!response.ok)
return {
item: emptySourceMapItem(
request,
"fetch_failed",
`Source-map server returned HTTP ${String(response.status)}.`,
),
bytesRead: 0,
};
const body = await boundedResponseText(response, maximumBytes);
return {
item: normalizeSourceMap(request, body.text, input),
bytesRead: body.bytes,
};
return emptySourceMapItem(
request,
"fetch_failed",
`Source-map server returned HTTP ${String(response.status)}.`,
);
return normalizeSourceMap(request, await response.text());
} catch (cause: unknown) {
if (signal?.aborted === true) throw cause;
return {
item: emptySourceMapItem(
request,
isLimitError(cause) ? "truncated" : "fetch_failed",
isLimitError(cause)
? "Source-map response exceeded the remaining approved byte budget."
: "Source-map fetch or validation failed.",
),
bytesRead: cause instanceof SourceMapByteLimitError ? cause.bytesRead : 0,
};
return emptySourceMapItem(
request,
"fetch_failed",
"Source-map fetch or validation failed.",
);
}
};
@@ -152,75 +100,37 @@ const fetchFollowingApprovedRedirects = async (
host: SourceMapFetchHost,
): Promise<Response | undefined> => {
let current = initialUrl;
for (let redirect = 0; redirect <= 5; redirect += 1) {
const visited = new Set<string>();
for (;;) {
if (!approvedUrl(current, allowedOrigins)) return undefined;
const response = await timedFetch(current, signal, host);
if (visited.has(current)) throw new Error("source_map_redirect_loop");
visited.add(current);
const response = await host.fetch(current, {
method: "GET",
headers: {
Accept: "application/json, application/source-map+json;q=0.9",
},
redirect: "manual",
credentials: "omit",
referrerPolicy: "no-referrer",
...(signal === undefined ? {} : { signal }),
});
if (response.status < 300 || response.status >= 400) return response;
const location = response.headers.get("location");
if (location === null) return response;
current = new URL(location, current).href;
}
throw new Error("source_map_redirect_limit");
};
const timedFetch = async (
url: string,
signal: AbortSignal | undefined,
host: SourceMapFetchHost,
): Promise<Response> => {
const timeout = AbortSignal.timeout(5_000);
return await host.fetch(url, {
method: "GET",
headers: { Accept: "application/json, application/source-map+json;q=0.9" },
redirect: "manual",
credentials: "omit",
referrerPolicy: "no-referrer",
signal: signal === undefined ? timeout : AbortSignal.any([signal, timeout]),
});
};
const boundedResponseText = async (
response: Response,
maximumBytes: number,
): Promise<{ readonly text: string; readonly bytes: number }> => {
const declared = Number(response.headers.get("content-length"));
if (Number.isFinite(declared) && declared > maximumBytes)
throw new SourceMapByteLimitError(0);
if (response.body === null) return { text: "", bytes: 0 };
const reader = response.body.getReader();
const decoder = new TextDecoder("utf-8", { fatal: true });
let bytes = 0;
let text = "";
for (;;) {
const chunk = await reader.read();
if (chunk.done) break;
bytes += chunk.value.byteLength;
if (bytes > maximumBytes) {
await reader.cancel();
throw new SourceMapByteLimitError(bytes);
}
text += decoder.decode(chunk.value, { stream: true });
}
text += decoder.decode();
return { text, bytes };
};
class SourceMapByteLimitError extends RangeError {
constructor(readonly bytesRead: number) {
super("source_map_bytes");
}
}
const normalizeSourceMap = (
request: WebSourceMapRequest,
text: string,
input: AnalyzeWebBundleInput,
): SourceMapItem => {
if (!validSourceMapEnvelope(text))
return emptySourceMapItem(
request,
"invalid",
"Source-map JSON is not a bounded version 3 map.",
"Source-map JSON is not a version 3 map.",
);
try {
const map = new AnyMap(text, request.fetchUrl);
@@ -239,7 +149,6 @@ const normalizeSourceMap = (
};
});
const mappings: ParsedSourceMapItem["mappings"] = [];
let totalMappings = 0;
eachMapping(map, (mapping) => {
if (
mapping.source === null ||
@@ -247,9 +156,6 @@ const normalizeSourceMap = (
mapping.originalColumn === null
)
return;
totalMappings += 1;
if (mappings.length >= input.analysis_limits.max_source_map_mappings)
return;
mappings.push({
generated_line: mapping.generatedLine,
generated_column: mapping.generatedColumn,
@@ -258,16 +164,10 @@ const normalizeSourceMap = (
),
original_line: mapping.originalLine,
original_column: mapping.originalColumn,
name: mapping.name?.slice(0, 1_024) ?? null,
name: mapping.name ?? null,
});
});
const modules = originalModuleEdges(originalSources);
const mappingTruncated = totalMappings > mappings.length;
const limitations = mappingTruncated
? [
`Mappings were truncated from ${String(totalMappings)} to ${String(mappings.length)}.`,
]
: [];
const parsed = {
...sourceMapContext(request),
artifact: createWebTextArtifact(text, "application/source-map+json"),
@@ -275,13 +175,7 @@ const normalizeSourceMap = (
original_module_edges: modules,
mappings,
};
return limitations.length === 0
? { ...parsed, status: "included", limitation: null }
: {
...parsed,
status: "truncated",
limitation: limitations.join(" "),
};
return { ...parsed, status: "included", limitation: null };
} catch {
return emptySourceMapItem(
request,
@@ -300,7 +194,7 @@ const originalModuleEdges = (
if (source.artifact === null) continue;
for (const detector of originalImportDetectors) {
for (const match of source.artifact.text.matchAll(detector.pattern)) {
const specifier = match[1]?.slice(0, 4_096);
const specifier = match[1];
if (specifier === undefined) continue;
const key = `${source.source}\0${detector.kind}\0${specifier}`;
if (seen.has(key)) continue;
@@ -327,19 +221,16 @@ const validSourceMapEnvelope = (text: string): boolean => {
if (!isRecord(parsed) || parsed.version !== 3) return false;
if (typeof parsed.mappings === "string")
return Array.isArray(parsed.sources) && Array.isArray(parsed.names);
if (!Array.isArray(parsed.sections) || parsed.sections.length > 10_000)
return false;
if (!Array.isArray(parsed.sections)) return false;
const pending: unknown[] = [...parsed.sections];
let visited = 0;
while (pending.length > 0) {
const section = pending.pop();
if (!isRecord(section) || !isRecord(section.offset) || !("map" in section))
return false;
visited += 1;
if (visited > 10_000) return false;
const map = section.map;
if (!isRecord(map) || map.version !== 3) return false;
if (Array.isArray(map.sections)) pending.push(...map.sections);
if (Array.isArray(map.sections))
for (const child of map.sections) pending.push(child);
else if (
typeof map.mappings !== "string" ||
!Array.isArray(map.sources) ||
@@ -387,14 +278,13 @@ const emptySourceMapItem = (
});
const sanitizeSource = (value: string): string => {
const bounded = value.slice(0, 4_096);
try {
const parsed = new URL(bounded);
const parsed = new URL(value);
return parsed.protocol === "http:" || parsed.protocol === "https:"
? sanitizeBrowserUrl(parsed.href).url
: `${parsed.protocol}//${parsed.host}${parsed.pathname}`;
} catch {
return bounded.split("#", 1)[0]?.split("?", 1)[0] ?? "";
return value.split("#", 1)[0]?.split("?", 1)[0] ?? "";
}
};
@@ -417,9 +307,6 @@ const sourceMediaType = (source: string | null): string =>
const isRecord = (value: unknown): value is Readonly<Record<string, unknown>> =>
typeof value === "object" && value !== null && !Array.isArray(value);
const isLimitError = (value: unknown): value is RangeError =>
value instanceof RangeError;
const originalImportDetectors = [
{
kind: "static_import" as const,