fix(filesystem): bound and verify admitted file and JSON reads

This commit is contained in:
morluto
2026-10-11 00:28:24 +08:00
parent 6d8f95e38d
commit abb8e81d3a
8 changed files with 268 additions and 59 deletions
+12 -5
View File
@@ -16,8 +16,7 @@ import {
AnalysisResourceConstraintError,
} from "../domain/analysisErrorCore.js";
import { err, ok, type Result } from "../domain/result.js";
import { parseUtf8Json } from "./Utf8JsonInput.js";
import { readRegularFile } from "./RegularFileRead.js";
import { readJsonInputFile } from "./JsonInputFile.js";
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
/** Request control and its owning operation for an interruptible atomic write. */
@@ -37,12 +36,20 @@ export const readJsonFile = async (
> => {
const requestedPath = resolve(path);
try {
const encoded = await readRegularFile(requestedPath);
const decoded = parseUtf8Json(encoded, "read_evidence_file", requestedPath);
const decoded = await readJsonInputFile(
requestedPath,
"read_evidence_file",
);
if (!decoded.ok) {
if (decoded.error instanceof AnalysisResourceConstraintError)
return err(decoded.error);
return err(
new EvidenceFileError("read", "invalid-json", {
cause: decoded.cause ?? new SyntaxError(decoded.error),
cause:
decoded.error.cause ??
new SyntaxError(
decoded.error.issues[0]?.message ?? decoded.error.message,
),
path: requestedPath,
}),
);
+372
View File
@@ -0,0 +1,372 @@
import { constants } from "node:buffer";
import type { FileHandle } from "node:fs/promises";
import { pipeline } from "node:stream/promises";
import { getHeapStatistics } from "node:v8";
import parser from "stream-json/parser.js";
import streamValues from "stream-json/streamers/stream-values.js";
import { RegularFileChangedError } from "../filesystem/RegularFile.js";
import { withRegularFile } from "./RegularFileRead.js";
import {
JSON_BYTE_ORDER_MARK_MESSAGE,
JSON_INPUT_RESOURCE_REMEDIATION,
parseUtf8Json,
} from "./Utf8JsonInput.js";
import {
AnalysisInputError,
AnalysisResourceConstraintError,
} from "../domain/analysisErrorCore.js";
import { err, ok, type Result } from "../domain/result.js";
type JsonInputFileFailure =
| AnalysisInputError
| AnalysisResourceConstraintError;
const READ_CHUNK_BYTES = 64 * 1024;
const PACKED_TOKEN_PART_CODE_UNITS = 64 * 1024;
// Retain native-parser speed for modest inputs while bounding its extra copies.
// This selects an implementation; larger files remain accepted through streaming.
const NATIVE_PARSE_READ_BUDGET = 8 * 1024 * 1024;
/** Keep native parsing only when input copies and value expansion fit live headroom. */
const nativeParseFits = (inputBytes: number): boolean =>
inputBytes <= NATIVE_PARSE_READ_BUDGET &&
// The assembler estimate reserves up to 128 bytes per two-byte container.
// Allow that 64x expansion plus byte/text copies before native JSON.parse,
// whose individual container allocations cannot be intercepted.
inputBytes * 72 + READ_CHUNK_BYTES <=
getHeapStatistics().total_available_size;
/** Parse strict UTF-8 JSON without requiring a whole-document string. */
export const readJsonInputFile = (
path: string,
operation: string,
signal?: AbortSignal,
): Promise<Result<unknown, JsonInputFileFailure>> =>
withRegularFile(
path,
async (handle, stats) => {
let found = false;
let value: unknown;
try {
const prefix = nativeParseFits(stats.size)
? await readPrefix(handle, stats.size + 1, signal)
: undefined;
if (prefix !== undefined) {
if (!prefix.complete || prefix.bytes.length !== stats.size)
throw new RegularFileChangedError(path);
const parsed = parseUtf8Json(
prefix.bytes,
operation,
path,
"json-file-input",
);
return parsed.ok
? ok(parsed.value)
: err(invalidJson(operation, parsed.error, parsed.cause));
}
await pipeline(
decodedChunks(handle, stats.size, path, signal),
parser.asStream({
jsonStreaming: false,
streamValues: false,
packKeys: false,
packStrings: false,
packNumbers: false,
}),
packJsonScalars(operation, stats.size, path),
admitJsonAssembly(operation, stats.size, path),
streamValues.asStream(),
async (rows: AsyncIterable<unknown>) => {
for await (const row of rows) {
if (
found ||
typeof row !== "object" ||
row === null ||
!("key" in row) ||
row.key !== 0 ||
!("value" in row)
)
throw new Error("Unexpected JSON parser result");
found = true;
value = row.value;
}
},
signal === undefined ? {} : { signal },
);
return found
? ok(value)
: err(invalidJson(operation, "JSON document is empty"));
} catch (cause: unknown) {
if (cause instanceof AnalysisResourceConstraintError) return err(cause);
if (
cause instanceof TypeError &&
"code" in cause &&
cause.code === "ERR_ENCODING_INVALID_ENCODED_DATA"
)
return err(
invalidJson(operation, "JSON input is not valid UTF-8", cause),
);
if (
cause instanceof SyntaxError ||
(cause instanceof Error && cause.message.startsWith("Parser "))
)
return err(invalidJson(operation, cause.message, cause));
if (
cause instanceof Error &&
((cause instanceof RangeError &&
cause.message === "Invalid string length") ||
("code" in cause && cause.code === "ERR_STRING_TOO_LONG"))
)
return err(
new AnalysisResourceConstraintError(
operation,
"memory",
"An individual JSON string or number token exceeds the runtime string limit",
{
input_path: path,
input_file_bytes: stats.size,
max_string_code_units: constants.MAX_STRING_LENGTH,
},
{
cause,
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
},
),
);
throw cause;
}
},
{ signal },
);
const packJsonScalars = (
operation: string,
inputFileBytes: number,
path: string,
) =>
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
let kind: "keyValue" | "stringValue" | "numberValue" | undefined;
let length = 0;
let value = "";
let parts: string[] = [];
let partLength = 0;
for await (const token of tokens) {
if (typeof token !== "object" || token === null || !("name" in token))
throw new Error("Unexpected JSON parser token");
switch (token.name) {
case "startKey":
case "startString":
kind = token.name === "startKey" ? "keyValue" : "stringValue";
break;
case "startNumber":
kind = "numberValue";
break;
case "numberChunk":
case "stringChunk":
if (
kind === undefined ||
!("value" in token) ||
typeof token.value !== "string"
)
throw new Error("Unexpected JSON scalar fragment");
length += token.value.length;
if (length > constants.MAX_STRING_LENGTH)
throw new RangeError("Invalid string length");
parts.push(token.value);
partLength += token.value.length;
if (partLength >= PACKED_TOKEN_PART_CODE_UNITS) {
// The tokenizer emits short fragments. Coalesce them before retaining
// a scalar so millions of substring/rope nodes cannot exhaust the heap
// before the native string-length constraint can be reported.
requireStringAssemblyHeadroom(
operation,
inputFileBytes,
path,
length,
);
value += parts.join("");
parts = [];
partLength = 0;
}
break;
case "endKey":
case "endString":
case "endNumber":
if (kind === undefined) throw new Error("Unexpected JSON scalar end");
requireStringAssemblyHeadroom(
operation,
inputFileBytes,
path,
length,
);
yield { name: kind, value: value + parts.join("") };
kind = undefined;
length = 0;
value = "";
parts = [];
partLength = 0;
break;
default:
yield token;
}
}
};
/** Admit the retained value graph before the upstream assembler grows it. */
const admitJsonAssembly = (
operation: string,
inputFileBytes: number,
path: string,
) =>
async function* (tokens: AsyncIterable<unknown>): AsyncGenerator<unknown> {
let projectedBytes = 0;
let nextCheck = 0;
for await (const token of tokens) {
if (typeof token !== "object" || token === null || !("name" in token))
throw new Error("Unexpected JSON assembly token");
if (token.name === "startObject" || token.name === "startArray") {
projectedBytes += 128;
} else if (token.name === "keyValue" || token.name === "stringValue") {
if (!("value" in token) || typeof token.value !== "string")
throw new Error("Unexpected packed JSON string");
projectedBytes += 64 + token.value.length * 2;
} else if (
token.name === "numberValue" ||
token.name === "nullValue" ||
token.name === "trueValue" ||
token.name === "falseValue"
) {
projectedBytes += 64;
}
if (projectedBytes >= nextCheck) {
const heap = getHeapStatistics();
// Conservative capacity for containers, property/array slots and UTF-16
// values. Keep another whole projection available for container growth,
// plus the next chunk of estimates between samples. This is admission
// against live runtime headroom, not a fixed document-size policy.
const requiredBytes = projectedBytes + READ_CHUNK_BYTES;
if (requiredBytes > heap.total_available_size)
throw new AnalysisResourceConstraintError(
operation,
"memory",
"Insufficient heap headroom to materialize the complete JSON value",
{
input_path: path,
input_file_bytes: inputFileBytes,
projected_value_bytes: projectedBytes,
assembly_headroom_bytes: requiredBytes,
available_heap_bytes: heap.total_available_size,
heap_size_limit_bytes: heap.heap_size_limit,
},
{ remediationAction: JSON_INPUT_RESOURCE_REMEDIATION },
);
nextCheck = projectedBytes + READ_CHUNK_BYTES;
}
yield token;
}
};
const requireStringAssemblyHeadroom = (
operation: string,
inputFileBytes: number,
path: string,
stringCodeUnits: number,
): void => {
if (stringCodeUnits < PACKED_TOKEN_PART_CODE_UNITS) return;
const heap = getHeapStatistics();
// Concatenation or key interning may flatten the retained rope. Reserve its
// UTF-16 representation and the next coalesced part before that allocation.
const requiredBytes = 2 * (stringCodeUnits + PACKED_TOKEN_PART_CODE_UNITS);
if (requiredBytes > heap.total_available_size)
throw new AnalysisResourceConstraintError(
operation,
"memory",
"Insufficient heap headroom to assemble a complete JSON string",
{
input_path: path,
input_file_bytes: inputFileBytes,
string_code_units: stringCodeUnits,
string_assembly_headroom_bytes: requiredBytes,
available_heap_bytes: heap.total_available_size,
heap_size_limit_bytes: heap.heap_size_limit,
},
{
remediationAction: JSON_INPUT_RESOURCE_REMEDIATION,
},
);
};
async function* decodedChunks(
handle: FileHandle,
admittedBytes: number,
path: string,
signal: AbortSignal | undefined,
): AsyncGenerator<string> {
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
let leading = true;
const checked = (text: string): string => {
if (leading && text.length > 0) {
leading = false;
if (text.startsWith("\uFEFF")) throw new JsonByteOrderMarkError();
}
return text;
};
const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES);
let observedBytes = 0;
while (true) {
signal?.throwIfAborted();
const read = await handle.read(
bytes,
0,
Math.min(bytes.length, admittedBytes - observedBytes + 1),
null,
);
signal?.throwIfAborted();
if (read.bytesRead === 0) break;
observedBytes += read.bytesRead;
if (observedBytes > admittedBytes) throw new RegularFileChangedError(path);
yield checked(
decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }),
);
}
if (observedBytes !== admittedBytes) throw new RegularFileChangedError(path);
yield checked(decoder.decode());
}
/** A streamed JSON file begins with a byte-order mark; reported as invalid JSON. */
class JsonByteOrderMarkError extends SyntaxError {
constructor() {
super(JSON_BYTE_ORDER_MARK_MESSAGE);
}
}
const readPrefix = async (
handle: FileHandle,
size: number,
signal: AbortSignal | undefined,
): Promise<{ readonly bytes: Buffer; readonly complete: boolean }> => {
const bytes = Buffer.allocUnsafe(size);
let offset = 0;
while (offset < bytes.length) {
signal?.throwIfAborted();
const read = await handle.read(bytes, offset, bytes.length - offset, null);
signal?.throwIfAborted();
if (read.bytesRead === 0)
return { bytes: bytes.subarray(0, offset), complete: true };
offset += read.bytesRead;
}
// One byte beyond the admitted extent proves the selected file changed.
return { bytes, complete: false };
};
const invalidJson = (
operation: string,
message: string,
cause?: unknown,
): AnalysisInputError =>
new AnalysisInputError(
operation,
cause === undefined ? undefined : { cause },
[{ path: [], reason: "invalid_format", expected: "JSON", message }],
);
+49 -6
View File
@@ -1,20 +1,44 @@
import type { Stats } from "node:fs";
import type { FileHandle } from "node:fs/promises";
import { lstat, stat, type FileHandle } from "node:fs/promises";
import { openRegularFile } from "../filesystem/RegularFile.js";
import {
openRegularFile,
sameRegularFileState,
RegularFileChangedError,
} from "../filesystem/RegularFile.js";
import { readBoundedFileBytes } from "../process/BoundedFileBytes.js";
/** Filesystem admission policy for one selected regular input. */
export interface RegularFileReadOptions {
readonly signal?: AbortSignal | undefined;
readonly symlinks?: "follow" | "reject";
}
/** Admit one regular-file handle, retain it through the consumer, and always close it. */
export const withRegularFile = async <Value>(
path: string,
read: (handle: FileHandle, stats: Stats) => Promise<Value>,
signal?: AbortSignal,
options: RegularFileReadOptions = {},
): Promise<Value> => {
const handle = await openRegularFile(path, { symlinks: "follow", signal });
const { signal } = options;
const handle = await openRegularFile(path, {
symlinks: options.symlinks ?? "follow",
signal,
});
try {
const stats = await handle.stat();
signal?.throwIfAborted();
const value = await read(handle, stats);
signal?.throwIfAborted();
const [opened, currentPath] = await Promise.all([
handle.stat(),
options.symlinks === "reject" ? lstat(path) : stat(path),
]);
if (
!sameRegularFileState(stats, opened) ||
!sameRegularFileState(stats, currentPath)
)
throw new RegularFileChangedError(path);
return value;
} finally {
await handle.close();
@@ -24,6 +48,25 @@ export const withRegularFile = async <Value>(
/** Admit a regular file without waiting for a pipe, then read its verified handle. */
export const readRegularFile = (
path: string,
signal?: AbortSignal,
options: RegularFileReadOptions = {},
): Promise<Buffer> =>
withRegularFile(path, (handle) => handle.readFile({ signal }), signal);
withRegularFile(
path,
async (handle, stats) => {
const bytes = await readBoundedFileBytes(
handle,
stats.size,
options.signal,
);
if (bytes === undefined || bytes.length !== stats.size)
throw new RegularFileChangedError(path);
return bytes;
},
options,
);
/** Read one admitted regular file as UTF-8 without waiting on a pipe. */
export const readRegularFileText = async (
path: string,
options: RegularFileReadOptions = {},
): Promise<string> => (await readRegularFile(path, options)).toString("utf8");
+6 -2
View File
@@ -27,7 +27,10 @@ import { WebScriptExportError } from "../domain/webScriptExportError.js";
import type { ExecutionOptions } from "./AnalysisProvider.js";
import { WEB_SCRIPT_EXPORT_PROVIDER as PROVIDER } from "./InvestigationProviders.js";
import { readRegularFile } from "./RegularFileRead.js";
import { NonRegularFileReadError } from "../filesystem/RegularFile.js";
import {
NonRegularFileReadError,
RegularFileChangedError,
} from "../filesystem/RegularFile.js";
const OPERATION = "export_web_scripts";
@@ -153,7 +156,7 @@ const readCapture = async (
signal: AbortSignal | undefined,
): Promise<Result<Buffer, AnalysisError>> => {
try {
return ok(await readRegularFile(path, signal));
return ok(await readRegularFile(path, { signal }));
} catch (cause: unknown) {
const code =
cause instanceof Error && "code" in cause ? String(cause.code) : "";
@@ -163,6 +166,7 @@ const readCapture = async (
);
if (
cause instanceof NonRegularFileReadError ||
cause instanceof RegularFileChangedError ||
code === "ENOENT" ||
code === "ENOTDIR" ||
code === "EISDIR" ||