From 9c4a68bc19a0f7d39557e3cc039f5877f6550f63 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Sat, 10 Oct 2026 03:39:35 +0800 Subject: [PATCH] fix(json): preserve runtime limits across JSON file readers (#1350) * fix(cli): preserve runtime JSON string-limit diagnostics * test(json): check actionable oversized Evidence recovery * fix(json): decode large UTF-8 inputs by character length * test(json): reject truncated UTF-8 after large input decode --- src/application/EvidenceBundleFiles.ts | 10 +- src/application/JsonFiles.ts | 25 ++-- src/application/Utf8JsonInput.ts | 78 ++++++++++++ .../binary/AnalysisSnapshotFiles.ts | 5 +- src/cliJsonInput.ts | 114 ++++++++---------- src/cliJsonInputStatus.test.ts | 1 + tests/boundary/cli/jsonInputSize.test.ts | 69 +++++++++++ .../boundary/filesystem/cliJsonInput.test.ts | 11 +- 8 files changed, 232 insertions(+), 81 deletions(-) create mode 100644 src/application/Utf8JsonInput.ts create mode 100644 tests/boundary/cli/jsonInputSize.test.ts diff --git a/src/application/EvidenceBundleFiles.ts b/src/application/EvidenceBundleFiles.ts index 487679070..4c7e310c7 100644 --- a/src/application/EvidenceBundleFiles.ts +++ b/src/application/EvidenceBundleFiles.ts @@ -9,7 +9,10 @@ import { EvidenceIntegrityError, } from "../domain/evidenceErrors.js"; import { err, ok, type Result } from "../domain/result.js"; -import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + type AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; import { parseProcessCapture } from "../domain/process/processCaptureParsing.js"; import { bufferedJsonParts, @@ -17,7 +20,10 @@ import { } from "../domain/jsonSerialization.js"; import { readJsonFile, writeTextParts } from "./JsonFiles.js"; -type EvidenceReadFailure = EvidenceFileError | EvidenceIntegrityError; +type EvidenceReadFailure = + | EvidenceFileError + | EvidenceIntegrityError + | AnalysisResourceConstraintError; type EvidenceWriteFailure = | EvidenceFileError | EvidenceIntegrityError diff --git a/src/application/JsonFiles.ts b/src/application/JsonFiles.ts index cdf80d7e8..8132255da 100644 --- a/src/application/JsonFiles.ts +++ b/src/application/JsonFiles.ts @@ -12,8 +12,12 @@ import { import { basename, dirname, resolve } from "node:path"; import { EvidenceFileError } from "../domain/evidenceErrors.js"; -import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; +import { + AnalysisCancelledError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; import { err, ok, type Result } from "../domain/result.js"; +import { parseUtf8Json } from "./Utf8JsonInput.js"; /** Request control and its owning operation for an interruptible atomic write. */ export interface TextWriteCancellation { @@ -27,7 +31,9 @@ type TextWriteResult = /** Read JSON data from a regular file at the caller-supplied path. */ export const readJsonFile = async ( path: string, -): Promise> => { +): Promise< + Result +> => { const requestedPath = resolve(path); try { const canonicalPath = await realpath(requestedPath); @@ -37,23 +43,18 @@ export const readJsonFile = async ( new EvidenceFileError("read", "not-file", { path: requestedPath }), ); const encoded = await readFile(canonicalPath); - let decoded: unknown; - try { - decoded = JSON.parse( - new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( - encoded, - ), - ); - } catch (cause: unknown) { + const decoded = parseUtf8Json(encoded, "read_evidence_file", requestedPath); + if (!decoded.ok) { return err( new EvidenceFileError("read", "invalid-json", { - cause, + cause: decoded.cause, path: requestedPath, }), ); } - return ok(decoded); + return ok(decoded.value); } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) return err(cause); return err( new EvidenceFileError("read", missingOrIo(cause), { cause, diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts new file mode 100644 index 000000000..4b9ecf10b --- /dev/null +++ b/src/application/Utf8JsonInput.ts @@ -0,0 +1,78 @@ +import { constants } from "node:buffer"; + +import { AnalysisResourceConstraintError } from "../domain/analysisErrorCore.js"; +import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js"; + +/** Decode JSON bytes without confusing a runtime string limit with malformed input. */ +export const parseUtf8Json = ( + bytes: Uint8Array, + operation: string, + path: string, + boundary = "json-input", +): SafeJsonParseResult => { + const stringLimitFailure = ( + cause?: unknown, + decodedUnits?: number, + ): AnalysisResourceConstraintError => + new AnalysisResourceConstraintError( + operation, + "memory", + "JSON input exceeds the runtime's maximum decoded string length", + { + boundary, + input_path: path, + input_bytes: bytes.byteLength, + max_string_code_units: constants.MAX_STRING_LENGTH, + ...(decodedUnits === undefined + ? {} + : { decoded_string_code_units_at_least: decodedUnits }), + }, + { + ...(cause === undefined ? {} : { cause }), + remediationAction: + "Provide a smaller JSON document. For Evidence-based workflows, re-analyze a smaller selection of the original target and use its Evidence; splitting JSON text alone does not produce a valid workflow input.", + }, + ); + let text: string; + try { + const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + if (bytes.byteLength <= constants.MAX_STRING_LENGTH) { + text = decoder.decode(bytes); + } else { + // Node 22's UTF-8 fast path bounds encoded bytes before decoding. Decode + // large inputs in chunks so valid multibyte text is bounded by its actual + // UTF-16 length. Streaming also preserves characters spanning chunks. + const parts: string[] = []; + let decodedUnits = 0; + const append = (part: string): void => { + decodedUnits += part.length; + if (decodedUnits > constants.MAX_STRING_LENGTH) + throw stringLimitFailure(undefined, decodedUnits); + parts.push(part); + }; + const chunkBytes = 1024 * 1024; + for (let offset = 0; offset < bytes.byteLength; offset += chunkBytes) + append( + decoder.decode(bytes.subarray(offset, offset + chunkBytes), { + stream: true, + }), + ); + append(decoder.decode()); + text = parts.join(""); + } + } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) throw cause; + const code = + typeof cause === "object" && cause !== null && "code" in cause + ? cause.code + : undefined; + if (code === "ERR_STRING_TOO_LONG") throw stringLimitFailure(cause); + if (code !== "ERR_ENCODING_INVALID_ENCODED_DATA") throw cause; + return { + ok: false, + error: "JSON input is not valid UTF-8", + cause, + }; + } + return safeParseJson(text); +}; diff --git a/src/application/binary/AnalysisSnapshotFiles.ts b/src/application/binary/AnalysisSnapshotFiles.ts index 2841e1dbf..87d910b5d 100644 --- a/src/application/binary/AnalysisSnapshotFiles.ts +++ b/src/application/binary/AnalysisSnapshotFiles.ts @@ -8,6 +8,7 @@ import { EvidenceIntegrityError, } from "../../domain/evidenceErrors.js"; import { err, ok, type Result } from "../../domain/result.js"; +import type { AnalysisResourceConstraintError } from "../../domain/analysisErrorCore.js"; import { readJsonFile, writeTextFile } from "../JsonFiles.js"; type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError; @@ -15,7 +16,9 @@ type SnapshotFailure = EvidenceFileError | EvidenceIntegrityError; /** Read and validate an analysis snapshot from the caller's path. */ export const readAnalysisSnapshot = async ( path: string, -): Promise> => { +): Promise< + Result +> => { const loaded = await readJsonFile(path); if (!loaded.ok) return loaded; try { diff --git a/src/cliJsonInput.ts b/src/cliJsonInput.ts index 74dd30bc9..a47d1e5f5 100644 --- a/src/cliJsonInput.ts +++ b/src/cliJsonInput.ts @@ -5,6 +5,7 @@ import { NonRegularFileReadError, readRegularFile, } from "./application/RegularFileRead.js"; +import { parseUtf8Json } from "./application/Utf8JsonInput.js"; import { AnalysisInputError, AnalysisResourceConstraintError, @@ -21,18 +22,33 @@ export const parseCliJsonInput = async ( | { readonly ok: true; readonly value: unknown } | { readonly ok: false; readonly error: JsonValue } > => { - const inline = parseJson(value); - if (inline.kind === "valid") return { ok: true, value: inline.value }; - if (inline.kind === "too-large") + const inline = safeParseJson(value); + if (inline.ok) return { ok: true, value: inline.value }; + if (isStringLengthLimit(inline.cause)) return jsonTooLargeError(undefined, operation); try { - // Read raw bytes so invalid UTF-8 is rejected by parseJson instead of + // Read raw bytes so invalid UTF-8 is rejected instead of // being silently replaced by lossy "utf8" decoding. - const parsed = parseJson(await readRegularFile(value)); - if (parsed.kind === "valid") return { ok: true, value: parsed.value }; - if (parsed.kind === "too-large") return jsonTooLargeError(value, operation); - return jsonFileError(value, operation, "invalid-json"); + const parsed = parseUtf8Json( + await readRegularFile(value), + operation, + value, + "cli-json-input", + ); + return parsed.ok + ? { ok: true, value: parsed.value } + : jsonFileError(value, operation, "invalid-json"); } catch (cause: unknown) { + if (cause instanceof AnalysisResourceConstraintError) + return { + ok: false, + error: { + error: "Application workflow failed", + ...projectAnalysisError(cause), + input_path: value, + input_reason: "too-large", + }, + }; if ( ["{", "["].includes(value.trimStart()[0] ?? "") && cannotBeAnExistingFile(cause) && @@ -93,39 +109,12 @@ const cannotBeAnExistingFile = (cause: unknown): boolean => const hasExplicitJsonFileExtension = (value: string): boolean => value.toLowerCase().endsWith(".json"); -type JsonParseResult = - | { readonly kind: "valid"; readonly value: unknown } - | { readonly kind: "invalid" } - | { readonly kind: "too-large" }; - -const parseJson = (value: string | Uint8Array): JsonParseResult => { - let text: string; - if (typeof value === "string") { - text = value; - } else { - try { - text = new TextDecoder("utf-8", { - fatal: true, - ignoreBOM: true, - }).decode(value); - } catch (cause: unknown) { - return isStringLengthLimit(cause) - ? { kind: "too-large" } - : { kind: "invalid" }; - } - } - const parsed = safeParseJson(text); - if (parsed.ok) return { kind: "valid", value: parsed.value }; - return isStringLengthLimit(parsed.cause) - ? { kind: "too-large" } - : { kind: "invalid" }; -}; - const isStringLengthLimit = (cause: unknown): boolean => cause instanceof Error && - /cannot create a string longer than|invalid string length/i.test( - cause.message, - ); + (("code" in cause && cause.code === "ERR_STRING_TOO_LONG") || + /cannot create a string longer than|invalid string length/i.test( + cause.message, + )); const inputError = (operation: string): JsonValue => ({ error: "Application workflow failed", @@ -181,29 +170,26 @@ const readFailureIssue = (path: string | undefined, cause: unknown) => { const jsonTooLargeError = ( path: string | undefined, operation: string, -): { readonly ok: false; readonly error: JsonValue } => { - const projection = projectAnalysisError( - new AnalysisResourceConstraintError( - operation, - "memory", - `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`, - { - boundary: "cli-json-input", - max_string_code_units: bufferConstants.MAX_STRING_LENGTH, - }, - { - remediationAction: - "Split the JSON into smaller inputs or rerun its producer on a smaller subset, then retry.", - }, +): { readonly ok: false; readonly error: JsonValue } => ({ + ok: false, + error: { + error: "Application workflow failed", + ...projectAnalysisError( + new AnalysisResourceConstraintError( + operation, + "memory", + `The JSON input exceeds this Node.js runtime's maximum string length (${bufferConstants.MAX_STRING_LENGTH} UTF-16 code units) and cannot be parsed as one value.`, + { + boundary: "cli-json-input", + max_string_code_units: bufferConstants.MAX_STRING_LENGTH, + }, + { + remediationAction: + "Provide a smaller JSON document or rerun its producer on a smaller selection. Splitting JSON text alone does not produce a valid workflow input.", + }, + ), ), - ); - return { - ok: false, - error: { - error: "Application workflow failed", - ...projection, - ...(path === undefined ? {} : { input_path: path }), - input_reason: "too-large", - }, - }; -}; + ...(path === undefined ? {} : { input_path: path }), + input_reason: "too-large", + }, +}); diff --git a/src/cliJsonInputStatus.test.ts b/src/cliJsonInputStatus.test.ts index 8f9238e00..f4bf8854d 100644 --- a/src/cliJsonInputStatus.test.ts +++ b/src/cliJsonInputStatus.test.ts @@ -18,6 +18,7 @@ describe("CLI JSON input failure recognition", () => { { input_reason: "invalid-json" }, { input_path: "input.json", input_reason: "invalid-json" }, { input_path: "missing.json", input_reason: "read-failed" }, + { input_path: "large.json", input_reason: "too-large" }, { input_path: "", input_reason: "read-failed" }, ])("recognizes a complete envelope with metadata %j", (metadata) => { const value = { ...envelope, ...metadata }; diff --git a/tests/boundary/cli/jsonInputSize.test.ts b/tests/boundary/cli/jsonInputSize.test.ts new file mode 100644 index 000000000..fce9412f2 --- /dev/null +++ b/tests/boundary/cli/jsonInputSize.test.ts @@ -0,0 +1,69 @@ +import { constants } from "node:buffer"; +import { open } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "preserves the runtime string limit through CLI and Evidence file inputs", + async ({ cli, processes }) => { + const root = await createTestTempDirectory("rea-json-string-limit-"); + const input = join(root, "oversized.json"); + const file = await open(input, "wx"); + try { + // Sparse zero bytes are valid UTF-8. Decoded length exceeds the runtime + // string limit before JSON syntax can be inspected; no fixture is retained. + await file.truncate(constants.MAX_STRING_LENGTH + 1); + } finally { + await file.close(); + } + const limits = { + input_path: input, + input_bytes: constants.MAX_STRING_LENGTH + 1, + max_string_code_units: constants.MAX_STRING_LENGTH, + }; + // Run readers sequentially in owned child processes, releasing each large + // read buffer when its process exits instead of retaining it in Vitest. + const result = await cli.run({ + arguments: ["trace-application-feature", input, "--json"], + }); + expect(result.json).toMatchObject({ + code: "resource_constraint", + category: "resource_constraint", + input_path: input, + input_reason: "too-large", + retryable: false, + remediation: { action: expect.stringContaining("re-analyze") }, + details: { + operation: "trace-application-feature", + resource: "memory", + reported_limits: limits, + }, + }); + expect(result.exitCode).toBe(1); + expect(result.stderr).toBe(""); + + const evidence = await processes.run(process.execPath, [ + "--input-type=module", + "-e", + `import { readJsonFile } from "./dist/application/JsonFiles.js"; + import { projectAnalysisError } from "./dist/domain/analysisErrorProjection.js"; + const result = await readJsonFile(process.argv[1]); + console.log(JSON.stringify(result.ok ? result : projectAnalysisError(result.error)));`, + input, + ]); + expect(JSON.parse(evidence.stdout)).toMatchObject({ + code: "resource_constraint", + details: { + operation: "read_evidence_file", + resource: "memory", + reported_limits: limits, + }, + }); + expect(evidence.stderr).toBe(""); + expect(evidence.exitCode).toBe(0); + }, +); diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts index 273168fdb..e550f88b8 100644 --- a/tests/boundary/filesystem/cliJsonInput.test.ts +++ b/tests/boundary/filesystem/cliJsonInput.test.ts @@ -1,6 +1,7 @@ import { execFile } from "node:child_process"; import { constants as bufferConstants } from "node:buffer"; import { + appendFile, chmod, open, symlink, @@ -119,8 +120,7 @@ describe("CLI JSON input", () => { }, }, remediation: { - action: - "Split the JSON into smaller inputs or rerun its producer on a smaller subset, then retry.", + action: expect.stringContaining("re-analyze a smaller selection"), }, }, }); @@ -137,6 +137,13 @@ describe("CLI JSON input", () => { if (typeof multibyteResult.value !== "string") throw new Error("Expected parsed JSON string"); expect(multibyteResult.value.length).toBe(characterCount); + + // Large-input streaming must flush and reject an incomplete UTF-8 suffix. + await appendFile(path, Buffer.from([0xc3])); + expect(await parseCliJsonInput(path, "test-input")).toMatchObject({ + ok: false, + error: { input_path: path, input_reason: "invalid-json" }, + }); }, 20_000); it("distinguishes malformed inline text from files and preserves bracket-prefixed paths", async () => {