From a936303723939478cbc8ebd7068ce32ef6eb1028 Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:37:41 -0700 Subject: [PATCH] fix(json): name a leading byte-order mark in rejected JSON input (#1510) Strict JSON file inputs reject a leading UTF-8 byte-order mark, but the diagnostic was the parser's own message, which quotes the invisible U+FEFF as the unexpected token: Unexpected token '', "{"invento"... is not valid JSON. Editors such as Windows PowerShell 5.1 write that mark by default, and nothing in the message showed it. CLI JSON files (whole and streamed), Evidence and snapshot JSON files, and process scenario files now report that the input begins with a UTF-8 byte-order mark and how to save it. Acceptance is unchanged. Co-authored-by: Claude Opus 5.5 --- src/application/Utf8JsonInput.ts | 9 ++++++ src/application/process/ProcessCli.test.ts | 11 +++++++ src/application/process/ProcessCli.ts | 3 ++ src/cliJsonFile.ts | 30 +++++++++++++++---- .../boundary/filesystem/cliJsonInput.test.ts | 16 ++++++++++ 5 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/application/Utf8JsonInput.ts b/src/application/Utf8JsonInput.ts index 86204e3c9..0e8ac8bb6 100644 --- a/src/application/Utf8JsonInput.ts +++ b/src/application/Utf8JsonInput.ts @@ -7,6 +7,13 @@ import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js"; export const JSON_INPUT_RESOURCE_REMEDIATION = "Provide a smaller valid JSON value. For Evidence workflows, re-analyze a smaller selection of the original target and use its complete Evidence; splitting JSON text or trimming Evidence fields does not produce valid workflow input."; +/** + * Strict JSON input keeps rejecting a leading byte-order mark, but names it: + * the parser's own message quotes the invisible U+FEFF as the bad token. + */ +export const JSON_BYTE_ORDER_MARK_MESSAGE = + "JSON input begins with a UTF-8 byte-order mark (EF BB BF), which strict JSON input does not accept. Save the file as UTF-8 without a byte-order mark."; + /** Decode JSON bytes without confusing a runtime string limit with malformed input. */ export const parseUtf8Json = ( bytes: Uint8Array, @@ -77,5 +84,7 @@ export const parseUtf8Json = ( cause, }; } + if (text.startsWith("\uFEFF")) + return { ok: false, error: JSON_BYTE_ORDER_MARK_MESSAGE, cause: undefined }; return safeParseJson(text); }; diff --git a/src/application/process/ProcessCli.test.ts b/src/application/process/ProcessCli.test.ts index 88ab756ad..34e777d31 100644 --- a/src/application/process/ProcessCli.test.ts +++ b/src/application/process/ProcessCli.test.ts @@ -6,6 +6,7 @@ import { captureProcessScenarioFile, projectProcessCliError, } from "./ProcessCli.js"; +import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../Utf8JsonInput.js"; const roots: string[] = []; afterEach(async () => { @@ -87,4 +88,14 @@ describe("process CLI input diagnostics", () => { }), }); }); + + it("names a leading byte-order mark instead of quoting it as a token", async () => { + const path = join(await fixture(), "scenario.json"); + await writeFile(path, '\uFEFF{"executable":"/bin/echo"}'); + expect(await captureProcessScenarioFile(path)).toEqual({ + error: "Process command failed", + category: "invalid_input", + message: `Process input file is not valid UTF-8 JSON: ${path} (${JSON_BYTE_ORDER_MARK_MESSAGE}). Repair the file, then try again.`, + }); + }); }); diff --git a/src/application/process/ProcessCli.ts b/src/application/process/ProcessCli.ts index 135953073..c7c4d479b 100644 --- a/src/application/process/ProcessCli.ts +++ b/src/application/process/ProcessCli.ts @@ -18,6 +18,7 @@ import { parseProcessCapture } from "../../domain/process/processCaptureParsing. import { captureProcessScenario } from "../../process/capture/ProcessHarness.js"; import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js"; import { createProcessCaptureEvidence } from "./ProcessEvidence.js"; +import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../Utf8JsonInput.js"; /** Safe process-command failure returned to the CLI adapter. */ export interface ProcessCliErrorOutput { @@ -181,6 +182,8 @@ const readJson = async (path: string): Promise => { fatal: true, ignoreBOM: true, }).decode(bytes); + if (text.startsWith("\uFEFF")) + throw new Error(JSON_BYTE_ORDER_MARK_MESSAGE); const parsed: unknown = JSON.parse(text); return parsed; } catch (cause: unknown) { diff --git a/src/cliJsonFile.ts b/src/cliJsonFile.ts index ea75f7c29..c0d474f69 100644 --- a/src/cliJsonFile.ts +++ b/src/cliJsonFile.ts @@ -8,6 +8,7 @@ import streamValues from "stream-json/streamers/stream-values.js"; import { withRegularFile } from "./application/RegularFileRead.js"; import { + JSON_BYTE_ORDER_MARK_MESSAGE, JSON_INPUT_RESOURCE_REMEDIATION, parseUtf8Json, } from "./application/Utf8JsonInput.js"; @@ -211,12 +212,22 @@ async function* decodedChunks( prefix?: Uint8Array, ): AsyncGenerator { const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }); + let leading = true; + const checked = (text: string): string => { + if (leading && text.length > 0) { + leading = false; + if (text.startsWith("\uFEFF")) throw new JsonByteOrderMarkError(); + } + return text; + }; if (prefix !== undefined) for (let offset = 0; offset < prefix.length; offset += READ_CHUNK_BYTES) { signal?.throwIfAborted(); - yield decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), { - stream: true, - }); + yield checked( + decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), { + stream: true, + }), + ); } const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES); while (true) { @@ -224,9 +235,18 @@ async function* decodedChunks( const read = await handle.read(bytes, 0, bytes.length, null); signal?.throwIfAborted(); if (read.bytesRead === 0) break; - yield decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }); + yield checked( + decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }), + ); + } + yield checked(decoder.decode()); +} + +/** A streamed JSON file begins with a byte-order mark; reported as invalid JSON. */ +class JsonByteOrderMarkError extends SyntaxError { + constructor() { + super(JSON_BYTE_ORDER_MARK_MESSAGE); } - yield decoder.decode(); } const readPrefix = async ( diff --git a/tests/boundary/filesystem/cliJsonInput.test.ts b/tests/boundary/filesystem/cliJsonInput.test.ts index 099c95b25..dd4809ea8 100644 --- a/tests/boundary/filesystem/cliJsonInput.test.ts +++ b/tests/boundary/filesystem/cliJsonInput.test.ts @@ -9,6 +9,7 @@ import { createCli } from "../../../src/cli.js"; import { parseCliJsonInput } from "../../../src/cliJsonInput.js"; import { readCliJsonFile } from "../../../src/cliJsonFile.js"; import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js"; +import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../../../src/application/Utf8JsonInput.js"; import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; @@ -160,6 +161,21 @@ describe("CLI JSON streamed input", () => { } }); + it("names a leading byte-order mark in small and streamed files", async () => { + const root = await createTestTempDirectory("rea-json-input-bom-"); + const path = join(root, "input.json"); + for (const body of ["{}", `${" ".repeat(9 * 1024 * 1024)}{}`]) { + await writeFile(path, `\uFEFF${body}`); + expect(await parseCliJsonInput(path, "test-input")).toMatchObject({ + ok: false, + error: { + input_reason: "invalid-json", + details: { issues: [{ message: JSON_BYTE_ORDER_MARK_MESSAGE }] }, + }, + }); + } + }); + it.each([Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), Buffer.from([0xe2, 0x82])])( "rejects malformed and incomplete UTF-8 at a read boundary (%j)", async (invalid) => {