fix(json): name a leading byte-order mark in rejected JSON input (#1510)
Strict JSON file inputs reject a leading UTF-8 byte-order mark, but the
diagnostic was the parser's own message, which quotes the invisible U+FEFF
as the unexpected token: Unexpected token '', "{"invento"... is not
valid JSON. Editors such as Windows PowerShell 5.1 write that mark by
default, and nothing in the message showed it.
CLI JSON files (whole and streamed), Evidence and snapshot JSON files, and
process scenario files now report that the input begins with a UTF-8
byte-order mark and how to save it. Acceptance is unchanged.
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
93ca276921
commit
a936303723
@@ -7,6 +7,13 @@ import { safeParseJson, type SafeJsonParseResult } from "../domain/safeJson.js";
|
||||
export const JSON_INPUT_RESOURCE_REMEDIATION =
|
||||
"Provide a smaller valid JSON value. For Evidence workflows, re-analyze a smaller selection of the original target and use its complete Evidence; splitting JSON text or trimming Evidence fields does not produce valid workflow input.";
|
||||
|
||||
/**
|
||||
* Strict JSON input keeps rejecting a leading byte-order mark, but names it:
|
||||
* the parser's own message quotes the invisible U+FEFF as the bad token.
|
||||
*/
|
||||
export const JSON_BYTE_ORDER_MARK_MESSAGE =
|
||||
"JSON input begins with a UTF-8 byte-order mark (EF BB BF), which strict JSON input does not accept. Save the file as UTF-8 without a byte-order mark.";
|
||||
|
||||
/** Decode JSON bytes without confusing a runtime string limit with malformed input. */
|
||||
export const parseUtf8Json = (
|
||||
bytes: Uint8Array,
|
||||
@@ -77,5 +84,7 @@ export const parseUtf8Json = (
|
||||
cause,
|
||||
};
|
||||
}
|
||||
if (text.startsWith("\uFEFF"))
|
||||
return { ok: false, error: JSON_BYTE_ORDER_MARK_MESSAGE, cause: undefined };
|
||||
return safeParseJson(text);
|
||||
};
|
||||
|
||||
@@ -6,6 +6,7 @@ import {
|
||||
captureProcessScenarioFile,
|
||||
projectProcessCliError,
|
||||
} from "./ProcessCli.js";
|
||||
import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../Utf8JsonInput.js";
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(async () => {
|
||||
@@ -87,4 +88,14 @@ describe("process CLI input diagnostics", () => {
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it("names a leading byte-order mark instead of quoting it as a token", async () => {
|
||||
const path = join(await fixture(), "scenario.json");
|
||||
await writeFile(path, '\uFEFF{"executable":"/bin/echo"}');
|
||||
expect(await captureProcessScenarioFile(path)).toEqual({
|
||||
error: "Process command failed",
|
||||
category: "invalid_input",
|
||||
message: `Process input file is not valid UTF-8 JSON: ${path} (${JSON_BYTE_ORDER_MARK_MESSAGE}). Repair the file, then try again.`,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -18,6 +18,7 @@ import { parseProcessCapture } from "../../domain/process/processCaptureParsing.
|
||||
import { captureProcessScenario } from "../../process/capture/ProcessHarness.js";
|
||||
import { PROCESS_PROVIDER } from "../../domain/process/processEvidenceProvider.js";
|
||||
import { createProcessCaptureEvidence } from "./ProcessEvidence.js";
|
||||
import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../Utf8JsonInput.js";
|
||||
|
||||
/** Safe process-command failure returned to the CLI adapter. */
|
||||
export interface ProcessCliErrorOutput {
|
||||
@@ -181,6 +182,8 @@ const readJson = async (path: string): Promise<unknown> => {
|
||||
fatal: true,
|
||||
ignoreBOM: true,
|
||||
}).decode(bytes);
|
||||
if (text.startsWith("\uFEFF"))
|
||||
throw new Error(JSON_BYTE_ORDER_MARK_MESSAGE);
|
||||
const parsed: unknown = JSON.parse(text);
|
||||
return parsed;
|
||||
} catch (cause: unknown) {
|
||||
|
||||
+25
-5
@@ -8,6 +8,7 @@ import streamValues from "stream-json/streamers/stream-values.js";
|
||||
|
||||
import { withRegularFile } from "./application/RegularFileRead.js";
|
||||
import {
|
||||
JSON_BYTE_ORDER_MARK_MESSAGE,
|
||||
JSON_INPUT_RESOURCE_REMEDIATION,
|
||||
parseUtf8Json,
|
||||
} from "./application/Utf8JsonInput.js";
|
||||
@@ -211,12 +212,22 @@ async function* decodedChunks(
|
||||
prefix?: Uint8Array,
|
||||
): AsyncGenerator<string> {
|
||||
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
let leading = true;
|
||||
const checked = (text: string): string => {
|
||||
if (leading && text.length > 0) {
|
||||
leading = false;
|
||||
if (text.startsWith("\uFEFF")) throw new JsonByteOrderMarkError();
|
||||
}
|
||||
return text;
|
||||
};
|
||||
if (prefix !== undefined)
|
||||
for (let offset = 0; offset < prefix.length; offset += READ_CHUNK_BYTES) {
|
||||
signal?.throwIfAborted();
|
||||
yield decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), {
|
||||
stream: true,
|
||||
});
|
||||
yield checked(
|
||||
decoder.decode(prefix.subarray(offset, offset + READ_CHUNK_BYTES), {
|
||||
stream: true,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const bytes = Buffer.allocUnsafe(READ_CHUNK_BYTES);
|
||||
while (true) {
|
||||
@@ -224,9 +235,18 @@ async function* decodedChunks(
|
||||
const read = await handle.read(bytes, 0, bytes.length, null);
|
||||
signal?.throwIfAborted();
|
||||
if (read.bytesRead === 0) break;
|
||||
yield decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true });
|
||||
yield checked(
|
||||
decoder.decode(bytes.subarray(0, read.bytesRead), { stream: true }),
|
||||
);
|
||||
}
|
||||
yield checked(decoder.decode());
|
||||
}
|
||||
|
||||
/** A streamed JSON file begins with a byte-order mark; reported as invalid JSON. */
|
||||
class JsonByteOrderMarkError extends SyntaxError {
|
||||
constructor() {
|
||||
super(JSON_BYTE_ORDER_MARK_MESSAGE);
|
||||
}
|
||||
yield decoder.decode();
|
||||
}
|
||||
|
||||
const readPrefix = async (
|
||||
|
||||
@@ -9,6 +9,7 @@ import { createCli } from "../../../src/cli.js";
|
||||
import { parseCliJsonInput } from "../../../src/cliJsonInput.js";
|
||||
import { readCliJsonFile } from "../../../src/cliJsonFile.js";
|
||||
import { analysisCliErrorEnvelopeSchema } from "../../../src/contracts/errorSchemas.js";
|
||||
import { JSON_BYTE_ORDER_MARK_MESSAGE } from "../../../src/application/Utf8JsonInput.js";
|
||||
import { readWithoutFifoWriter } from "../../fixtures/fifoInput.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
@@ -160,6 +161,21 @@ describe("CLI JSON streamed input", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("names a leading byte-order mark in small and streamed files", async () => {
|
||||
const root = await createTestTempDirectory("rea-json-input-bom-");
|
||||
const path = join(root, "input.json");
|
||||
for (const body of ["{}", `${" ".repeat(9 * 1024 * 1024)}{}`]) {
|
||||
await writeFile(path, `\uFEFF${body}`);
|
||||
expect(await parseCliJsonInput(path, "test-input")).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
input_reason: "invalid-json",
|
||||
details: { issues: [{ message: JSON_BYTE_ORDER_MARK_MESSAGE }] },
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it.each([Buffer.from([0xf0, 0x28, 0x8c, 0xbc]), Buffer.from([0xe2, 0x82])])(
|
||||
"rejects malformed and incomplete UTF-8 at a read boundary (%j)",
|
||||
async (invalid) => {
|
||||
|
||||
Reference in New Issue
Block a user