fix(javascript): hash graph and evidence JSON incrementally (#646)

This commit is contained in:
N0zoM1z0
2026-10-06 11:54:55 +08:00
committed by GitHub
parent 9b392bb2f8
commit 9f42495d11
9 changed files with 346 additions and 21 deletions
@@ -52,6 +52,23 @@ manifest and graph commitments, JavaScript Application Graph, static
Electron summary, reconstruction statistics, and explicit limitations. It does
not require a live Hopper, Ghidra, browser, or Electron process.
## Large results
Graph and Evidence identifiers hash canonical JSON incrementally, without
assembling a single string for the whole graph. The canonical bytes and existing
identifiers remain unchanged. The opt-in regression check is
`npm run verify:javascript:digests`;
it hashes a value larger than the running Node engine's single-string limit.
Output formatters and MCP transport serialization still assemble whole strings.
For a large CLI result, select the needed fields
before formatting, for example `--format json --filter-output
evidence_id,normalized_result.statistics`. Complete serialization of a result
beyond the engine's string limit is not established by this digest check.
Client framing limits also apply: the pinned Node MCP SDK's stdio transport
defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate
the complete response, including text and structured Evidence projections.
## What is reconstructed
The projector reuses the content-addressed artifact inventory and safe artifact
+1
View File
@@ -128,6 +128,7 @@
"verify:electron": "npm run build:cached && node scripts/verify-real-electron.mjs",
"verify:inspector": "npm run build:cached && node scripts/verify-real-javascript-runtime-observation.mjs",
"verify:javascript-runtime": "npm run build:cached && node scripts/verify-local-javascript-runtime.mjs",
"verify:javascript:digests": "npm run build:cached && node scripts/verify-large-javascript-digests.mjs",
"verify:application-workflows": "npm run build:cached && node scripts/verify-local-application-workflows.mjs",
"verify:managed": "npm run build:cached && node scripts/verify-managed-conformance.mjs",
"verify:agent": "npm run build:cached && node scripts/verify-agent-experience.mjs",
@@ -0,0 +1,38 @@
#!/usr/bin/env node
import assert from "node:assert/strict";
import { constants } from "node:buffer";
import { createHash } from "node:crypto";
import { canonicalDigest } from "../dist/domain/comparisonSemantics.js";
import { createEvidence, parseEvidence } from "../dist/domain/evidence.js";
// This opt-in check crosses the actual engine string limit; keep it outside
// routine source tests. Build first, then run this script with the local Node.
const leaf = "x".repeat(1024 * 1024);
const encodedLeaf = JSON.stringify(leaf);
const count = Math.ceil(constants.MAX_STRING_LENGTH / encodedLeaf.length) + 1;
const value = { payload: Array.from({ length: count }, () => leaf) };
const expected = createHash("sha256");
expected.update('{"payload":[');
for (let index = 0; index < count; index += 1) {
if (index > 0) expected.update(",");
expected.update(encodedLeaf);
}
expected.update("]}");
assert.equal(canonicalDigest(value), expected.digest("hex"));
const evidence = createEvidence(
undefined,
{ id: "large-json-fixture", name: "Large JSON fixture", version: "1" },
{ operation: "inspect_fixture", parameters: {}, result: value },
);
assert.equal(parseEvidence(evidence).evidence_id, evidence.evidence_id);
process.stdout.write(
`${JSON.stringify({
verified: true,
serialized_value_bytes: count * encodedLeaf.length + count - 1 + 14,
engine_string_limit: constants.MAX_STRING_LENGTH,
evidence_id: evidence.evidence_id,
})}\n`,
);
@@ -0,0 +1,87 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
describe("JavaScript application failure diagnostics", () => {
it("retains unexpected failures and their input identity in caller-visible diagnostics", async () => {
const inputPath = await createTestTempDirectory("rea-js-failure-");
await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
const cause = new RangeError("Invalid string length");
const result = await analyzeJavaScriptApplication(
{ input_path: inputPath, format: "directory" },
{
progress: {
report: async (event) => {
if (event.terminal) throw cause;
},
},
},
);
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected analysis failure");
expect(result.error.cause).toBe(cause);
expect(projectAnalysisError(result.error)).toMatchObject({
code: "execution_failure",
details: {
provider_id: "rea-javascript-application",
operation: "analyze_javascript_application",
diagnostics: {
input_path: inputPath,
error_name: "RangeError",
error_message: "Invalid string length",
},
},
});
});
it("keeps filesystem failures distinct from engine failures", async () => {
const root = await createTestTempDirectory("rea-js-missing-");
const result = await analyzeJavaScriptApplication({
input_path: join(root, "missing"),
format: "directory",
});
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected missing artifact failure");
expect(projectAnalysisError(result.error)).toMatchObject({
code: "artifact_operation_failed",
details: { operation: "analyze_javascript_application", reason: "io" },
});
});
it.each(["reported failure", null])(
"retains non-Error failures (%s)",
async (cause) => {
const inputPath = await createTestTempDirectory("rea-js-non-error-");
await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
const result = await analyzeJavaScriptApplication(
{ input_path: inputPath, format: "directory" },
{
progress: {
report: async (event) => {
if (event.terminal) throw cause;
},
},
},
);
expect(result.ok).toBe(false);
if (result.ok) throw new Error("Expected analysis failure");
expect(projectAnalysisError(result.error)).toMatchObject({
code: "execution_failure",
details: {
diagnostics: {
error_name: "UnknownError",
error_message:
typeof cause === "string"
? cause
: "JavaScript analysis failed with a non-Error value",
},
},
});
},
);
});
@@ -14,9 +14,11 @@ import { type AnalysisError } from "../domain/analysisErrorBase.js";
import type { Evidence } from "../domain/evidence.js";
import { projectInputIssues } from "../domain/inputIssueProjection.js";
import { err, ok, type Result } from "../domain/result.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
import type { ExecutionOptions } from "./AnalysisProvider.js";
import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js";
import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js";
import { JAVASCRIPT_APPLICATION_PROVIDER } from "./InvestigationProviders.js";
const OPERATION = "analyze_javascript_application" as const;
@@ -82,6 +84,42 @@ export const analyzeJavaScriptApplicationValidated = async (
{ cause },
),
);
return err(new ArtifactOperationError(OPERATION, "io"));
if (
cause instanceof Error &&
"code" in cause &&
typeof cause.code === "string" &&
FILESYSTEM_ERROR_CODES.has(cause.code)
)
return err(new ArtifactOperationError(OPERATION, "io"));
return err(
new ProviderAdapterError(JAVASCRIPT_APPLICATION_PROVIDER.id, OPERATION, {
cause,
diagnostics: {
input_path: input.input_path,
error_name: cause instanceof Error ? cause.name : "UnknownError",
error_message:
cause instanceof Error
? cause.message
: typeof cause === "string"
? cause
: "JavaScript analysis failed with a non-Error value",
},
}),
);
}
};
const FILESYSTEM_ERROR_CODES = new Set([
"ENOENT",
"EACCES",
"EPERM",
"ENOTDIR",
"EISDIR",
"EIO",
"ELOOP",
"ENAMETOOLONG",
"EMFILE",
"ENFILE",
"ENOSPC",
"EROFS",
]);
+83
View File
@@ -0,0 +1,83 @@
import { createHash } from "node:crypto";
import { fc, it } from "@fast-check/vitest";
import canonicalize from "canonicalize";
import { describe, expect } from "vitest";
import { digestCanonicalValue } from "./canonicalDigest.js";
const legacyDigest = (value: unknown): string => {
const encoded = canonicalize(value);
if (encoded === undefined) throw new TypeError("Unserializable fixture");
return createHash("sha256").update(encoded).digest("hex");
};
describe("incremental canonical digest", () => {
it.prop([fc.jsonValue()])(
"preserves canonicalize digests for JSON values",
(value) => {
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
},
);
it("preserves Unicode ordering, escaping, numeric forms and non-JSON normalization", () => {
const shared = { key: "value" };
const sparse: unknown[] = [undefined, Symbol("omit"), () => undefined];
sparse.length = 6;
const fixtures: unknown[] = [
{
"€": "\ud800",
"😀": "\udc00",
"\r": '\n\t"\\',
number: -0,
exponent: 1e30,
},
{ omitted: undefined, symbol: Symbol("omit"), fn: () => undefined },
sparse,
[shared, shared],
new Date("2026-10-06T00:00:00Z"),
{ toJSON: () => ({ b: 2, a: [1, null] }) },
{ nested: { toJSON: () => undefined } },
];
for (const value of fixtures)
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
});
it("rejects unsupported roots, non-finite numbers, BigInt and ancestor cycles", () => {
for (const value of [undefined, Symbol("root"), () => undefined])
expect(() => digestCanonicalValue(value, "Fixture")).toThrow(
"Fixture could not canonicalize data",
);
const cyclic: Record<string, unknown> = {};
cyclic.self = cyclic;
const recursiveJson = { toJSON: (): unknown => recursiveJson };
for (const value of [NaN, Infinity, -Infinity, 1n, cyclic, recursiveJson])
expect(() => digestCanonicalValue(value)).toThrow();
});
it("hashes repeated large leaves with the same byte sequence", () => {
const value = {
payload: Array.from({ length: 64 }, () => "x".repeat(16_384)),
};
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
});
it("preserves canonicalize property access and toJSON cycle behavior", () => {
const createAccessor = () => {
let reads = 0;
return {
get value() {
reads += 1;
return reads;
},
};
};
expect(digestCanonicalValue(createAccessor())).toBe(
legacyDigest(createAccessor()),
);
const shared = { toJSON: () => ({ answer: 42 }) };
expect(digestCanonicalValue([shared, shared])).toBe(
legacyDigest([shared, shared]),
);
});
});
+75
View File
@@ -0,0 +1,75 @@
import { createHash } from "node:crypto";
/** Hash canonicalize-compatible JSON without assembling an aggregate string. */
export const digestCanonicalValue = (
value: unknown,
context = "Comparison",
): string => {
const hash = createHash("sha256");
const encoded = emitCanonical(value, (part) => hash.update(part), new Set());
if (!encoded) throw new TypeError(`${context} could not canonicalize data`);
return hash.digest("hex");
};
type Emit = (part: string) => void;
const emitCanonical = (
value: unknown,
emit: Emit,
ancestors: Set<object>,
): boolean => {
if (typeof value === "number" && !Number.isFinite(value))
throw new Error(
Number.isNaN(value) ? "NaN is not allowed" : "Infinity is not allowed",
);
if (value === null || typeof value !== "object") {
const encoded = JSON.stringify(value);
if (encoded === undefined) return false;
emit(encoded);
return true;
}
if (ancestors.has(value)) throw new Error("Circular reference detected");
ancestors.add(value);
try {
if ("toJSON" in value && typeof value.toJSON === "function") {
const normalized: unknown = value.toJSON();
return emitCanonical(normalized, emit, ancestors);
}
if (Array.isArray(value)) {
emit("[");
const length = value.length;
for (let index = 0; index < length; index += 1) {
if (index > 0) emit(",");
// Preserve canonicalize's sparse-array and unsupported-value behavior.
if (!(index in value)) continue;
const item: unknown = value[index];
emitCanonical(
item === undefined || typeof item === "symbol" ? null : item,
emit,
ancestors,
);
}
emit("]");
} else {
emit("{");
let first = true;
for (const key of Object.keys(value).sort()) {
if (
Reflect.get(value, key) === undefined ||
typeof Reflect.get(value, key) === "symbol"
)
continue;
const item: unknown = Reflect.get(value, key);
if (!first) emit(",");
first = false;
emit(JSON.stringify(key));
emit(":");
if (!emitCanonical(item, emit, ancestors)) emit("undefined");
}
emit("}");
}
return true;
} finally {
ancestors.delete(value);
}
};
+3 -5
View File
@@ -1,7 +1,7 @@
import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { digestCanonicalValue } from "./canonicalDigest.js";
/** Whether an inventory can support a claim that an item is absent. */
export const absenceClaimable = (coverage: {
readonly status: string;
@@ -17,9 +17,7 @@ export const canonicalDigest = (
value: unknown,
context = "Comparison",
): string => {
return createHash("sha256")
.update(canonicalJson(value, context))
.digest("hex");
return digestCanonicalValue(value, context);
};
/** Canonical JSON used for deterministic tie breaking and stable digests. */
+3 -15
View File
@@ -1,8 +1,7 @@
import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import { z } from "zod";
import { digestCanonicalValue } from "./canonicalDigest.js";
import {
analysisProfileSchema,
type AnalysisProfileCommitment,
@@ -173,22 +172,11 @@ export interface EvidenceObservation {
readonly evidenceLinks?: readonly string[];
}
const sha256 = (value: string): string =>
createHash("sha256").update(value).digest("hex");
type WithoutEvidenceId<Record_> = Record_ extends unknown
? Omit<Record_, "evidence_id">
: never;
type EvidenceWithoutId = WithoutEvidenceId<Evidence>;
/** Serialize JSON according to RFC 8785 JSON Canonicalization Scheme. */
const canonicalJson = (value: JsonValue): string => {
const serialized = canonicalize(value);
if (serialized === undefined)
throw new TypeError("RFC 8785 canonicalization rejected a JSON value");
return serialized;
};
const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
subject:
evidence.subject === null
@@ -217,7 +205,7 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
/** Recompute the semantic identifier, excluding paths and raw payload bytes. */
const computeEvidenceId = (evidence: EvidenceWithoutId): string =>
`ev_${sha256(canonicalJson(semanticProjection(evidence)))}`;
`ev_${digestCanonicalValue(semanticProjection(evidence), "Evidence")}`;
/** Parse evidence and reject a syntactically valid but tampered semantic ID. */
export const parseEvidence = (input: unknown): Evidence => {