fix(javascript): hash graph and evidence JSON incrementally (#646)
This commit is contained in:
@@ -52,6 +52,23 @@ manifest and graph commitments, JavaScript Application Graph, static
|
||||
Electron summary, reconstruction statistics, and explicit limitations. It does
|
||||
not require a live Hopper, Ghidra, browser, or Electron process.
|
||||
|
||||
## Large results
|
||||
|
||||
Graph and Evidence identifiers hash canonical JSON incrementally, without
|
||||
assembling a single string for the whole graph. The canonical bytes and existing
|
||||
identifiers remain unchanged. The opt-in regression check is
|
||||
`npm run verify:javascript:digests`;
|
||||
it hashes a value larger than the running Node engine's single-string limit.
|
||||
|
||||
Output formatters and MCP transport serialization still assemble whole strings.
|
||||
For a large CLI result, select the needed fields
|
||||
before formatting, for example `--format json --filter-output
|
||||
evidence_id,normalized_result.statistics`. Complete serialization of a result
|
||||
beyond the engine's string limit is not established by this digest check.
|
||||
Client framing limits also apply: the pinned Node MCP SDK's stdio transport
|
||||
defaults to a 10 MiB buffer. Its caller-selected `maxBufferSize` must accommodate
|
||||
the complete response, including text and structured Evidence projections.
|
||||
|
||||
## What is reconstructed
|
||||
|
||||
The projector reuses the content-addressed artifact inventory and safe artifact
|
||||
|
||||
@@ -128,6 +128,7 @@
|
||||
"verify:electron": "npm run build:cached && node scripts/verify-real-electron.mjs",
|
||||
"verify:inspector": "npm run build:cached && node scripts/verify-real-javascript-runtime-observation.mjs",
|
||||
"verify:javascript-runtime": "npm run build:cached && node scripts/verify-local-javascript-runtime.mjs",
|
||||
"verify:javascript:digests": "npm run build:cached && node scripts/verify-large-javascript-digests.mjs",
|
||||
"verify:application-workflows": "npm run build:cached && node scripts/verify-local-application-workflows.mjs",
|
||||
"verify:managed": "npm run build:cached && node scripts/verify-managed-conformance.mjs",
|
||||
"verify:agent": "npm run build:cached && node scripts/verify-agent-experience.mjs",
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { constants } from "node:buffer";
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { canonicalDigest } from "../dist/domain/comparisonSemantics.js";
|
||||
import { createEvidence, parseEvidence } from "../dist/domain/evidence.js";
|
||||
|
||||
// This opt-in check crosses the actual engine string limit; keep it outside
|
||||
// routine source tests. Build first, then run this script with the local Node.
|
||||
const leaf = "x".repeat(1024 * 1024);
|
||||
const encodedLeaf = JSON.stringify(leaf);
|
||||
const count = Math.ceil(constants.MAX_STRING_LENGTH / encodedLeaf.length) + 1;
|
||||
const value = { payload: Array.from({ length: count }, () => leaf) };
|
||||
const expected = createHash("sha256");
|
||||
expected.update('{"payload":[');
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
if (index > 0) expected.update(",");
|
||||
expected.update(encodedLeaf);
|
||||
}
|
||||
expected.update("]}");
|
||||
assert.equal(canonicalDigest(value), expected.digest("hex"));
|
||||
|
||||
const evidence = createEvidence(
|
||||
undefined,
|
||||
{ id: "large-json-fixture", name: "Large JSON fixture", version: "1" },
|
||||
{ operation: "inspect_fixture", parameters: {}, result: value },
|
||||
);
|
||||
assert.equal(parseEvidence(evidence).evidence_id, evidence.evidence_id);
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
verified: true,
|
||||
serialized_value_bytes: count * encodedLeaf.length + count - 1 + 14,
|
||||
engine_string_limit: constants.MAX_STRING_LENGTH,
|
||||
evidence_id: evidence.evidence_id,
|
||||
})}\n`,
|
||||
);
|
||||
@@ -0,0 +1,87 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
|
||||
import { analyzeJavaScriptApplication } from "./JavaScriptApplicationService.js";
|
||||
|
||||
describe("JavaScript application failure diagnostics", () => {
|
||||
it("retains unexpected failures and their input identity in caller-visible diagnostics", async () => {
|
||||
const inputPath = await createTestTempDirectory("rea-js-failure-");
|
||||
await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
|
||||
const cause = new RangeError("Invalid string length");
|
||||
const result = await analyzeJavaScriptApplication(
|
||||
{ input_path: inputPath, format: "directory" },
|
||||
{
|
||||
progress: {
|
||||
report: async (event) => {
|
||||
if (event.terminal) throw cause;
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) throw new Error("Expected analysis failure");
|
||||
expect(result.error.cause).toBe(cause);
|
||||
expect(projectAnalysisError(result.error)).toMatchObject({
|
||||
code: "execution_failure",
|
||||
details: {
|
||||
provider_id: "rea-javascript-application",
|
||||
operation: "analyze_javascript_application",
|
||||
diagnostics: {
|
||||
input_path: inputPath,
|
||||
error_name: "RangeError",
|
||||
error_message: "Invalid string length",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps filesystem failures distinct from engine failures", async () => {
|
||||
const root = await createTestTempDirectory("rea-js-missing-");
|
||||
const result = await analyzeJavaScriptApplication({
|
||||
input_path: join(root, "missing"),
|
||||
format: "directory",
|
||||
});
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) throw new Error("Expected missing artifact failure");
|
||||
expect(projectAnalysisError(result.error)).toMatchObject({
|
||||
code: "artifact_operation_failed",
|
||||
details: { operation: "analyze_javascript_application", reason: "io" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["reported failure", null])(
|
||||
"retains non-Error failures (%s)",
|
||||
async (cause) => {
|
||||
const inputPath = await createTestTempDirectory("rea-js-non-error-");
|
||||
await writeFile(join(inputPath, "main.js"), "export const value = 1;\n");
|
||||
const result = await analyzeJavaScriptApplication(
|
||||
{ input_path: inputPath, format: "directory" },
|
||||
{
|
||||
progress: {
|
||||
report: async (event) => {
|
||||
if (event.terminal) throw cause;
|
||||
},
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(result.ok).toBe(false);
|
||||
if (result.ok) throw new Error("Expected analysis failure");
|
||||
expect(projectAnalysisError(result.error)).toMatchObject({
|
||||
code: "execution_failure",
|
||||
details: {
|
||||
diagnostics: {
|
||||
error_name: "UnknownError",
|
||||
error_message:
|
||||
typeof cause === "string"
|
||||
? cause
|
||||
: "JavaScript analysis failed with a non-Error value",
|
||||
},
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -14,9 +14,11 @@ import { type AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { Evidence } from "../domain/evidence.js";
|
||||
import { projectInputIssues } from "../domain/inputIssueProjection.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
|
||||
import type { ExecutionOptions } from "./AnalysisProvider.js";
|
||||
import { createJavaScriptApplicationEvidence } from "./JavaScriptApplicationEvidence.js";
|
||||
import { reconstructJavaScriptArtifact } from "./JavaScriptArtifactReconstruction.js";
|
||||
import { JAVASCRIPT_APPLICATION_PROVIDER } from "./InvestigationProviders.js";
|
||||
|
||||
const OPERATION = "analyze_javascript_application" as const;
|
||||
|
||||
@@ -82,6 +84,42 @@ export const analyzeJavaScriptApplicationValidated = async (
|
||||
{ cause },
|
||||
),
|
||||
);
|
||||
return err(new ArtifactOperationError(OPERATION, "io"));
|
||||
if (
|
||||
cause instanceof Error &&
|
||||
"code" in cause &&
|
||||
typeof cause.code === "string" &&
|
||||
FILESYSTEM_ERROR_CODES.has(cause.code)
|
||||
)
|
||||
return err(new ArtifactOperationError(OPERATION, "io"));
|
||||
return err(
|
||||
new ProviderAdapterError(JAVASCRIPT_APPLICATION_PROVIDER.id, OPERATION, {
|
||||
cause,
|
||||
diagnostics: {
|
||||
input_path: input.input_path,
|
||||
error_name: cause instanceof Error ? cause.name : "UnknownError",
|
||||
error_message:
|
||||
cause instanceof Error
|
||||
? cause.message
|
||||
: typeof cause === "string"
|
||||
? cause
|
||||
: "JavaScript analysis failed with a non-Error value",
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const FILESYSTEM_ERROR_CODES = new Set([
|
||||
"ENOENT",
|
||||
"EACCES",
|
||||
"EPERM",
|
||||
"ENOTDIR",
|
||||
"EISDIR",
|
||||
"EIO",
|
||||
"ELOOP",
|
||||
"ENAMETOOLONG",
|
||||
"EMFILE",
|
||||
"ENFILE",
|
||||
"ENOSPC",
|
||||
"EROFS",
|
||||
]);
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { fc, it } from "@fast-check/vitest";
|
||||
import canonicalize from "canonicalize";
|
||||
import { describe, expect } from "vitest";
|
||||
|
||||
import { digestCanonicalValue } from "./canonicalDigest.js";
|
||||
|
||||
const legacyDigest = (value: unknown): string => {
|
||||
const encoded = canonicalize(value);
|
||||
if (encoded === undefined) throw new TypeError("Unserializable fixture");
|
||||
return createHash("sha256").update(encoded).digest("hex");
|
||||
};
|
||||
|
||||
describe("incremental canonical digest", () => {
|
||||
it.prop([fc.jsonValue()])(
|
||||
"preserves canonicalize digests for JSON values",
|
||||
(value) => {
|
||||
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves Unicode ordering, escaping, numeric forms and non-JSON normalization", () => {
|
||||
const shared = { key: "value" };
|
||||
const sparse: unknown[] = [undefined, Symbol("omit"), () => undefined];
|
||||
sparse.length = 6;
|
||||
const fixtures: unknown[] = [
|
||||
{
|
||||
"€": "\ud800",
|
||||
"😀": "\udc00",
|
||||
"\r": '\n\t"\\',
|
||||
number: -0,
|
||||
exponent: 1e30,
|
||||
},
|
||||
{ omitted: undefined, symbol: Symbol("omit"), fn: () => undefined },
|
||||
sparse,
|
||||
[shared, shared],
|
||||
new Date("2026-10-06T00:00:00Z"),
|
||||
{ toJSON: () => ({ b: 2, a: [1, null] }) },
|
||||
{ nested: { toJSON: () => undefined } },
|
||||
];
|
||||
for (const value of fixtures)
|
||||
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
|
||||
});
|
||||
|
||||
it("rejects unsupported roots, non-finite numbers, BigInt and ancestor cycles", () => {
|
||||
for (const value of [undefined, Symbol("root"), () => undefined])
|
||||
expect(() => digestCanonicalValue(value, "Fixture")).toThrow(
|
||||
"Fixture could not canonicalize data",
|
||||
);
|
||||
const cyclic: Record<string, unknown> = {};
|
||||
cyclic.self = cyclic;
|
||||
const recursiveJson = { toJSON: (): unknown => recursiveJson };
|
||||
for (const value of [NaN, Infinity, -Infinity, 1n, cyclic, recursiveJson])
|
||||
expect(() => digestCanonicalValue(value)).toThrow();
|
||||
});
|
||||
|
||||
it("hashes repeated large leaves with the same byte sequence", () => {
|
||||
const value = {
|
||||
payload: Array.from({ length: 64 }, () => "x".repeat(16_384)),
|
||||
};
|
||||
expect(digestCanonicalValue(value)).toBe(legacyDigest(value));
|
||||
});
|
||||
|
||||
it("preserves canonicalize property access and toJSON cycle behavior", () => {
|
||||
const createAccessor = () => {
|
||||
let reads = 0;
|
||||
return {
|
||||
get value() {
|
||||
reads += 1;
|
||||
return reads;
|
||||
},
|
||||
};
|
||||
};
|
||||
expect(digestCanonicalValue(createAccessor())).toBe(
|
||||
legacyDigest(createAccessor()),
|
||||
);
|
||||
const shared = { toJSON: () => ({ answer: 42 }) };
|
||||
expect(digestCanonicalValue([shared, shared])).toBe(
|
||||
legacyDigest([shared, shared]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
/** Hash canonicalize-compatible JSON without assembling an aggregate string. */
|
||||
export const digestCanonicalValue = (
|
||||
value: unknown,
|
||||
context = "Comparison",
|
||||
): string => {
|
||||
const hash = createHash("sha256");
|
||||
const encoded = emitCanonical(value, (part) => hash.update(part), new Set());
|
||||
if (!encoded) throw new TypeError(`${context} could not canonicalize data`);
|
||||
return hash.digest("hex");
|
||||
};
|
||||
|
||||
type Emit = (part: string) => void;
|
||||
|
||||
const emitCanonical = (
|
||||
value: unknown,
|
||||
emit: Emit,
|
||||
ancestors: Set<object>,
|
||||
): boolean => {
|
||||
if (typeof value === "number" && !Number.isFinite(value))
|
||||
throw new Error(
|
||||
Number.isNaN(value) ? "NaN is not allowed" : "Infinity is not allowed",
|
||||
);
|
||||
if (value === null || typeof value !== "object") {
|
||||
const encoded = JSON.stringify(value);
|
||||
if (encoded === undefined) return false;
|
||||
emit(encoded);
|
||||
return true;
|
||||
}
|
||||
if (ancestors.has(value)) throw new Error("Circular reference detected");
|
||||
ancestors.add(value);
|
||||
try {
|
||||
if ("toJSON" in value && typeof value.toJSON === "function") {
|
||||
const normalized: unknown = value.toJSON();
|
||||
return emitCanonical(normalized, emit, ancestors);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
emit("[");
|
||||
const length = value.length;
|
||||
for (let index = 0; index < length; index += 1) {
|
||||
if (index > 0) emit(",");
|
||||
// Preserve canonicalize's sparse-array and unsupported-value behavior.
|
||||
if (!(index in value)) continue;
|
||||
const item: unknown = value[index];
|
||||
emitCanonical(
|
||||
item === undefined || typeof item === "symbol" ? null : item,
|
||||
emit,
|
||||
ancestors,
|
||||
);
|
||||
}
|
||||
emit("]");
|
||||
} else {
|
||||
emit("{");
|
||||
let first = true;
|
||||
for (const key of Object.keys(value).sort()) {
|
||||
if (
|
||||
Reflect.get(value, key) === undefined ||
|
||||
typeof Reflect.get(value, key) === "symbol"
|
||||
)
|
||||
continue;
|
||||
const item: unknown = Reflect.get(value, key);
|
||||
if (!first) emit(",");
|
||||
first = false;
|
||||
emit(JSON.stringify(key));
|
||||
emit(":");
|
||||
if (!emitCanonical(item, emit, ancestors)) emit("undefined");
|
||||
}
|
||||
emit("}");
|
||||
}
|
||||
return true;
|
||||
} finally {
|
||||
ancestors.delete(value);
|
||||
}
|
||||
};
|
||||
@@ -1,7 +1,7 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import canonicalize from "canonicalize";
|
||||
|
||||
import { digestCanonicalValue } from "./canonicalDigest.js";
|
||||
|
||||
/** Whether an inventory can support a claim that an item is absent. */
|
||||
export const absenceClaimable = (coverage: {
|
||||
readonly status: string;
|
||||
@@ -17,9 +17,7 @@ export const canonicalDigest = (
|
||||
value: unknown,
|
||||
context = "Comparison",
|
||||
): string => {
|
||||
return createHash("sha256")
|
||||
.update(canonicalJson(value, context))
|
||||
.digest("hex");
|
||||
return digestCanonicalValue(value, context);
|
||||
};
|
||||
|
||||
/** Canonical JSON used for deterministic tie breaking and stable digests. */
|
||||
|
||||
+3
-15
@@ -1,8 +1,7 @@
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import canonicalize from "canonicalize";
|
||||
import { z } from "zod";
|
||||
|
||||
import { digestCanonicalValue } from "./canonicalDigest.js";
|
||||
|
||||
import {
|
||||
analysisProfileSchema,
|
||||
type AnalysisProfileCommitment,
|
||||
@@ -173,22 +172,11 @@ export interface EvidenceObservation {
|
||||
readonly evidenceLinks?: readonly string[];
|
||||
}
|
||||
|
||||
const sha256 = (value: string): string =>
|
||||
createHash("sha256").update(value).digest("hex");
|
||||
|
||||
type WithoutEvidenceId<Record_> = Record_ extends unknown
|
||||
? Omit<Record_, "evidence_id">
|
||||
: never;
|
||||
type EvidenceWithoutId = WithoutEvidenceId<Evidence>;
|
||||
|
||||
/** Serialize JSON according to RFC 8785 JSON Canonicalization Scheme. */
|
||||
const canonicalJson = (value: JsonValue): string => {
|
||||
const serialized = canonicalize(value);
|
||||
if (serialized === undefined)
|
||||
throw new TypeError("RFC 8785 canonicalization rejected a JSON value");
|
||||
return serialized;
|
||||
};
|
||||
|
||||
const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
|
||||
subject:
|
||||
evidence.subject === null
|
||||
@@ -217,7 +205,7 @@ const semanticProjection = (evidence: EvidenceWithoutId): JsonValue => ({
|
||||
|
||||
/** Recompute the semantic identifier, excluding paths and raw payload bytes. */
|
||||
const computeEvidenceId = (evidence: EvidenceWithoutId): string =>
|
||||
`ev_${sha256(canonicalJson(semanticProjection(evidence)))}`;
|
||||
`ev_${digestCanonicalValue(semanticProjection(evidence), "Evidence")}`;
|
||||
|
||||
/** Parse evidence and reject a syntactically valid but tampered semantic ID. */
|
||||
export const parseEvidence = (input: unknown): Evidence => {
|
||||
|
||||
Reference in New Issue
Block a user