refactor(javascript): validate transferable file projections

This commit is contained in:
N0zoM1z0
2026-10-11 07:02:44 +08:00
parent 1b527f4898
commit 78f9da9a6a
9 changed files with 1100 additions and 32 deletions
@@ -8,6 +8,7 @@ import {
classifyParsedJavaScriptOpenReceiversSteps,
} from "../../domain/javascript/javascriptSemanticAnalysis.js";
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
import { projectJavaScriptModuleSemantics } from "../../domain/javascript/javascriptModuleSemanticIr.js";
import { parseJavaScriptSource } from "../../domain/javascript/javascriptSourceParser.js";
import { hasValidSourceMapContents } from "../../domain/sourceMapContents.js";
import { flattenSourceMapLeaves } from "../../domain/sourceMapEnvelope.js";
@@ -95,27 +96,7 @@ export const analyzeAndProjectJavaScriptArtifactFiles = async (
): Generator<void, JavaScriptModuleSemanticIr> {
const projection = projectSemantics(file, ir);
if (projection !== undefined) while (projection.next().done !== true) yield;
const programScopes = ir.scopes.filter(({ kind }) => kind === "program");
const programScopeId = programScopes[0]?.scopeId;
const localNames = new Set(
ir.moduleLinks.map(({ localName }) => localName),
);
const callableIds = new Set(
ir.moduleLinks.map(({ callableId }) => callableId),
);
return {
scopes: programScopes,
bindings: ir.bindings.filter(
({ name, scopeId }) =>
scopeId === programScopeId && localNames.has(name),
),
callables: ir.callables.filter(({ callableId }) =>
callableIds.has(callableId),
),
moduleLinks: ir.moduleLinks,
coverage: ir.coverage,
limitations: ir.limitations,
};
return projectJavaScriptModuleSemantics(ir);
};
const context = { state, projectSemantics: projectFileSemantics };
for (const [index, file] of fileSet.files.entries()) {
@@ -4,6 +4,7 @@ import type {
JavaScriptStaticAnalysis,
} from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
interface JavaScriptStructuredObservation {
@@ -89,15 +90,7 @@ export type JavaScriptJsonModuleObservation = JavaScriptStructuredObservation &
);
/** Module facts needed after file-local semantic projection has finished. */
export type JavaScriptModuleSemanticIr = Pick<
JavaScriptSemanticIr,
| "scopes"
| "bindings"
| "callables"
| "moduleLinks"
| "coverage"
| "limitations"
>;
export type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
/** One relevant file plus optional AST-only JavaScript facts. */
export interface AnalyzedJavaScriptArtifactFile<
@@ -55,6 +55,7 @@ import {
bindSemanticGraphApplicationNodes,
type SemanticGraphProjectionState as BuilderState,
} from "./JavaScriptSemanticGraphConstruction.js";
import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js";
interface BuilderInput {
readonly rootArtifactSha256: string;
@@ -126,6 +127,12 @@ export const buildJavaScriptSemanticGraph = ({
/** File-local semantic projection that does not retain consumed source IR. */
export interface JavaScriptSemanticGraphProjection {
/** Node capacity offered to the next independently analyzed source file. */
readonly remainingFileNodeBudget: () => number;
/** Adopt decoded file-local facts; final validation and proof remain in this realm. */
readonly adoptFileProjection: (
projection: JavaScriptSemanticFileProjection,
) => void;
readonly projectFile: (
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
@@ -252,6 +259,36 @@ export const createJavaScriptSemanticGraphProjection =
return graph;
};
return {
remainingFileNodeBudget: () =>
Math.min(
SEMANTIC_GRAPH_FILE_NODE_CEILING,
Math.max(0, SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size),
),
adoptFileProjection: (projection) => {
const newNodes = projection.nodes.filter(
({ node_id }) => !state.nodes.has(node_id),
);
if (
newNodes.length >
Math.min(
SEMANTIC_GRAPH_FILE_NODE_CEILING,
SEMANTIC_GRAPH_NODE_CEILING - state.nodes.size,
)
)
throw new RangeError(
"Transferred semantic nodes exceed the offered tree capacity",
);
for (const context of projection.evidenceContexts)
state.evidenceContexts.adopt(context);
for (const node of newNodes) state.nodes.set(node.node_id, node);
for (const relation of projection.relations)
state.relations.set(relation.relation_id, relation);
for (const unknown of projection.unknowns)
state.unknowns.set(unknown.unknown_id, unknown);
for (const root of projection.roots) state.roots.add(root);
fingerprints.push(...projection.fingerprints);
if (projection.truncated) truncatedFiles += 1;
},
projectFile: (file, ir) => {
const steps = projectSourceSteps(file, ir);
while (steps.next().done !== true);
@@ -272,6 +309,35 @@ export const createJavaScriptSemanticGraphProjection =
};
};
/** Project one source with the parent's remaining node capacity, retaining no full IR. */
export function* projectJavaScriptSemanticFileSteps(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
nodeBudget: number,
): Generator<void, JavaScriptSemanticFileProjection> {
if (
!Number.isSafeInteger(nodeBudget) ||
nodeBudget < 0 ||
nodeBudget > SEMANTIC_GRAPH_FILE_NODE_CEILING
)
throw new RangeError(
"File semantic projection requires a valid offered node capacity",
);
const state = emptyState();
state.fileNodeBudget = nodeBudget;
state.fileNodesDropped = false;
const fingerprints = yield* projectFileSteps(file, ir, state);
return {
roots: [...state.roots],
evidenceContexts: state.evidenceContexts.contexts,
nodes: [...state.nodes.values()],
relations: [...state.relations.values()],
unknowns: [...state.unknowns.values()],
fingerprints,
truncated: Boolean(state.fileNodesDropped),
};
}
function* projectFileSteps(
file: JavaScriptArtifactFile,
ir: JavaScriptSemanticIr,
@@ -0,0 +1,36 @@
import type { JavaScriptSemanticIr } from "./javascriptSemanticIr.js";
/** Module facts needed after file-local semantic projection has finished. */
export type JavaScriptModuleSemanticIr = Pick<
JavaScriptSemanticIr,
| "scopes"
| "bindings"
| "callables"
| "moduleLinks"
| "coverage"
| "limitations"
>;
/** Retain facts consumed by source-module and export-return composition. */
export const projectJavaScriptModuleSemantics = (
ir: JavaScriptSemanticIr,
): JavaScriptModuleSemanticIr => {
const scopes = ir.scopes.filter(({ kind }) => kind === "program");
const programScopeId = scopes[0]?.scopeId;
const localNames = new Set(ir.moduleLinks.map(({ localName }) => localName));
const callableIds = new Set(
ir.moduleLinks.map(({ callableId }) => callableId),
);
return {
scopes,
bindings: ir.bindings.filter(
({ name, scopeId }) => scopeId === programScopeId && localNames.has(name),
),
callables: ir.callables.filter(({ callableId }) =>
callableIds.has(callableId),
),
moduleLinks: ir.moduleLinks,
coverage: ir.coverage,
limitations: ir.limitations,
};
};
@@ -0,0 +1,18 @@
import type {
JavaScriptSemanticEvidenceContext,
JavaScriptSemanticFingerprint,
JavaScriptSemanticGraphNode,
JavaScriptSemanticGraphRelation,
JavaScriptSemanticGraphUnknown,
} from "./javascriptSemanticGraphSchemas.js";
/** File-local facts transported before binding them to the containing application graph. */
export interface JavaScriptSemanticFileProjection {
readonly roots: readonly string[];
readonly evidenceContexts: readonly JavaScriptSemanticEvidenceContext[];
readonly nodes: readonly JavaScriptSemanticGraphNode[];
readonly relations: readonly JavaScriptSemanticGraphRelation[];
readonly unknowns: readonly JavaScriptSemanticGraphUnknown[];
readonly fingerprints: readonly JavaScriptSemanticFingerprint[];
readonly truncated: boolean;
}
@@ -19,6 +19,7 @@ import {
JAVASCRIPT_SEMANTIC_RELATION_FAMILIES,
JAVASCRIPT_SEMANTIC_RELATION_FAMILY,
javaScriptSemanticFingerprintInputSchema,
javaScriptSemanticEvidenceContextSchema,
javaScriptSemanticGraphInputSchema,
javaScriptSemanticGraphRecordSchema,
javaScriptSemanticNodeInputSchema,
@@ -109,6 +110,33 @@ export class JavaScriptSemanticEvidenceContextRegistry {
this.#contexts.clear();
this.#idsByCanonicalContext.clear();
}
/** Validate and intern provenance decoded in this realm from an analysis worker. */
adopt(input: unknown): void {
const context = javaScriptSemanticEvidenceContextSchema.parse(input);
const { context_id: identifier, ...value } = context;
if (identifier !== evidenceContextId(value))
throw new TypeError(
"Transferred semantic evidence context identifier is stale",
);
const canonical = canonicalJson(
value,
"JavaScript semantic evidence context",
);
const existing = this.#contexts.get(identifier);
if (existing !== undefined) {
const { context_id: _existingIdentifier, ...existingValue } = existing;
if (
canonical !==
canonicalJson(existingValue, "JavaScript semantic evidence context")
)
throw new TypeError(
"Transferred semantic evidence context digest collision",
);
}
this.#contexts.set(identifier, context);
this.#idsByCanonicalContext.set(canonical, identifier);
}
}
/** Derive a semantic node ID from its kind and exact artifact identity. */
@@ -0,0 +1,276 @@
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { open, stat } from "node:fs/promises";
import { z } from "zod";
import type { JavaScriptArtifactFile } from "../../domain/javascript/javascriptArtifactFiles.js";
import type { JavaScriptModuleSemanticIr } from "../../domain/javascript/javascriptModuleSemanticIr.js";
import type { JavaScriptSemanticFileProjection } from "../../domain/javascript/javascriptSemanticFileProjection.js";
import { digestSchema } from "../../domain/digests.js";
import {
javaScriptWorkerGraphRecordSchema,
javaScriptWorkerModuleHeaderSchema,
javaScriptWorkerModuleRecordSchema,
javaScriptWorkerProjectionHeaderSchema,
} from "./JavaScriptAnalysisWorkerSchemas.js";
// Budgets bound transfer representation, independently of source length.
const FRAME_BYTES = 2 * 1024 * 1024;
const TRANSFER_BYTES = 512 * 1024 * 1024;
const BATCH_BYTES = 1024 * 1024;
const recordSchema = z.discriminatedUnion("kind", [
z.strictObject({
kind: z.literal("projection-header"),
value: javaScriptWorkerProjectionHeaderSchema,
}),
z.strictObject({
kind: z.literal("module-header"),
value: javaScriptWorkerModuleHeaderSchema,
}),
...javaScriptWorkerGraphRecordSchema.options,
...javaScriptWorkerModuleRecordSchema.options,
]);
/** Complete-file coordinates authenticated before adopting any worker facts. */
export const javaScriptAnalysisTransferDescriptorSchema = z.strictObject({
bytes: z.number().int().nonnegative().max(TRANSFER_BYTES),
sha256: digestSchema,
records: z.number().int().nonnegative(),
});
type Descriptor = z.output<typeof javaScriptAnalysisTransferDescriptorSchema>;
/** Encode validated facts in bounded records, without serializing a whole graph or IR. */
export const writeJavaScriptAnalysisTransfer = async (
path: string,
module: JavaScriptModuleSemanticIr,
projection: JavaScriptSemanticFileProjection,
): Promise<Descriptor> => {
const output = await open(path, "wx", 0o600);
const digest = createHash("sha256");
let bytes = 0;
let records = 0;
let batch: string[] = [];
let batchBytes = 0;
const flush = async (): Promise<void> => {
if (batch.length === 0) return;
const buffer = Buffer.from(batch.join(""));
// FileHandle.write can be short; writeFile completes the buffer.
await output.writeFile(buffer);
digest.update(buffer);
batch = [];
batchBytes = 0;
};
const append = async (input: unknown): Promise<void> => {
const record = recordSchema.parse(input);
const line = `${JSON.stringify(record)}\n`;
const size = Buffer.byteLength(line);
if (size > FRAME_BYTES || bytes + size > TRANSFER_BYTES)
throw new RangeError(
"JavaScript analysis transfer exceeds its record or file byte budget",
);
bytes += size;
records += 1;
batch.push(line);
batchBytes += size;
if (batchBytes >= BATCH_BYTES) await flush();
};
try {
await append({
kind: "projection-header",
value: { roots: projection.roots, truncated: projection.truncated },
});
await append({
kind: "module-header",
value: { coverage: module.coverage, limitations: module.limitations },
});
for (const value of module.scopes) await append({ kind: "scope", value });
for (const value of module.bindings)
await append({ kind: "binding", value });
for (const value of module.callables)
await append({ kind: "callable", value });
for (const value of module.moduleLinks)
await append({ kind: "module-link", value });
for (const value of projection.evidenceContexts)
await append({ kind: "evidence-context", value });
for (const value of projection.nodes) await append({ kind: "node", value });
for (const value of projection.relations)
await append({ kind: "relation", value });
for (const value of projection.unknowns)
await append({ kind: "unknown", value });
for (const value of projection.fingerprints)
await append({ kind: "fingerprint", value });
await flush();
return { bytes, sha256: digest.digest("hex"), records };
} finally {
await output.close();
}
};
/** Decode producer records into realm-local, source-bound facts; sealing follows in the consumer. */
export const readJavaScriptAnalysisTransfer = async (
path: string,
descriptor: Descriptor,
source: Pick<JavaScriptArtifactFile, "path" | "sha256">,
signal?: AbortSignal,
): Promise<{
readonly module: JavaScriptModuleSemanticIr;
readonly projection: JavaScriptSemanticFileProjection;
}> => {
const expected = javaScriptAnalysisTransferDescriptorSchema.parse(descriptor);
if ((await stat(path)).size !== expected.bytes)
throw new TypeError(
"JavaScript analysis transfer byte count does not match",
);
const scopes: JavaScriptModuleSemanticIr["scopes"][number][] = [];
const bindings: JavaScriptModuleSemanticIr["bindings"][number][] = [];
const callables: JavaScriptModuleSemanticIr["callables"][number][] = [];
const moduleLinks: JavaScriptModuleSemanticIr["moduleLinks"][number][] = [];
const evidenceContexts: JavaScriptSemanticFileProjection["evidenceContexts"][number][] =
[];
const nodes: JavaScriptSemanticFileProjection["nodes"][number][] = [];
const relations: JavaScriptSemanticFileProjection["relations"][number][] = [];
const unknowns: JavaScriptSemanticFileProjection["unknowns"][number][] = [];
const fingerprints: JavaScriptSemanticFileProjection["fingerprints"][number][] =
[];
let projectionHeader:
| z.output<typeof javaScriptWorkerProjectionHeaderSchema>
| undefined;
let moduleHeader:
| z.output<typeof javaScriptWorkerModuleHeaderSchema>
| undefined;
const identities = new Map<string, Set<string>>();
const retainIdentity = (kind: string, identifier: string): void => {
let seen = identities.get(kind);
if (seen === undefined) {
seen = new Set();
identities.set(kind, seen);
}
if (seen.has(identifier))
throw new TypeError(
`Duplicate JavaScript transfer ${kind} identity: ${identifier}`,
);
seen.add(identifier);
};
let records = 0;
const digest = createHash("sha256");
const stream = createReadStream(path, {
highWaterMark: 64 * 1024,
...(signal === undefined ? {} : { signal }),
});
let bytes = 0;
let pending = "";
stream.setEncoding("utf8");
try {
for await (const chunk of stream) {
signal?.throwIfAborted();
if (typeof chunk !== "string")
throw new TypeError("JavaScript transfer stream is not UTF-8 text");
bytes += Buffer.byteLength(chunk);
if (bytes > expected.bytes)
throw new RangeError("JavaScript transfer grew while being read");
digest.update(chunk);
pending += chunk;
let newline: number;
while ((newline = pending.indexOf("\n")) >= 0) {
const line = pending.slice(0, newline);
pending = pending.slice(newline + 1);
if (Buffer.byteLength(line) + 1 > FRAME_BYTES)
throw new RangeError(
"JavaScript transfer record exceeds its byte budget",
);
const input: unknown = JSON.parse(line);
const record = recordSchema.parse(input);
if (record.kind === "projection-header") {
if (records !== 0 || projectionHeader !== undefined)
throw new TypeError(
"Duplicate or late JavaScript projection header",
);
projectionHeader = record.value;
} else if (record.kind === "module-header") {
if (records !== 1 || moduleHeader !== undefined)
throw new TypeError("Duplicate or late JavaScript module header");
moduleHeader = record.value;
} else {
if (projectionHeader === undefined || moduleHeader === undefined)
throw new TypeError(
"JavaScript transfer facts precede their headers",
);
switch (record.kind) {
case "scope":
retainIdentity(record.kind, record.value.scopeId);
scopes.push(record.value);
break;
case "binding":
retainIdentity(record.kind, record.value.bindingId);
bindings.push(record.value);
break;
case "callable":
retainIdentity(record.kind, record.value.callableId);
callables.push(record.value);
break;
case "module-link":
moduleLinks.push(record.value);
break;
case "evidence-context":
retainIdentity(record.kind, record.value.context_id);
evidenceContexts.push(record.value);
break;
case "node": {
retainIdentity(record.kind, record.value.node_id);
if (
record.value.identity.artifact_sha256 !== source.sha256 ||
record.value.identity.module_path !== source.path
)
throw new TypeError(
"JavaScript transfer node belongs to a different artifact source",
);
nodes.push(record.value);
break;
}
case "relation":
retainIdentity(record.kind, record.value.relation_id);
relations.push(record.value);
break;
case "unknown":
retainIdentity(record.kind, record.value.unknown_id);
unknowns.push(record.value);
break;
case "fingerprint":
retainIdentity(record.kind, record.value.fingerprint_id);
fingerprints.push(record.value);
break;
}
}
records += 1;
}
if (Buffer.byteLength(pending) + 1 > FRAME_BYTES)
throw new RangeError(
"Unterminated JavaScript transfer record exceeds its byte budget",
);
}
} finally {
stream.destroy();
}
if (
pending !== "" ||
records !== expected.records ||
bytes !== expected.bytes ||
digest.digest("hex") !== expected.sha256
)
throw new TypeError(
"JavaScript analysis transfer is incomplete or its digest does not match",
);
if (projectionHeader === undefined || moduleHeader === undefined)
throw new TypeError("JavaScript analysis transfer is missing its headers");
return {
module: { ...moduleHeader, scopes, bindings, callables, moduleLinks },
projection: {
...projectionHeader,
evidenceContexts,
nodes,
relations,
unknowns,
fingerprints,
},
};
};
@@ -0,0 +1,582 @@
import { z } from "zod";
import { digestSchema } from "../../domain/digests.js";
import type { JavaScriptStaticAnalysis } from "../../domain/javascript/javascriptStaticAnalysisTypes.js";
import type {
JavaScriptSemanticProperty,
JavaScriptSemanticValue,
} from "../../domain/javascript/javascriptSemanticValueTypes.js";
import {
javaScriptSemanticNodeSchema,
javaScriptSemanticRelationSchema,
javaScriptSemanticUnknownSchema,
javaScriptSemanticFingerprintSchema,
javaScriptSemanticEvidenceContextSchema,
} from "../../domain/javascript/javascriptSemanticGraphSchemas.js";
const range =
javaScriptSemanticNodeSchema.shape.identity.shape.source_range.unwrap();
const count = z.number().int().nonnegative();
const strings = z.array(z.string());
const pathContext = z.enum(["module-specifier", "filesystem-expression"]);
const findingContext = {
module_key: z.string().nullable(),
module_runtime: z.string().optional(),
location: range,
};
const withRuntime = <Value extends { module_runtime?: string | undefined }>(
value: Value,
) => {
const { module_runtime, ...rest } = value;
return {
...rest,
...(module_runtime === undefined ? {} : { module_runtime }),
};
};
const staticValue = z.discriminatedUnion("status", [
z.strictObject({
status: z.literal("literal"),
value: z.union([z.string(), z.number(), z.boolean(), z.null()]),
expression: z.null(),
}),
z.strictObject({
status: z.literal("dynamic"),
value: z.null(),
expression: z.string(),
}),
]);
const nullHandler = { handler_kind: z.null(), handler_location: z.null() };
const missingHandler = {
handler_kind: z.literal("missing"),
handler_location: z.null(),
};
const locatedHandler = {
handler_kind: z.enum([
"inline-function",
"identifier",
"member-expression",
"dynamic-expression",
]),
handler_location: range,
};
const ipcChannel = <Shape extends z.ZodRawShape>(shape: Shape) =>
z.union([
z.strictObject({
...findingContext,
...shape,
channel: z.string(),
channel_expression: z.null(),
}),
z.strictObject({
...findingContext,
...shape,
channel: z.null(),
channel_expression: z.string(),
}),
]);
const ipc = z
.union([
ipcChannel({
side: z.literal("renderer"),
operation: z.enum(["send", "send-sync", "post-message", "send-to-host"]),
mode: z.literal("send"),
...nullHandler,
}),
ipcChannel({
side: z.literal("renderer"),
operation: z.literal("invoke"),
mode: z.literal("invoke"),
...nullHandler,
}),
ipcChannel({
side: z.enum(["renderer", "main"]),
operation: z.enum(["on", "once"]),
mode: z.literal("listen"),
...missingHandler,
}),
ipcChannel({
side: z.enum(["renderer", "main"]),
operation: z.enum(["on", "once"]),
mode: z.literal("listen"),
...locatedHandler,
}),
ipcChannel({
side: z.literal("main"),
operation: z.enum(["handle", "handle-once"]),
mode: z.literal("handle"),
...missingHandler,
}),
ipcChannel({
side: z.literal("main"),
operation: z.enum(["handle", "handle-once"]),
mode: z.literal("handle"),
...locatedHandler,
}),
])
.transform((value) => {
const { module_runtime, ...rest } = value;
return {
...rest,
...(module_runtime === undefined ? {} : { module_runtime }),
};
});
const windowShape = {
...findingContext,
options_status: z.enum(["object-literal", "dynamic", "missing"]),
web_preferences_status: z.enum(["object-literal", "dynamic", "missing"]),
web_preferences: z.array(
z.strictObject({ name: z.string(), value: staticValue }),
),
};
const bridgeShape = {
...findingContext,
world: z.enum(["main", "isolated"]),
world_id: staticValue.nullable(),
api_status: z.enum(["object-literal", "dynamic", "missing"]),
members: strings,
unknown_members: count,
};
const utilityShape = { ...findingContext, service_name: z.string().nullable() };
const electron = z.strictObject({
browser_windows: z.array(
z.union([
z
.strictObject({
...windowShape,
preload_path: z.string(),
preload_resolution_context: pathContext,
})
.transform(withRuntime),
z
.strictObject({
...windowShape,
preload_path: z.null(),
preload_resolution_context: z.null(),
})
.transform(withRuntime),
]),
),
context_bridge_apis: z.array(
z.union([
z
.strictObject({
...bridgeShape,
api_key: z.string(),
api_key_expression: z.null(),
})
.transform(withRuntime),
z
.strictObject({
...bridgeShape,
api_key: z.null(),
api_key_expression: z.string(),
})
.transform(withRuntime),
z
.strictObject({
...bridgeShape,
api_key: z.null(),
api_key_expression: z.null(),
})
.transform(withRuntime),
]),
),
ipc: z.array(ipc),
sender_validations: z.array(
z.strictObject({
...findingContext,
subject: z.enum([
"sender-url",
"sender-origin",
"sender-frame",
"sender-id",
"frame-id",
"process-id",
]),
mechanism: z.string(),
expected: staticValue,
enforcement: z.literal("unknown"),
}),
),
utility_processes: z.array(
z.union([
z
.strictObject({
...utilityShape,
module_path: z.string(),
module_resolution_context: pathContext,
module_expression: z.null(),
})
.transform(withRuntime),
z
.strictObject({
...utilityShape,
module_path: z.null(),
module_resolution_context: z.null(),
module_expression: z.string(),
})
.transform(withRuntime),
]),
),
native_addon_bindings: z.array(
z.strictObject({
...findingContext,
specifier: z.string(),
binding_kind: z.enum(["import", "require", "re-export"]),
module_kind: z.enum(["import", "require"]),
members: strings,
namespace_access: z.boolean().optional(),
dynamic_member_access: z.boolean().optional(),
}),
),
});
const referenceShape = {
...findingContext,
kind: z.enum([
"static-import",
"dynamic-import",
"require",
"worker",
"service-worker",
]),
};
/** Exact producer facts recoverable before a later semantic worker failure. */
export const javaScriptWorkerStaticAnalysisSchema = z
.strictObject({
parse_status: z.enum(["complete", "partial", "failed"]),
parse_error_count: count,
visited_ast_nodes: count,
references: z.array(
z.union([
z
.strictObject({
...referenceShape,
specifier: z.string(),
expression: z.null(),
})
.transform(withRuntime),
z
.strictObject({
...referenceShape,
specifier: z.null(),
expression: z.string(),
})
.transform(withRuntime),
]),
),
endpoints: z.array(
z.strictObject({
...findingContext,
kind: z.enum(["route", "network"]),
value: z.string(),
mechanism: z.string(),
}),
),
storage: z.array(
z.strictObject({
...findingContext,
kind: z.enum([
"local-storage",
"session-storage",
"indexed-db",
"cache-storage",
"sqlite",
]),
name: z.string().nullable(),
mechanism: z.string(),
}),
),
bundler_registrations: z.array(
z.strictObject({
bundler: z.enum(["webpack", "rspack", "esbuild"]),
runtime: z.string(),
chunk_keys: strings,
unknown_chunk_keys: count,
runtime_require_name: z.string().nullable(),
runtime_module_cache_status: z.enum(["observed", "not-observed"]),
entry_module_keys: strings,
unknown_entry_module_keys: count,
async_chunk_keys: strings,
unknown_async_chunk_keys: count,
modules: z.array(
z.strictObject({
module_key: z.string(),
factory_require_name: z.string().nullable(),
source_sha256: digestSchema,
exports: strings,
location: range,
structural_fingerprint_sha256: digestSchema,
structural_fingerprint_algorithm: z.literal("babel-ast-v1"),
}),
),
location: range,
}),
),
role_paths: z.array(
z.strictObject({
...findingContext,
role: z.enum(["preload", "renderer"]),
path: z.string(),
resolution_context: pathContext,
mechanism: z.string(),
}),
),
source_map_urls: z.array(
z.strictObject({ declared_url: z.string(), location: range }),
),
vendors: strings,
electron,
limitations: strings,
})
.transform((analysis) => ({
...analysis,
endpoints: analysis.endpoints.map(withRuntime),
storage: analysis.storage.map(withRuntime),
role_paths: analysis.role_paths.map(withRuntime),
electron: {
...analysis.electron,
sender_validations: analysis.electron.sender_validations.map(withRuntime),
native_addon_bindings: analysis.electron.native_addon_bindings.map(
(value) => {
const { namespace_access, dynamic_member_access, ...rest } =
withRuntime(value);
return {
...rest,
...(namespace_access === undefined ? {} : { namespace_access }),
...(dynamic_member_access === undefined
? {}
: { dynamic_member_access }),
};
},
),
},
})) satisfies z.ZodType<JavaScriptStaticAnalysis>;
const resourceLimit = z.enum([
"primitive-candidates",
"primitive-bytes",
"expression-depth",
]);
const primitive = z.union([z.string(), z.number(), z.boolean(), z.null()]);
const property: z.ZodType<JavaScriptSemanticProperty> = z.lazy(() =>
z.strictObject({
name: z.string(),
value: semanticValue,
presence: z.enum(["present", "absent", "unknown-coverage"]),
}),
);
const semanticValue: z.ZodType<JavaScriptSemanticValue> = z.lazy(() =>
z
.discriminatedUnion("status", [
z.strictObject({ status: z.literal("literal"), value: primitive }),
z.strictObject({
status: z.literal("union"),
values: z.array(primitive),
}),
z.discriminatedUnion("unknownProperties", [
z.strictObject({
status: z.literal("object"),
properties: z.array(property),
unknownProperties: z.literal(false),
omittedProperties: z.literal(0),
}),
z.strictObject({
status: z.literal("object"),
properties: z.array(property),
unknownProperties: z.literal(true),
omittedProperties: count.nullable(),
}),
]),
z.discriminatedUnion("unknownItems", [
z.strictObject({
status: z.literal("array"),
items: z.array(property),
unknownItems: z.literal(false),
omittedItems: z.literal(0),
}),
z.strictObject({
status: z.literal("array"),
items: z.array(property),
unknownItems: z.literal(true),
omittedItems: count.nullable(),
}),
]),
z.strictObject({
status: z.enum(["unknown", "ambiguous", "cycle"]),
reason: z.string(),
resourceLimit: resourceLimit.optional(),
}),
])
.transform((value) => {
if (
value.status === "literal" ||
value.status === "union" ||
value.status === "object" ||
value.status === "array"
)
return value;
const { resourceLimit: limit, ...rest } = value;
return {
...rest,
...(limit === undefined ? {} : { resourceLimit: limit }),
};
}),
);
const origin = z.strictObject({ specifier: z.string(), importedPath: strings });
const provenance = z.discriminatedUnion("status", [
z.strictObject({
status: z.literal("module"),
origins: z.tuple([origin]),
reason: z.null(),
}),
z.strictObject({
status: z.literal("local"),
origins: z.tuple([]),
reason: z.null(),
}),
z.strictObject({
status: z.literal("ambiguous"),
origins: z.array(origin),
reason: z.string(),
}),
z.strictObject({
status: z.enum(["unknown", "cycle"]),
origins: z.tuple([]),
reason: z.string(),
}),
]);
const coverage = z
.discriminatedUnion("status", [
z.strictObject({
status: z.literal("complete"),
omittedCount: z.literal(0),
resourceLimits: z.array(resourceLimit).optional(),
}),
z.strictObject({
status: z.literal("partial"),
omittedCount: z.union([z.literal(0), z.null()]),
resourceLimits: z.array(resourceLimit).optional(),
}),
z.strictObject({
status: z.literal("failed"),
omittedCount: z.null(),
resourceLimits: z.array(resourceLimit).optional(),
}),
])
.transform((value) => {
const { resourceLimits, ...rest } = value;
return {
...rest,
...(resourceLimits === undefined ? {} : { resourceLimits }),
};
});
const moduleLink = z.strictObject({
kind: z.enum(["import", "require", "export", "re-export", "commonjs-export"]),
specifier: z.string().nullable(),
importedName: z.string().nullable(),
localName: z.string().nullable(),
exportedName: z.string().nullable(),
callableId: z.string().nullable(),
location: range,
});
const definitionKind = z.enum([
"import",
"variable",
"parameter",
"function",
"class",
"catch",
"assignment",
]);
const scope = z.strictObject({
scopeId: z.string(),
parentScopeId: z.string().nullable(),
kind: z.enum([
"program",
"function",
"block",
"static-block",
"class",
"catch",
]),
location: range,
bindingsComplete: z.boolean(),
bindingIds: strings,
});
const binding = z.strictObject({
bindingId: z.string(),
scopeId: z.string(),
name: z.string(),
kind: definitionKind,
mutable: z.boolean(),
definitions: z.array(
z.strictObject({ kind: definitionKind, location: range }),
),
value: semanticValue,
provenance,
});
const callable = z.strictObject({
callableId: z.string(),
kind: z.enum(["function", "class", "method"]),
name: z.string().nullable(),
containerScopeId: z.string(),
bodyScopeId: z.string().nullable(),
location: range,
returnSites: z.array(
z.strictObject({
returnSiteId: z.string(),
location: range,
identityReferenceLocation: range.nullable(),
value: semanticValue,
}),
),
returnCoverage: z.strictObject({
retainedCount: count,
status: z.enum(["complete", "partial"]),
omittedCount: z.union([z.literal(0), z.null()]),
}),
});
/** Validate each transferred graph fact independently of its emitting realm. */
export const javaScriptWorkerGraphRecordSchema = z.discriminatedUnion("kind", [
z.strictObject({
kind: z.literal("evidence-context"),
value: javaScriptSemanticEvidenceContextSchema,
}),
z.strictObject({
kind: z.literal("node"),
value: javaScriptSemanticNodeSchema,
}),
z.strictObject({
kind: z.literal("relation"),
value: javaScriptSemanticRelationSchema,
}),
z.strictObject({
kind: z.literal("unknown"),
value: javaScriptSemanticUnknownSchema,
}),
z.strictObject({
kind: z.literal("fingerprint"),
value: javaScriptSemanticFingerprintSchema,
}),
]);
/** Portable per-file header, before any graph binding or immutable proof. */
export const javaScriptWorkerProjectionHeaderSchema = z.strictObject({
roots: z.array(javaScriptSemanticNodeSchema.shape.node_id),
truncated: z.boolean(),
});
/** Module facts are transferred individually to avoid a whole-IR JSON string. */
export const javaScriptWorkerModuleRecordSchema = z.discriminatedUnion("kind", [
z.strictObject({ kind: z.literal("scope"), value: scope }),
z.strictObject({ kind: z.literal("binding"), value: binding }),
z.strictObject({ kind: z.literal("callable"), value: callable }),
z.strictObject({ kind: z.literal("module-link"), value: moduleLink }),
]);
/** Coverage and limitations are independent of module collection sizes. */
export const javaScriptWorkerModuleHeaderSchema = z.strictObject({
coverage,
limitations: strings,
});
@@ -3,7 +3,7 @@ import {
primitiveByteExpansionSource,
} from "../../fixtures/javascriptPrimitiveExpansion.js";
import { createHash } from "node:crypto";
import { mkdir, rm, writeFile } from "node:fs/promises";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { createPackageWithOptions } from "@electron/asar";
@@ -23,7 +23,15 @@ import { buildJavaScriptArtifactGraph } from "../../../src/application/javascrip
import {
buildJavaScriptSemanticGraph,
createJavaScriptSemanticGraphProjection,
projectJavaScriptSemanticFileSteps,
} from "../../../src/application/javascript/JavaScriptSemanticGraphBuilder.js";
import { projectJavaScriptModuleSemantics } from "../../../src/domain/javascript/javascriptModuleSemanticIr.js";
import { completeJavaScriptAnalysisSteps } from "../../../src/application/javascript/JavaScriptAnalysisControl.js";
import {
writeJavaScriptAnalysisTransfer,
readJavaScriptAnalysisTransfer,
} from "../../../src/javascript/analysis/JavaScriptAnalysisTransfer.js";
import { javaScriptWorkerStaticAnalysisSchema } from "../../../src/javascript/analysis/JavaScriptAnalysisWorkerSchemas.js";
import { createJavaScriptArtifactReader } from "../../../src/artifacts/javascript/JavaScriptArtifactReader.js";
import {
scanCanonicalArtifactInventory,
@@ -31,7 +39,10 @@ import {
} from "../../fixtures/artifactInventory.js";
import { parseJavaScriptApplicationGraph } from "../../../src/domain/javascript/javascriptApplicationGraph.js";
import { javascriptApplicationAnalysisResultSchema } from "../../../src/domain/javascript/javascriptApplicationAnalysis.js";
import { createJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraph.js";
import {
createJavaScriptSemanticGraph,
isValidatedImmutableJavaScriptSemanticGraph,
} from "../../../src/domain/javascript/javascriptSemanticGraph.js";
import { parseJavaScriptSemanticGraph } from "../../../src/domain/javascript/javascriptSemanticGraphSerialization.js";
import { writeJavaScriptArtifactFixture } from "../../fixtures/javascriptArtifactApplication.js";
@@ -61,6 +72,7 @@ it("preserves graph commitments and export shapes when consuming file-local IR",
import { readFile } from "node:fs";
export { readFile as read };
export function create(value) { return { value, nested: { enabled: true } }; }
export function configured(condition) { return { values: [1, null, undefined], choice: condition ? 1 : 2, ...external }; }
function unexported() { return "private"; }
`,
);
@@ -96,6 +108,7 @@ it("preserves graph commitments and export shapes when consuming file-local IR",
);
expect(exported?.semantic?.ir.callables.map(({ name }) => name)).toEqual([
"create",
"configured",
]);
expect(
full.files
@@ -103,6 +116,81 @@ it("preserves graph commitments and export shapes when consuming file-local IR",
?.semantic?.ir.callables.map(({ name }) => name),
).toContain("unexported");
expect(parseJavaScriptSemanticGraph(semantic)).toEqual(semantic);
const transferred = createJavaScriptSemanticGraphProjection();
const wireRoot = await createTestTempDirectory("rea-javascript-transfer-");
let rejectedInvalidTransfer = false;
for (const [index, analyzed] of full.files.entries()) {
if (analyzed.javascript !== null) {
const raw: unknown = JSON.parse(JSON.stringify(analyzed.javascript));
expect(javaScriptWorkerStaticAnalysisSchema.parse(raw)).toEqual(
analyzed.javascript,
);
}
if (analyzed.semantic === null) continue;
const fragment = await completeJavaScriptAnalysisSteps(
projectJavaScriptSemanticFileSteps(
analyzed.file,
analyzed.semantic.ir,
transferred.remainingFileNodeBudget(),
),
);
const module = projectJavaScriptModuleSemantics(analyzed.semantic.ir);
const wirePath = join(wireRoot, `${String(index)}.jsonl`);
const descriptor = await writeJavaScriptAnalysisTransfer(
wirePath,
module,
fragment,
);
const decoded = await readJavaScriptAnalysisTransfer(
wirePath,
descriptor,
analyzed.file,
);
expect(decoded.module).toEqual(module);
transferred.adoptFileProjection(decoded.projection);
if (!rejectedInvalidTransfer) {
await expect(
readJavaScriptAnalysisTransfer(wirePath, descriptor, {
...analyzed.file,
sha256: "0".repeat(64),
}),
).rejects.toThrow("different artifact source");
await expect(
readJavaScriptAnalysisTransfer(
wirePath,
{ ...descriptor, sha256: "0".repeat(64) },
analyzed.file,
),
).rejects.toThrow("digest does not match");
const wire = await readFile(wirePath, "utf8");
const duplicate = wire
.split("\n")
.find((line) => line.startsWith('{"kind":"node"'));
expect(duplicate).toBeDefined();
if (duplicate === undefined)
throw new Error("Fixture emitted no semantic node record");
const invalid = `${wire}${duplicate}\n`;
const invalidPath = join(wireRoot, "duplicate.jsonl");
await writeFile(invalidPath, invalid);
await expect(
readJavaScriptAnalysisTransfer(
invalidPath,
{
bytes: Buffer.byteLength(invalid),
records: descriptor.records + 1,
sha256: createHash("sha256").update(invalid).digest("hex"),
},
analyzed.file,
),
).rejects.toThrow("Duplicate JavaScript transfer node identity");
rejectedInvalidTransfer = true;
}
}
const resealed = await completeJavaScriptAnalysisSteps(
transferred.finishImmutableSteps(snapshot.manifest.root_sha256, graph),
);
expect(resealed).toEqual(semantic);
expect(isValidatedImmutableJavaScriptSemanticGraph(resealed)).toBe(true);
} finally {
await reader.close();
}