refactor(artifacts): remove selectors and traversal quotas

This commit is contained in:
morluto
2026-09-29 12:15:49 +08:00
parent fe342faa83
commit 8c24272ee7
31 changed files with 268 additions and 666 deletions
+23 -126
View File
@@ -8,7 +8,6 @@ import {
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactLimits,
type ArtifactReader,
} from "../artifacts/ArtifactReader.js";
import { DirectoryArtifactReader } from "../artifacts/DirectoryArtifactReader.js";
@@ -24,35 +23,28 @@ import {
} from "../domain/artifactGraph.js";
import type { BinaryTarget } from "../domain/binaryTarget.js";
import { scanArtifactInventory } from "./ArtifactInventory.js";
import {
digestCanonical,
toOutputLimits,
} from "./ArtifactGraphConstruction.js";
import { digestCanonical } from "./ArtifactGraphConstruction.js";
/** Already-approved extraction request from a caller-owned policy boundary. */
export interface ArtifactExtractionInput {
readonly inputPath: string;
readonly inputFormat: BinaryTarget["format"];
readonly outputRoot: string;
readonly selection:
| { readonly paths: readonly string[] }
| { readonly occurrenceIds: readonly string[] };
readonly limits: ArtifactLimits;
}
/** Extract selected inventory occurrences into an exclusively owned absent root. */
/** Extract every regular inventory occurrence into an exclusively owned absent root. */
export const extractArtifact = async (
input: ArtifactExtractionInput,
signal?: AbortSignal,
): Promise<ArtifactExtractionResult> => {
validateSelection(input.selection, input.limits);
const sourcePath = await realpath(input.inputPath);
const snapshot = await scanArtifactInventory(sourcePath, input.limits, {
const snapshot = await scanArtifactInventory(sourcePath, {
signal,
});
const selectedOccurrences = resolveSelection(
snapshot.occurrences,
input.selection,
const selectedOccurrences = snapshot.occurrences.filter(
(occurrence) =>
(occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") &&
occurrence.logical_path !== ".",
);
const selectedIds = new Set(
selectedOccurrences.map(({ occurrence_id: id }) => id),
@@ -130,25 +122,25 @@ const materializeSelection = async ({
selected.map((item) => [item.occurrence.logical_path, item]),
);
const reader = await createReader(sourcePath, input.inputFormat);
const output = await SafeOutputTree.create(input.outputRoot, input.limits);
const output = await SafeOutputTree.create(input.outputRoot);
let readerClosed = false;
const extracted: ExtractedOccurrence[] = [];
try {
const found = new Set<string>();
const materialized: SelectedOccurrence[] = [];
const registry = new ArtifactPathRegistry();
let entryCount = 0;
for await (const entry of reader.entries(signal)) {
entryCount += 1;
if (entryCount > input.limits.maxEntries)
throw new ArtifactReaderFailure(
"limit",
"Artifact entry limit exceeded during extraction",
);
const path = normalizeArtifactPath(entry.path, input.limits);
const path = normalizeArtifactPath(entry.path);
registry.add(path, entry.kind);
const selectedItem = byPath.get(path);
if (selectedItem === undefined) continue;
preflight(entry, input.limits);
if (selectedItem === undefined) {
if (entry.kind === "file" || entry.kind === "slice")
throw new ArtifactReaderFailure(
"integrity",
`Regular artifact entry is missing from inventory: ${path}`,
);
continue;
}
preflight(entry);
const stream = await reader.open(entry, signal);
const written = await output.write(
path,
@@ -163,13 +155,8 @@ const materializeSelection = async ({
bytes_written: written.bytesWritten,
created: true,
});
found.add(path);
materialized.push(selectedItem);
}
if (found.size !== selected.length)
throw new ArtifactReaderFailure(
"integrity",
"Selected inventory occurrence disappeared before extraction",
);
await reader.close();
readerClosed = true;
extracted.sort((left, right) =>
@@ -178,7 +165,7 @@ const materializeSelection = async ({
const result = createExtractionResult(
input,
inventory,
selected,
materialized,
extracted,
);
await output.commit();
@@ -214,10 +201,9 @@ const createExtractionResult = (
artifacts: extracted,
containment_verified: true,
cleanup: { attempted: false, verified: true, residual_paths: [] },
limits: toOutputLimits(input.limits),
provenance: [],
limitations: [
"Only caller-selected regular file occurrences were materialized.",
"All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file.",
],
});
};
@@ -272,99 +258,10 @@ const createReader = async (
);
};
type ArtifactSelection = ArtifactExtractionInput["selection"];
const validateSelection = (
selection: ArtifactSelection,
limits: ArtifactLimits,
): void => {
const values =
"paths" in selection ? selection.paths : selection.occurrenceIds;
if (values.length === 0)
throw new ArtifactReaderFailure(
"limit",
"Extraction requires at least one selected path or occurrence",
);
const normalized =
"paths" in selection
? values.map((path) => normalizeArtifactPath(path, limits))
: values;
if (new Set(normalized).size !== normalized.length)
throw new ArtifactReaderFailure(
"path",
"Extraction selection contains duplicates",
);
};
const resolveSelection = (
occurrences: readonly ArtifactOccurrence[],
selection: ArtifactSelection,
): ArtifactOccurrence[] => {
if ("occurrenceIds" in selection) {
const byId = new Map(
occurrences.map((occurrence) => [occurrence.occurrence_id, occurrence]),
);
return selection.occurrenceIds.map((id) => {
const occurrence = byId.get(id);
if (occurrence === undefined)
throw new ArtifactReaderFailure(
"unavailable",
`Selected artifact occurrence was not found: ${id}`,
);
return occurrence;
});
}
const byPath = new Map<string, ArtifactOccurrence[]>();
for (const occurrence of occurrences) {
const matches = byPath.get(occurrence.logical_path);
if (matches === undefined)
byPath.set(occurrence.logical_path, [occurrence]);
else matches.push(occurrence);
}
return selection.paths.map((path) => {
const normalizedPath = path.normalize("NFC");
const matches = byPath.get(normalizedPath) ?? [];
if (matches.length === 0)
throw new ArtifactReaderFailure(
"unavailable",
`Selected artifact path was not found: ${path}`,
);
if (matches.length > 1)
throw new ArtifactReaderFailure(
"path",
`Selected artifact path is ambiguous: ${path}`,
);
const match = matches[0];
if (match === undefined)
throw new ArtifactReaderFailure(
"unavailable",
`Selected artifact path was not found: ${path}`,
);
return match;
});
};
const preflight = (entry: ArtifactEntry, limits: ArtifactLimits): void => {
const preflight = (entry: ArtifactEntry): void => {
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
throw new ArtifactReaderFailure(
"format",
`Selected artifact entry cannot be read: ${entry.path}`,
);
if (entry.declaredSize !== null && entry.declaredSize > limits.maxEntryBytes)
throw new ArtifactReaderFailure(
"limit",
`Selected artifact exceeds byte limit: ${entry.path}`,
);
if (
entry.declaredSize !== null &&
entry.compressedSize !== null &&
(entry.compressedSize === 0
? entry.declaredSize > 0
: entry.declaredSize / entry.compressedSize > limits.maxCompressionRatio)
)
throw new ArtifactReaderFailure(
"limit",
`Selected artifact exceeds compression ratio limit: ${entry.path}`,
);
};
+2 -14
View File
@@ -2,10 +2,7 @@ import { createHash } from "node:crypto";
import canonicalize from "canonicalize";
import type {
ArtifactEntry,
ArtifactLimits,
} from "../artifacts/ArtifactReader.js";
import type { ArtifactEntry } from "../artifacts/ArtifactReader.js";
import type {
ArtifactCommand,
ArtifactEdge,
@@ -55,7 +52,7 @@ export const createOccurrence = (
compressed_size: entry.compressedSize,
executable: entry.executable,
encrypted: entry.encrypted,
hash_status: entry.encrypted ? "unavailable" : "not-hashed-limit",
hash_status: entry.encrypted ? "unavailable" : "not-hashed",
source_location:
entry.byteOffset === null || entry.declaredSize === null
? null
@@ -318,15 +315,6 @@ export const classifyArtifactContent = (
return byPath;
};
export const toOutputLimits = (limits: ArtifactLimits) => ({
max_entries: limits.maxEntries,
max_total_bytes: limits.maxTotalBytes,
max_entry_bytes: limits.maxEntryBytes,
max_compression_ratio: limits.maxCompressionRatio,
max_depth: limits.maxDepth,
max_path_bytes: limits.maxPathBytes,
});
export const digestCanonical = (value: unknown): string => {
const encoded = canonicalize(value);
if (encoded === undefined)
+3 -7
View File
@@ -1,6 +1,5 @@
import { realpath } from "node:fs/promises";
import type { ArtifactLimits } from "../artifacts/ArtifactReader.js";
import {
artifactInventoryResultSchema,
type ArtifactInventoryResult,
@@ -26,38 +25,35 @@ export type {
/** Inventory one local artifact without extracting or mounting it. */
export const inventoryArtifact = async (
inputPath: string,
limits: ArtifactLimits,
options: {
readonly signal?: AbortSignal;
readonly nativeMount?: ArtifactNativeMountPolicy;
readonly integrity?: ArtifactIntegrityPolicy;
} = {},
): Promise<ArtifactInventoryResult> => {
const snapshot = await scanArtifactInventory(inputPath, limits, options);
const snapshot = await scanArtifactInventory(inputPath, options);
return artifactInventoryResultSchema.parse(snapshot);
};
/** Inventory one artifact and return every graph collection in one response. */
export const inventoryArtifactFully = async (
inputPath: string,
limits: ArtifactLimits,
options: {
readonly signal?: AbortSignal;
readonly nativeMount?: ArtifactNativeMountPolicy;
readonly integrity?: ArtifactIntegrityPolicy;
} = {},
): Promise<ArtifactInventoryResult> => {
const snapshot = await scanArtifactInventory(inputPath, limits, options);
const snapshot = await scanArtifactInventory(inputPath, options);
return artifactInventoryResultSchema.parse(snapshot);
};
/** Scan an artifact once and retain the complete immutable graph for projection. */
export const scanArtifactInventory = async (
inputPath: string,
limits: ArtifactLimits,
options: ArtifactInventoryOptions = {},
): Promise<ArtifactInventorySnapshot> => {
abortIfNeeded(options.signal);
const path = await realpath(inputPath);
return scanCanonicalArtifactInventory(path, limits, options);
return scanCanonicalArtifactInventory(path, options);
};
+1 -44
View File
@@ -1,11 +1,7 @@
import { createHash } from "node:crypto";
import type { Readable } from "node:stream";
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactLimits,
} from "../../artifacts/ArtifactReader.js";
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
import { streamChunkToBuffer } from "../../artifacts/StreamBytes.js";
export type HashResult = {
@@ -22,40 +18,8 @@ export const abortIfNeeded = (signal?: AbortSignal): void => {
);
};
export const preflightEntry = (
entry: ArtifactEntry,
limits: ArtifactLimits,
): void => {
if (entry.declaredSize !== null && entry.declaredSize > limits.maxEntryBytes)
throw new ArtifactReaderFailure(
"limit",
`Entry exceeds byte limit: ${entry.path}`,
);
if (
entry.declaredSize !== null &&
entry.compressedSize !== null &&
entry.compressedSize === 0 &&
entry.declaredSize > 0
)
throw new ArtifactReaderFailure(
"limit",
`Invalid compression ratio: ${entry.path}`,
);
if (
entry.declaredSize !== null &&
entry.compressedSize !== null &&
entry.compressedSize > 0 &&
entry.declaredSize / entry.compressedSize > limits.maxCompressionRatio
)
throw new ArtifactReaderFailure(
"limit",
`Compression ratio exceeds limit: ${entry.path}`,
);
};
export const hashReadable = async (
stream: Readable,
maximum: number,
signal?: AbortSignal,
): Promise<HashResult> => {
const hash = createHash("sha256");
@@ -66,13 +30,6 @@ export const hashReadable = async (
abortIfNeeded(signal);
const chunk = streamChunkToBuffer(raw);
bytes += chunk.length;
if (bytes > maximum) {
stream.destroy();
throw new ArtifactReaderFailure(
"limit",
"Observed entry bytes exceed limit",
);
}
hash.update(chunk);
if (prefixBytes < 16) {
const selected = chunk.subarray(0, 16 - prefixBytes);
@@ -4,7 +4,6 @@ import type { Stats } from "node:fs";
import {
ArtifactReaderFailure,
type ArtifactLimits,
type ArtifactReader,
} from "../../artifacts/ArtifactReader.js";
import {
@@ -21,7 +20,6 @@ import {
materializeDirectoryNodes,
rekeyOccurrences,
rootOccurrenceFor,
toOutputLimits,
type MutableOccurrence,
} from "../ArtifactGraphConstruction.js";
import { classifyRoot } from "./classify.js";
@@ -40,7 +38,6 @@ import {
export const scanCanonicalArtifactInventory = async (
path: string,
limits: ArtifactLimits,
options: ArtifactInventoryOptions = {},
): Promise<ArtifactInventorySnapshot> => {
const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
@@ -48,11 +45,7 @@ export const scanCanonicalArtifactInventory = async (
const rootFormat = await classifyRoot(path, metadata.isDirectory());
const rootDigest = metadata.isDirectory()
? null
: await hashReadable(
createReadStream(path),
limits.maxTotalBytes,
options.signal,
);
: await hashReadable(createReadStream(path), options.signal);
const reader = await createReader(
path,
rootFormat,
@@ -63,7 +56,6 @@ export const scanCanonicalArtifactInventory = async (
try {
const { nodes, occurrences, pendingContradictions } = await scanReader(
reader,
limits,
options.signal,
integrity,
);
@@ -73,7 +65,6 @@ export const scanCanonicalArtifactInventory = async (
rootFormat,
rootDigest,
reader,
limits,
signal: options.signal,
nodes,
occurrences,
@@ -90,7 +81,6 @@ interface SnapshotBuildInput {
readonly rootFormat: ArtifactNode["format"];
readonly rootDigest: HashResult | null;
readonly reader: ArtifactReader | undefined;
readonly limits: ArtifactLimits;
readonly signal: AbortSignal | undefined;
readonly nodes: Map<string, ArtifactNode>;
readonly occurrences: MutableOccurrence[];
@@ -100,8 +90,7 @@ interface SnapshotBuildInput {
const buildInventorySnapshot = async (
input: SnapshotBuildInput,
): Promise<ArtifactInventorySnapshot> => {
const { path, metadata, rootFormat, rootDigest, reader, limits, signal } =
input;
const { path, metadata, rootFormat, rootDigest, reader, signal } = input;
materializeDirectoryNodes(input.occurrences, input.nodes);
const rootNode = createRootNode({
path,
@@ -140,7 +129,7 @@ const buildInventorySnapshot = async (
const orderedOccurrences = sortOccurrences(input.occurrences);
const orderedEdges = sortEdges(edges);
await verifyRootDigest(path, rootDigest, limits.maxTotalBytes, signal);
await verifyRootDigest(path, rootDigest, signal);
const graphSha256 = digestCanonical({
nodes: orderedNodes,
@@ -160,7 +149,6 @@ const buildInventorySnapshot = async (
nodes: orderedNodes,
occurrences: orderedOccurrences,
edges: orderedEdges,
limits: toOutputLimits(limits),
provenance: reader?.provenance() ?? [],
integrity_contradictions: integrityContradictions,
limitations: buildLimitations(rootFormat, reader, integrityContradictions),
@@ -210,15 +198,10 @@ const buildIntegrityContradictions = (
const verifyRootDigest = async (
path: string,
rootDigest: HashResult | null,
maxTotalBytes: number,
signal: AbortSignal | undefined,
): Promise<void> => {
if (rootDigest === null) return;
const verified = await hashReadable(
createReadStream(path),
maxTotalBytes,
signal,
);
const verified = await hashReadable(createReadStream(path), signal);
if (
verified.sha256 !== rootDigest.sha256 ||
verified.bytes !== rootDigest.bytes
+83 -75
View File
@@ -5,7 +5,6 @@ import {
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactLimits,
type ArtifactReader,
} from "../../artifacts/ArtifactReader.js";
import { AsarArtifactReader } from "../../artifacts/AsarArtifactReader.js";
@@ -21,7 +20,7 @@ import {
STRICT_INTEGRITY_POLICY,
type ArtifactIntegrityPolicy,
} from "./types.js";
import { hashReadable, preflightEntry } from "./hash.js";
import { hashReadable } from "./hash.js";
export interface PendingIntegrityContradiction {
readonly logicalPath: string;
@@ -42,7 +41,6 @@ const emptyScan = (): {
interface ScanContext {
readonly reader: ArtifactReader;
readonly limits: ArtifactLimits;
readonly signal: AbortSignal | undefined;
readonly integrity: ArtifactIntegrityPolicy;
readonly nodes: Map<string, ArtifactNode>;
@@ -50,12 +48,10 @@ interface ScanContext {
readonly pendingContradictions: PendingIntegrityContradiction[];
readonly occurrenceByPath: Map<string, MutableOccurrence>;
readonly registry: ArtifactPathRegistry;
totalBytes: number;
}
export const scanReader = async (
reader: ArtifactReader | undefined,
limits: ArtifactLimits,
signal?: AbortSignal,
integrity: ArtifactIntegrityPolicy = STRICT_INTEGRITY_POLICY,
): Promise<{
@@ -68,7 +64,6 @@ export const scanReader = async (
return { nodes, occurrences, pendingContradictions };
const context: ScanContext = {
reader,
limits,
signal,
integrity,
nodes,
@@ -76,7 +71,6 @@ export const scanReader = async (
pendingContradictions,
occurrenceByPath: new Map<string, MutableOccurrence>(),
registry: new ArtifactPathRegistry(),
totalBytes: 0,
};
await visitArtifactEntries(context, reader, "");
return { nodes, occurrences, pendingContradictions };
@@ -87,53 +81,93 @@ const visitArtifactEntries = async (
currentReader: ArtifactReader,
prefix: string,
): Promise<void> => {
for await (const entry of currentReader.entries(context.signal)) {
if (context.occurrences.length >= context.limits.maxEntries)
throw new ArtifactReaderFailure("limit", "Artifact entry limit exceeded");
const logicalPath = normalizeArtifactPath(
prefix.length === 0 ? entry.path : `${prefix}/${entry.path}`,
context.limits,
);
const expandableAsar = isExpandableAsar(entry, logicalPath);
context.registry.add(
logicalPath,
expandableAsar ? "directory" : entry.kind,
);
preflightEntry(entry, context.limits);
const parent = nearestParent(logicalPath, context.occurrenceByPath);
const occurrence = createOccurrence(
entry,
logicalPath,
parent?.occurrence_id ?? null,
);
let digested:
| { readonly node: ArtifactNode; readonly mismatched: boolean }
| undefined;
try {
digested = await digestArtifactEntry(
context,
currentReader,
const stack: Array<{
readonly reader: ArtifactReader;
readonly prefix: string;
readonly iterator: AsyncIterator<ArtifactEntry>;
readonly owned: boolean;
}> = [
{
reader: currentReader,
prefix,
iterator: currentReader.entries(context.signal)[Symbol.asyncIterator](),
owned: false,
},
];
try {
while (stack.length > 0) {
const frame = stack.at(-1);
if (frame === undefined) break;
const next = await frame.iterator.next();
if (next.done) {
stack.pop();
if (frame.owned) await frame.reader.close();
continue;
}
const entry = next.value;
const logicalPath = normalizeArtifactPath(
frame.prefix.length === 0
? entry.path
: `${frame.prefix}/${entry.path}`,
);
const expandableAsar = isExpandableAsar(entry, logicalPath);
context.registry.add(
logicalPath,
expandableAsar ? "directory" : entry.kind,
);
const parent = nearestParent(logicalPath, context.occurrenceByPath);
const occurrence = createOccurrence(
entry,
logicalPath,
parent?.occurrence_id ?? null,
);
} catch (cause: unknown) {
if (!isUnavailableUnpackedEntry(cause, entry)) throw cause;
occurrence.hash_status = "unavailable";
occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION);
}
if (digested !== undefined) {
context.nodes.set(digested.node.artifact_id, digested.node);
occurrence.artifact_id = digested.node.artifact_id;
occurrence.hash_status = digested.mismatched ? "mismatched" : "verified";
if (digested.mismatched)
occurrence.limitations.push(
"Declared integrity metadata contradicts observed bytes.",
let digested:
| { readonly node: ArtifactNode; readonly mismatched: boolean }
| undefined;
try {
digested = await digestArtifactEntry(
context,
frame.reader,
entry,
logicalPath,
);
} catch (cause: unknown) {
if (!isUnavailableUnpackedEntry(cause, entry)) throw cause;
occurrence.hash_status = "unavailable";
occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION);
}
if (digested !== undefined) {
context.nodes.set(digested.node.artifact_id, digested.node);
occurrence.artifact_id = digested.node.artifact_id;
occurrence.hash_status = digested.mismatched
? "mismatched"
: "verified";
if (digested.mismatched)
occurrence.limitations.push(
"Declared integrity metadata contradicts observed bytes.",
);
}
context.occurrences.push(occurrence);
context.occurrenceByPath.set(logicalPath, occurrence);
if (expandableAsar && digested?.mismatched !== true) {
const nested = new AsarArtifactReader(entry.adapterKey);
stack.push({
reader: nested,
prefix: logicalPath,
iterator: nested.entries(context.signal)[Symbol.asyncIterator](),
owned: true,
});
}
}
context.occurrences.push(occurrence);
context.occurrenceByPath.set(logicalPath, occurrence);
if (expandableAsar && digested?.mismatched !== true)
await visitNestedAsar(context, entry.adapterKey, logicalPath);
} finally {
await Promise.allSettled(
stack
.filter(({ owned }) => owned)
.map(async ({ reader, iterator }) => {
await iterator.return?.();
await reader.close();
}),
);
}
};
@@ -147,23 +181,10 @@ const digestArtifactEntry = async (
> => {
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
return undefined;
const remainingBytes = context.limits.maxTotalBytes - context.totalBytes;
if (remainingBytes <= 0)
throw new ArtifactReaderFailure(
"limit",
"Artifact total byte limit exceeded",
);
if (entry.declaredSize !== null && entry.declaredSize > remainingBytes)
throw new ArtifactReaderFailure(
"limit",
"Declared artifact bytes exceed remaining cumulative limit",
);
const digest = await hashReadable(
await currentReader.open(entry, context.signal),
Math.min(context.limits.maxEntryBytes, remainingBytes),
context.signal,
);
context.totalBytes += digest.bytes;
const mismatched =
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
if (mismatched && entry.declaredSha256 !== null) {
@@ -209,19 +230,6 @@ const digestArtifactEntry = async (
};
};
const visitNestedAsar = async (
context: ScanContext,
adapterKey: string,
logicalPath: string,
): Promise<void> => {
const nested = new AsarArtifactReader(adapterKey);
try {
await visitArtifactEntries(context, nested, logicalPath);
} finally {
await nested.close();
}
};
const isExpandableAsar = (entry: ArtifactEntry, logicalPath: string): boolean =>
entry.kind === "file" &&
logicalPath.toLowerCase().endsWith(".asar") &&
@@ -35,7 +35,6 @@ export interface ArtifactInventorySnapshot {
readonly nodes: readonly ArtifactNode[];
readonly occurrences: ArtifactInventoryResult["occurrences"];
readonly edges: ArtifactInventoryResult["edges"];
readonly limits: ArtifactInventoryResult["limits"];
readonly provenance: ReadonlyArray<
ArtifactInventoryResult["provenance"][number]
>;
@@ -1,6 +1,5 @@
import { realpath } from "node:fs/promises";
import type { ArtifactLimits } from "../artifacts/ArtifactReader.js";
import {
scanCanonicalArtifactInventory,
type ArtifactInventoryOptions,
@@ -10,9 +9,8 @@ import {
/** Resolve, authorize, and scan one artifact without a second path resolution. */
export const scanAuthorizedArtifactInventory = async (
inputPath: string,
limits: ArtifactLimits,
options: ArtifactInventoryOptions = {},
): Promise<ArtifactInventorySnapshot> => {
const path = await realpath(inputPath);
return scanCanonicalArtifactInventory(path, limits, options);
return scanCanonicalArtifactInventory(path, options);
};
+73 -49
View File
@@ -148,56 +148,80 @@ const visitReader = async (
containerSha256: string,
context: ReadContext,
): Promise<void> => {
for await (const entry of reader.entries(context.signal)) {
abortIfNeeded(context.signal);
const path = normalizeArtifactPath(
prefix === "" ? entry.path : `${prefix}/${entry.path}`,
{
maxDepth: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxDepth,
maxPathBytes:
JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxPathBytes,
},
);
const nestedAsar = isFilesystemAsar(entry, path);
context.registry.add(path, nestedAsar ? "directory" : entry.kind);
if (nestedAsar) {
await visitNestedAsar(entry, path, context);
continue;
}
const expected = context.expected.get(path);
if (expected === undefined) continue;
const selection = context.selections.get(path);
const text = await readTextIfSelected(context, {
const stack: Array<{
readonly reader: ArtifactReader;
readonly prefix: string;
readonly containerSha256: string;
readonly iterator: AsyncIterator<ArtifactEntry>;
readonly owned: boolean;
}> = [
{
reader,
entry,
expected,
selection,
});
context.files.push({
path,
container_sha256: containerSha256,
sha256: expected.sha256,
bytes: expected.bytes,
inventory_artifact_id: expected.inventoryArtifactId,
kind: expected.kind,
unpacked: entry.unpacked,
text,
});
}
};
const visitNestedAsar = async (
entry: ArtifactEntry,
path: string,
context: ReadContext,
): Promise<void> => {
const inventory = expectedContainer(path, context);
context.containers.push(inventory);
const nested = new AsarArtifactReader(entry.adapterKey);
prefix,
containerSha256,
iterator: reader.entries(context.signal)[Symbol.asyncIterator](),
owned: false,
},
];
try {
await visitReader(nested, path, inventory.sha256, context);
while (stack.length > 0) {
const frame = stack.at(-1);
if (frame === undefined) break;
const next = await frame.iterator.next();
if (next.done) {
stack.pop();
if (frame.owned) await frame.reader.close();
continue;
}
const entry = next.value;
abortIfNeeded(context.signal);
const path = normalizeArtifactPath(
frame.prefix === "" ? entry.path : `${frame.prefix}/${entry.path}`,
);
const nestedAsar = isFilesystemAsar(entry, path);
context.registry.add(path, nestedAsar ? "directory" : entry.kind);
if (nestedAsar) {
const inventory = expectedContainer(path, context);
context.containers.push(inventory);
const nested = new AsarArtifactReader(entry.adapterKey);
stack.push({
reader: nested,
prefix: path,
containerSha256: inventory.sha256,
iterator: nested.entries(context.signal)[Symbol.asyncIterator](),
owned: true,
});
continue;
}
const expected = context.expected.get(path);
if (expected === undefined) continue;
const selection = context.selections.get(path);
const text = await readTextIfSelected(context, {
reader: frame.reader,
entry,
expected,
selection,
});
context.files.push({
path,
container_sha256: frame.containerSha256,
sha256: expected.sha256,
bytes: expected.bytes,
inventory_artifact_id: expected.inventoryArtifactId,
kind: expected.kind,
unpacked: entry.unpacked,
text,
});
}
} finally {
await nested.close();
await Promise.allSettled(
stack
.filter(({ owned }) => owned)
.map(async ({ reader, iterator }) => {
await iterator.return?.();
await reader.close();
}),
);
}
};
@@ -333,8 +357,8 @@ const readBounded = async (
if (bytes > maximum) {
stream.destroy();
throw new ArtifactReaderFailure(
"limit",
"Selected JavaScript artifact text exceeded its byte limit",
"integrity",
"Artifact entry grew after inventory and exceeded the selected text read size",
);
}
chunks.push(chunk);
@@ -204,16 +204,6 @@ const graphLimitations = (
};
const reconstructionLimits = () => [
{
name: "max-entries",
value: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxEntries,
unit: "items" as const,
},
{
name: "max-total-artifact-bytes",
value: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxTotalBytes,
unit: "bytes" as const,
},
{
name: "max-text-files",
value: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.maxTextFiles,
@@ -14,7 +14,6 @@ import { analyzeJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"
import { readJavaScriptArtifactFiles } from "./JavaScriptArtifactFiles.js";
import { buildJavaScriptArtifactGraph } from "./JavaScriptArtifactGraphBuilder.js";
import {
artifactLimitsForReconstruction,
javascriptArtifactReconstructionInputSchema,
type JavaScriptArtifactReconstructionInput,
} from "./JavaScriptArtifactReconstructionInput.js";
@@ -59,11 +58,7 @@ export const reconstructJavaScriptArtifact = async (
abortIfNeeded(signal);
const path = await realpath(input.input_path);
const format = await resolveFormat(path, input);
const snapshot = await scanCanonicalArtifactInventory(
path,
artifactLimitsForReconstruction(),
{ signal },
);
const snapshot = await scanCanonicalArtifactInventory(path, { signal });
if (snapshot.manifest.root_format !== format)
throw new ArtifactReaderFailure(
"format",
@@ -1,17 +1,7 @@
import { z } from "zod";
import type { ArtifactLimits } from "../artifacts/ArtifactReader.js";
/** Resource safeguards for local artifact traversal and static parsing. */
export const JAVASCRIPT_APPLICATION_RESOURCE_LIMITS = {
artifact: {
maxEntries: 8_000,
maxTotalBytes: 512 * 1_024 * 1_024,
maxEntryBytes: 128 * 1_024 * 1_024,
maxCompressionRatio: 1_000,
maxDepth: 64,
maxPathBytes: 4_096,
} satisfies ArtifactLimits,
maxTextFiles: 5_000,
maxTotalTextBytes: 128 * 1_024 * 1_024,
maxTextFileBytes: 8 * 1_024 * 1_024,
@@ -29,7 +19,3 @@ export const javascriptArtifactReconstructionInputSchema = z.strictObject({
export type JavaScriptArtifactReconstructionInput = z.infer<
typeof javascriptArtifactReconstructionInputSchema
>;
/** Project private resource safeguards into the artifact traversal policy. */
export const artifactLimitsForReconstruction = (): ArtifactLimits =>
JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact;