refactor(artifacts): remove selectors and traversal quotas
This commit is contained in:
@@ -8,7 +8,6 @@ import {
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactLimits,
|
||||
type ArtifactReader,
|
||||
} from "../artifacts/ArtifactReader.js";
|
||||
import { DirectoryArtifactReader } from "../artifacts/DirectoryArtifactReader.js";
|
||||
@@ -24,35 +23,28 @@ import {
|
||||
} from "../domain/artifactGraph.js";
|
||||
import type { BinaryTarget } from "../domain/binaryTarget.js";
|
||||
import { scanArtifactInventory } from "./ArtifactInventory.js";
|
||||
import {
|
||||
digestCanonical,
|
||||
toOutputLimits,
|
||||
} from "./ArtifactGraphConstruction.js";
|
||||
import { digestCanonical } from "./ArtifactGraphConstruction.js";
|
||||
|
||||
/** Already-approved extraction request from a caller-owned policy boundary. */
|
||||
export interface ArtifactExtractionInput {
|
||||
readonly inputPath: string;
|
||||
readonly inputFormat: BinaryTarget["format"];
|
||||
readonly outputRoot: string;
|
||||
readonly selection:
|
||||
| { readonly paths: readonly string[] }
|
||||
| { readonly occurrenceIds: readonly string[] };
|
||||
readonly limits: ArtifactLimits;
|
||||
}
|
||||
|
||||
/** Extract selected inventory occurrences into an exclusively owned absent root. */
|
||||
/** Extract every regular inventory occurrence into an exclusively owned absent root. */
|
||||
export const extractArtifact = async (
|
||||
input: ArtifactExtractionInput,
|
||||
signal?: AbortSignal,
|
||||
): Promise<ArtifactExtractionResult> => {
|
||||
validateSelection(input.selection, input.limits);
|
||||
const sourcePath = await realpath(input.inputPath);
|
||||
const snapshot = await scanArtifactInventory(sourcePath, input.limits, {
|
||||
const snapshot = await scanArtifactInventory(sourcePath, {
|
||||
signal,
|
||||
});
|
||||
const selectedOccurrences = resolveSelection(
|
||||
snapshot.occurrences,
|
||||
input.selection,
|
||||
const selectedOccurrences = snapshot.occurrences.filter(
|
||||
(occurrence) =>
|
||||
(occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") &&
|
||||
occurrence.logical_path !== ".",
|
||||
);
|
||||
const selectedIds = new Set(
|
||||
selectedOccurrences.map(({ occurrence_id: id }) => id),
|
||||
@@ -130,25 +122,25 @@ const materializeSelection = async ({
|
||||
selected.map((item) => [item.occurrence.logical_path, item]),
|
||||
);
|
||||
const reader = await createReader(sourcePath, input.inputFormat);
|
||||
const output = await SafeOutputTree.create(input.outputRoot, input.limits);
|
||||
const output = await SafeOutputTree.create(input.outputRoot);
|
||||
let readerClosed = false;
|
||||
const extracted: ExtractedOccurrence[] = [];
|
||||
try {
|
||||
const found = new Set<string>();
|
||||
const materialized: SelectedOccurrence[] = [];
|
||||
const registry = new ArtifactPathRegistry();
|
||||
let entryCount = 0;
|
||||
for await (const entry of reader.entries(signal)) {
|
||||
entryCount += 1;
|
||||
if (entryCount > input.limits.maxEntries)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"Artifact entry limit exceeded during extraction",
|
||||
);
|
||||
const path = normalizeArtifactPath(entry.path, input.limits);
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
registry.add(path, entry.kind);
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) continue;
|
||||
preflight(entry, input.limits);
|
||||
if (selectedItem === undefined) {
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await reader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
@@ -163,13 +155,8 @@ const materializeSelection = async ({
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
found.add(path);
|
||||
materialized.push(selectedItem);
|
||||
}
|
||||
if (found.size !== selected.length)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Selected inventory occurrence disappeared before extraction",
|
||||
);
|
||||
await reader.close();
|
||||
readerClosed = true;
|
||||
extracted.sort((left, right) =>
|
||||
@@ -178,7 +165,7 @@ const materializeSelection = async ({
|
||||
const result = createExtractionResult(
|
||||
input,
|
||||
inventory,
|
||||
selected,
|
||||
materialized,
|
||||
extracted,
|
||||
);
|
||||
await output.commit();
|
||||
@@ -214,10 +201,9 @@ const createExtractionResult = (
|
||||
artifacts: extracted,
|
||||
containment_verified: true,
|
||||
cleanup: { attempted: false, verified: true, residual_paths: [] },
|
||||
limits: toOutputLimits(input.limits),
|
||||
provenance: [],
|
||||
limitations: [
|
||||
"Only caller-selected regular file occurrences were materialized.",
|
||||
"All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file.",
|
||||
],
|
||||
});
|
||||
};
|
||||
@@ -272,99 +258,10 @@ const createReader = async (
|
||||
);
|
||||
};
|
||||
|
||||
type ArtifactSelection = ArtifactExtractionInput["selection"];
|
||||
|
||||
const validateSelection = (
|
||||
selection: ArtifactSelection,
|
||||
limits: ArtifactLimits,
|
||||
): void => {
|
||||
const values =
|
||||
"paths" in selection ? selection.paths : selection.occurrenceIds;
|
||||
if (values.length === 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"Extraction requires at least one selected path or occurrence",
|
||||
);
|
||||
const normalized =
|
||||
"paths" in selection
|
||||
? values.map((path) => normalizeArtifactPath(path, limits))
|
||||
: values;
|
||||
if (new Set(normalized).size !== normalized.length)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Extraction selection contains duplicates",
|
||||
);
|
||||
};
|
||||
|
||||
const resolveSelection = (
|
||||
occurrences: readonly ArtifactOccurrence[],
|
||||
selection: ArtifactSelection,
|
||||
): ArtifactOccurrence[] => {
|
||||
if ("occurrenceIds" in selection) {
|
||||
const byId = new Map(
|
||||
occurrences.map((occurrence) => [occurrence.occurrence_id, occurrence]),
|
||||
);
|
||||
return selection.occurrenceIds.map((id) => {
|
||||
const occurrence = byId.get(id);
|
||||
if (occurrence === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`Selected artifact occurrence was not found: ${id}`,
|
||||
);
|
||||
return occurrence;
|
||||
});
|
||||
}
|
||||
|
||||
const byPath = new Map<string, ArtifactOccurrence[]>();
|
||||
for (const occurrence of occurrences) {
|
||||
const matches = byPath.get(occurrence.logical_path);
|
||||
if (matches === undefined)
|
||||
byPath.set(occurrence.logical_path, [occurrence]);
|
||||
else matches.push(occurrence);
|
||||
}
|
||||
return selection.paths.map((path) => {
|
||||
const normalizedPath = path.normalize("NFC");
|
||||
const matches = byPath.get(normalizedPath) ?? [];
|
||||
if (matches.length === 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`Selected artifact path was not found: ${path}`,
|
||||
);
|
||||
if (matches.length > 1)
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Selected artifact path is ambiguous: ${path}`,
|
||||
);
|
||||
const match = matches[0];
|
||||
if (match === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`Selected artifact path was not found: ${path}`,
|
||||
);
|
||||
return match;
|
||||
});
|
||||
};
|
||||
|
||||
const preflight = (entry: ArtifactEntry, limits: ArtifactLimits): void => {
|
||||
const preflight = (entry: ArtifactEntry): void => {
|
||||
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Selected artifact entry cannot be read: ${entry.path}`,
|
||||
);
|
||||
if (entry.declaredSize !== null && entry.declaredSize > limits.maxEntryBytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`Selected artifact exceeds byte limit: ${entry.path}`,
|
||||
);
|
||||
if (
|
||||
entry.declaredSize !== null &&
|
||||
entry.compressedSize !== null &&
|
||||
(entry.compressedSize === 0
|
||||
? entry.declaredSize > 0
|
||||
: entry.declaredSize / entry.compressedSize > limits.maxCompressionRatio)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`Selected artifact exceeds compression ratio limit: ${entry.path}`,
|
||||
);
|
||||
};
|
||||
|
||||
@@ -2,10 +2,7 @@ import { createHash } from "node:crypto";
|
||||
|
||||
import canonicalize from "canonicalize";
|
||||
|
||||
import type {
|
||||
ArtifactEntry,
|
||||
ArtifactLimits,
|
||||
} from "../artifacts/ArtifactReader.js";
|
||||
import type { ArtifactEntry } from "../artifacts/ArtifactReader.js";
|
||||
import type {
|
||||
ArtifactCommand,
|
||||
ArtifactEdge,
|
||||
@@ -55,7 +52,7 @@ export const createOccurrence = (
|
||||
compressed_size: entry.compressedSize,
|
||||
executable: entry.executable,
|
||||
encrypted: entry.encrypted,
|
||||
hash_status: entry.encrypted ? "unavailable" : "not-hashed-limit",
|
||||
hash_status: entry.encrypted ? "unavailable" : "not-hashed",
|
||||
source_location:
|
||||
entry.byteOffset === null || entry.declaredSize === null
|
||||
? null
|
||||
@@ -318,15 +315,6 @@ export const classifyArtifactContent = (
|
||||
return byPath;
|
||||
};
|
||||
|
||||
export const toOutputLimits = (limits: ArtifactLimits) => ({
|
||||
max_entries: limits.maxEntries,
|
||||
max_total_bytes: limits.maxTotalBytes,
|
||||
max_entry_bytes: limits.maxEntryBytes,
|
||||
max_compression_ratio: limits.maxCompressionRatio,
|
||||
max_depth: limits.maxDepth,
|
||||
max_path_bytes: limits.maxPathBytes,
|
||||
});
|
||||
|
||||
export const digestCanonical = (value: unknown): string => {
|
||||
const encoded = canonicalize(value);
|
||||
if (encoded === undefined)
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { realpath } from "node:fs/promises";
|
||||
|
||||
import type { ArtifactLimits } from "../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
artifactInventoryResultSchema,
|
||||
type ArtifactInventoryResult,
|
||||
@@ -26,38 +25,35 @@ export type {
|
||||
/** Inventory one local artifact without extracting or mounting it. */
|
||||
export const inventoryArtifact = async (
|
||||
inputPath: string,
|
||||
limits: ArtifactLimits,
|
||||
options: {
|
||||
readonly signal?: AbortSignal;
|
||||
readonly nativeMount?: ArtifactNativeMountPolicy;
|
||||
readonly integrity?: ArtifactIntegrityPolicy;
|
||||
} = {},
|
||||
): Promise<ArtifactInventoryResult> => {
|
||||
const snapshot = await scanArtifactInventory(inputPath, limits, options);
|
||||
const snapshot = await scanArtifactInventory(inputPath, options);
|
||||
return artifactInventoryResultSchema.parse(snapshot);
|
||||
};
|
||||
|
||||
/** Inventory one artifact and return every graph collection in one response. */
|
||||
export const inventoryArtifactFully = async (
|
||||
inputPath: string,
|
||||
limits: ArtifactLimits,
|
||||
options: {
|
||||
readonly signal?: AbortSignal;
|
||||
readonly nativeMount?: ArtifactNativeMountPolicy;
|
||||
readonly integrity?: ArtifactIntegrityPolicy;
|
||||
} = {},
|
||||
): Promise<ArtifactInventoryResult> => {
|
||||
const snapshot = await scanArtifactInventory(inputPath, limits, options);
|
||||
const snapshot = await scanArtifactInventory(inputPath, options);
|
||||
return artifactInventoryResultSchema.parse(snapshot);
|
||||
};
|
||||
|
||||
/** Scan an artifact once and retain the complete immutable graph for projection. */
|
||||
export const scanArtifactInventory = async (
|
||||
inputPath: string,
|
||||
limits: ArtifactLimits,
|
||||
options: ArtifactInventoryOptions = {},
|
||||
): Promise<ArtifactInventorySnapshot> => {
|
||||
abortIfNeeded(options.signal);
|
||||
const path = await realpath(inputPath);
|
||||
return scanCanonicalArtifactInventory(path, limits, options);
|
||||
return scanCanonicalArtifactInventory(path, options);
|
||||
};
|
||||
|
||||
@@ -1,11 +1,7 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { Readable } from "node:stream";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactLimits,
|
||||
} from "../../artifacts/ArtifactReader.js";
|
||||
import { ArtifactReaderFailure } from "../../artifacts/ArtifactReader.js";
|
||||
import { streamChunkToBuffer } from "../../artifacts/StreamBytes.js";
|
||||
|
||||
export type HashResult = {
|
||||
@@ -22,40 +18,8 @@ export const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
);
|
||||
};
|
||||
|
||||
export const preflightEntry = (
|
||||
entry: ArtifactEntry,
|
||||
limits: ArtifactLimits,
|
||||
): void => {
|
||||
if (entry.declaredSize !== null && entry.declaredSize > limits.maxEntryBytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`Entry exceeds byte limit: ${entry.path}`,
|
||||
);
|
||||
if (
|
||||
entry.declaredSize !== null &&
|
||||
entry.compressedSize !== null &&
|
||||
entry.compressedSize === 0 &&
|
||||
entry.declaredSize > 0
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`Invalid compression ratio: ${entry.path}`,
|
||||
);
|
||||
if (
|
||||
entry.declaredSize !== null &&
|
||||
entry.compressedSize !== null &&
|
||||
entry.compressedSize > 0 &&
|
||||
entry.declaredSize / entry.compressedSize > limits.maxCompressionRatio
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`Compression ratio exceeds limit: ${entry.path}`,
|
||||
);
|
||||
};
|
||||
|
||||
export const hashReadable = async (
|
||||
stream: Readable,
|
||||
maximum: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<HashResult> => {
|
||||
const hash = createHash("sha256");
|
||||
@@ -66,13 +30,6 @@ export const hashReadable = async (
|
||||
abortIfNeeded(signal);
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
if (bytes > maximum) {
|
||||
stream.destroy();
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"Observed entry bytes exceed limit",
|
||||
);
|
||||
}
|
||||
hash.update(chunk);
|
||||
if (prefixBytes < 16) {
|
||||
const selected = chunk.subarray(0, 16 - prefixBytes);
|
||||
|
||||
@@ -4,7 +4,6 @@ import type { Stats } from "node:fs";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactLimits,
|
||||
type ArtifactReader,
|
||||
} from "../../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
@@ -21,7 +20,6 @@ import {
|
||||
materializeDirectoryNodes,
|
||||
rekeyOccurrences,
|
||||
rootOccurrenceFor,
|
||||
toOutputLimits,
|
||||
type MutableOccurrence,
|
||||
} from "../ArtifactGraphConstruction.js";
|
||||
import { classifyRoot } from "./classify.js";
|
||||
@@ -40,7 +38,6 @@ import {
|
||||
|
||||
export const scanCanonicalArtifactInventory = async (
|
||||
path: string,
|
||||
limits: ArtifactLimits,
|
||||
options: ArtifactInventoryOptions = {},
|
||||
): Promise<ArtifactInventorySnapshot> => {
|
||||
const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
|
||||
@@ -48,11 +45,7 @@ export const scanCanonicalArtifactInventory = async (
|
||||
const rootFormat = await classifyRoot(path, metadata.isDirectory());
|
||||
const rootDigest = metadata.isDirectory()
|
||||
? null
|
||||
: await hashReadable(
|
||||
createReadStream(path),
|
||||
limits.maxTotalBytes,
|
||||
options.signal,
|
||||
);
|
||||
: await hashReadable(createReadStream(path), options.signal);
|
||||
const reader = await createReader(
|
||||
path,
|
||||
rootFormat,
|
||||
@@ -63,7 +56,6 @@ export const scanCanonicalArtifactInventory = async (
|
||||
try {
|
||||
const { nodes, occurrences, pendingContradictions } = await scanReader(
|
||||
reader,
|
||||
limits,
|
||||
options.signal,
|
||||
integrity,
|
||||
);
|
||||
@@ -73,7 +65,6 @@ export const scanCanonicalArtifactInventory = async (
|
||||
rootFormat,
|
||||
rootDigest,
|
||||
reader,
|
||||
limits,
|
||||
signal: options.signal,
|
||||
nodes,
|
||||
occurrences,
|
||||
@@ -90,7 +81,6 @@ interface SnapshotBuildInput {
|
||||
readonly rootFormat: ArtifactNode["format"];
|
||||
readonly rootDigest: HashResult | null;
|
||||
readonly reader: ArtifactReader | undefined;
|
||||
readonly limits: ArtifactLimits;
|
||||
readonly signal: AbortSignal | undefined;
|
||||
readonly nodes: Map<string, ArtifactNode>;
|
||||
readonly occurrences: MutableOccurrence[];
|
||||
@@ -100,8 +90,7 @@ interface SnapshotBuildInput {
|
||||
const buildInventorySnapshot = async (
|
||||
input: SnapshotBuildInput,
|
||||
): Promise<ArtifactInventorySnapshot> => {
|
||||
const { path, metadata, rootFormat, rootDigest, reader, limits, signal } =
|
||||
input;
|
||||
const { path, metadata, rootFormat, rootDigest, reader, signal } = input;
|
||||
materializeDirectoryNodes(input.occurrences, input.nodes);
|
||||
const rootNode = createRootNode({
|
||||
path,
|
||||
@@ -140,7 +129,7 @@ const buildInventorySnapshot = async (
|
||||
const orderedOccurrences = sortOccurrences(input.occurrences);
|
||||
const orderedEdges = sortEdges(edges);
|
||||
|
||||
await verifyRootDigest(path, rootDigest, limits.maxTotalBytes, signal);
|
||||
await verifyRootDigest(path, rootDigest, signal);
|
||||
|
||||
const graphSha256 = digestCanonical({
|
||||
nodes: orderedNodes,
|
||||
@@ -160,7 +149,6 @@ const buildInventorySnapshot = async (
|
||||
nodes: orderedNodes,
|
||||
occurrences: orderedOccurrences,
|
||||
edges: orderedEdges,
|
||||
limits: toOutputLimits(limits),
|
||||
provenance: reader?.provenance() ?? [],
|
||||
integrity_contradictions: integrityContradictions,
|
||||
limitations: buildLimitations(rootFormat, reader, integrityContradictions),
|
||||
@@ -210,15 +198,10 @@ const buildIntegrityContradictions = (
|
||||
const verifyRootDigest = async (
|
||||
path: string,
|
||||
rootDigest: HashResult | null,
|
||||
maxTotalBytes: number,
|
||||
signal: AbortSignal | undefined,
|
||||
): Promise<void> => {
|
||||
if (rootDigest === null) return;
|
||||
const verified = await hashReadable(
|
||||
createReadStream(path),
|
||||
maxTotalBytes,
|
||||
signal,
|
||||
);
|
||||
const verified = await hashReadable(createReadStream(path), signal);
|
||||
if (
|
||||
verified.sha256 !== rootDigest.sha256 ||
|
||||
verified.bytes !== rootDigest.bytes
|
||||
|
||||
@@ -5,7 +5,6 @@ import {
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactLimits,
|
||||
type ArtifactReader,
|
||||
} from "../../artifacts/ArtifactReader.js";
|
||||
import { AsarArtifactReader } from "../../artifacts/AsarArtifactReader.js";
|
||||
@@ -21,7 +20,7 @@ import {
|
||||
STRICT_INTEGRITY_POLICY,
|
||||
type ArtifactIntegrityPolicy,
|
||||
} from "./types.js";
|
||||
import { hashReadable, preflightEntry } from "./hash.js";
|
||||
import { hashReadable } from "./hash.js";
|
||||
|
||||
export interface PendingIntegrityContradiction {
|
||||
readonly logicalPath: string;
|
||||
@@ -42,7 +41,6 @@ const emptyScan = (): {
|
||||
|
||||
interface ScanContext {
|
||||
readonly reader: ArtifactReader;
|
||||
readonly limits: ArtifactLimits;
|
||||
readonly signal: AbortSignal | undefined;
|
||||
readonly integrity: ArtifactIntegrityPolicy;
|
||||
readonly nodes: Map<string, ArtifactNode>;
|
||||
@@ -50,12 +48,10 @@ interface ScanContext {
|
||||
readonly pendingContradictions: PendingIntegrityContradiction[];
|
||||
readonly occurrenceByPath: Map<string, MutableOccurrence>;
|
||||
readonly registry: ArtifactPathRegistry;
|
||||
totalBytes: number;
|
||||
}
|
||||
|
||||
export const scanReader = async (
|
||||
reader: ArtifactReader | undefined,
|
||||
limits: ArtifactLimits,
|
||||
signal?: AbortSignal,
|
||||
integrity: ArtifactIntegrityPolicy = STRICT_INTEGRITY_POLICY,
|
||||
): Promise<{
|
||||
@@ -68,7 +64,6 @@ export const scanReader = async (
|
||||
return { nodes, occurrences, pendingContradictions };
|
||||
const context: ScanContext = {
|
||||
reader,
|
||||
limits,
|
||||
signal,
|
||||
integrity,
|
||||
nodes,
|
||||
@@ -76,7 +71,6 @@ export const scanReader = async (
|
||||
pendingContradictions,
|
||||
occurrenceByPath: new Map<string, MutableOccurrence>(),
|
||||
registry: new ArtifactPathRegistry(),
|
||||
totalBytes: 0,
|
||||
};
|
||||
await visitArtifactEntries(context, reader, "");
|
||||
return { nodes, occurrences, pendingContradictions };
|
||||
@@ -87,53 +81,93 @@ const visitArtifactEntries = async (
|
||||
currentReader: ArtifactReader,
|
||||
prefix: string,
|
||||
): Promise<void> => {
|
||||
for await (const entry of currentReader.entries(context.signal)) {
|
||||
if (context.occurrences.length >= context.limits.maxEntries)
|
||||
throw new ArtifactReaderFailure("limit", "Artifact entry limit exceeded");
|
||||
const logicalPath = normalizeArtifactPath(
|
||||
prefix.length === 0 ? entry.path : `${prefix}/${entry.path}`,
|
||||
context.limits,
|
||||
);
|
||||
const expandableAsar = isExpandableAsar(entry, logicalPath);
|
||||
context.registry.add(
|
||||
logicalPath,
|
||||
expandableAsar ? "directory" : entry.kind,
|
||||
);
|
||||
preflightEntry(entry, context.limits);
|
||||
const parent = nearestParent(logicalPath, context.occurrenceByPath);
|
||||
const occurrence = createOccurrence(
|
||||
entry,
|
||||
logicalPath,
|
||||
parent?.occurrence_id ?? null,
|
||||
);
|
||||
let digested:
|
||||
| { readonly node: ArtifactNode; readonly mismatched: boolean }
|
||||
| undefined;
|
||||
try {
|
||||
digested = await digestArtifactEntry(
|
||||
context,
|
||||
currentReader,
|
||||
const stack: Array<{
|
||||
readonly reader: ArtifactReader;
|
||||
readonly prefix: string;
|
||||
readonly iterator: AsyncIterator<ArtifactEntry>;
|
||||
readonly owned: boolean;
|
||||
}> = [
|
||||
{
|
||||
reader: currentReader,
|
||||
prefix,
|
||||
iterator: currentReader.entries(context.signal)[Symbol.asyncIterator](),
|
||||
owned: false,
|
||||
},
|
||||
];
|
||||
try {
|
||||
while (stack.length > 0) {
|
||||
const frame = stack.at(-1);
|
||||
if (frame === undefined) break;
|
||||
const next = await frame.iterator.next();
|
||||
if (next.done) {
|
||||
stack.pop();
|
||||
if (frame.owned) await frame.reader.close();
|
||||
continue;
|
||||
}
|
||||
const entry = next.value;
|
||||
const logicalPath = normalizeArtifactPath(
|
||||
frame.prefix.length === 0
|
||||
? entry.path
|
||||
: `${frame.prefix}/${entry.path}`,
|
||||
);
|
||||
const expandableAsar = isExpandableAsar(entry, logicalPath);
|
||||
context.registry.add(
|
||||
logicalPath,
|
||||
expandableAsar ? "directory" : entry.kind,
|
||||
);
|
||||
const parent = nearestParent(logicalPath, context.occurrenceByPath);
|
||||
const occurrence = createOccurrence(
|
||||
entry,
|
||||
logicalPath,
|
||||
parent?.occurrence_id ?? null,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!isUnavailableUnpackedEntry(cause, entry)) throw cause;
|
||||
occurrence.hash_status = "unavailable";
|
||||
occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION);
|
||||
}
|
||||
if (digested !== undefined) {
|
||||
context.nodes.set(digested.node.artifact_id, digested.node);
|
||||
occurrence.artifact_id = digested.node.artifact_id;
|
||||
occurrence.hash_status = digested.mismatched ? "mismatched" : "verified";
|
||||
if (digested.mismatched)
|
||||
occurrence.limitations.push(
|
||||
"Declared integrity metadata contradicts observed bytes.",
|
||||
let digested:
|
||||
| { readonly node: ArtifactNode; readonly mismatched: boolean }
|
||||
| undefined;
|
||||
try {
|
||||
digested = await digestArtifactEntry(
|
||||
context,
|
||||
frame.reader,
|
||||
entry,
|
||||
logicalPath,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!isUnavailableUnpackedEntry(cause, entry)) throw cause;
|
||||
occurrence.hash_status = "unavailable";
|
||||
occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION);
|
||||
}
|
||||
if (digested !== undefined) {
|
||||
context.nodes.set(digested.node.artifact_id, digested.node);
|
||||
occurrence.artifact_id = digested.node.artifact_id;
|
||||
occurrence.hash_status = digested.mismatched
|
||||
? "mismatched"
|
||||
: "verified";
|
||||
if (digested.mismatched)
|
||||
occurrence.limitations.push(
|
||||
"Declared integrity metadata contradicts observed bytes.",
|
||||
);
|
||||
}
|
||||
context.occurrences.push(occurrence);
|
||||
context.occurrenceByPath.set(logicalPath, occurrence);
|
||||
if (expandableAsar && digested?.mismatched !== true) {
|
||||
const nested = new AsarArtifactReader(entry.adapterKey);
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: logicalPath,
|
||||
iterator: nested.entries(context.signal)[Symbol.asyncIterator](),
|
||||
owned: true,
|
||||
});
|
||||
}
|
||||
}
|
||||
context.occurrences.push(occurrence);
|
||||
context.occurrenceByPath.set(logicalPath, occurrence);
|
||||
if (expandableAsar && digested?.mismatched !== true)
|
||||
await visitNestedAsar(context, entry.adapterKey, logicalPath);
|
||||
} finally {
|
||||
await Promise.allSettled(
|
||||
stack
|
||||
.filter(({ owned }) => owned)
|
||||
.map(async ({ reader, iterator }) => {
|
||||
await iterator.return?.();
|
||||
await reader.close();
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -147,23 +181,10 @@ const digestArtifactEntry = async (
|
||||
> => {
|
||||
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
|
||||
return undefined;
|
||||
const remainingBytes = context.limits.maxTotalBytes - context.totalBytes;
|
||||
if (remainingBytes <= 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"Artifact total byte limit exceeded",
|
||||
);
|
||||
if (entry.declaredSize !== null && entry.declaredSize > remainingBytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"Declared artifact bytes exceed remaining cumulative limit",
|
||||
);
|
||||
const digest = await hashReadable(
|
||||
await currentReader.open(entry, context.signal),
|
||||
Math.min(context.limits.maxEntryBytes, remainingBytes),
|
||||
context.signal,
|
||||
);
|
||||
context.totalBytes += digest.bytes;
|
||||
const mismatched =
|
||||
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
|
||||
if (mismatched && entry.declaredSha256 !== null) {
|
||||
@@ -209,19 +230,6 @@ const digestArtifactEntry = async (
|
||||
};
|
||||
};
|
||||
|
||||
const visitNestedAsar = async (
|
||||
context: ScanContext,
|
||||
adapterKey: string,
|
||||
logicalPath: string,
|
||||
): Promise<void> => {
|
||||
const nested = new AsarArtifactReader(adapterKey);
|
||||
try {
|
||||
await visitArtifactEntries(context, nested, logicalPath);
|
||||
} finally {
|
||||
await nested.close();
|
||||
}
|
||||
};
|
||||
|
||||
const isExpandableAsar = (entry: ArtifactEntry, logicalPath: string): boolean =>
|
||||
entry.kind === "file" &&
|
||||
logicalPath.toLowerCase().endsWith(".asar") &&
|
||||
|
||||
@@ -35,7 +35,6 @@ export interface ArtifactInventorySnapshot {
|
||||
readonly nodes: readonly ArtifactNode[];
|
||||
readonly occurrences: ArtifactInventoryResult["occurrences"];
|
||||
readonly edges: ArtifactInventoryResult["edges"];
|
||||
readonly limits: ArtifactInventoryResult["limits"];
|
||||
readonly provenance: ReadonlyArray<
|
||||
ArtifactInventoryResult["provenance"][number]
|
||||
>;
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { realpath } from "node:fs/promises";
|
||||
|
||||
import type { ArtifactLimits } from "../artifacts/ArtifactReader.js";
|
||||
import {
|
||||
scanCanonicalArtifactInventory,
|
||||
type ArtifactInventoryOptions,
|
||||
@@ -10,9 +9,8 @@ import {
|
||||
/** Resolve, authorize, and scan one artifact without a second path resolution. */
|
||||
export const scanAuthorizedArtifactInventory = async (
|
||||
inputPath: string,
|
||||
limits: ArtifactLimits,
|
||||
options: ArtifactInventoryOptions = {},
|
||||
): Promise<ArtifactInventorySnapshot> => {
|
||||
const path = await realpath(inputPath);
|
||||
return scanCanonicalArtifactInventory(path, limits, options);
|
||||
return scanCanonicalArtifactInventory(path, options);
|
||||
};
|
||||
|
||||
@@ -148,56 +148,80 @@ const visitReader = async (
|
||||
containerSha256: string,
|
||||
context: ReadContext,
|
||||
): Promise<void> => {
|
||||
for await (const entry of reader.entries(context.signal)) {
|
||||
abortIfNeeded(context.signal);
|
||||
const path = normalizeArtifactPath(
|
||||
prefix === "" ? entry.path : `${prefix}/${entry.path}`,
|
||||
{
|
||||
maxDepth: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxDepth,
|
||||
maxPathBytes:
|
||||
JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxPathBytes,
|
||||
},
|
||||
);
|
||||
const nestedAsar = isFilesystemAsar(entry, path);
|
||||
context.registry.add(path, nestedAsar ? "directory" : entry.kind);
|
||||
if (nestedAsar) {
|
||||
await visitNestedAsar(entry, path, context);
|
||||
continue;
|
||||
}
|
||||
const expected = context.expected.get(path);
|
||||
if (expected === undefined) continue;
|
||||
const selection = context.selections.get(path);
|
||||
const text = await readTextIfSelected(context, {
|
||||
const stack: Array<{
|
||||
readonly reader: ArtifactReader;
|
||||
readonly prefix: string;
|
||||
readonly containerSha256: string;
|
||||
readonly iterator: AsyncIterator<ArtifactEntry>;
|
||||
readonly owned: boolean;
|
||||
}> = [
|
||||
{
|
||||
reader,
|
||||
entry,
|
||||
expected,
|
||||
selection,
|
||||
});
|
||||
context.files.push({
|
||||
path,
|
||||
container_sha256: containerSha256,
|
||||
sha256: expected.sha256,
|
||||
bytes: expected.bytes,
|
||||
inventory_artifact_id: expected.inventoryArtifactId,
|
||||
kind: expected.kind,
|
||||
unpacked: entry.unpacked,
|
||||
text,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const visitNestedAsar = async (
|
||||
entry: ArtifactEntry,
|
||||
path: string,
|
||||
context: ReadContext,
|
||||
): Promise<void> => {
|
||||
const inventory = expectedContainer(path, context);
|
||||
context.containers.push(inventory);
|
||||
const nested = new AsarArtifactReader(entry.adapterKey);
|
||||
prefix,
|
||||
containerSha256,
|
||||
iterator: reader.entries(context.signal)[Symbol.asyncIterator](),
|
||||
owned: false,
|
||||
},
|
||||
];
|
||||
try {
|
||||
await visitReader(nested, path, inventory.sha256, context);
|
||||
while (stack.length > 0) {
|
||||
const frame = stack.at(-1);
|
||||
if (frame === undefined) break;
|
||||
const next = await frame.iterator.next();
|
||||
if (next.done) {
|
||||
stack.pop();
|
||||
if (frame.owned) await frame.reader.close();
|
||||
continue;
|
||||
}
|
||||
const entry = next.value;
|
||||
abortIfNeeded(context.signal);
|
||||
const path = normalizeArtifactPath(
|
||||
frame.prefix === "" ? entry.path : `${frame.prefix}/${entry.path}`,
|
||||
);
|
||||
const nestedAsar = isFilesystemAsar(entry, path);
|
||||
context.registry.add(path, nestedAsar ? "directory" : entry.kind);
|
||||
if (nestedAsar) {
|
||||
const inventory = expectedContainer(path, context);
|
||||
context.containers.push(inventory);
|
||||
const nested = new AsarArtifactReader(entry.adapterKey);
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: path,
|
||||
containerSha256: inventory.sha256,
|
||||
iterator: nested.entries(context.signal)[Symbol.asyncIterator](),
|
||||
owned: true,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const expected = context.expected.get(path);
|
||||
if (expected === undefined) continue;
|
||||
const selection = context.selections.get(path);
|
||||
const text = await readTextIfSelected(context, {
|
||||
reader: frame.reader,
|
||||
entry,
|
||||
expected,
|
||||
selection,
|
||||
});
|
||||
context.files.push({
|
||||
path,
|
||||
container_sha256: frame.containerSha256,
|
||||
sha256: expected.sha256,
|
||||
bytes: expected.bytes,
|
||||
inventory_artifact_id: expected.inventoryArtifactId,
|
||||
kind: expected.kind,
|
||||
unpacked: entry.unpacked,
|
||||
text,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await nested.close();
|
||||
await Promise.allSettled(
|
||||
stack
|
||||
.filter(({ owned }) => owned)
|
||||
.map(async ({ reader, iterator }) => {
|
||||
await iterator.return?.();
|
||||
await reader.close();
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -333,8 +357,8 @@ const readBounded = async (
|
||||
if (bytes > maximum) {
|
||||
stream.destroy();
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
"Selected JavaScript artifact text exceeded its byte limit",
|
||||
"integrity",
|
||||
"Artifact entry grew after inventory and exceeded the selected text read size",
|
||||
);
|
||||
}
|
||||
chunks.push(chunk);
|
||||
|
||||
@@ -204,16 +204,6 @@ const graphLimitations = (
|
||||
};
|
||||
|
||||
const reconstructionLimits = () => [
|
||||
{
|
||||
name: "max-entries",
|
||||
value: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxEntries,
|
||||
unit: "items" as const,
|
||||
},
|
||||
{
|
||||
name: "max-total-artifact-bytes",
|
||||
value: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact.maxTotalBytes,
|
||||
unit: "bytes" as const,
|
||||
},
|
||||
{
|
||||
name: "max-text-files",
|
||||
value: JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.maxTextFiles,
|
||||
|
||||
@@ -14,7 +14,6 @@ import { analyzeJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"
|
||||
import { readJavaScriptArtifactFiles } from "./JavaScriptArtifactFiles.js";
|
||||
import { buildJavaScriptArtifactGraph } from "./JavaScriptArtifactGraphBuilder.js";
|
||||
import {
|
||||
artifactLimitsForReconstruction,
|
||||
javascriptArtifactReconstructionInputSchema,
|
||||
type JavaScriptArtifactReconstructionInput,
|
||||
} from "./JavaScriptArtifactReconstructionInput.js";
|
||||
@@ -59,11 +58,7 @@ export const reconstructJavaScriptArtifact = async (
|
||||
abortIfNeeded(signal);
|
||||
const path = await realpath(input.input_path);
|
||||
const format = await resolveFormat(path, input);
|
||||
const snapshot = await scanCanonicalArtifactInventory(
|
||||
path,
|
||||
artifactLimitsForReconstruction(),
|
||||
{ signal },
|
||||
);
|
||||
const snapshot = await scanCanonicalArtifactInventory(path, { signal });
|
||||
if (snapshot.manifest.root_format !== format)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
|
||||
@@ -1,17 +1,7 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import type { ArtifactLimits } from "../artifacts/ArtifactReader.js";
|
||||
|
||||
/** Resource safeguards for local artifact traversal and static parsing. */
|
||||
export const JAVASCRIPT_APPLICATION_RESOURCE_LIMITS = {
|
||||
artifact: {
|
||||
maxEntries: 8_000,
|
||||
maxTotalBytes: 512 * 1_024 * 1_024,
|
||||
maxEntryBytes: 128 * 1_024 * 1_024,
|
||||
maxCompressionRatio: 1_000,
|
||||
maxDepth: 64,
|
||||
maxPathBytes: 4_096,
|
||||
} satisfies ArtifactLimits,
|
||||
maxTextFiles: 5_000,
|
||||
maxTotalTextBytes: 128 * 1_024 * 1_024,
|
||||
maxTextFileBytes: 8 * 1_024 * 1_024,
|
||||
@@ -29,7 +19,3 @@ export const javascriptArtifactReconstructionInputSchema = z.strictObject({
|
||||
export type JavaScriptArtifactReconstructionInput = z.infer<
|
||||
typeof javascriptArtifactReconstructionInputSchema
|
||||
>;
|
||||
|
||||
/** Project private resource safeguards into the artifact traversal policy. */
|
||||
export const artifactLimitsForReconstruction = (): ArtifactLimits =>
|
||||
JAVASCRIPT_APPLICATION_RESOURCE_LIMITS.artifact;
|
||||
|
||||
Reference in New Issue
Block a user