leefer/rea
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(artifacts): admit bounded reads and preserve source identity

Browse Source 
Validation: integrated changes passed formatting, typecheck, lint, resolved module-boundary verification and git diff --check. Tests and real-provider workflows were not run.
This commit is contained in:
morluto
2026-10-11 00:03:18 +08:00
parent 2a0b58e440
commit 6d4cefc4f7
21 changed files with 502 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/application/AppBundleExecutable.ts
+9 -8
View File
@@ -1,5 +1,5 @@
import { execFile } from "node:child_process";
import { lstat, readdir, readFile, realpath, stat } from "node:fs/promises";
import { lstat, opendir, readFile, realpath, stat } from "node:fs/promises";
import { join } from "node:path";
import { promisify } from "node:util";
@@ -156,13 +156,14 @@ const wrappedBundle = async (bundle: string): Promise<string | undefined> => {
if (isAbsence(cause)) return undefined;
throw cause;
}
const apps = (await readdir(wrapper, { withFileTypes: true })).filter(
(entry) => entry.isDirectory() && entry.name.toLowerCase().endsWith(".app"),
);
const [app] = apps;
return apps.length === 1 && app !== undefined
? join(wrapper, app.name)
: undefined;
let app: string | undefined;
for await (const entry of await opendir(wrapper)) {
if (!entry.isDirectory() || !entry.name.toLowerCase().endsWith(".app"))
continue;
if (app !== undefined) return undefined;
app = join(wrapper, entry.name);
}
return app;
};
const resolveLayoutExecutable = async (
src/application/BinaryTargetResolver.ts
+10 -3
View File
@@ -1,4 +1,4 @@
import { constants } from "node:fs";
import { constants, type Stats } from "node:fs";
import { createHash } from "node:crypto";
import {
access,
@@ -248,7 +248,7 @@ const isArchiveFormat = (
const sha256Handle = async (
handle: FileHandle,
path: string,
initial: Awaited<ReturnType<FileHandle["stat"]>>,
initial: Stats,
signal?: AbortSignal,
): Promise<string> => {
const hash = createHash("sha256");
@@ -256,11 +256,18 @@ const sha256Handle = async (
let position = 0;
while (true) {
throwIfTargetResolutionCancelled(signal);
const observed = await handle.read(chunk, 0, chunk.length, position);
const observed = await handle.read(
chunk,
0,
Math.min(chunk.length, initial.size - position + 1),
position,
);
throwIfTargetResolutionCancelled(signal);
if (observed.bytesRead === 0) break;
hash.update(chunk.subarray(0, observed.bytesRead));
position += observed.bytesRead;
if (position > initial.size)
throw new BinaryTargetError(path, "target grew while being read");
}
throwIfTargetResolutionCancelled(signal);
const [opened, currentPath] = await Promise.all([handle.stat(), lstat(path)]);
src/application/JsonFiles.ts
+1 -1
View File
@@ -42,7 +42,7 @@ export const readJsonFile = async (
if (!decoded.ok) {
return err(
new EvidenceFileError("read", "invalid-json", {
cause: decoded.cause,
cause: decoded.cause ?? new SyntaxError(decoded.error),
path: requestedPath,
}),
);
src/artifacts/AsarArtifactReader.ts
+12
View File
@@ -27,6 +27,7 @@ import {
type ArtifactReader,
} from "./ArtifactReader.js";
import { closeAsarHandle, readValidatedAsarEntry } from "./AsarEntryStream.js";
import { admitAsarHeader } from "./AsarHeader.js";
/**
* Official Electron ASAR adapter with range-streamed member reads.
@@ -57,6 +58,17 @@ export class AsarArtifactReader implements ArtifactReader {
const snapshotPath = this.#snapshotPath();
this.#resetArchiveState();
this.#entries.clear();
const headerHandle = await open(
snapshotPath,
constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
);
try {
const admitted = await admitAsarHeader(headerHandle);
this.#archiveSize = admitted.archiveBytes;
} finally {
await headerHandle.close();
}
abortIfNeeded(signal);
paths = listPackage(snapshotPath, { isPack: false }).sort((left, right) =>
left.localeCompare(right, "en"),
);
src/artifacts/AsarHeader.ts
+48
View File
@@ -0,0 +1,48 @@
import type { FileHandle } from "node:fs/promises";
import { ArtifactReaderFailure } from "./ArtifactReader.js";
/** Shared resource budget for the eagerly decoded ASAR header, not member bytes. */
export const MAX_ASAR_HEADER_BYTES = 16 * 1024 * 1024;
/** Admit both Chromium pickle extents before an upstream decoder allocates them. */
export const admitAsarHeader = async (
handle: FileHandle,
): Promise<{
readonly archiveBytes: number;
readonly headerBytes: number;
readonly jsonBytes: number;
}> => {
const observed = await handle.stat();
if (!observed.isFile())
throw new ArtifactReaderFailure(
"format",
"ASAR container is not a regular file",
);
const prefix = Buffer.alloc(16);
const read = await handle.read(prefix, 0, prefix.length, 0);
if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4)
throw new ArtifactReaderFailure(
"format",
"Truncated or invalid ASAR size pickle",
);
const headerBytes = prefix.readUInt32LE(4);
const payloadBytes = prefix.readUInt32LE(8);
const jsonBytes = prefix.readUInt32LE(12);
if (
headerBytes < 8 ||
headerBytes > observed.size - 8 ||
payloadBytes !== headerBytes - 4 ||
jsonBytes > payloadBytes - 4
)
throw new ArtifactReaderFailure(
"format",
"ASAR header leaves its container or has invalid pickle extents",
);
if (headerBytes > MAX_ASAR_HEADER_BYTES)
throw new ArtifactReaderFailure(
"limit",
`ASAR header exceeds the ${String(MAX_ASAR_HEADER_BYTES)} byte decode budget`,
);
return { archiveBytes: observed.size, headerBytes, jsonBytes };
};
src/artifacts/DirectoryArtifactReader.ts
+42 -44
View File
@@ -3,15 +3,13 @@ import { lstat, open, opendir, realpath } from "node:fs/promises";
import { isAbsolute, join, relative, sep } from "node:path";
import type { Readable } from "node:stream";
import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactReader,
} from "./ArtifactReader.js";
/** Deterministic, non-symlink-following directory reader. */
/** Non-symlink-following directory reader. */
export class DirectoryArtifactReader implements ArtifactReader {
readonly format = "directory" as const;
readonly #rootPromise: Promise<string>;
@@ -38,13 +36,52 @@ export class DirectoryArtifactReader implements ArtifactReader {
await withDirectoryIoContext("inspect directory", directory, () =>
assertContainedDirectory(root, directory),
);
const children: string[] = [];
const handle = await opendir(directory).catch((cause: unknown) => {
throw directoryIoFailure("open directory", directory, cause);
});
const directories: string[] = [];
try {
for await (const child of handle) children.push(child.name);
for await (const child of handle) {
abortIfNeeded(signal);
const absolute = join(directory, child.name);
const metadata = await lstat(absolute).catch((cause: unknown) => {
throw directoryIoFailure("inspect entry", absolute, cause);
});
const path = relative(root, absolute).split(sep).join("/");
const kind = metadata.isSymbolicLink()
? "symlink"
: metadata.isDirectory()
? "directory"
: metadata.isFile()
? "file"
: undefined;
if (kind === undefined) continue;
yield {
path,
kind,
declaredSize: kind === "file" ? metadata.size : null,
compressedSize: null,
executable: (metadata.mode & 0o111) !== 0,
encrypted: false,
byteOffset: null,
declaredSha256: null,
unpacked: false,
limitations:
kind === "symlink"
? ["Symlink target was not followed or disclosed."]
: [],
adapterKey: absolute,
...(kind === "file"
? {
sourceIdentity: { device: metadata.dev, inode: metadata.ino },
}
: {}),
};
if (kind === "directory" && (descendInto?.(path) ?? true))
directories.push(absolute);
}
} catch (cause: unknown) {
if (cause instanceof ArtifactReaderFailure) throw cause;
throw directoryIoFailure("read directory", directory, cause);
} finally {
// best-effort cleanup: directory-handle close must not mask traversal.
@@ -53,45 +90,6 @@ export class DirectoryArtifactReader implements ArtifactReader {
await withDirectoryIoContext("inspect directory", directory, () =>
assertContainedDirectory(root, directory),
);
children.sort(compareUnicodeCodePoints);
const directories: string[] = [];
for (const name of children) {
abortIfNeeded(signal);
const absolute = join(directory, name);
const metadata = await lstat(absolute).catch((cause: unknown) => {
throw directoryIoFailure("inspect entry", absolute, cause);
});
const path = relative(root, absolute).split(sep).join("/");
const kind = metadata.isSymbolicLink()
? "symlink"
: metadata.isDirectory()
? "directory"
: metadata.isFile()
? "file"
: undefined;
if (kind === undefined) continue;
if (kind === "directory" && (descendInto?.(path) ?? true))
directories.push(absolute);
yield {
path,
kind,
declaredSize: kind === "file" ? metadata.size : null,
compressedSize: null,
executable: (metadata.mode & 0o111) !== 0,
encrypted: false,
byteOffset: null,
declaredSha256: null,
unpacked: false,
limitations:
kind === "symlink"
? ["Symlink target was not followed or disclosed."]
: [],
adapterKey: absolute,
...(kind === "file"
? { sourceIdentity: { device: metadata.dev, inode: metadata.ino } }
: {}),
};
}
directories.reverse();
pending.push(...directories);
}
src/artifacts/SafeOutputTree.ts
+93 -46
View File
@@ -105,7 +105,26 @@ export class SafeOutputTree {
}
return new SafeOutputTree(canonicalOutput, platform);
} catch (cause: unknown) {
await rm(canonicalOutput, { recursive: true, force: true });
let removalFailure: unknown;
try {
await rm(canonicalOutput, { recursive: true, force: true });
} catch (cleanupCause: unknown) {
removalFailure = cleanupCause;
}
let absent = false;
try {
absent = await isAbsent(canonicalOutput);
} catch (cleanupCause: unknown) {
removalFailure ??= cleanupCause;
}
if (!absent)
throw ArtifactReaderFailure.withCleanup(cause, {
reason:
removalFailure instanceof Error
? removalFailure.message
: "Extraction output root remains after setup failure",
resources: [canonicalOutput],
});
throw cause;
}
}
@@ -122,56 +141,84 @@ export class SafeOutputTree {
async write(
relativePath: string,
source: Readable,
expectedSha256: string,
expected: { readonly sha256: string; readonly bytes: number },
signal?: AbortSignal,
): Promise<SafeOutputFile> {
this.#assertWritable();
const path = normalizeArtifactPath(relativePath);
this.#registry.add(path, "file");
const destination = await this.#prepareParent(path);
const handle = await open(
destination,
constants.O_CREAT |
constants.O_EXCL |
constants.O_WRONLY |
constants.O_NOFOLLOW,
0o600,
).catch(async (cause: unknown) => {
await throwIfDestinationCaseCollision(dirname(destination), path, cause);
throw new ArtifactReaderFailure(
"path",
`Could not exclusively create extraction path: ${path}`,
{ cause },
);
});
const hash = createHash("sha256");
let bytes = 0;
try {
for await (const raw of source) {
abortIfNeeded(signal);
const chunk = streamChunkToBuffer(raw);
bytes += chunk.length;
hash.update(chunk);
await writeAll(handle, chunk);
this.#assertWritable();
if (!Number.isSafeInteger(expected.bytes) || expected.bytes < 0)
throw new ArtifactReaderFailure(
"format",
`Invalid expected extraction size for ${relativePath}`,
);
const path = normalizeArtifactPath(relativePath);
this.#registry.add(path, "file");
const destination = await this.#prepareParent(path);
const handle = await open(
destination,
constants.O_CREAT |
constants.O_EXCL |
constants.O_WRONLY |
constants.O_NOFOLLOW,
0o600,
).catch(async (cause: unknown) => {
await throwIfDestinationCaseCollision(
dirname(destination),
path,
cause,
);
throw new ArtifactReaderFailure(
"path",
`Could not exclusively create extraction path: ${path}`,
{ cause },
);
});
const hash = createHash("sha256");
let bytes = 0;
try {
for await (const raw of source) {
abortIfNeeded(signal);
const chunk = streamChunkToBuffer(raw);
if (chunk.length > expected.bytes - bytes)
throw new ArtifactReaderFailure(
"integrity",
`Extracted content exceeds the inventoried size: ${path}`,
);
bytes += chunk.length;
hash.update(chunk);
await writeAll(handle, chunk);
}
if (bytes !== expected.bytes)
throw new ArtifactReaderFailure(
"integrity",
`Extracted content size disagrees with inventory: ${path}`,
);
const sha256 = hash.digest("hex");
if (sha256 !== expected.sha256)
throw new ArtifactReaderFailure(
"integrity",
`Extracted content disagrees with inventory: ${path}`,
);
await handle.sync();
await handle.close();
const readback = await hashFile(destination, bytes, signal);
if (readback.sha256 !== sha256 || readback.bytes !== bytes)
throw new ArtifactReaderFailure(
"integrity",
`Durable readback verification failed: ${path}`,
);
return { relativePath: path, sha256, bytesWritten: bytes };
} catch (cause: unknown) {
// best-effort cleanup: file-handle close must not mask the write failure.
await handle.close().catch(() => undefined);
throw cause;
}
const sha256 = hash.digest("hex");
if (sha256 !== expectedSha256)
throw new ArtifactReaderFailure(
"integrity",
`Extracted content disagrees with inventory: ${path}`,
);
await handle.sync();
await handle.close();
const readback = await hashFile(destination, bytes, signal);
if (readback.sha256 !== sha256 || readback.bytes !== bytes)
throw new ArtifactReaderFailure(
"integrity",
`Durable readback verification failed: ${path}`,
);
return { relativePath: path, sha256, bytesWritten: bytes };
} catch (cause: unknown) {
// best-effort cleanup: file-handle close must not mask the write failure.
await handle.close().catch(() => undefined);
try {
source.destroy();
} catch {
// Preserve the write refusal or failure as the caller-visible error.
}
throw cause;
}
}
src/artifacts/ZipArtifactReader.ts
+47 -6
View File
@@ -1,4 +1,4 @@
import { open, type FileHandle } from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
import { PassThrough, type Readable } from "node:stream";
import { once } from "node:events";
@@ -10,20 +10,57 @@ import {
type ArtifactReader,
} from "./ArtifactReader.js";
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
import {
NonRegularFileReadError,
openRegularFile,
sameRegularFileState,
type StableRegularFileDescriptor,
} from "../filesystem/RegularFile.js";
class NodeFileReader extends Reader<string> {
#handle: FileHandle | undefined;
#initPromise: Promise<void> | undefined;
readonly #ownsHandle: boolean;
constructor(
private readonly path: string,
private readonly maximumReadBytes?: number,
private readonly admitted?: StableRegularFileDescriptor,
) {
super(path);
this.#ownsHandle = admitted === undefined;
}
override async init(): Promise<void> {
this.#handle = await open(this.path, "r");
this.size = (await this.#handle.stat()).size;
override init(): Promise<void> {
this.#initPromise ??= this.#initialize();
return this.#initPromise;
}
async #initialize(): Promise<void> {
try {
this.#handle =
this.admitted?.handle ??
(await openRegularFile(this.path, { symlinks: "reject" }));
} catch (cause: unknown) {
if (cause instanceof NonRegularFileReadError)
throw new ArtifactReaderFailure(
"format",
`ZIP source is not a regular file: ${this.path}`,
{ cause },
);
throw cause;
}
const metadata = await this.#handle.stat();
if (
!metadata.isFile() ||
(this.admitted !== undefined &&
!sameRegularFileState(this.admitted.initial, metadata))
)
throw new ArtifactReaderFailure(
"integrity",
`ZIP source identity changed before reading: ${this.path}`,
);
this.size = metadata.size;
}
override async readUint8Array(
@@ -47,7 +84,10 @@ class NodeFileReader extends Reader<string> {
}
async closeHandle(): Promise<void> {
await this.#handle?.close();
if (!this.#ownsHandle) return;
await this.#initPromise?.catch(() => undefined);
if (this.#handle === undefined) return;
await this.#handle.close();
this.#handle = undefined;
}
}
@@ -64,9 +104,10 @@ export class ZipArtifactReader implements ArtifactReader {
path: string,
format: ZipPackageFormat,
maximumMetadataReadBytes?: number,
admitted?: StableRegularFileDescriptor,
) {
this.format = format;
this.#source = new NodeFileReader(path, maximumMetadataReadBytes);
this.#source = new NodeFileReader(path, maximumMetadataReadBytes, admitted);
this.#reader = new ZipReader(this.#source, {
checkSignature: true,
checkOverlappingEntry: true,
src/artifacts/apple/InterfaceBuilderAnalysis.ts
+65 -10
View File
@@ -25,6 +25,7 @@ import {
type NibHierarchyNode,
} from "./NibViewHierarchy.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
import { decodeXmlPlistText } from "../../domain/propertyListXmlText.js";
@@ -49,7 +50,6 @@ export const analyzeInterfaceBuilderBundle = async (input: {
const prototypeKeyOmissions = new Map<string, number>();
let omitted = 0;
let attempted = 0;
let candidates = 0;
let aggregateInputBytes = 0;
const decodeBudget = new InterfaceBuilderDecodeBudget(
MAX_AGGREGATE_DECODE_BYTES,
@@ -60,15 +60,16 @@ export const analyzeInterfaceBuilderBundle = async (input: {
else unreportedOmittedArchives += 1;
};
try {
for await (const entry of reader.entries(input.signal)) {
if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path))
continue;
candidates += 1;
if (candidates > limits.max_documents) {
omitted += 1;
recordOmittedArchive(entry.path, "max_documents");
continue;
}
const selection = await selectInterfaceBuilderCandidates(
reader,
input.signal,
limits.max_documents,
);
omitted = selection.omitted;
for (const entry of selection.omittedSamples)
recordOmittedArchive(entry.path, "max_documents");
unreportedOmittedArchives += selection.unreportedOmitted;
for (const entry of selection.selected) {
try {
const remainingInputBytes =
MAX_AGGREGATE_INPUT_BYTES - aggregateInputBytes;
@@ -167,6 +168,60 @@ export const analyzeInterfaceBuilderBundle = async (input: {
});
};
/** Select lexically earliest document paths independent of host traversal order. */
const selectInterfaceBuilderCandidates = async (
reader: DirectoryArtifactReader,
signal: AbortSignal | undefined,
maximumDocuments: number,
): Promise<{
readonly selected: readonly ArtifactEntry[];
readonly omittedSamples: readonly ArtifactEntry[];
readonly omitted: number;
readonly unreportedOmitted: number;
}> => {
const capacity = maximumDocuments + MAX_REPORTED_OMITTED_ARCHIVES;
const candidates: ArtifactEntry[] = [];
let total = 0;
for await (const entry of reader.entries(signal)) {
if (entry.kind !== "file" || !isInterfaceBuilderArchive(entry.path))
continue;
total += 1;
retainCandidate(candidates, entry, capacity);
}
const omittedSamples = candidates.slice(maximumDocuments, capacity);
return {
selected: candidates.slice(0, maximumDocuments),
omittedSamples,
omitted: Math.max(0, total - maximumDocuments),
unreportedOmitted: Math.max(
0,
total - maximumDocuments - omittedSamples.length,
),
};
};
/** Keep the earliest bounded path set; directory iteration order is host-defined. */
const retainCandidate = (
candidates: ArtifactEntry[],
entry: ArtifactEntry,
maximum: number,
): void => {
let low = 0;
let high = candidates.length;
while (low < high) {
const middle = Math.floor((low + high) / 2);
const existing = candidates[middle];
if (
existing !== undefined &&
compareUnicodeCodePoints(existing.path, entry.path) < 0
)
low = middle + 1;
else high = middle;
}
candidates.splice(low, 0, entry);
if (candidates.length > maximum) candidates.pop();
};
const finalizeHierarchyCoverage = (
result: ReturnType<typeof buildInterfaceBuilderAnalysis>,
counts: {
src/artifacts/apple/KeyedArchiveReader.ts
+12 -13
View File
@@ -147,29 +147,28 @@ export const inspectBundleKeyedArchive = async (input: {
try {
// Match inventory spelling while retaining the raw path for I/O and evidence.
const wanted = selected.path.normalize("NFC");
const matches: ArtifactEntry[] = [];
for await (const entry of reader.entries(input.signal, (directory) =>
let entry: ArtifactEntry | undefined;
for await (const candidate of reader.entries(input.signal, (directory) =>
wanted.startsWith(`${directory.normalize("NFC")}/`),
)) {
if (entry.path.normalize("NFC") !== wanted) continue;
if (entry.kind !== "file")
if (candidate.path.normalize("NFC") !== wanted) continue;
if (candidate.kind !== "file")
throw archivePathError(
"invalid_value",
`Archive path selects a ${entry.kind}, not a regular file: ${selected.path}`,
`Archive path selects a ${candidate.kind}, not a regular file: ${selected.path}`,
);
if ((entry.declaredSize ?? 0) > MAX_BYTES)
if ((candidate.declaredSize ?? 0) > MAX_BYTES)
throw new ArtifactReaderFailure(
"limit",
"Keyed archive exceeds 64 MiB",
);
matches.push(entry);
if (entry !== undefined)
throw archivePathError(
"invalid_value",
`Archive path matches multiple Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`,
);
entry = candidate;
}
if (matches.length > 1)
throw archivePathError(
"invalid_value",
`Archive path matches ${String(matches.length)} Unicode-equivalent bundle entries; selection is ambiguous: ${selected.path}`,
);
const entry = matches[0];
if (entry === undefined)
throw archivePathError(
"invalid_value",
src/artifacts/extraction/ArtifactExtraction.ts
+4 -1
View File
@@ -215,7 +215,10 @@ const materializeSelection = async ({
const written = await output.write(
path,
stream,
selectedItem.node.sha256,
{
sha256: selectedItem.node.sha256,
bytes: selectedItem.node.size,
},
signal,
);
extracted.push({
src/artifacts/inventory/classify.ts
+49 -7
View File
@@ -5,6 +5,7 @@ import {
} from "../../filesystem/RegularFile.js";
import type { FileHandle } from "node:fs/promises";
import type { Stats } from "node:fs";
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";
@@ -17,18 +18,46 @@ import { ArtifactReaderFailure } from "../ArtifactReader.js";
import type { HashResult } from "../ArtifactHash.js";
import { hashStableRootArtifactHandle } from "./hashStableRootArtifact.js";
interface RootClassification {
readonly format: ArtifactOccurrence["artifact_format"];
readonly digest: HashResult | null;
}
type RootInventoryClassification = RootClassification & {
readonly zipSource: StableRegularFileDescriptor | undefined;
};
/** Classify and hash one file root through the same stable open descriptor. */
export const classifyAndHashRoot = async (
path: string,
directory: boolean,
expectedMetadata: Stats,
signal?: AbortSignal,
): Promise<{
readonly format: ArtifactOccurrence["artifact_format"];
readonly digest: HashResult | null;
}> => {
if (directory) return { format: "directory", digest: null };
): Promise<RootClassification> => {
const result = await classifyAndHashRootForInventory(
path,
directory,
expectedMetadata,
signal,
);
try {
return { format: result.format, digest: result.digest };
} finally {
await result.zipSource?.handle.close();
}
};
/** Classify a root and retain its admitted ZIP descriptor for child inventory. */
export const classifyAndHashRootForInventory = async (
path: string,
directory: boolean,
expectedMetadata: Stats,
signal?: AbortSignal,
): Promise<RootInventoryClassification> => {
if (directory)
return { format: "directory", digest: null, zipSource: undefined };
const handle = await openRootFile(path, signal);
let transferHandle = false;
try {
const initial = await handle.stat();
if (!sameRegularFileState(expectedMetadata, initial))
@@ -43,12 +72,25 @@ export const classifyAndHashRoot = async (
initial,
signal,
);
return { format, digest };
const zipSource = isZipFormat(format);
if (zipSource) transferHandle = true;
return {
format,
digest,
zipSource: zipSource ? { handle, initial } : undefined,
};
} finally {
await handle.close();
if (!transferHandle) await handle.close();
}
};
const isZipFormat = (format: ArtifactOccurrence["artifact_format"]): boolean =>
format === "zip" ||
format === "ipa" ||
format === "apk" ||
format === "msix" ||
format === "appx";
const openRootFile = async (
path: string,
signal?: AbortSignal,
src/artifacts/inventory/hashStableRootArtifact.ts
+5 -2
View File
@@ -1,4 +1,5 @@
import { lstat } from "node:fs/promises";
import type { Stats } from "node:fs";
import type { FileHandle } from "node:fs/promises";
import { ArtifactReaderFailure } from "../ArtifactReader.js";
@@ -45,11 +46,13 @@ export const hashStableRootArtifact = async (
export const hashStableRootArtifactHandle = async (
path: string,
handle: FileHandle,
initial: Awaited<ReturnType<FileHandle["stat"]>>,
initial: Stats,
signal?: AbortSignal,
): Promise<HashResult> => {
const digest = await hashReadable(
handle.createReadStream({ autoClose: false }),
// Read at most the admitted extent plus one byte to detect growth without
// following an actively appended file indefinitely.
handle.createReadStream({ autoClose: false, start: 0, end: initial.size }),
signal,
);
abortIfNeeded(signal);
src/artifacts/inventory/reader.ts
+3 -1
View File
@@ -6,12 +6,14 @@ import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
import { ZipArtifactReader } from "../ZipArtifactReader.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
import { ZIP_NON_ENTRY_TAIL_LIMITATION } from "../../domain/zipPackageFormat.js";
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
export const createReader = async (
path: string,
format: ArtifactOccurrence["artifact_format"],
environment: Readonly<NodeJS.ProcessEnv>,
signal?: AbortSignal,
zipSource?: StableRegularFileDescriptor,
): Promise<ArtifactReader | undefined> => {
switch (format) {
case "directory":
@@ -21,7 +23,7 @@ export const createReader = async (
case "apk":
case "msix":
case "appx":
return new ZipArtifactReader(path, format);
return new ZipArtifactReader(path, format, undefined, zipSource);
case "asar":
return new AsarArtifactReader(path);
case "mach-o-universal":
src/artifacts/inventory/scanCanonical.ts
+58 -12
View File
@@ -5,6 +5,7 @@ import { AsarArtifactReader } from "../AsarArtifactReader.js";
import { lstat } from "node:fs/promises";
import type { Stats } from "node:fs";
import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.js";
import {
ArtifactReaderFailure,
@@ -31,9 +32,12 @@ import {
artifactGraphDigest,
artifactManifestId,
} from "../../domain/artifactIdentity.js";
import { classifyAndHashRoot } from "./classify.js";
import { classifyAndHashRootForInventory } from "./classify.js";
import { type HashResult } from "../ArtifactHash.js";
import { hashStableRootArtifact } from "./hashStableRootArtifact.js";
import {
hashStableRootArtifact,
hashStableRootArtifactHandle,
} from "./hashStableRootArtifact.js";
import { createReader, inventoryLimitations } from "./reader.js";
import {
scanReader,
@@ -51,18 +55,29 @@ export const scanCanonicalArtifactInventory = async (
): Promise<ArtifactInventorySnapshot> => {
const integrity = options.integrity ?? STRICT_INTEGRITY_POLICY;
const metadata = await lstat(path);
const { format: rootFormat, digest: rootDigest } = await classifyAndHashRoot(
const {
format: rootFormat,
digest: rootDigest,
zipSource,
} = await classifyAndHashRootForInventory(
path,
metadata.isDirectory(),
metadata,
options.signal,
);
const reader = await readerFactory(
path,
rootFormat,
options.environment ?? {},
options.signal,
);
let reader: ArtifactReader | undefined;
let readerCreationFailure: { readonly cause: unknown } | undefined;
try {
reader = await readerFactory(
path,
rootFormat,
options.environment ?? {},
options.signal,
zipSource,
);
} catch (cause: unknown) {
readerCreationFailure = { cause };
}
const ownedReaders: ArtifactReader[] = reader === undefined ? [] : [reader];
let outcome:
| {
@@ -71,6 +86,7 @@ export const scanCanonicalArtifactInventory = async (
}
| { readonly kind: "failed"; readonly cause: unknown };
try {
if (readerCreationFailure !== undefined) throw readerCreationFailure.cause;
if (reader instanceof AsarArtifactReader)
await reader.prepareContainer(rootDigest?.sha256, options.signal);
const {
@@ -86,6 +102,7 @@ export const scanCanonicalArtifactInventory = async (
metadata,
rootFormat,
rootDigest,
zipSource,
reader,
signal: options.signal,
nodes,
@@ -116,6 +133,24 @@ export const scanCanonicalArtifactInventory = async (
);
}
}
if (zipSource !== undefined) {
try {
await zipSource.handle.close();
} catch (cleanupCause: unknown) {
const cleanup = ArtifactReaderFailure.cleanupObservation(
cleanupCause,
`root ZIP descriptor for ${path}`,
);
cleanupFailure = ArtifactReaderFailure.withCleanup(
cleanupFailure ??
(outcome.kind === "failed" ? outcome.cause : cleanupCause),
cleanup,
outcome.kind === "completed"
? { kind: "artifact-inventory", inventory: outcome.snapshot }
: undefined,
);
}
}
if (cleanupFailure !== undefined) throw cleanupFailure;
if (outcome.kind === "failed") throw outcome.cause;
return outcome.snapshot;
@@ -126,6 +161,7 @@ interface SnapshotBuildInput {
readonly metadata: Stats;
readonly rootFormat: ArtifactOccurrence["artifact_format"];
readonly rootDigest: HashResult | null;
readonly zipSource: StableRegularFileDescriptor | undefined;
readonly reader: ArtifactReader | undefined;
readonly signal: AbortSignal | undefined;
readonly nodes: Map<string, ArtifactNode>;
@@ -137,7 +173,8 @@ interface SnapshotBuildInput {
const buildInventorySnapshot = async (
input: SnapshotBuildInput,
): Promise<ArtifactInventorySnapshot> => {
const { path, metadata, rootFormat, rootDigest, reader, signal } = input;
const { path, metadata, rootFormat, rootDigest, zipSource, reader, signal } =
input;
materializeDirectoryNodes(input.occurrences, input.nodes);
const rootNode = createRootNode({
digest: rootDigest,
@@ -181,7 +218,7 @@ const buildInventorySnapshot = async (
ordinal,
}));
await verifyRootDigest(path, rootDigest, signal);
await verifyRootDigest(path, rootDigest, zipSource, signal);
const graphSha256 = artifactGraphDigest({
nodes: orderedNodes,
@@ -256,10 +293,19 @@ const buildIntegrityContradictions = (
const verifyRootDigest = async (
path: string,
rootDigest: HashResult | null,
zipSource: StableRegularFileDescriptor | undefined,
signal: AbortSignal | undefined,
): Promise<void> => {
if (rootDigest === null) return;
const verified = await hashStableRootArtifact(path, signal);
const verified =
zipSource === undefined
? await hashStableRootArtifact(path, signal)
: await hashStableRootArtifactHandle(
path,
zipSource.handle,
zipSource.initial,
signal,
);
if (
verified.sha256 !== rootDigest.sha256 ||
verified.bytes !== rootDigest.bytes
src/browser/ElectronFileScope.ts
+8 -19
View File
@@ -3,6 +3,11 @@ import { open, realpath, stat } from "node:fs/promises";
import { isAbsolute, join, sep } from "node:path";
import { fileURLToPath } from "node:url";
import {
admitAsarHeader,
MAX_ASAR_HEADER_BYTES,
} from "../artifacts/AsarHeader.js";
/** Resolve one local file URL while rejecting remote hosts and encoded separators. */
export const authorizedElectronFile = async (
value: string,
@@ -77,7 +82,6 @@ const authorizedAsarMember = async (
type ArchiveIdentity = Pick<Stats, "dev" | "ino" | "size" | "mtimeMs">;
// This is a retained-memory budget for JSON headers, shared across all archives.
const MAX_HEADER_BYTES = 16 * 1024 * 1024;
const MAX_CACHED_HEADERS = 8;
const headerCache = new Map<
string,
@@ -105,23 +109,8 @@ const asarFiles = async (
const opened = await handle.stat();
if (!opened.isFile() || archiveIdentity(opened) !== identity)
throw new Error("ASAR identity changed before header read");
const prefix = Buffer.alloc(16);
const read = await handle.read(prefix, 0, prefix.length, 0);
if (read.bytesRead !== prefix.length || prefix.readUInt32LE(0) !== 4)
throw new Error("Truncated or invalid ASAR size pickle");
const headerBytes = prefix.readUInt32LE(4);
const payloadBytes = prefix.readUInt32LE(8);
jsonBytes = prefix.readUInt32LE(12);
if (
headerBytes < 8 ||
headerBytes > opened.size - 8 ||
payloadBytes !== headerBytes - 4 ||
jsonBytes > payloadBytes - 4 ||
jsonBytes > MAX_HEADER_BYTES
)
throw new Error(
"ASAR JSON header leaves the file or exceeds its memory budget",
);
const admitted = await admitAsarHeader(handle);
jsonBytes = admitted.jsonBytes;
const json = Buffer.alloc(jsonBytes);
let offset = 0;
while (offset < json.length) {
@@ -151,7 +140,7 @@ const asarFiles = async (
[...headerCache.values()].reduce((sum, entry) => sum + entry.bytes, 0);
while (
headerCache.size > MAX_CACHED_HEADERS ||
retainedBytes() > MAX_HEADER_BYTES
retainedBytes() > MAX_ASAR_HEADER_BYTES
)
headerCache.delete(headerCache.keys().next().value ?? archive);
return files;
src/browser/assets/PublishWebScripts.ts
+7 -2
View File
@@ -76,7 +76,12 @@ export const publishWebScripts = async (
bytes: bytes.length,
},
});
await tree.write("manifest.json", Readable.from([bytes]), digest, signal);
await tree.write(
"manifest.json",
Readable.from([bytes]),
{ sha256: digest, bytes: bytes.byteLength },
signal,
);
signal?.throwIfAborted();
await tree.commit();
return result;
@@ -109,7 +114,7 @@ const publishPlannedScript = async (
tree.write(
`files/${relativePath}`,
Readable.from([content.bytes]),
content.sha256,
{ sha256: content.sha256, bytes: content.bytes.byteLength },
signal,
);
try {
src/filesystem/RegularFile.ts
+6
View File
@@ -16,6 +16,12 @@ export class NonRegularFileReadError extends Error {
}
}
/** One admitted file descriptor and the metadata that identified its contents. */
export interface StableRegularFileDescriptor {
readonly handle: FileHandle;
readonly initial: Stats;
}
/** Open and verify the selected file descriptor without blocking on pipes. */
export const openRegularFile = async (
path: string,
src/firmware/FirmwarePublication.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export const publishFirmwareExtraction = async (context: {
await tree.write(
entry.relativePath,
createReadStream(entry.path),
sha256,
{ sha256, bytes: entry.size },
context.signal,
);
files.push({
src/javascript/recovery/WakaruPublication.ts
+6 -1
View File
@@ -50,7 +50,12 @@ export const publishWakaruArtifacts = async (
const { tree, source, parsed, staging, signal } = context;
const write = async (path: string, bytes: Buffer) => {
const sha256 = recoveryDigest(bytes);
const file = await tree.write(path, Readable.from([bytes]), sha256, signal);
const file = await tree.write(
path,
Readable.from([bytes]),
{ sha256, bytes: bytes.byteLength },
signal,
);
return {
path: join(tree.outputRoot, file.relativePath),
sha256: file.sha256,
tests/boundary/filesystem/safeOutputTree.test.ts
+16 -12
View File
@@ -43,11 +43,10 @@ describe("safe artifact output tree", () => {
const output = join(parent, "published");
const tree = await SafeOutputTree.create(output);
await expect(
tree.write(
"nested/file.txt",
Readable.from(Buffer.from("unexpected")),
"0".repeat(64),
),
tree.write("nested/file.txt", Readable.from(Buffer.from("unexpected")), {
sha256: "0".repeat(64),
bytes: Buffer.byteLength("unexpected"),
}),
).rejects.toThrow(/disagrees/u);
expect(await tree.rollback()).toMatchObject({
status: "complete",
@@ -64,11 +63,10 @@ describe("safe artifact output tree", () => {
const output = join(parent, "published");
const tree = await SafeOutputTree.create(output);
const bytes = Buffer.from("retained until rollback can retry");
await tree.write(
"nested/file.txt",
Readable.from(bytes),
createHash("sha256").update(bytes).digest("hex"),
);
await tree.write("nested/file.txt", Readable.from(bytes), {
sha256: createHash("sha256").update(bytes).digest("hex"),
bytes: bytes.byteLength,
});
try {
await chmod(output, 0o500);
const failure = await tree.rollback().catch((cause: unknown) => cause);
@@ -90,7 +88,10 @@ describe("safe artifact output tree", () => {
const bytes = Buffer.from("visible before seal");
const digest = createHash("sha256").update(bytes).digest("hex");
await tree.write("file.txt", Readable.from(bytes), digest);
await tree.write("file.txt", Readable.from(bytes), {
sha256: digest,
bytes: bytes.byteLength,
});
expect(await readFile(join(output, "file.txt"), "utf8")).toBe(
"visible before seal",
);
@@ -123,7 +124,10 @@ describe("safe artifact output tree", () => {
const bytes = Buffer.from("windows output");
const digest = createHash("sha256").update(bytes).digest("hex");
await tree.write("nested/file.txt", Readable.from(bytes), digest);
await tree.write("nested/file.txt", Readable.from(bytes), {
sha256: digest,
bytes: bytes.byteLength,
});
await tree.commit();
expect(await readFile(join(output, "nested", "file.txt"), "utf8")).toBe(
"windows output",