fix(artifacts): reuse current classification and region reads

This commit is contained in:
64579525
2026-10-13 15:38:40 +08:00
175 changed files with 6746 additions and 1348 deletions
+3 -1
View File
@@ -23,10 +23,11 @@ export const abortIfNeeded = (signal?: AbortSignal): void => {
);
};
/** Hash every stream byte and retain its bounded classification prefix. */
/** Hash every stream byte, optionally feeding the same bytes to a scan aggregate. */
export const hashReadable = async (
stream: Readable,
signal?: AbortSignal,
observeChunk?: (chunk: Buffer) => void,
): Promise<HashResult> => {
const hash = createHash("sha256");
const prefixes: Buffer[] = [];
@@ -37,6 +38,7 @@ export const hashReadable = async (
const chunk = streamChunkToBuffer(raw);
bytes += chunk.length;
hash.update(chunk);
observeChunk?.(chunk);
if (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) {
const selected = chunk.subarray(
0,
-1
View File
@@ -187,7 +187,6 @@ class ArtifactClient implements AnalysisClient {
const result = await extractArtifact(
{
inputPath: this.target.sourcePath ?? this.target.path,
inputFormat: this.target.format,
outputRoot: parsed.output_root,
environment: this.environment,
integrityPolicy: parsed.integrity_policy,
+15 -25
View File
@@ -1,5 +1,4 @@
import { readFileHandlePrefix } from "../filesystem/readFileHandlePrefix.js";
import { createHash } from "node:crypto";
import { Readable } from "node:stream";
import {
parseInesLayout,
@@ -27,6 +26,9 @@ import { streamChunkToBuffer } from "./StreamBytes.js";
export const INES_LIMITATION =
"Classic iNES file regions are observed; mapper execution, CPU bank mappings, CHR-RAM contents, 6502 analysis and emulator behavior are not observed. Trailing bytes are retained without interpretation. NES 2.0 and UNIF are unsupported.";
/** Actual concatenated bytes observed by the inventory's ordered region scan. */
export type InesRegionScan = Omit<InesInventory["reconstruction"], "verified">;
/** Read positional regions from a descriptor retained by the artifact resource scope. */
export class InesArtifactReader implements ArtifactReader {
readonly format = "ines" as const;
@@ -120,35 +122,25 @@ export class InesArtifactReader implements ArtifactReader {
);
}
/** Hash actual ordered region bytes; per-region digests alone cannot prove concatenation. */
/** Validate the completed region scan without rereading its payload. */
async verifyReconstruction(
expectedSha256: string,
observed: InesRegionScan,
signal?: AbortSignal,
): Promise<InesInventory> {
abortIfNeeded(signal);
const layout = this.#layout;
if (layout === undefined || this.#entries.size !== layout.regions.length)
throw new ArtifactReaderFailure("integrity", "Incomplete iNES inventory");
const hash = createHash("sha256");
let bytes = 0;
for (const region of layout.regions) {
if (region.offset !== bytes)
throw new ArtifactReaderFailure(
"integrity",
"Noncontiguous iNES regions",
);
const entry = this.#entries.get(region.path);
if (entry === undefined)
throw new ArtifactReaderFailure("integrity", "Missing iNES region");
for await (const raw of await this.open(entry, signal)) {
const chunk = streamChunkToBuffer(raw);
hash.update(chunk);
bytes += chunk.length;
}
}
abortIfNeeded(signal);
const sha256 = hash.digest("hex");
if (bytes !== this.source.initial.size || sha256 !== expectedSha256)
await this.#verifySource(signal);
if (
observed.region_order.length !== layout.regions.length ||
layout.regions.some(
(region, index) => observed.region_order[index] !== region.path,
) ||
observed.bytes !== this.source.initial.size ||
observed.sha256 !== expectedSha256
)
throw new ArtifactReaderFailure(
"integrity",
"iNES reconstruction differs from the source",
@@ -156,9 +148,7 @@ export class InesArtifactReader implements ArtifactReader {
return {
header: layout.header,
reconstruction: {
region_order: layout.regions.map(({ path }) => path),
bytes,
sha256,
...observed,
verified: true,
},
};
+3 -1
View File
@@ -81,7 +81,9 @@ export class MachOSliceArtifactReader implements ArtifactReader {
if (!captured.ok)
throw new ArtifactReaderFailure(
captured.error.reason === "cancelled" ? "cancelled" : "unavailable",
"lipo could not enumerate universal Mach-O slices",
captured.error.detail === null
? "lipo could not enumerate universal Mach-O slices"
: `lipo could not enumerate universal Mach-O slices: ${captured.error.detail}`,
{
cause: captured.error,
...(captured.error.cleanupFailure === null &&
+22 -11
View File
@@ -280,9 +280,11 @@ export class SafeOutputTree {
}
/** Sync the owned output tree and prevent further writes through this instance. */
async commit(): Promise<void> {
async commit(signal?: AbortSignal): Promise<void> {
this.#assertWritable();
abortIfNeeded(signal);
await this.#assertOwnedDirectories();
abortIfNeeded(signal);
// Windows has no directory fsync; file contents are already synced in write().
if (this.platform === "win32") {
this.#published = true;
@@ -315,6 +317,7 @@ export class SafeOutputTree {
);
await output.handle.sync();
await parent.handle.sync();
abortIfNeeded(signal);
this.#published = true;
} catch (cause: unknown) {
sync = { kind: "failed", cause };
@@ -336,23 +339,26 @@ export class SafeOutputTree {
closeFailure,
this.#outputRoot,
);
if (sync.kind === "failed")
throw ArtifactReaderFailure.withCleanup(
new ArtifactReaderFailure(
"path",
"Could not durably sync extraction output",
{ cause: sync.cause },
),
cleanup,
);
if (sync.kind === "failed") {
const primary = isCancellationFailure(sync.cause)
? sync.cause
: new ArtifactReaderFailure(
"path",
"Could not durably sync extraction output",
{ cause: sync.cause },
);
throw ArtifactReaderFailure.withCleanup(primary, cleanup);
}
throw ArtifactReaderFailure.withCleanup(closeFailure, cleanup);
}
if (sync.kind === "failed")
if (sync.kind === "failed") {
if (isCancellationFailure(sync.cause)) throw sync.cause;
throw new ArtifactReaderFailure(
"path",
"Could not durably sync extraction output",
{ cause: sync.cause },
);
}
}
/** Remove only this operation's unsealed tree and verify absence. */
@@ -590,6 +596,11 @@ const abortIfNeeded = (signal?: AbortSignal): void => {
);
};
const isCancellationFailure = (
cause: unknown,
): cause is ArtifactReaderFailure =>
cause instanceof ArtifactReaderFailure && cause.reason === "cancelled";
const isAlreadyExists = (cause: unknown): boolean =>
cause instanceof Error && "code" in cause && cause.code === "EEXIST";
+12 -20
View File
@@ -1,4 +1,4 @@
import { lstat, realpath } from "node:fs/promises";
import { realpath } from "node:fs/promises";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
import { digestCanonicalValue } from "../../domain/canonicalDigest.js";
@@ -37,10 +37,7 @@ import type {
ArtifactResourceOwner,
ArtifactResourceScope,
} from "../ArtifactResourceScope.js";
import {
isMachOFormat,
type BinaryTarget,
} from "../../domain/binaryTargetTypes.js";
import { isMachOFormat } from "../../domain/binaryTargetTypes.js";
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import { scanArtifactInventoryInScope } from "../inventory/ArtifactInventory.js";
import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrityPolicy.js";
@@ -48,7 +45,6 @@ import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrity
/** Local extraction input with the output root chosen by the adapter. */
export interface ArtifactExtractionInput {
readonly inputPath: string;
readonly inputFormat: BinaryTarget["format"];
readonly outputRoot: string;
readonly environment: Readonly<NodeJS.ProcessEnv>;
readonly integrityPolicy: ArtifactIntegrityPolicyName;
@@ -69,12 +65,13 @@ const extractArtifactInScope = async (
// Cancellation wins over every later refusal, including unsupported formats.
abortIfNeeded(signal);
const sourcePath = await realpath(input.inputPath);
await requireExtractableFormat(sourcePath, input);
const snapshot = await scanArtifactInventoryInScope(sourcePath, {
resourceScope: input.resourceScope,
signal,
environment: input.environment,
integrity: { mode: input.integrityPolicy },
admitRootFormat: (format) =>
requireExtractableFormat(format, input.inputPath),
});
return materializeArtifactInventoryInScope(
input,
@@ -259,7 +256,7 @@ const materializeSelection = async ({
compareUnicodeCodePoints(left.relative_path, right.relative_path),
);
completed = createExtractionResult(input, inventory, selected, extracted);
await output.commit();
await output.commit(signal);
return completed;
} catch (cause: unknown) {
const cleanupFailures = await releaseExtractionOwners(
@@ -412,18 +409,14 @@ interface ExtractionInventory {
}
/**
* Refuse a format without an extraction reader before inventory work starts.
* The target kind, not the host, is unsupported by extraction. The error keeps
* the caller's spelling of the path; I/O uses the canonical path.
* Admit the scanner's format, preserving the caller's path in a refusal.
*/
const requireExtractableFormat = async (
path: string,
input: Pick<ArtifactExtractionInput, "inputPath" | "inputFormat">,
): Promise<boolean> => {
const format = input.inputFormat;
const directory = (await lstat(path)).isDirectory();
const requireExtractableFormat = (
format: ArtifactGraphManifest["root_format"],
inputPath: string,
): void => {
if (
!directory &&
format !== "directory" &&
format !== "asar" &&
format !== "ines" &&
!isMachOFormat(format) &&
@@ -431,10 +424,9 @@ const requireExtractableFormat = async (
)
throw new AnalysisUnsupportedTargetError(
"extract_artifact",
input.inputPath,
inputPath,
`Artifact format has no extraction reader: ${format}`,
);
return directory;
};
const createReader = async (
+14 -3
View File
@@ -53,7 +53,10 @@ export const classifyAndHashRoot = async (
path: string,
directory: boolean,
expectedMetadata: Stats,
options: Pick<ArtifactInventoryOptions, "resourceScope" | "signal">,
options: Pick<
ArtifactInventoryOptions,
"resourceScope" | "signal" | "admitRootFormat"
>,
): Promise<RootClassification> => {
return options.resourceScope.run(async () => {
const result = await classifyAndHashRootForInventory(
@@ -88,10 +91,16 @@ export const classifyAndHashRootForInventory = async (
path: string,
directory: boolean,
expectedMetadata: Stats,
options: Pick<ArtifactInventoryOptions, "resourceScope" | "signal">,
options: Pick<
ArtifactInventoryOptions,
"resourceScope" | "signal" | "admitRootFormat"
>,
): Promise<RootInventoryClassification> => {
if (directory)
abortIfNeeded(options.signal);
if (directory) {
options.admitRootFormat?.("directory");
return { format: "directory", digest: null, rootSource: undefined };
}
const ownedHandle = await openRootArtifact(
path,
options.resourceScope,
@@ -123,6 +132,8 @@ export const classifyAndHashRootForInventory = async (
initial,
options.signal,
);
abortIfNeeded(options.signal);
options.admitRootFormat?.(format);
const digest = await hashStableRootArtifactHandle(
path,
handle,
+22 -4
View File
@@ -1,8 +1,12 @@
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
import type { InesInventory } from "../../domain/inesContainer.js";
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
import { AsarArtifactReader } from "../AsarArtifactReader.js";
import { InesArtifactReader } from "../InesArtifactReader.js";
import {
InesArtifactReader,
type InesRegionScan,
} from "../InesArtifactReader.js";
import { lstat } from "node:fs/promises";
import type { Stats } from "node:fs";
@@ -100,6 +104,7 @@ export const scanCanonicalArtifactInventoryInScope = async (
occurrences,
pendingContradictions,
caseCollisionLimitation,
inesRegionScan,
} = await scanReader(reader, ownedReaders, options.signal, integrity);
outcome = {
kind: "completed",
@@ -116,6 +121,7 @@ export const scanCanonicalArtifactInventoryInScope = async (
occurrences,
pendingContradictions,
caseCollisionLimitation,
inesRegionScan,
}),
};
} catch (cause: unknown) {
@@ -197,6 +203,7 @@ interface SnapshotBuildInput {
readonly occurrences: MutableOccurrence[];
readonly pendingContradictions: PendingIntegrityContradiction[];
readonly caseCollisionLimitation: string | undefined;
readonly inesRegionScan: InesRegionScan | undefined;
}
const buildInventorySnapshot = async (
@@ -275,11 +282,22 @@ const buildInventorySnapshot = async (
integrityContradictions,
input.caseCollisionLimitation,
);
let ines: InesInventory | undefined;
if (reader instanceof InesArtifactReader) {
if (rootDigest === null || input.inesRegionScan === undefined)
throw new ArtifactReaderFailure(
"integrity",
"Missing iNES reconstruction scan",
);
ines = await reader.verifyReconstruction(
rootDigest.sha256,
input.inesRegionScan,
signal,
);
}
return {
manifest,
...(reader instanceof InesArtifactReader && rootDigest !== null
? { ines: await reader.verifyReconstruction(rootDigest.sha256, signal) }
: {}),
...(ines === undefined ? {} : { ines }),
nodes: orderedNodes,
occurrences: orderedOccurrences,
edges: orderedEdges,
+38
View File
@@ -1,4 +1,5 @@
import { isAbsolute } from "node:path";
import { createHash, type Hash } from "node:crypto";
import {
annotateEnglishUnicodeCaseCollisions,
@@ -11,6 +12,10 @@ import {
type ArtifactReader,
} from "../ArtifactReader.js";
import { AsarArtifactReader } from "../AsarArtifactReader.js";
import {
InesArtifactReader,
type InesRegionScan,
} from "../InesArtifactReader.js";
import type { ArtifactNode } from "../../domain/artifactGraph.js";
import {
classifyArtifactContent,
@@ -55,6 +60,13 @@ interface ScanContext {
readonly registry: ArtifactPathRegistry;
readonly expandedContainerIds: Set<string>;
readonly ownedReaders: ArtifactReader[];
readonly reconstruction:
| {
readonly hash: Hash;
readonly regionOrder: string[];
bytes: number;
}
| undefined;
}
export const scanReader = async (
@@ -67,6 +79,7 @@ export const scanReader = async (
readonly occurrences: MutableOccurrence[];
readonly pendingContradictions: PendingIntegrityContradiction[];
readonly caseCollisionLimitation: string | undefined;
readonly inesRegionScan: InesRegionScan | undefined;
}> => {
const { nodes, occurrences, pendingContradictions } = emptyScan();
if (reader === undefined)
@@ -75,6 +88,7 @@ export const scanReader = async (
occurrences,
pendingContradictions,
caseCollisionLimitation: undefined,
inesRegionScan: undefined,
};
const context: ScanContext = {
reader,
@@ -87,6 +101,10 @@ export const scanReader = async (
registry: new ArtifactPathRegistry(),
expandedContainerIds: new Set<string>(),
ownedReaders,
reconstruction:
reader instanceof InesArtifactReader
? { hash: createHash("sha256"), regionOrder: [], bytes: 0 }
: undefined,
};
await visitArtifactEntries(context, reader, "");
// Archive directories may appear after their children. Resolve containment
@@ -101,6 +119,14 @@ export const scanReader = async (
occurrences,
pendingContradictions,
caseCollisionLimitation: annotateEnglishUnicodeCaseCollisions(occurrences),
inesRegionScan:
context.reconstruction === undefined
? undefined
: {
region_order: context.reconstruction.regionOrder,
bytes: context.reconstruction.bytes,
sha256: context.reconstruction.hash.digest("hex"),
},
};
};
@@ -233,10 +259,22 @@ const digestArtifactEntry = async (
> => {
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
return undefined;
const reconstruction =
currentReader === context.reader ? context.reconstruction : undefined;
if (reconstruction !== undefined && entry.byteOffset !== reconstruction.bytes)
throw new ArtifactReaderFailure("integrity", "Noncontiguous iNES regions");
const digest = await hashReadable(
await currentReader.open(entry, context.signal),
context.signal,
reconstruction === undefined
? undefined
: (chunk) => {
// Hash the same bytes used for each node, in physical region order.
reconstruction.hash.update(chunk);
reconstruction.bytes += chunk.length;
},
);
reconstruction?.regionOrder.push(entry.path);
const mismatched =
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
if (mismatched && entry.declaredSha256 !== null) {
+5
View File
@@ -1,4 +1,5 @@
import type { ArtifactResourceScope } from "../ArtifactResourceScope.js";
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
/** Resolved integrity behavior admitted to the artifact scanner. */
export type ArtifactIntegrityPolicy =
@@ -15,4 +16,8 @@ export interface ArtifactInventoryOptions {
readonly signal?: AbortSignal | undefined;
readonly integrity?: ArtifactIntegrityPolicy | undefined;
readonly environment?: Readonly<NodeJS.ProcessEnv> | undefined;
/** Admit the current root before payload hashing or native child inventory. */
readonly admitRootFormat?: (
format: ArtifactOccurrence["artifact_format"],
) => void;
}