fix(artifacts): reuse current classification and region reads
This commit is contained in:
@@ -23,10 +23,11 @@ export const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
);
|
||||
};
|
||||
|
||||
/** Hash every stream byte and retain its bounded classification prefix. */
|
||||
/** Hash every stream byte, optionally feeding the same bytes to a scan aggregate. */
|
||||
export const hashReadable = async (
|
||||
stream: Readable,
|
||||
signal?: AbortSignal,
|
||||
observeChunk?: (chunk: Buffer) => void,
|
||||
): Promise<HashResult> => {
|
||||
const hash = createHash("sha256");
|
||||
const prefixes: Buffer[] = [];
|
||||
@@ -37,6 +38,7 @@ export const hashReadable = async (
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
hash.update(chunk);
|
||||
observeChunk?.(chunk);
|
||||
if (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) {
|
||||
const selected = chunk.subarray(
|
||||
0,
|
||||
|
||||
@@ -187,7 +187,6 @@ class ArtifactClient implements AnalysisClient {
|
||||
const result = await extractArtifact(
|
||||
{
|
||||
inputPath: this.target.sourcePath ?? this.target.path,
|
||||
inputFormat: this.target.format,
|
||||
outputRoot: parsed.output_root,
|
||||
environment: this.environment,
|
||||
integrityPolicy: parsed.integrity_policy,
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
import { readFileHandlePrefix } from "../filesystem/readFileHandlePrefix.js";
|
||||
import { createHash } from "node:crypto";
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
parseInesLayout,
|
||||
@@ -27,6 +26,9 @@ import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
export const INES_LIMITATION =
|
||||
"Classic iNES file regions are observed; mapper execution, CPU bank mappings, CHR-RAM contents, 6502 analysis and emulator behavior are not observed. Trailing bytes are retained without interpretation. NES 2.0 and UNIF are unsupported.";
|
||||
|
||||
/** Actual concatenated bytes observed by the inventory's ordered region scan. */
|
||||
export type InesRegionScan = Omit<InesInventory["reconstruction"], "verified">;
|
||||
|
||||
/** Read positional regions from a descriptor retained by the artifact resource scope. */
|
||||
export class InesArtifactReader implements ArtifactReader {
|
||||
readonly format = "ines" as const;
|
||||
@@ -120,35 +122,25 @@ export class InesArtifactReader implements ArtifactReader {
|
||||
);
|
||||
}
|
||||
|
||||
/** Hash actual ordered region bytes; per-region digests alone cannot prove concatenation. */
|
||||
/** Validate the completed region scan without rereading its payload. */
|
||||
async verifyReconstruction(
|
||||
expectedSha256: string,
|
||||
observed: InesRegionScan,
|
||||
signal?: AbortSignal,
|
||||
): Promise<InesInventory> {
|
||||
abortIfNeeded(signal);
|
||||
const layout = this.#layout;
|
||||
if (layout === undefined || this.#entries.size !== layout.regions.length)
|
||||
throw new ArtifactReaderFailure("integrity", "Incomplete iNES inventory");
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
for (const region of layout.regions) {
|
||||
if (region.offset !== bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Noncontiguous iNES regions",
|
||||
);
|
||||
const entry = this.#entries.get(region.path);
|
||||
if (entry === undefined)
|
||||
throw new ArtifactReaderFailure("integrity", "Missing iNES region");
|
||||
for await (const raw of await this.open(entry, signal)) {
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
hash.update(chunk);
|
||||
bytes += chunk.length;
|
||||
}
|
||||
}
|
||||
abortIfNeeded(signal);
|
||||
const sha256 = hash.digest("hex");
|
||||
if (bytes !== this.source.initial.size || sha256 !== expectedSha256)
|
||||
await this.#verifySource(signal);
|
||||
if (
|
||||
observed.region_order.length !== layout.regions.length ||
|
||||
layout.regions.some(
|
||||
(region, index) => observed.region_order[index] !== region.path,
|
||||
) ||
|
||||
observed.bytes !== this.source.initial.size ||
|
||||
observed.sha256 !== expectedSha256
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"iNES reconstruction differs from the source",
|
||||
@@ -156,9 +148,7 @@ export class InesArtifactReader implements ArtifactReader {
|
||||
return {
|
||||
header: layout.header,
|
||||
reconstruction: {
|
||||
region_order: layout.regions.map(({ path }) => path),
|
||||
bytes,
|
||||
sha256,
|
||||
...observed,
|
||||
verified: true,
|
||||
},
|
||||
};
|
||||
|
||||
@@ -81,7 +81,9 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
if (!captured.ok)
|
||||
throw new ArtifactReaderFailure(
|
||||
captured.error.reason === "cancelled" ? "cancelled" : "unavailable",
|
||||
"lipo could not enumerate universal Mach-O slices",
|
||||
captured.error.detail === null
|
||||
? "lipo could not enumerate universal Mach-O slices"
|
||||
: `lipo could not enumerate universal Mach-O slices: ${captured.error.detail}`,
|
||||
{
|
||||
cause: captured.error,
|
||||
...(captured.error.cleanupFailure === null &&
|
||||
|
||||
@@ -280,9 +280,11 @@ export class SafeOutputTree {
|
||||
}
|
||||
|
||||
/** Sync the owned output tree and prevent further writes through this instance. */
|
||||
async commit(): Promise<void> {
|
||||
async commit(signal?: AbortSignal): Promise<void> {
|
||||
this.#assertWritable();
|
||||
abortIfNeeded(signal);
|
||||
await this.#assertOwnedDirectories();
|
||||
abortIfNeeded(signal);
|
||||
// Windows has no directory fsync; file contents are already synced in write().
|
||||
if (this.platform === "win32") {
|
||||
this.#published = true;
|
||||
@@ -315,6 +317,7 @@ export class SafeOutputTree {
|
||||
);
|
||||
await output.handle.sync();
|
||||
await parent.handle.sync();
|
||||
abortIfNeeded(signal);
|
||||
this.#published = true;
|
||||
} catch (cause: unknown) {
|
||||
sync = { kind: "failed", cause };
|
||||
@@ -336,23 +339,26 @@ export class SafeOutputTree {
|
||||
closeFailure,
|
||||
this.#outputRoot,
|
||||
);
|
||||
if (sync.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Could not durably sync extraction output",
|
||||
{ cause: sync.cause },
|
||||
),
|
||||
cleanup,
|
||||
);
|
||||
if (sync.kind === "failed") {
|
||||
const primary = isCancellationFailure(sync.cause)
|
||||
? sync.cause
|
||||
: new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Could not durably sync extraction output",
|
||||
{ cause: sync.cause },
|
||||
);
|
||||
throw ArtifactReaderFailure.withCleanup(primary, cleanup);
|
||||
}
|
||||
throw ArtifactReaderFailure.withCleanup(closeFailure, cleanup);
|
||||
}
|
||||
if (sync.kind === "failed")
|
||||
if (sync.kind === "failed") {
|
||||
if (isCancellationFailure(sync.cause)) throw sync.cause;
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
"Could not durably sync extraction output",
|
||||
{ cause: sync.cause },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Remove only this operation's unsealed tree and verify absence. */
|
||||
@@ -590,6 +596,11 @@ const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
);
|
||||
};
|
||||
|
||||
const isCancellationFailure = (
|
||||
cause: unknown,
|
||||
): cause is ArtifactReaderFailure =>
|
||||
cause instanceof ArtifactReaderFailure && cause.reason === "cancelled";
|
||||
|
||||
const isAlreadyExists = (cause: unknown): boolean =>
|
||||
cause instanceof Error && "code" in cause && cause.code === "EEXIST";
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { lstat, realpath } from "node:fs/promises";
|
||||
import { realpath } from "node:fs/promises";
|
||||
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
import { digestCanonicalValue } from "../../domain/canonicalDigest.js";
|
||||
@@ -37,10 +37,7 @@ import type {
|
||||
ArtifactResourceOwner,
|
||||
ArtifactResourceScope,
|
||||
} from "../ArtifactResourceScope.js";
|
||||
import {
|
||||
isMachOFormat,
|
||||
type BinaryTarget,
|
||||
} from "../../domain/binaryTargetTypes.js";
|
||||
import { isMachOFormat } from "../../domain/binaryTargetTypes.js";
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
import { scanArtifactInventoryInScope } from "../inventory/ArtifactInventory.js";
|
||||
import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrityPolicy.js";
|
||||
@@ -48,7 +45,6 @@ import type { ArtifactIntegrityPolicyName } from "../../domain/artifactIntegrity
|
||||
/** Local extraction input with the output root chosen by the adapter. */
|
||||
export interface ArtifactExtractionInput {
|
||||
readonly inputPath: string;
|
||||
readonly inputFormat: BinaryTarget["format"];
|
||||
readonly outputRoot: string;
|
||||
readonly environment: Readonly<NodeJS.ProcessEnv>;
|
||||
readonly integrityPolicy: ArtifactIntegrityPolicyName;
|
||||
@@ -69,12 +65,13 @@ const extractArtifactInScope = async (
|
||||
// Cancellation wins over every later refusal, including unsupported formats.
|
||||
abortIfNeeded(signal);
|
||||
const sourcePath = await realpath(input.inputPath);
|
||||
await requireExtractableFormat(sourcePath, input);
|
||||
const snapshot = await scanArtifactInventoryInScope(sourcePath, {
|
||||
resourceScope: input.resourceScope,
|
||||
signal,
|
||||
environment: input.environment,
|
||||
integrity: { mode: input.integrityPolicy },
|
||||
admitRootFormat: (format) =>
|
||||
requireExtractableFormat(format, input.inputPath),
|
||||
});
|
||||
return materializeArtifactInventoryInScope(
|
||||
input,
|
||||
@@ -259,7 +256,7 @@ const materializeSelection = async ({
|
||||
compareUnicodeCodePoints(left.relative_path, right.relative_path),
|
||||
);
|
||||
completed = createExtractionResult(input, inventory, selected, extracted);
|
||||
await output.commit();
|
||||
await output.commit(signal);
|
||||
return completed;
|
||||
} catch (cause: unknown) {
|
||||
const cleanupFailures = await releaseExtractionOwners(
|
||||
@@ -412,18 +409,14 @@ interface ExtractionInventory {
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse a format without an extraction reader before inventory work starts.
|
||||
* The target kind, not the host, is unsupported by extraction. The error keeps
|
||||
* the caller's spelling of the path; I/O uses the canonical path.
|
||||
* Admit the scanner's format, preserving the caller's path in a refusal.
|
||||
*/
|
||||
const requireExtractableFormat = async (
|
||||
path: string,
|
||||
input: Pick<ArtifactExtractionInput, "inputPath" | "inputFormat">,
|
||||
): Promise<boolean> => {
|
||||
const format = input.inputFormat;
|
||||
const directory = (await lstat(path)).isDirectory();
|
||||
const requireExtractableFormat = (
|
||||
format: ArtifactGraphManifest["root_format"],
|
||||
inputPath: string,
|
||||
): void => {
|
||||
if (
|
||||
!directory &&
|
||||
format !== "directory" &&
|
||||
format !== "asar" &&
|
||||
format !== "ines" &&
|
||||
!isMachOFormat(format) &&
|
||||
@@ -431,10 +424,9 @@ const requireExtractableFormat = async (
|
||||
)
|
||||
throw new AnalysisUnsupportedTargetError(
|
||||
"extract_artifact",
|
||||
input.inputPath,
|
||||
inputPath,
|
||||
`Artifact format has no extraction reader: ${format}`,
|
||||
);
|
||||
return directory;
|
||||
};
|
||||
|
||||
const createReader = async (
|
||||
|
||||
@@ -53,7 +53,10 @@ export const classifyAndHashRoot = async (
|
||||
path: string,
|
||||
directory: boolean,
|
||||
expectedMetadata: Stats,
|
||||
options: Pick<ArtifactInventoryOptions, "resourceScope" | "signal">,
|
||||
options: Pick<
|
||||
ArtifactInventoryOptions,
|
||||
"resourceScope" | "signal" | "admitRootFormat"
|
||||
>,
|
||||
): Promise<RootClassification> => {
|
||||
return options.resourceScope.run(async () => {
|
||||
const result = await classifyAndHashRootForInventory(
|
||||
@@ -88,10 +91,16 @@ export const classifyAndHashRootForInventory = async (
|
||||
path: string,
|
||||
directory: boolean,
|
||||
expectedMetadata: Stats,
|
||||
options: Pick<ArtifactInventoryOptions, "resourceScope" | "signal">,
|
||||
options: Pick<
|
||||
ArtifactInventoryOptions,
|
||||
"resourceScope" | "signal" | "admitRootFormat"
|
||||
>,
|
||||
): Promise<RootInventoryClassification> => {
|
||||
if (directory)
|
||||
abortIfNeeded(options.signal);
|
||||
if (directory) {
|
||||
options.admitRootFormat?.("directory");
|
||||
return { format: "directory", digest: null, rootSource: undefined };
|
||||
}
|
||||
const ownedHandle = await openRootArtifact(
|
||||
path,
|
||||
options.resourceScope,
|
||||
@@ -123,6 +132,8 @@ export const classifyAndHashRootForInventory = async (
|
||||
initial,
|
||||
options.signal,
|
||||
);
|
||||
abortIfNeeded(options.signal);
|
||||
options.admitRootFormat?.(format);
|
||||
const digest = await hashStableRootArtifactHandle(
|
||||
path,
|
||||
handle,
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySnapshot.js";
|
||||
import type { InesInventory } from "../../domain/inesContainer.js";
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import { InesArtifactReader } from "../InesArtifactReader.js";
|
||||
import {
|
||||
InesArtifactReader,
|
||||
type InesRegionScan,
|
||||
} from "../InesArtifactReader.js";
|
||||
|
||||
import { lstat } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
@@ -100,6 +104,7 @@ export const scanCanonicalArtifactInventoryInScope = async (
|
||||
occurrences,
|
||||
pendingContradictions,
|
||||
caseCollisionLimitation,
|
||||
inesRegionScan,
|
||||
} = await scanReader(reader, ownedReaders, options.signal, integrity);
|
||||
outcome = {
|
||||
kind: "completed",
|
||||
@@ -116,6 +121,7 @@ export const scanCanonicalArtifactInventoryInScope = async (
|
||||
occurrences,
|
||||
pendingContradictions,
|
||||
caseCollisionLimitation,
|
||||
inesRegionScan,
|
||||
}),
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
@@ -197,6 +203,7 @@ interface SnapshotBuildInput {
|
||||
readonly occurrences: MutableOccurrence[];
|
||||
readonly pendingContradictions: PendingIntegrityContradiction[];
|
||||
readonly caseCollisionLimitation: string | undefined;
|
||||
readonly inesRegionScan: InesRegionScan | undefined;
|
||||
}
|
||||
|
||||
const buildInventorySnapshot = async (
|
||||
@@ -275,11 +282,22 @@ const buildInventorySnapshot = async (
|
||||
integrityContradictions,
|
||||
input.caseCollisionLimitation,
|
||||
);
|
||||
let ines: InesInventory | undefined;
|
||||
if (reader instanceof InesArtifactReader) {
|
||||
if (rootDigest === null || input.inesRegionScan === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Missing iNES reconstruction scan",
|
||||
);
|
||||
ines = await reader.verifyReconstruction(
|
||||
rootDigest.sha256,
|
||||
input.inesRegionScan,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
return {
|
||||
manifest,
|
||||
...(reader instanceof InesArtifactReader && rootDigest !== null
|
||||
? { ines: await reader.verifyReconstruction(rootDigest.sha256, signal) }
|
||||
: {}),
|
||||
...(ines === undefined ? {} : { ines }),
|
||||
nodes: orderedNodes,
|
||||
occurrences: orderedOccurrences,
|
||||
edges: orderedEdges,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { isAbsolute } from "node:path";
|
||||
import { createHash, type Hash } from "node:crypto";
|
||||
|
||||
import {
|
||||
annotateEnglishUnicodeCaseCollisions,
|
||||
@@ -11,6 +12,10 @@ import {
|
||||
type ArtifactReader,
|
||||
} from "../ArtifactReader.js";
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import {
|
||||
InesArtifactReader,
|
||||
type InesRegionScan,
|
||||
} from "../InesArtifactReader.js";
|
||||
import type { ArtifactNode } from "../../domain/artifactGraph.js";
|
||||
import {
|
||||
classifyArtifactContent,
|
||||
@@ -55,6 +60,13 @@ interface ScanContext {
|
||||
readonly registry: ArtifactPathRegistry;
|
||||
readonly expandedContainerIds: Set<string>;
|
||||
readonly ownedReaders: ArtifactReader[];
|
||||
readonly reconstruction:
|
||||
| {
|
||||
readonly hash: Hash;
|
||||
readonly regionOrder: string[];
|
||||
bytes: number;
|
||||
}
|
||||
| undefined;
|
||||
}
|
||||
|
||||
export const scanReader = async (
|
||||
@@ -67,6 +79,7 @@ export const scanReader = async (
|
||||
readonly occurrences: MutableOccurrence[];
|
||||
readonly pendingContradictions: PendingIntegrityContradiction[];
|
||||
readonly caseCollisionLimitation: string | undefined;
|
||||
readonly inesRegionScan: InesRegionScan | undefined;
|
||||
}> => {
|
||||
const { nodes, occurrences, pendingContradictions } = emptyScan();
|
||||
if (reader === undefined)
|
||||
@@ -75,6 +88,7 @@ export const scanReader = async (
|
||||
occurrences,
|
||||
pendingContradictions,
|
||||
caseCollisionLimitation: undefined,
|
||||
inesRegionScan: undefined,
|
||||
};
|
||||
const context: ScanContext = {
|
||||
reader,
|
||||
@@ -87,6 +101,10 @@ export const scanReader = async (
|
||||
registry: new ArtifactPathRegistry(),
|
||||
expandedContainerIds: new Set<string>(),
|
||||
ownedReaders,
|
||||
reconstruction:
|
||||
reader instanceof InesArtifactReader
|
||||
? { hash: createHash("sha256"), regionOrder: [], bytes: 0 }
|
||||
: undefined,
|
||||
};
|
||||
await visitArtifactEntries(context, reader, "");
|
||||
// Archive directories may appear after their children. Resolve containment
|
||||
@@ -101,6 +119,14 @@ export const scanReader = async (
|
||||
occurrences,
|
||||
pendingContradictions,
|
||||
caseCollisionLimitation: annotateEnglishUnicodeCaseCollisions(occurrences),
|
||||
inesRegionScan:
|
||||
context.reconstruction === undefined
|
||||
? undefined
|
||||
: {
|
||||
region_order: context.reconstruction.regionOrder,
|
||||
bytes: context.reconstruction.bytes,
|
||||
sha256: context.reconstruction.hash.digest("hex"),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
@@ -233,10 +259,22 @@ const digestArtifactEntry = async (
|
||||
> => {
|
||||
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
|
||||
return undefined;
|
||||
const reconstruction =
|
||||
currentReader === context.reader ? context.reconstruction : undefined;
|
||||
if (reconstruction !== undefined && entry.byteOffset !== reconstruction.bytes)
|
||||
throw new ArtifactReaderFailure("integrity", "Noncontiguous iNES regions");
|
||||
const digest = await hashReadable(
|
||||
await currentReader.open(entry, context.signal),
|
||||
context.signal,
|
||||
reconstruction === undefined
|
||||
? undefined
|
||||
: (chunk) => {
|
||||
// Hash the same bytes used for each node, in physical region order.
|
||||
reconstruction.hash.update(chunk);
|
||||
reconstruction.bytes += chunk.length;
|
||||
},
|
||||
);
|
||||
reconstruction?.regionOrder.push(entry.path);
|
||||
const mismatched =
|
||||
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
|
||||
if (mismatched && entry.declaredSha256 !== null) {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { ArtifactResourceScope } from "../ArtifactResourceScope.js";
|
||||
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
|
||||
|
||||
/** Resolved integrity behavior admitted to the artifact scanner. */
|
||||
export type ArtifactIntegrityPolicy =
|
||||
@@ -15,4 +16,8 @@ export interface ArtifactInventoryOptions {
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly integrity?: ArtifactIntegrityPolicy | undefined;
|
||||
readonly environment?: Readonly<NodeJS.ProcessEnv> | undefined;
|
||||
/** Admit the current root before payload hashing or native child inventory. */
|
||||
readonly admitRootFormat?: (
|
||||
format: ArtifactOccurrence["artifact_format"],
|
||||
) => void;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user