feat(artifacts): inspect classic iNES container regions
This commit is contained in:
@@ -32,7 +32,7 @@ import {
|
||||
} from "../domain/analysisErrorCore.js";
|
||||
import { ArtifactOperationError } from "../domain/artifactOperationError.js";
|
||||
import type { AnalysisError } from "../domain/analysisErrorBase.js";
|
||||
import type { JsonValue } from "../domain/jsonValue.js";
|
||||
import { jsonValueSchema, type JsonValue } from "../domain/jsonValue.js";
|
||||
import { interfaceBuilderLimitsSchema } from "../domain/apple/interfaceBuilderGraph.js";
|
||||
import { err, ok, type Result } from "../domain/result.js";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
@@ -349,7 +349,7 @@ class ArtifactClient implements AnalysisClient {
|
||||
const inventoryEvidence = createEvidence(subject, ARTIFACT_GRAPH_PROVIDER, {
|
||||
operation: "inventory_artifact",
|
||||
parameters: inventoryParameters,
|
||||
result: inventory,
|
||||
result: jsonValueSchema.parse(inventory),
|
||||
rawResult: null,
|
||||
limitations: inventory.limitations,
|
||||
// The nested inventory already names every occurrence's logical path.
|
||||
|
||||
@@ -92,7 +92,7 @@ export const sameArtifactEntry = (
|
||||
|
||||
/** Read-only adapter over one directory, archive, or virtual container. */
|
||||
export interface ArtifactReader {
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "file";
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "ines" | "file";
|
||||
entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry>;
|
||||
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable>;
|
||||
provenance(): readonly ArtifactCommand[];
|
||||
|
||||
@@ -0,0 +1,192 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { lstat } from "node:fs/promises";
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
parseInesLayout,
|
||||
type InesInventory,
|
||||
type InesLayout,
|
||||
} from "../domain/inesContainer.js";
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
sameRegularFileState,
|
||||
type StableRegularFileDescriptor,
|
||||
} from "../filesystem/RegularFile.js";
|
||||
import { readFileHandleChunks } from "../filesystem/readFileHandleChunks.js";
|
||||
import type { OwnedFileHandle } from "../filesystem/OwnedFileHandle.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
copyArtifactEntry,
|
||||
sameArtifactEntry,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
import { abortIfNeeded } from "./ArtifactHash.js";
|
||||
import { streamChunkToBuffer } from "./StreamBytes.js";
|
||||
|
||||
export const INES_LIMITATION =
|
||||
"Classic iNES file regions are observed; mapper execution, CPU bank mappings, CHR-RAM contents, 6502 analysis and emulator behavior are not observed. Trailing bytes are retained without interpretation. NES 2.0 and UNIF are unsupported.";
|
||||
|
||||
/** Read positional regions from a descriptor retained by the artifact resource scope. */
|
||||
export class InesArtifactReader implements ArtifactReader {
|
||||
readonly format = "ines" as const;
|
||||
readonly #entries = new Map<string, ArtifactEntry>();
|
||||
#layout: InesLayout | undefined;
|
||||
#closed = false;
|
||||
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
private readonly source: StableRegularFileDescriptor,
|
||||
private readonly ownedSource?: OwnedFileHandle,
|
||||
) {}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
abortIfNeeded(signal);
|
||||
await this.#verifySource();
|
||||
const prefix = Buffer.alloc(16);
|
||||
const { bytesRead } = await this.source.handle.read(prefix, 0, 16, 0);
|
||||
abortIfNeeded(signal);
|
||||
const parsed = parseInesLayout(
|
||||
prefix.subarray(0, bytesRead),
|
||||
this.source.initial.size,
|
||||
);
|
||||
if (!parsed.ok) throw new ArtifactReaderFailure("format", parsed.error);
|
||||
this.#layout = parsed.value;
|
||||
await this.#verifySource();
|
||||
this.#entries.clear();
|
||||
for (const region of parsed.value.regions) {
|
||||
abortIfNeeded(signal);
|
||||
const entry: ArtifactEntry = {
|
||||
path: region.path,
|
||||
kind: "file",
|
||||
declaredSize: region.length,
|
||||
compressedSize: null,
|
||||
executable: false,
|
||||
encrypted: false,
|
||||
byteOffset: region.offset,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: [],
|
||||
adapterKey: region.path,
|
||||
};
|
||||
this.#entries.set(entry.adapterKey, copyArtifactEntry(entry));
|
||||
yield entry;
|
||||
}
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
abortIfNeeded(signal);
|
||||
const produced = this.#entries.get(entry.adapterKey);
|
||||
if (produced === undefined || !sameArtifactEntry(entry, produced))
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"iNES region does not match this reader's inventory",
|
||||
);
|
||||
await this.#verifySource();
|
||||
const offset = produced.byteOffset;
|
||||
const size = produced.declaredSize;
|
||||
if (offset === null || size === null || size <= 0)
|
||||
throw new ArtifactReaderFailure("integrity", "Invalid iNES region range");
|
||||
const stream = readFileHandleChunks(this.source.handle, {
|
||||
start: offset,
|
||||
end: offset + size - 1,
|
||||
signal,
|
||||
});
|
||||
const verify = () => this.#verifySource();
|
||||
return Readable.from(
|
||||
(async function* () {
|
||||
let bytes = 0;
|
||||
try {
|
||||
for await (const raw of stream) {
|
||||
abortIfNeeded(signal);
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
yield chunk;
|
||||
}
|
||||
if (bytes !== size)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Truncated iNES region",
|
||||
);
|
||||
await verify();
|
||||
} catch (cause: unknown) {
|
||||
abortIfNeeded(signal);
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
throw new ArtifactReaderFailure("io", "Could not read iNES region", {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
})(),
|
||||
{ objectMode: false },
|
||||
);
|
||||
}
|
||||
|
||||
/** Hash actual ordered region bytes; per-region digests alone cannot prove concatenation. */
|
||||
async verifyReconstruction(
|
||||
expectedSha256: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<InesInventory> {
|
||||
const layout = this.#layout;
|
||||
if (layout === undefined || this.#entries.size !== layout.regions.length)
|
||||
throw new ArtifactReaderFailure("integrity", "Incomplete iNES inventory");
|
||||
const hash = createHash("sha256");
|
||||
let bytes = 0;
|
||||
for (const region of layout.regions) {
|
||||
if (region.offset !== bytes)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"Noncontiguous iNES regions",
|
||||
);
|
||||
const entry = this.#entries.get(region.path);
|
||||
if (entry === undefined)
|
||||
throw new ArtifactReaderFailure("integrity", "Missing iNES region");
|
||||
for await (const raw of await this.open(entry, signal)) {
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
hash.update(chunk);
|
||||
bytes += chunk.length;
|
||||
}
|
||||
}
|
||||
const sha256 = hash.digest("hex");
|
||||
if (bytes !== this.source.initial.size || sha256 !== expectedSha256)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"iNES reconstruction differs from the source",
|
||||
);
|
||||
return {
|
||||
header: layout.header,
|
||||
reconstruction: {
|
||||
region_order: layout.regions.map(({ path }) => path),
|
||||
bytes,
|
||||
sha256,
|
||||
verified: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
provenance(): readonly ArtifactCommand[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
close(): Promise<void> {
|
||||
// Inventory borrows its admitting scope's descriptor. Extraction owns its
|
||||
// separately opened descriptor and retains this reader if close fails.
|
||||
this.#closed = true;
|
||||
return this.ownedSource?.close() ?? Promise.resolve();
|
||||
}
|
||||
|
||||
async #verifySource(): Promise<void> {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure("unavailable", "iNES reader is closed");
|
||||
const [opened, selected] = await Promise.all([
|
||||
this.source.handle.stat(),
|
||||
lstat(this.path),
|
||||
]);
|
||||
if (
|
||||
!sameRegularFileState(this.source.initial, opened) ||
|
||||
!sameRegularFileState(this.source.initial, selected)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"iNES source changed during reading",
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,8 @@ import { lstat, realpath } from "node:fs/promises";
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
import { digestCanonicalValue } from "../../domain/canonicalDigest.js";
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import { InesArtifactReader } from "../InesArtifactReader.js";
|
||||
import { openRootArtifact } from "../inventory/hashStableRootArtifact.js";
|
||||
import {
|
||||
ArtifactPathRegistry,
|
||||
normalizeArtifactPath,
|
||||
@@ -421,6 +423,7 @@ const requireExtractableFormat = async (
|
||||
if (
|
||||
!directory &&
|
||||
format !== "asar" &&
|
||||
format !== "ines" &&
|
||||
!isMachOFormat(format) &&
|
||||
!isZipFormat(format)
|
||||
)
|
||||
@@ -436,13 +439,38 @@ const createReader = async (
|
||||
path: string,
|
||||
input: Pick<
|
||||
ArtifactExtractionInput,
|
||||
"inputPath" | "inputFormat" | "environment"
|
||||
"inputPath" | "inputFormat" | "environment" | "resourceScope"
|
||||
>,
|
||||
): Promise<ArtifactReader> => {
|
||||
const format = input.inputFormat;
|
||||
if (await requireExtractableFormat(path, input))
|
||||
return new DirectoryArtifactReader(path);
|
||||
if (format === "asar") return new AsarArtifactReader(path);
|
||||
if (format === "ines") {
|
||||
const owner = await openRootArtifact(path, input.resourceScope);
|
||||
try {
|
||||
return new InesArtifactReader(
|
||||
path,
|
||||
{
|
||||
handle: owner.handle,
|
||||
initial: await owner.handle.stat(),
|
||||
},
|
||||
owner,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
const attempt = await input.resourceScope.release({
|
||||
kind: "file-handle",
|
||||
handle: owner,
|
||||
resource: `iNES source ${path}`,
|
||||
});
|
||||
if (attempt.kind === "failed")
|
||||
throw ArtifactReaderFailure.withCleanup(
|
||||
cause,
|
||||
ArtifactReaderFailure.cleanupObservation(attempt.cause, path),
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
if (isZipFormat(format)) return new ZipArtifactReader(path, format);
|
||||
return new MachOSliceArtifactReader(path, input.environment);
|
||||
};
|
||||
|
||||
@@ -19,6 +19,10 @@ import {
|
||||
type ZipPackageFormat,
|
||||
} from "../../domain/zipPackageFormat.js";
|
||||
import { mzWindowsHeaderOffset, parseDosMzHeader } from "../../domain/dosMz.js";
|
||||
import {
|
||||
nesContainerFormat,
|
||||
parseInesLayout,
|
||||
} from "../../domain/inesContainer.js";
|
||||
|
||||
/** Mutable internal occurrence used until root-bound IDs are known. */
|
||||
export interface MutableOccurrence {
|
||||
@@ -219,7 +223,7 @@ export const rootOccurrenceFor = (
|
||||
artifact_kind:
|
||||
metadata.format === "directory" ||
|
||||
isZipPackageFormat(metadata.format) ||
|
||||
["asar", "dmg", "pkg"].includes(metadata.format)
|
||||
["asar", "ines", "dmg", "pkg"].includes(metadata.format)
|
||||
? "container"
|
||||
: artifactRoleForFormat(metadata.path, metadata.format).kind,
|
||||
artifact_format: metadata.format,
|
||||
@@ -419,6 +423,7 @@ const artifactRoleForFormat = (
|
||||
format: zipPackageFormatForPath(path) ?? format,
|
||||
};
|
||||
if (format === "plist") return { kind: "plist", format };
|
||||
if (format === "ines") return { kind: "container", format };
|
||||
return {
|
||||
kind: ["native-addon", "dynamic-library"].includes(byPath.kind)
|
||||
? byPath.kind
|
||||
@@ -432,6 +437,13 @@ export const classifyArtifactBytes = (
|
||||
prefix: Buffer,
|
||||
fileSize = prefix.length,
|
||||
): ArtifactNode["format"] => {
|
||||
// A header-only region has the signature but not the declared cartridge
|
||||
// extent. Keep it as a file instead of inventing another container.
|
||||
if (
|
||||
nesContainerFormat(prefix) === "ines" &&
|
||||
parseInesLayout(prefix, fileSize).ok
|
||||
)
|
||||
return "ines";
|
||||
if (hasZipSignature(prefix)) return "zip";
|
||||
if (prefix.subarray(0, 8).equals(Buffer.from("bplist00"))) return "plist";
|
||||
if (prefix.length >= 4) {
|
||||
|
||||
@@ -5,6 +5,10 @@ import type { StableRegularFileDescriptor } from "../../filesystem/RegularFile.j
|
||||
import type { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
|
||||
|
||||
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
|
||||
import {
|
||||
nesContainerFormat,
|
||||
parseInesLayout,
|
||||
} from "../../domain/inesContainer.js";
|
||||
import { ARTIFACT_CLASSIFICATION_PREFIX_BYTES } from "../ArtifactHash.js";
|
||||
import type { ArtifactOccurrence } from "../../domain/artifactGraph.js";
|
||||
import {
|
||||
@@ -71,7 +75,7 @@ export const classifyAndHashRoot = async (
|
||||
});
|
||||
};
|
||||
|
||||
/** Classify a root and retain its admitted ZIP descriptor for child inventory. */
|
||||
/** Classify a root and retain its admitted container descriptor for child inventory. */
|
||||
export const classifyAndHashRootForInventory = async (
|
||||
path: string,
|
||||
directory: boolean,
|
||||
@@ -112,7 +116,9 @@ export const classifyAndHashRootForInventory = async (
|
||||
options.signal,
|
||||
);
|
||||
const retainSource =
|
||||
isZipPackageFormat(format) || format === "mach-o-universal";
|
||||
isZipPackageFormat(format) ||
|
||||
format === "mach-o-universal" ||
|
||||
format === "ines";
|
||||
outcome = {
|
||||
kind: "completed",
|
||||
value: {
|
||||
@@ -148,6 +154,11 @@ const classifyRootFormat = async (
|
||||
const magic = Buffer.alloc(ARTIFACT_CLASSIFICATION_PREFIX_BYTES);
|
||||
const observed = await handle.read(magic, 0, magic.length, 0);
|
||||
const prefix = magic.subarray(0, observed.bytesRead);
|
||||
if (nesContainerFormat(prefix) !== undefined) {
|
||||
const parsed = parseInesLayout(prefix, (await handle.stat()).size);
|
||||
if (!parsed.ok) throw new ArtifactReaderFailure("format", parsed.error);
|
||||
return "ines";
|
||||
}
|
||||
const lower = path.toLowerCase();
|
||||
if (hasZipSignature(prefix)) return zipRootPackageFormat(lower);
|
||||
if (
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import { InesArtifactReader, INES_LIMITATION } from "../InesArtifactReader.js";
|
||||
import { ArtifactReaderFailure } from "../ArtifactReader.js";
|
||||
import type { ArtifactReader } from "../ArtifactReader.js";
|
||||
import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
|
||||
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
|
||||
@@ -20,6 +22,13 @@ export const createReader = (
|
||||
if (isZipPackageFormat(format))
|
||||
return new ZipArtifactReader(path, format, undefined, rootSource);
|
||||
switch (format) {
|
||||
case "ines":
|
||||
if (rootSource === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"iNES inventory requires its admitted source descriptor",
|
||||
);
|
||||
return new InesArtifactReader(path, rootSource);
|
||||
case "directory":
|
||||
return new DirectoryArtifactReader(path);
|
||||
case "asar":
|
||||
@@ -40,6 +49,7 @@ export const inventoryLimitations = (
|
||||
format: ArtifactOccurrence["artifact_format"],
|
||||
reader: ArtifactReader | undefined,
|
||||
): string[] => {
|
||||
if (format === "ines") return [INES_LIMITATION];
|
||||
if (reader !== undefined)
|
||||
return isZipPackageFormat(reader.format)
|
||||
? [ZIP_NON_ENTRY_TAIL_LIMITATION]
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { ArtifactInventorySnapshot } from "../../domain/artifactInventorySn
|
||||
import { compareUnicodeCodePoints } from "../../domain/unicodeCodePointOrder.js";
|
||||
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import { InesArtifactReader } from "../InesArtifactReader.js";
|
||||
|
||||
import { lstat } from "node:fs/promises";
|
||||
import type { Stats } from "node:fs";
|
||||
@@ -270,6 +271,9 @@ const buildInventorySnapshot = async (
|
||||
});
|
||||
return {
|
||||
manifest,
|
||||
...(reader instanceof InesArtifactReader && rootDigest !== null
|
||||
? { ines: await reader.verifyReconstruction(rootDigest.sha256, signal) }
|
||||
: {}),
|
||||
nodes: orderedNodes,
|
||||
occurrences: orderedOccurrences,
|
||||
edges: orderedEdges,
|
||||
|
||||
Reference in New Issue
Block a user