chore: merge captured main into iNES container support

This commit is contained in:
morluto
2026-10-13 21:45:12 +08:00
252 changed files with 12483 additions and 1405 deletions
+10 -2
View File
@@ -40,6 +40,7 @@ import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
import { artifactCapabilities } from "./ArtifactProviderMetadata.js";
import { ARTIFACT_GRAPH_PROVIDER } from "../application/InvestigationProviders.js";
import { createEvidence } from "../domain/evidence.js";
import { freezeJsonSnapshot } from "../domain/immutableJson.js";
import { createArtifactInspection } from "../domain/artifactInspection.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
@@ -340,7 +341,11 @@ class ArtifactClient implements AnalysisClient {
total: 1,
message: "inventory substep started",
});
const inventory = await this.inventory(inventoryParameters, options);
// The freshly parsed inventory is exclusively owned here. Sealing it lets
// the nested Evidence share it instead of cloning and re-digesting it.
const inventory = freezeJsonSnapshot(
await this.inventory(inventoryParameters, options),
);
const subject = subjectFor(
this.target.sourcePath ?? this.target.path,
inventory.manifest,
@@ -354,7 +359,10 @@ class ArtifactClient implements AnalysisClient {
// The nested inventory already names every occurrence's logical path.
locations: [],
});
const result = createArtifactInspection(inventoryEvidence);
// The returned execution and its Evidence then reuse this owned report.
const result = freezeJsonSnapshot(
createArtifactInspection(inventoryEvidence),
);
await options?.progress?.report({
phase: "inspect_artifact.inventory",
completed: 1,
+2 -2
View File
@@ -8,7 +8,7 @@ import {
type NativeCommandRunner,
} from "../native/CommandRunner.js";
import {
parseLipoArchitectures,
parseLipoOutput,
type LipoArchitecture,
} from "../native/parsers/lipo.js";
import type { MachoSlice } from "../domain/apple/dylibResolution.js";
@@ -114,7 +114,7 @@ export class MachOSliceArtifactReader implements ArtifactReader {
};
await this.#verifySource(source);
const fileSize = source.initial.size;
const architectures = parseLipoArchitectures(captured.value.stdout);
const { architectures } = parseLipoOutput(captured.value.stdout, this.path);
const structural = await readStructuralSlices(source.handle, fileSize);
await this.#verifySource(source);
if (structural.status === "malformed")
+79 -6
View File
@@ -2,7 +2,29 @@ import type { FileHandle } from "node:fs/promises";
import { PassThrough, type Readable } from "node:stream";
import { once } from "node:events";
import { Reader, ZipReader, type Entry, type FileEntry } from "@zip.js/zip.js";
import {
ERR_AMBIGUOUS_ARCHIVE,
ERR_BAD_FORMAT,
ERR_CENTRAL_DIRECTORY_NOT_FOUND,
ERR_ENCRYPTED_CENTRAL_DIRECTORY,
ERR_ENTRY_DATA_OUT_OF_BOUNDS,
ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND,
ERR_EOCDR_NOT_FOUND,
ERR_EXTRAFIELD_ZIP64_NOT_FOUND,
ERR_INVALID_COMPRESSED_DATA,
ERR_INVALID_CRC32,
ERR_INVALID_UNCOMPRESSED_SIZE,
ERR_LOCAL_FILE_HEADER_NOT_FOUND,
ERR_OVERLAPPING_ENTRY,
ERR_SPLIT_ZIP_FILE,
ERR_UNSUPPORTED_COMPRESSION,
ERR_UNSUPPORTED_ENCRYPTION,
ERR_UNSUPPORTED_UINT64,
Reader,
ZipReader,
type Entry,
type FileEntry,
} from "@zip.js/zip.js";
import {
ArtifactReaderFailure,
@@ -159,9 +181,9 @@ export class ZipArtifactReader implements ArtifactReader {
let index = 0;
// A traversal owns its accounting, including overlapping or repeated passes.
let declaredFileBytes = 0;
for await (const entry of this.#reader.getEntriesGenerator({
decodeText: decodeZipEntryText,
})) {
for await (const entry of typedZipFailures(
this.#reader.getEntriesGenerator({ decodeText: decodeZipEntryText }),
)) {
abortIfNeeded(signal);
// ZIP member identity is its directory occurrence, not its filename.
// Logical path uniqueness belongs to the artifact inventory boundary.
@@ -327,7 +349,7 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => {
output.end();
},
abort: (reason) => {
output.destroy(toError(reason));
output.destroy(toError(typedZipFailure(reason)));
},
});
void entry
@@ -337,10 +359,61 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => {
checkOverlappingEntry: true,
onprogress: () => abortIfNeeded(signal),
})
.catch((cause: unknown) => output.destroy(toError(cause)));
.catch((cause: unknown) => output.destroy(toError(typedZipFailure(cause))));
return output;
};
/** zip.js reports archive faults as plain errors identified by message. */
const ZIP_FAILURE_REASONS: ReadonlyMap<string, "format" | "unavailable"> =
new Map([
...[
ERR_AMBIGUOUS_ARCHIVE,
ERR_BAD_FORMAT,
ERR_CENTRAL_DIRECTORY_NOT_FOUND,
ERR_ENTRY_DATA_OUT_OF_BOUNDS,
ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND,
ERR_EOCDR_NOT_FOUND,
ERR_EXTRAFIELD_ZIP64_NOT_FOUND,
ERR_INVALID_COMPRESSED_DATA,
ERR_INVALID_CRC32,
ERR_INVALID_UNCOMPRESSED_SIZE,
ERR_LOCAL_FILE_HEADER_NOT_FOUND,
ERR_OVERLAPPING_ENTRY,
ERR_UNSUPPORTED_UINT64,
].map((message) => [message, "format"] as const),
...[
ERR_ENCRYPTED_CENTRAL_DIRECTORY,
ERR_SPLIT_ZIP_FILE,
ERR_UNSUPPORTED_COMPRESSION,
ERR_UNSUPPORTED_ENCRYPTION,
].map((message) => [message, "unavailable"] as const),
]);
/** Type a zip.js archive fault so consumers do not report it as file I/O. */
const typedZipFailure = (cause: unknown): unknown => {
if (!(cause instanceof Error) || cause instanceof ArtifactReaderFailure)
return cause;
const reason = ZIP_FAILURE_REASONS.get(cause.message);
if (reason === undefined) return cause;
return new ArtifactReaderFailure(
reason,
reason === "format"
? `ZIP archive is malformed: ${cause.message}`
: `ZIP archive uses an unsupported feature: ${cause.message}`,
{ cause },
);
};
async function* typedZipFailures<T>(
source: AsyncIterable<T>,
): AsyncIterable<T> {
try {
yield* source;
} catch (cause: unknown) {
throw typedZipFailure(cause);
}
}
const isSymlink = (entry: Entry): boolean =>
entry.unixMode !== undefined && (entry.unixMode & 0o170000) === 0o120000;
@@ -56,6 +56,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
const omittedArchives: Array<{ path: string; reason: string }> = [];
let unreportedOmittedArchives = 0;
const incompleteHierarchies = new Map<string, number>();
const unresolvedConnections = new Map<string, number>();
const prototypeKeyOmissions = new Map<string, number | null>();
let omitted = 0;
let attempted = 0;
@@ -136,6 +137,8 @@ export const analyzeInterfaceBuilderBundle = async (input: {
: { value: nib.raw, omittedPrototypeKeys: 0 };
if (nib !== null && nib.omitted > 0)
incompleteHierarchies.set(entry.path, nib.omitted);
if (nib !== null && nib.unresolvedConnections > 0)
unresolvedConnections.set(entry.path, nib.unresolvedConnections);
if (omittedPrototypeKeys !== 0)
prototypeKeyOmissions.set(entry.path, omittedPrototypeKeys);
const documentHash = createHash("sha256").update(bytes).digest("hex");
@@ -170,6 +173,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
});
return finalizeHierarchyCoverage(result, {
incompleteHierarchies,
unresolvedConnections,
prototypeKeyOmissions,
invalid,
omittedArchives,
@@ -237,6 +241,7 @@ const finalizeHierarchyCoverage = (
result: ReturnType<typeof buildInterfaceBuilderAnalysis>,
counts: {
incompleteHierarchies: ReadonlyMap<string, number>;
unresolvedConnections: ReadonlyMap<string, number>;
prototypeKeyOmissions: ReadonlyMap<string, number | null>;
invalid: readonly string[];
omittedArchives: readonly { path: string; reason: string }[];
@@ -247,6 +252,7 @@ const finalizeHierarchyCoverage = (
) => {
const {
incompleteHierarchies,
unresolvedConnections,
prototypeKeyOmissions,
invalid,
omittedArchives,
@@ -260,6 +266,13 @@ const finalizeHierarchyCoverage = (
...result,
documents: result.documents.map((document) => ({
...document,
// The dictionary parser only sees connectors that were projected, so the
// omission recorded where an ownerless connector was dropped has to be
// added here for this producer to report the same document count as the
// keyed-archive producer for the same recognized connector.
connection_count:
document.connection_count +
(unresolvedConnections.get(document.relative_path) ?? 0),
hierarchy_complete:
document.hierarchy_complete &&
!incompleteHierarchies.has(document.relative_path),
@@ -268,15 +281,32 @@ const finalizeHierarchyCoverage = (
...result.graph,
coverage: [
...result.graph.coverage.map((facet) => {
const omitted = incompleteHierarchies.get(
const omittedHierarchy = incompleteHierarchies.get(
facet.facet.replace(/^hierarchy:/u, ""),
);
return facet.facet.startsWith("hierarchy:") && omitted !== undefined
if (
facet.facet.startsWith("hierarchy:") &&
omittedHierarchy !== undefined
)
return {
...facet,
status: "partial" as const,
reason: "serialized_view_hierarchy_incomplete",
omitted: facet.omitted + omittedHierarchy,
};
const omittedConnection = unresolvedConnections.get(
facet.facet.replace(/^connections:/u, ""),
);
return facet.facet.startsWith("connections:") &&
omittedConnection !== undefined
? {
...facet,
status: "partial" as const,
reason: "serialized_view_hierarchy_incomplete",
omitted: facet.omitted + omitted,
// A connection dropped by the limit is recoverable by raising
// it, so that cause keeps the reason the graph already named.
// Only a facet with no other cause reports the omission.
reason: facet.reason ?? "connection_unresolved",
omitted: facet.omitted + omittedConnection,
}
: facet;
}),
@@ -306,7 +336,8 @@ const finalizeHierarchyCoverage = (
result.graph.truncated ||
counts.omitted > 0 ||
archiveDecodePartial ||
incompleteHierarchies.size > 0,
incompleteHierarchies.size > 0 ||
unresolvedConnections.size > 0,
},
limitations: [
...result.limitations,
@@ -316,6 +347,11 @@ const finalizeHierarchyCoverage = (
: [
"Some serialized hierarchy links could not be projected within the bounded hierarchy; their parentage remains unknown.",
]),
...(unresolvedConnections.size === 0
? []
: [
"Some recognized NIBArchive connectors declare no resolvable owner; they carry no connection evidence and are reported as omissions.",
]),
...(invalid.length === 0
? []
: [
@@ -496,6 +532,10 @@ const projectNibArchive = (
);
const connections: Record<string, JsonValue[]> = {};
const hierarchy: NibHierarchyNode[] = [];
// A recognized connector whose owner reference cannot be resolved produces no
// connection entry, so the omission has to be counted here: the projected
// dictionary carries no trace of it for a downstream consumer to find.
let unresolvedConnections = 0;
const reference = (value: JsonValue | undefined): number | null => {
if (typeof value !== "object" || value === null || Array.isArray(value))
return null;
@@ -512,7 +552,10 @@ const projectNibArchive = (
const values = object.values;
const connectionFields = interfaceBuilderConnectionFieldNames(className);
const source = reference(values[connectionFields.source]);
if (source === null) continue;
if (source === null) {
unresolvedConnections += 1;
continue;
}
const destination = reference(values[connectionFields.destination]);
const label = referencedString(values[connectionFields.label]);
const type = className.includes("Outlet")
@@ -608,6 +651,7 @@ const projectNibArchive = (
);
return {
omitted: viewHierarchy.omitted + hierarchyOmitted + merged.omitted,
unresolvedConnections,
raw: jsonValueSchema.parse({
"com.apple.ibtool.document.objects": objects,
"com.apple.ibtool.document.connections": connections,
@@ -0,0 +1,103 @@
import { describe, expect, it } from "vitest";
import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js";
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
const fat = (wide = false): Buffer =>
Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide));
describe("bounded FAT inventory classification", () => {
it("requires the complete table but not slice bytes in the retained prefix", () => {
const bytes = fat();
for (const length of [4, 7, 8, 27]) {
expect(
classifyArtifactContent(
"native",
bytes.subarray(0, length),
bytes.length,
),
).toEqual({ kind: "unknown", format: "unknown" });
}
expect(
classifyArtifactContent(
"native.class",
bytes.subarray(0, 28),
bytes.length,
),
).toEqual({ kind: "executable", format: "mach-o-universal" });
});
it.each([
["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)],
["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)],
["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)],
[
"slice beyond file",
(bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20),
],
["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)],
["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)],
] as const)("keeps %s unknown", (_label, mutate) => {
const bytes = fat();
mutate(bytes);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "unknown",
format: "unknown",
});
});
it("does not infer Mach-O from an out-of-bounds architecture count", () => {
const bytes = fat();
bytes.writeUInt32BE(0xffffffff, 4);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "resource",
format: "file",
});
});
it("does not lose unsafe FAT64 range values to number conversion", () => {
const bytes = fat(true);
bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "unknown",
format: "unknown",
});
});
it("requires valid ranges for every declared slice", () => {
const bytes = Buffer.from(
fatImage([
{ cpu: CPU.arm64, bytes: machoImage({}) },
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
]),
);
bytes.writeUInt32BE(bytes.length, 40);
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "unknown",
format: "unknown",
});
});
it("requires observed slice magic for an unknown CPU type", () => {
const cpu = { type: 0x12345678, subtype: 0 };
const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }]));
expect(
classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length),
).toEqual({ kind: "unknown", format: "unknown" });
expect(classifyArtifactContent("native", bytes)).toEqual({
kind: "executable",
format: "mach-o-universal",
});
});
it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])(
"keeps invalid or contradictory file size %s unknown",
(size) => {
const bytes = fat();
expect(classifyArtifactContent("native", bytes, size)).toEqual({
kind: "unknown",
format: "unknown",
});
},
);
});
@@ -37,6 +37,7 @@ interface ExpectedFile {
interface ExpectedContainer {
readonly hash_status: ArtifactOccurrence["hash_status"];
readonly format: ArtifactOccurrence["artifact_format"];
readonly inventory: JavaScriptArtifactContainer | null;
}
@@ -219,6 +220,9 @@ const visitReader = async (
"integrity",
`Nested ASAR was not integrity-verified: ${path}`,
);
// Inventory expands an ASAR-named file only when its bytes are an
// ASAR. One it recorded as ordinary bytes has no members to read.
if (container.format !== "asar") continue;
const inventory = container.inventory;
if (inventory === null)
throw new ArtifactReaderFailure(
@@ -395,6 +399,7 @@ const expectedInventory = (
if (isNestedAsar)
containers.set(occurrence.logical_path, {
hash_status: occurrence.hash_status,
format: occurrence.artifact_format,
inventory: null,
});
continue;
@@ -412,6 +417,7 @@ const expectedInventory = (
if (isNestedAsar)
containers.set(occurrence.logical_path, {
hash_status: occurrence.hash_status,
format: occurrence.artifact_format,
inventory: {
path: occurrence.logical_path,
sha256: node.sha256,