chore: merge captured main into iNES container support
This commit is contained in:
@@ -40,6 +40,7 @@ import { ArtifactResourceScope } from "./ArtifactResourceScope.js";
|
||||
import { artifactCapabilities } from "./ArtifactProviderMetadata.js";
|
||||
import { ARTIFACT_GRAPH_PROVIDER } from "../application/InvestigationProviders.js";
|
||||
import { createEvidence } from "../domain/evidence.js";
|
||||
import { freezeJsonSnapshot } from "../domain/immutableJson.js";
|
||||
import { createArtifactInspection } from "../domain/artifactInspection.js";
|
||||
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
|
||||
|
||||
@@ -340,7 +341,11 @@ class ArtifactClient implements AnalysisClient {
|
||||
total: 1,
|
||||
message: "inventory substep started",
|
||||
});
|
||||
const inventory = await this.inventory(inventoryParameters, options);
|
||||
// The freshly parsed inventory is exclusively owned here. Sealing it lets
|
||||
// the nested Evidence share it instead of cloning and re-digesting it.
|
||||
const inventory = freezeJsonSnapshot(
|
||||
await this.inventory(inventoryParameters, options),
|
||||
);
|
||||
const subject = subjectFor(
|
||||
this.target.sourcePath ?? this.target.path,
|
||||
inventory.manifest,
|
||||
@@ -354,7 +359,10 @@ class ArtifactClient implements AnalysisClient {
|
||||
// The nested inventory already names every occurrence's logical path.
|
||||
locations: [],
|
||||
});
|
||||
const result = createArtifactInspection(inventoryEvidence);
|
||||
// The returned execution and its Evidence then reuse this owned report.
|
||||
const result = freezeJsonSnapshot(
|
||||
createArtifactInspection(inventoryEvidence),
|
||||
);
|
||||
await options?.progress?.report({
|
||||
phase: "inspect_artifact.inventory",
|
||||
completed: 1,
|
||||
|
||||
@@ -8,7 +8,7 @@ import {
|
||||
type NativeCommandRunner,
|
||||
} from "../native/CommandRunner.js";
|
||||
import {
|
||||
parseLipoArchitectures,
|
||||
parseLipoOutput,
|
||||
type LipoArchitecture,
|
||||
} from "../native/parsers/lipo.js";
|
||||
import type { MachoSlice } from "../domain/apple/dylibResolution.js";
|
||||
@@ -114,7 +114,7 @@ export class MachOSliceArtifactReader implements ArtifactReader {
|
||||
};
|
||||
await this.#verifySource(source);
|
||||
const fileSize = source.initial.size;
|
||||
const architectures = parseLipoArchitectures(captured.value.stdout);
|
||||
const { architectures } = parseLipoOutput(captured.value.stdout, this.path);
|
||||
const structural = await readStructuralSlices(source.handle, fileSize);
|
||||
await this.#verifySource(source);
|
||||
if (structural.status === "malformed")
|
||||
|
||||
@@ -2,7 +2,29 @@ import type { FileHandle } from "node:fs/promises";
|
||||
import { PassThrough, type Readable } from "node:stream";
|
||||
import { once } from "node:events";
|
||||
|
||||
import { Reader, ZipReader, type Entry, type FileEntry } from "@zip.js/zip.js";
|
||||
import {
|
||||
ERR_AMBIGUOUS_ARCHIVE,
|
||||
ERR_BAD_FORMAT,
|
||||
ERR_CENTRAL_DIRECTORY_NOT_FOUND,
|
||||
ERR_ENCRYPTED_CENTRAL_DIRECTORY,
|
||||
ERR_ENTRY_DATA_OUT_OF_BOUNDS,
|
||||
ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND,
|
||||
ERR_EOCDR_NOT_FOUND,
|
||||
ERR_EXTRAFIELD_ZIP64_NOT_FOUND,
|
||||
ERR_INVALID_COMPRESSED_DATA,
|
||||
ERR_INVALID_CRC32,
|
||||
ERR_INVALID_UNCOMPRESSED_SIZE,
|
||||
ERR_LOCAL_FILE_HEADER_NOT_FOUND,
|
||||
ERR_OVERLAPPING_ENTRY,
|
||||
ERR_SPLIT_ZIP_FILE,
|
||||
ERR_UNSUPPORTED_COMPRESSION,
|
||||
ERR_UNSUPPORTED_ENCRYPTION,
|
||||
ERR_UNSUPPORTED_UINT64,
|
||||
Reader,
|
||||
ZipReader,
|
||||
type Entry,
|
||||
type FileEntry,
|
||||
} from "@zip.js/zip.js";
|
||||
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
@@ -159,9 +181,9 @@ export class ZipArtifactReader implements ArtifactReader {
|
||||
let index = 0;
|
||||
// A traversal owns its accounting, including overlapping or repeated passes.
|
||||
let declaredFileBytes = 0;
|
||||
for await (const entry of this.#reader.getEntriesGenerator({
|
||||
decodeText: decodeZipEntryText,
|
||||
})) {
|
||||
for await (const entry of typedZipFailures(
|
||||
this.#reader.getEntriesGenerator({ decodeText: decodeZipEntryText }),
|
||||
)) {
|
||||
abortIfNeeded(signal);
|
||||
// ZIP member identity is its directory occurrence, not its filename.
|
||||
// Logical path uniqueness belongs to the artifact inventory boundary.
|
||||
@@ -327,7 +349,7 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => {
|
||||
output.end();
|
||||
},
|
||||
abort: (reason) => {
|
||||
output.destroy(toError(reason));
|
||||
output.destroy(toError(typedZipFailure(reason)));
|
||||
},
|
||||
});
|
||||
void entry
|
||||
@@ -337,10 +359,61 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => {
|
||||
checkOverlappingEntry: true,
|
||||
onprogress: () => abortIfNeeded(signal),
|
||||
})
|
||||
.catch((cause: unknown) => output.destroy(toError(cause)));
|
||||
.catch((cause: unknown) => output.destroy(toError(typedZipFailure(cause))));
|
||||
return output;
|
||||
};
|
||||
|
||||
/** zip.js reports archive faults as plain errors identified by message. */
|
||||
const ZIP_FAILURE_REASONS: ReadonlyMap<string, "format" | "unavailable"> =
|
||||
new Map([
|
||||
...[
|
||||
ERR_AMBIGUOUS_ARCHIVE,
|
||||
ERR_BAD_FORMAT,
|
||||
ERR_CENTRAL_DIRECTORY_NOT_FOUND,
|
||||
ERR_ENTRY_DATA_OUT_OF_BOUNDS,
|
||||
ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND,
|
||||
ERR_EOCDR_NOT_FOUND,
|
||||
ERR_EXTRAFIELD_ZIP64_NOT_FOUND,
|
||||
ERR_INVALID_COMPRESSED_DATA,
|
||||
ERR_INVALID_CRC32,
|
||||
ERR_INVALID_UNCOMPRESSED_SIZE,
|
||||
ERR_LOCAL_FILE_HEADER_NOT_FOUND,
|
||||
ERR_OVERLAPPING_ENTRY,
|
||||
ERR_UNSUPPORTED_UINT64,
|
||||
].map((message) => [message, "format"] as const),
|
||||
...[
|
||||
ERR_ENCRYPTED_CENTRAL_DIRECTORY,
|
||||
ERR_SPLIT_ZIP_FILE,
|
||||
ERR_UNSUPPORTED_COMPRESSION,
|
||||
ERR_UNSUPPORTED_ENCRYPTION,
|
||||
].map((message) => [message, "unavailable"] as const),
|
||||
]);
|
||||
|
||||
/** Type a zip.js archive fault so consumers do not report it as file I/O. */
|
||||
const typedZipFailure = (cause: unknown): unknown => {
|
||||
if (!(cause instanceof Error) || cause instanceof ArtifactReaderFailure)
|
||||
return cause;
|
||||
const reason = ZIP_FAILURE_REASONS.get(cause.message);
|
||||
if (reason === undefined) return cause;
|
||||
return new ArtifactReaderFailure(
|
||||
reason,
|
||||
reason === "format"
|
||||
? `ZIP archive is malformed: ${cause.message}`
|
||||
: `ZIP archive uses an unsupported feature: ${cause.message}`,
|
||||
{ cause },
|
||||
);
|
||||
};
|
||||
|
||||
async function* typedZipFailures<T>(
|
||||
source: AsyncIterable<T>,
|
||||
): AsyncIterable<T> {
|
||||
try {
|
||||
yield* source;
|
||||
} catch (cause: unknown) {
|
||||
throw typedZipFailure(cause);
|
||||
}
|
||||
}
|
||||
|
||||
const isSymlink = (entry: Entry): boolean =>
|
||||
entry.unixMode !== undefined && (entry.unixMode & 0o170000) === 0o120000;
|
||||
|
||||
|
||||
@@ -56,6 +56,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
|
||||
const omittedArchives: Array<{ path: string; reason: string }> = [];
|
||||
let unreportedOmittedArchives = 0;
|
||||
const incompleteHierarchies = new Map<string, number>();
|
||||
const unresolvedConnections = new Map<string, number>();
|
||||
const prototypeKeyOmissions = new Map<string, number | null>();
|
||||
let omitted = 0;
|
||||
let attempted = 0;
|
||||
@@ -136,6 +137,8 @@ export const analyzeInterfaceBuilderBundle = async (input: {
|
||||
: { value: nib.raw, omittedPrototypeKeys: 0 };
|
||||
if (nib !== null && nib.omitted > 0)
|
||||
incompleteHierarchies.set(entry.path, nib.omitted);
|
||||
if (nib !== null && nib.unresolvedConnections > 0)
|
||||
unresolvedConnections.set(entry.path, nib.unresolvedConnections);
|
||||
if (omittedPrototypeKeys !== 0)
|
||||
prototypeKeyOmissions.set(entry.path, omittedPrototypeKeys);
|
||||
const documentHash = createHash("sha256").update(bytes).digest("hex");
|
||||
@@ -170,6 +173,7 @@ export const analyzeInterfaceBuilderBundle = async (input: {
|
||||
});
|
||||
return finalizeHierarchyCoverage(result, {
|
||||
incompleteHierarchies,
|
||||
unresolvedConnections,
|
||||
prototypeKeyOmissions,
|
||||
invalid,
|
||||
omittedArchives,
|
||||
@@ -237,6 +241,7 @@ const finalizeHierarchyCoverage = (
|
||||
result: ReturnType<typeof buildInterfaceBuilderAnalysis>,
|
||||
counts: {
|
||||
incompleteHierarchies: ReadonlyMap<string, number>;
|
||||
unresolvedConnections: ReadonlyMap<string, number>;
|
||||
prototypeKeyOmissions: ReadonlyMap<string, number | null>;
|
||||
invalid: readonly string[];
|
||||
omittedArchives: readonly { path: string; reason: string }[];
|
||||
@@ -247,6 +252,7 @@ const finalizeHierarchyCoverage = (
|
||||
) => {
|
||||
const {
|
||||
incompleteHierarchies,
|
||||
unresolvedConnections,
|
||||
prototypeKeyOmissions,
|
||||
invalid,
|
||||
omittedArchives,
|
||||
@@ -260,6 +266,13 @@ const finalizeHierarchyCoverage = (
|
||||
...result,
|
||||
documents: result.documents.map((document) => ({
|
||||
...document,
|
||||
// The dictionary parser only sees connectors that were projected, so the
|
||||
// omission recorded where an ownerless connector was dropped has to be
|
||||
// added here for this producer to report the same document count as the
|
||||
// keyed-archive producer for the same recognized connector.
|
||||
connection_count:
|
||||
document.connection_count +
|
||||
(unresolvedConnections.get(document.relative_path) ?? 0),
|
||||
hierarchy_complete:
|
||||
document.hierarchy_complete &&
|
||||
!incompleteHierarchies.has(document.relative_path),
|
||||
@@ -268,15 +281,32 @@ const finalizeHierarchyCoverage = (
|
||||
...result.graph,
|
||||
coverage: [
|
||||
...result.graph.coverage.map((facet) => {
|
||||
const omitted = incompleteHierarchies.get(
|
||||
const omittedHierarchy = incompleteHierarchies.get(
|
||||
facet.facet.replace(/^hierarchy:/u, ""),
|
||||
);
|
||||
return facet.facet.startsWith("hierarchy:") && omitted !== undefined
|
||||
if (
|
||||
facet.facet.startsWith("hierarchy:") &&
|
||||
omittedHierarchy !== undefined
|
||||
)
|
||||
return {
|
||||
...facet,
|
||||
status: "partial" as const,
|
||||
reason: "serialized_view_hierarchy_incomplete",
|
||||
omitted: facet.omitted + omittedHierarchy,
|
||||
};
|
||||
const omittedConnection = unresolvedConnections.get(
|
||||
facet.facet.replace(/^connections:/u, ""),
|
||||
);
|
||||
return facet.facet.startsWith("connections:") &&
|
||||
omittedConnection !== undefined
|
||||
? {
|
||||
...facet,
|
||||
status: "partial" as const,
|
||||
reason: "serialized_view_hierarchy_incomplete",
|
||||
omitted: facet.omitted + omitted,
|
||||
// A connection dropped by the limit is recoverable by raising
|
||||
// it, so that cause keeps the reason the graph already named.
|
||||
// Only a facet with no other cause reports the omission.
|
||||
reason: facet.reason ?? "connection_unresolved",
|
||||
omitted: facet.omitted + omittedConnection,
|
||||
}
|
||||
: facet;
|
||||
}),
|
||||
@@ -306,7 +336,8 @@ const finalizeHierarchyCoverage = (
|
||||
result.graph.truncated ||
|
||||
counts.omitted > 0 ||
|
||||
archiveDecodePartial ||
|
||||
incompleteHierarchies.size > 0,
|
||||
incompleteHierarchies.size > 0 ||
|
||||
unresolvedConnections.size > 0,
|
||||
},
|
||||
limitations: [
|
||||
...result.limitations,
|
||||
@@ -316,6 +347,11 @@ const finalizeHierarchyCoverage = (
|
||||
: [
|
||||
"Some serialized hierarchy links could not be projected within the bounded hierarchy; their parentage remains unknown.",
|
||||
]),
|
||||
...(unresolvedConnections.size === 0
|
||||
? []
|
||||
: [
|
||||
"Some recognized NIBArchive connectors declare no resolvable owner; they carry no connection evidence and are reported as omissions.",
|
||||
]),
|
||||
...(invalid.length === 0
|
||||
? []
|
||||
: [
|
||||
@@ -496,6 +532,10 @@ const projectNibArchive = (
|
||||
);
|
||||
const connections: Record<string, JsonValue[]> = {};
|
||||
const hierarchy: NibHierarchyNode[] = [];
|
||||
// A recognized connector whose owner reference cannot be resolved produces no
|
||||
// connection entry, so the omission has to be counted here: the projected
|
||||
// dictionary carries no trace of it for a downstream consumer to find.
|
||||
let unresolvedConnections = 0;
|
||||
const reference = (value: JsonValue | undefined): number | null => {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
return null;
|
||||
@@ -512,7 +552,10 @@ const projectNibArchive = (
|
||||
const values = object.values;
|
||||
const connectionFields = interfaceBuilderConnectionFieldNames(className);
|
||||
const source = reference(values[connectionFields.source]);
|
||||
if (source === null) continue;
|
||||
if (source === null) {
|
||||
unresolvedConnections += 1;
|
||||
continue;
|
||||
}
|
||||
const destination = reference(values[connectionFields.destination]);
|
||||
const label = referencedString(values[connectionFields.label]);
|
||||
const type = className.includes("Outlet")
|
||||
@@ -608,6 +651,7 @@ const projectNibArchive = (
|
||||
);
|
||||
return {
|
||||
omitted: viewHierarchy.omitted + hierarchyOmitted + merged.omitted,
|
||||
unresolvedConnections,
|
||||
raw: jsonValueSchema.parse({
|
||||
"com.apple.ibtool.document.objects": objects,
|
||||
"com.apple.ibtool.document.connections": connections,
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js";
|
||||
import { classifyArtifactContent } from "./ArtifactGraphConstruction.js";
|
||||
|
||||
const fat = (wide = false): Buffer =>
|
||||
Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide));
|
||||
|
||||
describe("bounded FAT inventory classification", () => {
|
||||
it("requires the complete table but not slice bytes in the retained prefix", () => {
|
||||
const bytes = fat();
|
||||
for (const length of [4, 7, 8, 27]) {
|
||||
expect(
|
||||
classifyArtifactContent(
|
||||
"native",
|
||||
bytes.subarray(0, length),
|
||||
bytes.length,
|
||||
),
|
||||
).toEqual({ kind: "unknown", format: "unknown" });
|
||||
}
|
||||
expect(
|
||||
classifyArtifactContent(
|
||||
"native.class",
|
||||
bytes.subarray(0, 28),
|
||||
bytes.length,
|
||||
),
|
||||
).toEqual({ kind: "executable", format: "mach-o-universal" });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)],
|
||||
["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)],
|
||||
["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)],
|
||||
[
|
||||
"slice beyond file",
|
||||
(bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20),
|
||||
],
|
||||
["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)],
|
||||
["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)],
|
||||
] as const)("keeps %s unknown", (_label, mutate) => {
|
||||
const bytes = fat();
|
||||
mutate(bytes);
|
||||
expect(classifyArtifactContent("native", bytes)).toEqual({
|
||||
kind: "unknown",
|
||||
format: "unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not infer Mach-O from an out-of-bounds architecture count", () => {
|
||||
const bytes = fat();
|
||||
bytes.writeUInt32BE(0xffffffff, 4);
|
||||
expect(classifyArtifactContent("native", bytes)).toEqual({
|
||||
kind: "resource",
|
||||
format: "file",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not lose unsafe FAT64 range values to number conversion", () => {
|
||||
const bytes = fat(true);
|
||||
bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16);
|
||||
expect(classifyArtifactContent("native", bytes)).toEqual({
|
||||
kind: "unknown",
|
||||
format: "unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires valid ranges for every declared slice", () => {
|
||||
const bytes = Buffer.from(
|
||||
fatImage([
|
||||
{ cpu: CPU.arm64, bytes: machoImage({}) },
|
||||
{ cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) },
|
||||
]),
|
||||
);
|
||||
bytes.writeUInt32BE(bytes.length, 40);
|
||||
expect(classifyArtifactContent("native", bytes)).toEqual({
|
||||
kind: "unknown",
|
||||
format: "unknown",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires observed slice magic for an unknown CPU type", () => {
|
||||
const cpu = { type: 0x12345678, subtype: 0 };
|
||||
const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }]));
|
||||
expect(
|
||||
classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length),
|
||||
).toEqual({ kind: "unknown", format: "unknown" });
|
||||
expect(classifyArtifactContent("native", bytes)).toEqual({
|
||||
kind: "executable",
|
||||
format: "mach-o-universal",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])(
|
||||
"keeps invalid or contradictory file size %s unknown",
|
||||
(size) => {
|
||||
const bytes = fat();
|
||||
expect(classifyArtifactContent("native", bytes, size)).toEqual({
|
||||
kind: "unknown",
|
||||
format: "unknown",
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -37,6 +37,7 @@ interface ExpectedFile {
|
||||
|
||||
interface ExpectedContainer {
|
||||
readonly hash_status: ArtifactOccurrence["hash_status"];
|
||||
readonly format: ArtifactOccurrence["artifact_format"];
|
||||
readonly inventory: JavaScriptArtifactContainer | null;
|
||||
}
|
||||
|
||||
@@ -219,6 +220,9 @@ const visitReader = async (
|
||||
"integrity",
|
||||
`Nested ASAR was not integrity-verified: ${path}`,
|
||||
);
|
||||
// Inventory expands an ASAR-named file only when its bytes are an
|
||||
// ASAR. One it recorded as ordinary bytes has no members to read.
|
||||
if (container.format !== "asar") continue;
|
||||
const inventory = container.inventory;
|
||||
if (inventory === null)
|
||||
throw new ArtifactReaderFailure(
|
||||
@@ -395,6 +399,7 @@ const expectedInventory = (
|
||||
if (isNestedAsar)
|
||||
containers.set(occurrence.logical_path, {
|
||||
hash_status: occurrence.hash_status,
|
||||
format: occurrence.artifact_format,
|
||||
inventory: null,
|
||||
});
|
||||
continue;
|
||||
@@ -412,6 +417,7 @@ const expectedInventory = (
|
||||
if (isNestedAsar)
|
||||
containers.set(occurrence.logical_path, {
|
||||
hash_status: occurrence.hash_status,
|
||||
format: occurrence.artifact_format,
|
||||
inventory: {
|
||||
path: occurrence.logical_path,
|
||||
sha256: node.sha256,
|
||||
|
||||
Reference in New Issue
Block a user