From 9b38f3a0af485e9945883e83e8123224fbebd035 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Thu, 8 Oct 2026 17:08:17 +0800 Subject: [PATCH] fix(artifacts): preserve unavailable members and stored checksum evidence --- src/artifacts/ArtifactReader.ts | 12 ++ src/artifacts/InstallerPackage.fixture.ts | 8 +- src/artifacts/InstallerPackageReaders.test.ts | 5 +- src/artifacts/XarArtifactReader.ts | 64 ++++++++- .../extraction/ArtifactExtraction.ts | 16 ++- src/artifacts/inventory/scanReader.ts | 22 ++- .../installerPackageRecovery.test.ts | 128 ++++++++++++++++++ 7 files changed, 241 insertions(+), 14 deletions(-) create mode 100644 tests/boundary/filesystem/installerPackageRecovery.test.ts diff --git a/src/artifacts/ArtifactReader.ts b/src/artifacts/ArtifactReader.ts index 3f0faeaf3..c23512263 100644 --- a/src/artifacts/ArtifactReader.ts +++ b/src/artifacts/ArtifactReader.ts @@ -35,11 +35,23 @@ export interface ArtifactEntry { }; } +/** Checksum evidence for a member representation the scanner cannot hash itself. */ +export interface ArtifactChecksumObservation { + readonly representation: "stored"; + readonly algorithm: string; + readonly declared: string; + readonly observed: string; +} + /** Read-only adapter over one directory, archive, or virtual container. */ export interface ArtifactReader { readonly format: "directory" | ZipPackageFormat | "asar" | "pkg" | "file"; entries(signal?: AbortSignal): AsyncIterable; open(entry: ArtifactEntry, signal?: AbortSignal): Promise; + /** Checksums observed after the complete member stream has been consumed. */ + integrityObservations?( + entry: ArtifactEntry, + ): readonly ArtifactChecksumObservation[]; provenance(): readonly ArtifactCommand[]; close(): Promise; } diff --git a/src/artifacts/InstallerPackage.fixture.ts b/src/artifacts/InstallerPackage.fixture.ts index bc8fa9562..e44612cd5 100644 --- a/src/artifacts/InstallerPackage.fixture.ts +++ b/src/artifacts/InstallerPackage.fixture.ts @@ -11,6 +11,8 @@ export interface XarFixtureMember { readonly mode?: string; /** Replace the declared extracted checksum, to model tampering. */ readonly extractedSha1?: string; + /** Replace the declared stored checksum while retaining decodable content. */ + readonly archivedSha1?: string; /** Store these bytes instead of encoding `data`, to model corrupt members. */ readonly archived?: Uint8Array; readonly children?: readonly XarFixtureMember[]; @@ -29,6 +31,8 @@ export const xarArchive = ( readonly corruptTocChecksum?: boolean; /** Declared TOC checksum size; the real digest is 20 bytes. */ readonly tocChecksumSize?: number; + /** Override the fixed-header decoded length without altering the TOC. */ + readonly tocDecodedSize?: number; } = {}, ): Uint8Array => { const heap: Uint8Array[] = [new Uint8Array(20)]; @@ -51,7 +55,7 @@ export const xarArchive = ( : member.encoding === "bzip2" ? "application/x-bzip2" : "application/octet-stream"; - data = `${sha1(archived)}${member.extractedSha1 ?? sha1(member.data)}${member.data.length}${offset}${archived.length}`; + data = `${member.archivedSha1 ?? sha1(archived)}${member.extractedSha1 ?? sha1(member.data)}${member.data.length}${offset}${archived.length}`; } const link = type === "symlink" @@ -74,7 +78,7 @@ export const xarArchive = ( header.writeUInt16BE(28, 4); header.writeUInt16BE(1, 6); header.writeBigUInt64BE(BigInt(compressed.length), 8); - header.writeBigUInt64BE(BigInt(toc.length), 16); + header.writeBigUInt64BE(BigInt(options.tocDecodedSize ?? toc.length), 16); header.writeUInt32BE(1, 24); return Buffer.concat([header, compressed, ...heap]); }; diff --git a/src/artifacts/InstallerPackageReaders.test.ts b/src/artifacts/InstallerPackageReaders.test.ts index 22b647374..d465b1aae 100644 --- a/src/artifacts/InstallerPackageReaders.test.ts +++ b/src/artifacts/InstallerPackageReaders.test.ts @@ -146,7 +146,10 @@ describe("xar installer package reader", () => { ); const [entry] = await collect(bzip); if (entry === undefined) throw new Error("missing entry"); - await expect(bzip.open(entry)).rejects.toMatchObject({ reason: "format" }); + expect(entry.contentUnavailable).toBe(true); + await expect(bzip.open(entry)).rejects.toMatchObject({ + reason: "unavailable", + }); await bzip.close(); const notXar = new XarArtifactReader( await writePackage(Buffer.from("not an archive at all")), diff --git a/src/artifacts/XarArtifactReader.ts b/src/artifacts/XarArtifactReader.ts index e58977a93..fb3eb9d0e 100644 --- a/src/artifacts/XarArtifactReader.ts +++ b/src/artifacts/XarArtifactReader.ts @@ -14,6 +14,7 @@ import type { ArtifactCommand } from "../domain/artifactGraph.js"; import { ArtifactReaderFailure, type ArtifactEntry, + type ArtifactChecksumObservation, type ArtifactReader, } from "./ArtifactReader.js"; @@ -177,6 +178,13 @@ const memberLimitations = ( : [ `Member declares unsupported archived checksum ${unsupportedArchived.style}; stored-byte integrity not verified.`, ]), + ...(member.data !== undefined && + member.data.encoding !== "application/octet-stream" && + member.data.encoding !== "application/x-gzip" + ? [ + `Unsupported xar encoding ${member.data.encoding}; content is unavailable and was not expanded.`, + ] + : []), ...classified.limitations, ]; @@ -311,6 +319,9 @@ class ChecksumVerifier extends Transform { readonly algorithm: string; readonly value: string; }, + private readonly record?: ( + observation: ArtifactChecksumObservation, + ) => void, ) { super(); this.#hash = createHash(expected.algorithm); @@ -327,6 +338,16 @@ class ChecksumVerifier extends Transform { override _flush(done: TransformCallback): void { const observed = this.#hash.digest("hex"); + if (this.record !== undefined) { + this.record({ + representation: "stored", + algorithm: this.expected.algorithm, + declared: this.expected.value, + observed, + }); + done(); + return; + } done( observed === this.expected.value ? null @@ -352,6 +373,7 @@ export class XarArtifactReader implements ArtifactReader { #handle: FileHandle | undefined; #heap = 0; #members: readonly XarMember[] = []; + readonly #integrity = new Map(); readonly #verifyChecksums: boolean; @@ -359,8 +381,8 @@ export class XarArtifactReader implements ArtifactReader { * @param options.verifyChecksums Fail `open()` streams whose bytes disagree * with the member's extracted checksum (default). Inventory passes false and * verifies the declared checksum itself under the caller's integrity policy. - * Archived checksums are always verified: the scanner only observes decoded - * bytes, so it cannot apply its policy to the stored stream. + * Archived checksums are always computed. When inventory owns the policy, + * integrityObservations exposes their declared/observed values after reading. */ constructor( private readonly path: string, @@ -404,6 +426,11 @@ export class XarArtifactReader implements ArtifactReader { const inflated = inflateSync(compressed, { maxOutputLength: MAX_TOC_BYTES, }); + if (BigInt(inflated.length) !== tocSize) + throw new ArtifactReaderFailure( + "format", + `xar TOC decoded ${inflated.length} bytes, expected ${tocSize}`, + ); try { xml = new TextDecoder("utf-8", { fatal: true }).decode(inflated); } catch (cause: unknown) { @@ -593,10 +620,15 @@ export class XarArtifactReader implements ArtifactReader { assertRawSize(member, data); const unsupportedChecksum = data?.unsupportedChecksum; const unsupportedArchived = data?.unsupportedArchivedChecksum; + const unsupportedEncoding = + data !== undefined && + data.encoding !== "application/octet-stream" && + data.encoding !== "application/x-gzip"; const classified = member.kind === "file" && unsupportedChecksum === undefined && - unsupportedArchived === undefined + unsupportedArchived === undefined && + !unsupportedEncoding ? await this.#nestedArchive(member) : { nested: undefined as XarNestedArchive | undefined, @@ -622,6 +654,11 @@ export class XarArtifactReader implements ArtifactReader { classified, ), adapterKey: member.path, + ...(unsupportedEncoding || + unsupportedChecksum !== undefined || + unsupportedArchived !== undefined + ? { contentUnavailable: true } + : {}), ...(classified.nested === undefined ? {} : { nestedArchive: classified.nested }), @@ -630,6 +667,7 @@ export class XarArtifactReader implements ArtifactReader { async open(entry: ArtifactEntry, signal?: AbortSignal): Promise { cancelled(signal); + this.#integrity.delete(entry.adapterKey); await this.#load(signal); const member = this.#members.find(({ path }) => path === entry.adapterKey); if (member === undefined || member.kind !== "file") @@ -675,8 +713,19 @@ export class XarArtifactReader implements ArtifactReader { const archivedVerifier = archived === undefined ? undefined - : new ChecksumVerifier(`${entry.path} (archived)`, archived); + : new ChecksumVerifier( + `${entry.path} (archived)`, + archived, + this.#verifyChecksums + ? undefined + : (observation) => { + this.#integrity.set(entry.adapterKey, [observation]); + }, + ); const raw = Readable.from(this.#chunks(data.offset, data.length, signal)); + raw.on("error", (cause: unknown) => + archivedVerifier?.destroy(memberFailure(entry.path, cause)), + ); const archivedChecked = archivedVerifier === undefined ? raw : raw.pipe(archivedVerifier); let decoded: Readable; @@ -739,6 +788,13 @@ export class XarArtifactReader implements ArtifactReader { return []; } + /** Stored-byte checksums computed while consuming the member stream. */ + integrityObservations( + entry: ArtifactEntry, + ): readonly ArtifactChecksumObservation[] { + return this.#integrity.get(entry.adapterKey) ?? []; + } + async close(): Promise { const handle = this.#handle; this.#handle = undefined; diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index 6e7fe767e..38dc9568a 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -46,6 +46,7 @@ export const extractArtifact = async ( const selectedOccurrences = snapshot.occurrences.filter( (occurrence) => (occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") && + occurrence.hash_status !== "unavailable" && occurrence.logical_path !== ".", ); const selectedIds = new Set( @@ -65,6 +66,9 @@ export const extractArtifact = async ( manifest: snapshot.manifest, occurrences, nodes, + unavailablePaths: snapshot.occurrences + .filter(({ hash_status }) => hash_status === "unavailable") + .map(({ logical_path }) => logical_path), }; const selected = selectedOccurrences.map((occurrence) => { if ( @@ -181,6 +185,11 @@ const materializeSelection = async ({ } const selectedItem = byPath.get(path); if (selectedItem === undefined) { + if ( + entry.contentUnavailable === true && + inventory.unavailablePaths.includes(path) + ) + continue; if (entry.kind === "file" || entry.kind === "slice") throw new ArtifactReaderFailure( "integrity", @@ -263,7 +272,11 @@ const createExtractionResult = ( cleanup: { attempted: false, verified: true, residual_paths: [] }, provenance: [], limitations: [ - "All regular files in the active artifact were materialized, including members of expanded PKG Payload/Scripts archives.", + "All available selected regular files were materialized; expanded PKG Payload/Scripts archives are represented by their extracted children.", + ...inventory.unavailablePaths.map( + (path) => + `Unavailable member ${path} was not extracted; see its inventory occurrence for the reason.`, + ), ], }); }; @@ -272,6 +285,7 @@ interface LoadedInventory { readonly manifest: ArtifactGraphManifest; readonly occurrences: ReadonlyMap; readonly nodes: ReadonlyMap; + readonly unavailablePaths: readonly string[]; } const collectOccurrences = ( diff --git a/src/artifacts/inventory/scanReader.ts b/src/artifacts/inventory/scanReader.ts index c695e9a4f..378a56b3c 100644 --- a/src/artifacts/inventory/scanReader.ts +++ b/src/artifacts/inventory/scanReader.ts @@ -235,7 +235,11 @@ const digestArtifactEntry = async ( entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256; const checksumMismatch = checksum !== undefined && checksum.value !== digest.also; - const mismatched = sha256Mismatch || checksumMismatch; + const storedMismatches = ( + currentReader.integrityObservations?.(entry) ?? [] + ).filter(({ declared, observed }) => declared !== observed); + const mismatched = + sha256Mismatch || checksumMismatch || storedMismatches.length > 0; if (mismatched && context.integrity.mode === "fail") throw new ArtifactReaderFailure( "integrity", @@ -277,11 +281,17 @@ const digestArtifactEntry = async ( : [], }), mismatched, - mismatchDetails: checksumMismatch - ? [ - `Declared ${checksum.algorithm} ${checksum.value} disagrees with observed ${digest.also ?? "unavailable"}.`, - ] - : [], + mismatchDetails: [ + ...(checksumMismatch + ? [ + `Declared ${checksum.algorithm} ${checksum.value} disagrees with observed ${digest.also ?? "unavailable"}.`, + ] + : []), + ...storedMismatches.map( + ({ algorithm, declared, observed }) => + `Declared stored ${algorithm} ${declared} disagrees with observed ${observed}.`, + ), + ], }; }; diff --git a/tests/boundary/filesystem/installerPackageRecovery.test.ts b/tests/boundary/filesystem/installerPackageRecovery.test.ts new file mode 100644 index 000000000..7f0082a7b --- /dev/null +++ b/tests/boundary/filesystem/installerPackageRecovery.test.ts @@ -0,0 +1,128 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { deflateSync } from "node:zlib"; + +import { expect, it } from "vitest"; + +import { extractArtifact } from "../../../src/artifacts/extraction/ArtifactExtraction.js"; +import { + MODE, + gzipCpio, + xarArchive, +} from "../../../src/artifacts/InstallerPackage.fixture.js"; +import { scanArtifactInventory } from "../../../src/artifacts/inventory/ArtifactInventory.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("extracts regular nested siblings while retaining skipped special-node evidence", async () => { + const directory = await createTestTempDirectory("rea-pkg-extract-special-"); + const inputPath = join(directory, "Installer.pkg"); + const outputRoot = join(directory, "extracted"); + await writeFile( + inputPath, + xarArchive([ + { + name: "Payload", + data: gzipCpio([ + { name: "./fifo", mode: MODE.fifo }, + { name: "./usr/bin/tool", mode: MODE.executable, data: "tool bytes" }, + ]), + }, + ]), + ); + const extracted = await extractArtifact({ + inputPath, + outputRoot, + inputFormat: "pkg", + }); + expect(await readFile(join(outputRoot, "Payload/usr/bin/tool"), "utf8")).toBe( + "tool bytes", + ); + expect(extracted.artifacts.map(({ relative_path }) => relative_path)).toEqual( + ["Payload/usr/bin/tool"], + ); + expect(extracted.extraction_manifest.selected_occurrence_ids).toHaveLength(1); + expect(extracted.limitations).toContainEqual( + expect.stringContaining("Payload/fifo was not extracted"), + ); +}); + +it("retains unsupported-encoding occurrences under the default inventory policy", async () => { + const directory = await createTestTempDirectory("rea-pkg-unsupported-"); + const path = join(directory, "Installer.pkg"); + await writeFile( + path, + xarArchive([ + { name: "encoded", encoding: "bzip2", data: Buffer.from("opaque") }, + { name: "plain", data: Buffer.from("available") }, + ]), + ); + const inventory = await scanArtifactInventory(path); + expect( + inventory.occurrences.find( + ({ logical_path }) => logical_path === "encoded", + ), + ).toMatchObject({ + hash_status: "unavailable", + artifact_id: null, + limitations: [ + expect.stringContaining("Unsupported xar encoding application/x-bzip2"), + ], + }); + expect( + inventory.occurrences.find(({ logical_path }) => logical_path === "plain"), + ).toMatchObject({ hash_status: "verified" }); +}); + +it("rejects a header whose decoded TOC length contradicts the actual data", async () => { + const directory = await createTestTempDirectory("rea-pkg-toc-size-"); + const path = join(directory, "Installer.pkg"); + await writeFile( + path, + xarArchive([{ name: "plain", data: Buffer.from("ok") }], { + tocDecodedSize: 1, + }), + ); + await expect(scanArtifactInventory(path)).rejects.toMatchObject({ + reason: "format", + message: expect.stringMatching(/xar TOC decoded \d+ bytes, expected 1/u), + }); +}); + +it("records stored-checksum contradictions without losing decoded content identity", async () => { + const directory = await createTestTempDirectory("rea-pkg-stored-integrity-"); + const path = join(directory, "Installer.pkg"); + const decoded = Buffer.from("same decoded bytes"); + const archivedSha1 = "0".repeat(40); + await writeFile( + path, + xarArchive([ + { name: "encoded", encoding: "zlib", data: decoded, archivedSha1 }, + { name: "plain", data: Buffer.from("ok") }, + ]), + ); + const inventory = await scanArtifactInventory(path, { + integrity: { mode: "record-and-continue" }, + }); + const entry = inventory.occurrences.find( + ({ logical_path }) => logical_path === "encoded", + ); + expect(entry).toMatchObject({ hash_status: "mismatched" }); + expect( + inventory.nodes.find( + ({ artifact_id }) => artifact_id === entry?.artifact_id, + ), + ).toMatchObject({ + size: decoded.length, + sha256: createHash("sha256").update(decoded).digest("hex"), + }); + const observed = createHash("sha1") + .update(deflateSync(decoded)) + .digest("hex"); + expect(entry?.limitations).toContain( + `Declared stored sha1 ${archivedSha1} disagrees with observed ${observed}.`, + ); + await expect(scanArtifactInventory(path)).rejects.toMatchObject({ + reason: "integrity", + }); +});