fix(artifacts): preserve unavailable members and stored checksum evidence

This commit is contained in:
morluto
2026-10-08 17:08:17 +08:00
parent c541052ca9
commit 9b38f3a0af
7 changed files with 241 additions and 14 deletions
+12
View File
@@ -35,11 +35,23 @@ export interface ArtifactEntry {
};
}
/** Checksum evidence for a member representation the scanner cannot hash itself. */
export interface ArtifactChecksumObservation {
readonly representation: "stored";
readonly algorithm: string;
readonly declared: string;
readonly observed: string;
}
/** Read-only adapter over one directory, archive, or virtual container. */
export interface ArtifactReader {
readonly format: "directory" | ZipPackageFormat | "asar" | "pkg" | "file";
entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry>;
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable>;
/** Checksums observed after the complete member stream has been consumed. */
integrityObservations?(
entry: ArtifactEntry,
): readonly ArtifactChecksumObservation[];
provenance(): readonly ArtifactCommand[];
close(): Promise<void>;
}
+6 -2
View File
@@ -11,6 +11,8 @@ export interface XarFixtureMember {
readonly mode?: string;
/** Replace the declared extracted checksum, to model tampering. */
readonly extractedSha1?: string;
/** Replace the declared stored checksum while retaining decodable content. */
readonly archivedSha1?: string;
/** Store these bytes instead of encoding `data`, to model corrupt members. */
readonly archived?: Uint8Array;
readonly children?: readonly XarFixtureMember[];
@@ -29,6 +31,8 @@ export const xarArchive = (
readonly corruptTocChecksum?: boolean;
/** Declared TOC checksum size; the real digest is 20 bytes. */
readonly tocChecksumSize?: number;
/** Override the fixed-header decoded length without altering the TOC. */
readonly tocDecodedSize?: number;
} = {},
): Uint8Array => {
const heap: Uint8Array[] = [new Uint8Array(20)];
@@ -51,7 +55,7 @@ export const xarArchive = (
: member.encoding === "bzip2"
? "application/x-bzip2"
: "application/octet-stream";
data = `<data><archived-checksum style="sha1">${sha1(archived)}</archived-checksum><extracted-checksum style="sha1">${member.extractedSha1 ?? sha1(member.data)}</extracted-checksum><encoding style="${style}"/><size>${member.data.length}</size><offset>${offset}</offset><length>${archived.length}</length></data>`;
data = `<data><archived-checksum style="sha1">${member.archivedSha1 ?? sha1(archived)}</archived-checksum><extracted-checksum style="sha1">${member.extractedSha1 ?? sha1(member.data)}</extracted-checksum><encoding style="${style}"/><size>${member.data.length}</size><offset>${offset}</offset><length>${archived.length}</length></data>`;
}
const link =
type === "symlink"
@@ -74,7 +78,7 @@ export const xarArchive = (
header.writeUInt16BE(28, 4);
header.writeUInt16BE(1, 6);
header.writeBigUInt64BE(BigInt(compressed.length), 8);
header.writeBigUInt64BE(BigInt(toc.length), 16);
header.writeBigUInt64BE(BigInt(options.tocDecodedSize ?? toc.length), 16);
header.writeUInt32BE(1, 24);
return Buffer.concat([header, compressed, ...heap]);
};
@@ -146,7 +146,10 @@ describe("xar installer package reader", () => {
);
const [entry] = await collect(bzip);
if (entry === undefined) throw new Error("missing entry");
await expect(bzip.open(entry)).rejects.toMatchObject({ reason: "format" });
expect(entry.contentUnavailable).toBe(true);
await expect(bzip.open(entry)).rejects.toMatchObject({
reason: "unavailable",
});
await bzip.close();
const notXar = new XarArtifactReader(
await writePackage(Buffer.from("not an archive at all")),
+60 -4
View File
@@ -14,6 +14,7 @@ import type { ArtifactCommand } from "../domain/artifactGraph.js";
import {
ArtifactReaderFailure,
type ArtifactEntry,
type ArtifactChecksumObservation,
type ArtifactReader,
} from "./ArtifactReader.js";
@@ -177,6 +178,13 @@ const memberLimitations = (
: [
`Member declares unsupported archived checksum ${unsupportedArchived.style}; stored-byte integrity not verified.`,
]),
...(member.data !== undefined &&
member.data.encoding !== "application/octet-stream" &&
member.data.encoding !== "application/x-gzip"
? [
`Unsupported xar encoding ${member.data.encoding}; content is unavailable and was not expanded.`,
]
: []),
...classified.limitations,
];
@@ -311,6 +319,9 @@ class ChecksumVerifier extends Transform {
readonly algorithm: string;
readonly value: string;
},
private readonly record?: (
observation: ArtifactChecksumObservation,
) => void,
) {
super();
this.#hash = createHash(expected.algorithm);
@@ -327,6 +338,16 @@ class ChecksumVerifier extends Transform {
override _flush(done: TransformCallback): void {
const observed = this.#hash.digest("hex");
if (this.record !== undefined) {
this.record({
representation: "stored",
algorithm: this.expected.algorithm,
declared: this.expected.value,
observed,
});
done();
return;
}
done(
observed === this.expected.value
? null
@@ -352,6 +373,7 @@ export class XarArtifactReader implements ArtifactReader {
#handle: FileHandle | undefined;
#heap = 0;
#members: readonly XarMember[] = [];
readonly #integrity = new Map<string, ArtifactChecksumObservation[]>();
readonly #verifyChecksums: boolean;
@@ -359,8 +381,8 @@ export class XarArtifactReader implements ArtifactReader {
* @param options.verifyChecksums Fail `open()` streams whose bytes disagree
* with the member's extracted checksum (default). Inventory passes false and
* verifies the declared checksum itself under the caller's integrity policy.
* Archived checksums are always verified: the scanner only observes decoded
* bytes, so it cannot apply its policy to the stored stream.
* Archived checksums are always computed. When inventory owns the policy,
* integrityObservations exposes their declared/observed values after reading.
*/
constructor(
private readonly path: string,
@@ -404,6 +426,11 @@ export class XarArtifactReader implements ArtifactReader {
const inflated = inflateSync(compressed, {
maxOutputLength: MAX_TOC_BYTES,
});
if (BigInt(inflated.length) !== tocSize)
throw new ArtifactReaderFailure(
"format",
`xar TOC decoded ${inflated.length} bytes, expected ${tocSize}`,
);
try {
xml = new TextDecoder("utf-8", { fatal: true }).decode(inflated);
} catch (cause: unknown) {
@@ -593,10 +620,15 @@ export class XarArtifactReader implements ArtifactReader {
assertRawSize(member, data);
const unsupportedChecksum = data?.unsupportedChecksum;
const unsupportedArchived = data?.unsupportedArchivedChecksum;
const unsupportedEncoding =
data !== undefined &&
data.encoding !== "application/octet-stream" &&
data.encoding !== "application/x-gzip";
const classified =
member.kind === "file" &&
unsupportedChecksum === undefined &&
unsupportedArchived === undefined
unsupportedArchived === undefined &&
!unsupportedEncoding
? await this.#nestedArchive(member)
: {
nested: undefined as XarNestedArchive | undefined,
@@ -622,6 +654,11 @@ export class XarArtifactReader implements ArtifactReader {
classified,
),
adapterKey: member.path,
...(unsupportedEncoding ||
unsupportedChecksum !== undefined ||
unsupportedArchived !== undefined
? { contentUnavailable: true }
: {}),
...(classified.nested === undefined
? {}
: { nestedArchive: classified.nested }),
@@ -630,6 +667,7 @@ export class XarArtifactReader implements ArtifactReader {
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
cancelled(signal);
this.#integrity.delete(entry.adapterKey);
await this.#load(signal);
const member = this.#members.find(({ path }) => path === entry.adapterKey);
if (member === undefined || member.kind !== "file")
@@ -675,8 +713,19 @@ export class XarArtifactReader implements ArtifactReader {
const archivedVerifier =
archived === undefined
? undefined
: new ChecksumVerifier(`${entry.path} (archived)`, archived);
: new ChecksumVerifier(
`${entry.path} (archived)`,
archived,
this.#verifyChecksums
? undefined
: (observation) => {
this.#integrity.set(entry.adapterKey, [observation]);
},
);
const raw = Readable.from(this.#chunks(data.offset, data.length, signal));
raw.on("error", (cause: unknown) =>
archivedVerifier?.destroy(memberFailure(entry.path, cause)),
);
const archivedChecked =
archivedVerifier === undefined ? raw : raw.pipe(archivedVerifier);
let decoded: Readable;
@@ -739,6 +788,13 @@ export class XarArtifactReader implements ArtifactReader {
return [];
}
/** Stored-byte checksums computed while consuming the member stream. */
integrityObservations(
entry: ArtifactEntry,
): readonly ArtifactChecksumObservation[] {
return this.#integrity.get(entry.adapterKey) ?? [];
}
async close(): Promise<void> {
const handle = this.#handle;
this.#handle = undefined;
+15 -1
View File
@@ -46,6 +46,7 @@ export const extractArtifact = async (
const selectedOccurrences = snapshot.occurrences.filter(
(occurrence) =>
(occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") &&
occurrence.hash_status !== "unavailable" &&
occurrence.logical_path !== ".",
);
const selectedIds = new Set(
@@ -65,6 +66,9 @@ export const extractArtifact = async (
manifest: snapshot.manifest,
occurrences,
nodes,
unavailablePaths: snapshot.occurrences
.filter(({ hash_status }) => hash_status === "unavailable")
.map(({ logical_path }) => logical_path),
};
const selected = selectedOccurrences.map((occurrence) => {
if (
@@ -181,6 +185,11 @@ const materializeSelection = async ({
}
const selectedItem = byPath.get(path);
if (selectedItem === undefined) {
if (
entry.contentUnavailable === true &&
inventory.unavailablePaths.includes(path)
)
continue;
if (entry.kind === "file" || entry.kind === "slice")
throw new ArtifactReaderFailure(
"integrity",
@@ -263,7 +272,11 @@ const createExtractionResult = (
cleanup: { attempted: false, verified: true, residual_paths: [] },
provenance: [],
limitations: [
"All regular files in the active artifact were materialized, including members of expanded PKG Payload/Scripts archives.",
"All available selected regular files were materialized; expanded PKG Payload/Scripts archives are represented by their extracted children.",
...inventory.unavailablePaths.map(
(path) =>
`Unavailable member ${path} was not extracted; see its inventory occurrence for the reason.`,
),
],
});
};
@@ -272,6 +285,7 @@ interface LoadedInventory {
readonly manifest: ArtifactGraphManifest;
readonly occurrences: ReadonlyMap<string, ArtifactOccurrence>;
readonly nodes: ReadonlyMap<string, ArtifactNode>;
readonly unavailablePaths: readonly string[];
}
const collectOccurrences = (
+16 -6
View File
@@ -235,7 +235,11 @@ const digestArtifactEntry = async (
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
const checksumMismatch =
checksum !== undefined && checksum.value !== digest.also;
const mismatched = sha256Mismatch || checksumMismatch;
const storedMismatches = (
currentReader.integrityObservations?.(entry) ?? []
).filter(({ declared, observed }) => declared !== observed);
const mismatched =
sha256Mismatch || checksumMismatch || storedMismatches.length > 0;
if (mismatched && context.integrity.mode === "fail")
throw new ArtifactReaderFailure(
"integrity",
@@ -277,11 +281,17 @@ const digestArtifactEntry = async (
: [],
}),
mismatched,
mismatchDetails: checksumMismatch
? [
`Declared ${checksum.algorithm} ${checksum.value} disagrees with observed ${digest.also ?? "unavailable"}.`,
]
: [],
mismatchDetails: [
...(checksumMismatch
? [
`Declared ${checksum.algorithm} ${checksum.value} disagrees with observed ${digest.also ?? "unavailable"}.`,
]
: []),
...storedMismatches.map(
({ algorithm, declared, observed }) =>
`Declared stored ${algorithm} ${declared} disagrees with observed ${observed}.`,
),
],
};
};
@@ -0,0 +1,128 @@
import { createHash } from "node:crypto";
import { readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { deflateSync } from "node:zlib";
import { expect, it } from "vitest";
import { extractArtifact } from "../../../src/artifacts/extraction/ArtifactExtraction.js";
import {
MODE,
gzipCpio,
xarArchive,
} from "../../../src/artifacts/InstallerPackage.fixture.js";
import { scanArtifactInventory } from "../../../src/artifacts/inventory/ArtifactInventory.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
it("extracts regular nested siblings while retaining skipped special-node evidence", async () => {
const directory = await createTestTempDirectory("rea-pkg-extract-special-");
const inputPath = join(directory, "Installer.pkg");
const outputRoot = join(directory, "extracted");
await writeFile(
inputPath,
xarArchive([
{
name: "Payload",
data: gzipCpio([
{ name: "./fifo", mode: MODE.fifo },
{ name: "./usr/bin/tool", mode: MODE.executable, data: "tool bytes" },
]),
},
]),
);
const extracted = await extractArtifact({
inputPath,
outputRoot,
inputFormat: "pkg",
});
expect(await readFile(join(outputRoot, "Payload/usr/bin/tool"), "utf8")).toBe(
"tool bytes",
);
expect(extracted.artifacts.map(({ relative_path }) => relative_path)).toEqual(
["Payload/usr/bin/tool"],
);
expect(extracted.extraction_manifest.selected_occurrence_ids).toHaveLength(1);
expect(extracted.limitations).toContainEqual(
expect.stringContaining("Payload/fifo was not extracted"),
);
});
it("retains unsupported-encoding occurrences under the default inventory policy", async () => {
const directory = await createTestTempDirectory("rea-pkg-unsupported-");
const path = join(directory, "Installer.pkg");
await writeFile(
path,
xarArchive([
{ name: "encoded", encoding: "bzip2", data: Buffer.from("opaque") },
{ name: "plain", data: Buffer.from("available") },
]),
);
const inventory = await scanArtifactInventory(path);
expect(
inventory.occurrences.find(
({ logical_path }) => logical_path === "encoded",
),
).toMatchObject({
hash_status: "unavailable",
artifact_id: null,
limitations: [
expect.stringContaining("Unsupported xar encoding application/x-bzip2"),
],
});
expect(
inventory.occurrences.find(({ logical_path }) => logical_path === "plain"),
).toMatchObject({ hash_status: "verified" });
});
it("rejects a header whose decoded TOC length contradicts the actual data", async () => {
const directory = await createTestTempDirectory("rea-pkg-toc-size-");
const path = join(directory, "Installer.pkg");
await writeFile(
path,
xarArchive([{ name: "plain", data: Buffer.from("ok") }], {
tocDecodedSize: 1,
}),
);
await expect(scanArtifactInventory(path)).rejects.toMatchObject({
reason: "format",
message: expect.stringMatching(/xar TOC decoded \d+ bytes, expected 1/u),
});
});
it("records stored-checksum contradictions without losing decoded content identity", async () => {
const directory = await createTestTempDirectory("rea-pkg-stored-integrity-");
const path = join(directory, "Installer.pkg");
const decoded = Buffer.from("same decoded bytes");
const archivedSha1 = "0".repeat(40);
await writeFile(
path,
xarArchive([
{ name: "encoded", encoding: "zlib", data: decoded, archivedSha1 },
{ name: "plain", data: Buffer.from("ok") },
]),
);
const inventory = await scanArtifactInventory(path, {
integrity: { mode: "record-and-continue" },
});
const entry = inventory.occurrences.find(
({ logical_path }) => logical_path === "encoded",
);
expect(entry).toMatchObject({ hash_status: "mismatched" });
expect(
inventory.nodes.find(
({ artifact_id }) => artifact_id === entry?.artifact_id,
),
).toMatchObject({
size: decoded.length,
sha256: createHash("sha256").update(decoded).digest("hex"),
});
const observed = createHash("sha1")
.update(deflateSync(decoded))
.digest("hex");
expect(entry?.limitations).toContain(
`Declared stored sha1 ${archivedSha1} disagrees with observed ${observed}.`,
);
await expect(scanArtifactInventory(path)).rejects.toMatchObject({
reason: "integrity",
});
});