fix(artifacts): preserve unavailable members and stored checksum evidence
This commit is contained in:
@@ -35,11 +35,23 @@ export interface ArtifactEntry {
|
||||
};
|
||||
}
|
||||
|
||||
/** Checksum evidence for a member representation the scanner cannot hash itself. */
|
||||
export interface ArtifactChecksumObservation {
|
||||
readonly representation: "stored";
|
||||
readonly algorithm: string;
|
||||
readonly declared: string;
|
||||
readonly observed: string;
|
||||
}
|
||||
|
||||
/** Read-only adapter over one directory, archive, or virtual container. */
|
||||
export interface ArtifactReader {
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "pkg" | "file";
|
||||
entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry>;
|
||||
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable>;
|
||||
/** Checksums observed after the complete member stream has been consumed. */
|
||||
integrityObservations?(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[];
|
||||
provenance(): readonly ArtifactCommand[];
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ export interface XarFixtureMember {
|
||||
readonly mode?: string;
|
||||
/** Replace the declared extracted checksum, to model tampering. */
|
||||
readonly extractedSha1?: string;
|
||||
/** Replace the declared stored checksum while retaining decodable content. */
|
||||
readonly archivedSha1?: string;
|
||||
/** Store these bytes instead of encoding `data`, to model corrupt members. */
|
||||
readonly archived?: Uint8Array;
|
||||
readonly children?: readonly XarFixtureMember[];
|
||||
@@ -29,6 +31,8 @@ export const xarArchive = (
|
||||
readonly corruptTocChecksum?: boolean;
|
||||
/** Declared TOC checksum size; the real digest is 20 bytes. */
|
||||
readonly tocChecksumSize?: number;
|
||||
/** Override the fixed-header decoded length without altering the TOC. */
|
||||
readonly tocDecodedSize?: number;
|
||||
} = {},
|
||||
): Uint8Array => {
|
||||
const heap: Uint8Array[] = [new Uint8Array(20)];
|
||||
@@ -51,7 +55,7 @@ export const xarArchive = (
|
||||
: member.encoding === "bzip2"
|
||||
? "application/x-bzip2"
|
||||
: "application/octet-stream";
|
||||
data = `<data><archived-checksum style="sha1">${sha1(archived)}</archived-checksum><extracted-checksum style="sha1">${member.extractedSha1 ?? sha1(member.data)}</extracted-checksum><encoding style="${style}"/><size>${member.data.length}</size><offset>${offset}</offset><length>${archived.length}</length></data>`;
|
||||
data = `<data><archived-checksum style="sha1">${member.archivedSha1 ?? sha1(archived)}</archived-checksum><extracted-checksum style="sha1">${member.extractedSha1 ?? sha1(member.data)}</extracted-checksum><encoding style="${style}"/><size>${member.data.length}</size><offset>${offset}</offset><length>${archived.length}</length></data>`;
|
||||
}
|
||||
const link =
|
||||
type === "symlink"
|
||||
@@ -74,7 +78,7 @@ export const xarArchive = (
|
||||
header.writeUInt16BE(28, 4);
|
||||
header.writeUInt16BE(1, 6);
|
||||
header.writeBigUInt64BE(BigInt(compressed.length), 8);
|
||||
header.writeBigUInt64BE(BigInt(toc.length), 16);
|
||||
header.writeBigUInt64BE(BigInt(options.tocDecodedSize ?? toc.length), 16);
|
||||
header.writeUInt32BE(1, 24);
|
||||
return Buffer.concat([header, compressed, ...heap]);
|
||||
};
|
||||
|
||||
@@ -146,7 +146,10 @@ describe("xar installer package reader", () => {
|
||||
);
|
||||
const [entry] = await collect(bzip);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
await expect(bzip.open(entry)).rejects.toMatchObject({ reason: "format" });
|
||||
expect(entry.contentUnavailable).toBe(true);
|
||||
await expect(bzip.open(entry)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
await bzip.close();
|
||||
const notXar = new XarArtifactReader(
|
||||
await writePackage(Buffer.from("not an archive at all")),
|
||||
|
||||
@@ -14,6 +14,7 @@ import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactChecksumObservation,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
|
||||
@@ -177,6 +178,13 @@ const memberLimitations = (
|
||||
: [
|
||||
`Member declares unsupported archived checksum ${unsupportedArchived.style}; stored-byte integrity not verified.`,
|
||||
]),
|
||||
...(member.data !== undefined &&
|
||||
member.data.encoding !== "application/octet-stream" &&
|
||||
member.data.encoding !== "application/x-gzip"
|
||||
? [
|
||||
`Unsupported xar encoding ${member.data.encoding}; content is unavailable and was not expanded.`,
|
||||
]
|
||||
: []),
|
||||
...classified.limitations,
|
||||
];
|
||||
|
||||
@@ -311,6 +319,9 @@ class ChecksumVerifier extends Transform {
|
||||
readonly algorithm: string;
|
||||
readonly value: string;
|
||||
},
|
||||
private readonly record?: (
|
||||
observation: ArtifactChecksumObservation,
|
||||
) => void,
|
||||
) {
|
||||
super();
|
||||
this.#hash = createHash(expected.algorithm);
|
||||
@@ -327,6 +338,16 @@ class ChecksumVerifier extends Transform {
|
||||
|
||||
override _flush(done: TransformCallback): void {
|
||||
const observed = this.#hash.digest("hex");
|
||||
if (this.record !== undefined) {
|
||||
this.record({
|
||||
representation: "stored",
|
||||
algorithm: this.expected.algorithm,
|
||||
declared: this.expected.value,
|
||||
observed,
|
||||
});
|
||||
done();
|
||||
return;
|
||||
}
|
||||
done(
|
||||
observed === this.expected.value
|
||||
? null
|
||||
@@ -352,6 +373,7 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
#handle: FileHandle | undefined;
|
||||
#heap = 0;
|
||||
#members: readonly XarMember[] = [];
|
||||
readonly #integrity = new Map<string, ArtifactChecksumObservation[]>();
|
||||
|
||||
readonly #verifyChecksums: boolean;
|
||||
|
||||
@@ -359,8 +381,8 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
* @param options.verifyChecksums Fail `open()` streams whose bytes disagree
|
||||
* with the member's extracted checksum (default). Inventory passes false and
|
||||
* verifies the declared checksum itself under the caller's integrity policy.
|
||||
* Archived checksums are always verified: the scanner only observes decoded
|
||||
* bytes, so it cannot apply its policy to the stored stream.
|
||||
* Archived checksums are always computed. When inventory owns the policy,
|
||||
* integrityObservations exposes their declared/observed values after reading.
|
||||
*/
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
@@ -404,6 +426,11 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
const inflated = inflateSync(compressed, {
|
||||
maxOutputLength: MAX_TOC_BYTES,
|
||||
});
|
||||
if (BigInt(inflated.length) !== tocSize)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC decoded ${inflated.length} bytes, expected ${tocSize}`,
|
||||
);
|
||||
try {
|
||||
xml = new TextDecoder("utf-8", { fatal: true }).decode(inflated);
|
||||
} catch (cause: unknown) {
|
||||
@@ -593,10 +620,15 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
assertRawSize(member, data);
|
||||
const unsupportedChecksum = data?.unsupportedChecksum;
|
||||
const unsupportedArchived = data?.unsupportedArchivedChecksum;
|
||||
const unsupportedEncoding =
|
||||
data !== undefined &&
|
||||
data.encoding !== "application/octet-stream" &&
|
||||
data.encoding !== "application/x-gzip";
|
||||
const classified =
|
||||
member.kind === "file" &&
|
||||
unsupportedChecksum === undefined &&
|
||||
unsupportedArchived === undefined
|
||||
unsupportedArchived === undefined &&
|
||||
!unsupportedEncoding
|
||||
? await this.#nestedArchive(member)
|
||||
: {
|
||||
nested: undefined as XarNestedArchive | undefined,
|
||||
@@ -622,6 +654,11 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
classified,
|
||||
),
|
||||
adapterKey: member.path,
|
||||
...(unsupportedEncoding ||
|
||||
unsupportedChecksum !== undefined ||
|
||||
unsupportedArchived !== undefined
|
||||
? { contentUnavailable: true }
|
||||
: {}),
|
||||
...(classified.nested === undefined
|
||||
? {}
|
||||
: { nestedArchive: classified.nested }),
|
||||
@@ -630,6 +667,7 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
cancelled(signal);
|
||||
this.#integrity.delete(entry.adapterKey);
|
||||
await this.#load(signal);
|
||||
const member = this.#members.find(({ path }) => path === entry.adapterKey);
|
||||
if (member === undefined || member.kind !== "file")
|
||||
@@ -675,8 +713,19 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
const archivedVerifier =
|
||||
archived === undefined
|
||||
? undefined
|
||||
: new ChecksumVerifier(`${entry.path} (archived)`, archived);
|
||||
: new ChecksumVerifier(
|
||||
`${entry.path} (archived)`,
|
||||
archived,
|
||||
this.#verifyChecksums
|
||||
? undefined
|
||||
: (observation) => {
|
||||
this.#integrity.set(entry.adapterKey, [observation]);
|
||||
},
|
||||
);
|
||||
const raw = Readable.from(this.#chunks(data.offset, data.length, signal));
|
||||
raw.on("error", (cause: unknown) =>
|
||||
archivedVerifier?.destroy(memberFailure(entry.path, cause)),
|
||||
);
|
||||
const archivedChecked =
|
||||
archivedVerifier === undefined ? raw : raw.pipe(archivedVerifier);
|
||||
let decoded: Readable;
|
||||
@@ -739,6 +788,13 @@ export class XarArtifactReader implements ArtifactReader {
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Stored-byte checksums computed while consuming the member stream. */
|
||||
integrityObservations(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[] {
|
||||
return this.#integrity.get(entry.adapterKey) ?? [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
const handle = this.#handle;
|
||||
this.#handle = undefined;
|
||||
|
||||
@@ -46,6 +46,7 @@ export const extractArtifact = async (
|
||||
const selectedOccurrences = snapshot.occurrences.filter(
|
||||
(occurrence) =>
|
||||
(occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") &&
|
||||
occurrence.hash_status !== "unavailable" &&
|
||||
occurrence.logical_path !== ".",
|
||||
);
|
||||
const selectedIds = new Set(
|
||||
@@ -65,6 +66,9 @@ export const extractArtifact = async (
|
||||
manifest: snapshot.manifest,
|
||||
occurrences,
|
||||
nodes,
|
||||
unavailablePaths: snapshot.occurrences
|
||||
.filter(({ hash_status }) => hash_status === "unavailable")
|
||||
.map(({ logical_path }) => logical_path),
|
||||
};
|
||||
const selected = selectedOccurrences.map((occurrence) => {
|
||||
if (
|
||||
@@ -181,6 +185,11 @@ const materializeSelection = async ({
|
||||
}
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (
|
||||
entry.contentUnavailable === true &&
|
||||
inventory.unavailablePaths.includes(path)
|
||||
)
|
||||
continue;
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
@@ -263,7 +272,11 @@ const createExtractionResult = (
|
||||
cleanup: { attempted: false, verified: true, residual_paths: [] },
|
||||
provenance: [],
|
||||
limitations: [
|
||||
"All regular files in the active artifact were materialized, including members of expanded PKG Payload/Scripts archives.",
|
||||
"All available selected regular files were materialized; expanded PKG Payload/Scripts archives are represented by their extracted children.",
|
||||
...inventory.unavailablePaths.map(
|
||||
(path) =>
|
||||
`Unavailable member ${path} was not extracted; see its inventory occurrence for the reason.`,
|
||||
),
|
||||
],
|
||||
});
|
||||
};
|
||||
@@ -272,6 +285,7 @@ interface LoadedInventory {
|
||||
readonly manifest: ArtifactGraphManifest;
|
||||
readonly occurrences: ReadonlyMap<string, ArtifactOccurrence>;
|
||||
readonly nodes: ReadonlyMap<string, ArtifactNode>;
|
||||
readonly unavailablePaths: readonly string[];
|
||||
}
|
||||
|
||||
const collectOccurrences = (
|
||||
|
||||
@@ -235,7 +235,11 @@ const digestArtifactEntry = async (
|
||||
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
|
||||
const checksumMismatch =
|
||||
checksum !== undefined && checksum.value !== digest.also;
|
||||
const mismatched = sha256Mismatch || checksumMismatch;
|
||||
const storedMismatches = (
|
||||
currentReader.integrityObservations?.(entry) ?? []
|
||||
).filter(({ declared, observed }) => declared !== observed);
|
||||
const mismatched =
|
||||
sha256Mismatch || checksumMismatch || storedMismatches.length > 0;
|
||||
if (mismatched && context.integrity.mode === "fail")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
@@ -277,11 +281,17 @@ const digestArtifactEntry = async (
|
||||
: [],
|
||||
}),
|
||||
mismatched,
|
||||
mismatchDetails: checksumMismatch
|
||||
? [
|
||||
`Declared ${checksum.algorithm} ${checksum.value} disagrees with observed ${digest.also ?? "unavailable"}.`,
|
||||
]
|
||||
: [],
|
||||
mismatchDetails: [
|
||||
...(checksumMismatch
|
||||
? [
|
||||
`Declared ${checksum.algorithm} ${checksum.value} disagrees with observed ${digest.also ?? "unavailable"}.`,
|
||||
]
|
||||
: []),
|
||||
...storedMismatches.map(
|
||||
({ algorithm, declared, observed }) =>
|
||||
`Declared stored ${algorithm} ${declared} disagrees with observed ${observed}.`,
|
||||
),
|
||||
],
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { deflateSync } from "node:zlib";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { extractArtifact } from "../../../src/artifacts/extraction/ArtifactExtraction.js";
|
||||
import {
|
||||
MODE,
|
||||
gzipCpio,
|
||||
xarArchive,
|
||||
} from "../../../src/artifacts/InstallerPackage.fixture.js";
|
||||
import { scanArtifactInventory } from "../../../src/artifacts/inventory/ArtifactInventory.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
it("extracts regular nested siblings while retaining skipped special-node evidence", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-extract-special-");
|
||||
const inputPath = join(directory, "Installer.pkg");
|
||||
const outputRoot = join(directory, "extracted");
|
||||
await writeFile(
|
||||
inputPath,
|
||||
xarArchive([
|
||||
{
|
||||
name: "Payload",
|
||||
data: gzipCpio([
|
||||
{ name: "./fifo", mode: MODE.fifo },
|
||||
{ name: "./usr/bin/tool", mode: MODE.executable, data: "tool bytes" },
|
||||
]),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const extracted = await extractArtifact({
|
||||
inputPath,
|
||||
outputRoot,
|
||||
inputFormat: "pkg",
|
||||
});
|
||||
expect(await readFile(join(outputRoot, "Payload/usr/bin/tool"), "utf8")).toBe(
|
||||
"tool bytes",
|
||||
);
|
||||
expect(extracted.artifacts.map(({ relative_path }) => relative_path)).toEqual(
|
||||
["Payload/usr/bin/tool"],
|
||||
);
|
||||
expect(extracted.extraction_manifest.selected_occurrence_ids).toHaveLength(1);
|
||||
expect(extracted.limitations).toContainEqual(
|
||||
expect.stringContaining("Payload/fifo was not extracted"),
|
||||
);
|
||||
});
|
||||
|
||||
it("retains unsupported-encoding occurrences under the default inventory policy", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-unsupported-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{ name: "encoded", encoding: "bzip2", data: Buffer.from("opaque") },
|
||||
{ name: "plain", data: Buffer.from("available") },
|
||||
]),
|
||||
);
|
||||
const inventory = await scanArtifactInventory(path);
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "encoded",
|
||||
),
|
||||
).toMatchObject({
|
||||
hash_status: "unavailable",
|
||||
artifact_id: null,
|
||||
limitations: [
|
||||
expect.stringContaining("Unsupported xar encoding application/x-bzip2"),
|
||||
],
|
||||
});
|
||||
expect(
|
||||
inventory.occurrences.find(({ logical_path }) => logical_path === "plain"),
|
||||
).toMatchObject({ hash_status: "verified" });
|
||||
});
|
||||
|
||||
it("rejects a header whose decoded TOC length contradicts the actual data", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-toc-size-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([{ name: "plain", data: Buffer.from("ok") }], {
|
||||
tocDecodedSize: 1,
|
||||
}),
|
||||
);
|
||||
await expect(scanArtifactInventory(path)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringMatching(/xar TOC decoded \d+ bytes, expected 1/u),
|
||||
});
|
||||
});
|
||||
|
||||
it("records stored-checksum contradictions without losing decoded content identity", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-stored-integrity-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
const decoded = Buffer.from("same decoded bytes");
|
||||
const archivedSha1 = "0".repeat(40);
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{ name: "encoded", encoding: "zlib", data: decoded, archivedSha1 },
|
||||
{ name: "plain", data: Buffer.from("ok") },
|
||||
]),
|
||||
);
|
||||
const inventory = await scanArtifactInventory(path, {
|
||||
integrity: { mode: "record-and-continue" },
|
||||
});
|
||||
const entry = inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "encoded",
|
||||
);
|
||||
expect(entry).toMatchObject({ hash_status: "mismatched" });
|
||||
expect(
|
||||
inventory.nodes.find(
|
||||
({ artifact_id }) => artifact_id === entry?.artifact_id,
|
||||
),
|
||||
).toMatchObject({
|
||||
size: decoded.length,
|
||||
sha256: createHash("sha256").update(decoded).digest("hex"),
|
||||
});
|
||||
const observed = createHash("sha1")
|
||||
.update(deflateSync(decoded))
|
||||
.digest("hex");
|
||||
expect(entry?.limitations).toContain(
|
||||
`Declared stored sha1 ${archivedSha1} disagrees with observed ${observed}.`,
|
||||
);
|
||||
await expect(scanArtifactInventory(path)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user