Compare commits
39
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7946c3ad4 | ||
|
|
0ade49ae90 | ||
|
|
c9aa907495 | ||
|
|
a7fdb8b626 | ||
|
|
938d7adf70 | ||
|
|
ae44e01c04 | ||
|
|
f1f0a986fd | ||
|
|
6dda47c98c | ||
|
|
23578197cc | ||
|
|
64d2878882 | ||
|
|
31570a22fb | ||
|
|
3e9d38ccd8 | ||
|
|
c47693bb23 | ||
|
|
1d27061a65 | ||
|
|
d9a39e5b48 | ||
|
|
3664a4a57e | ||
|
|
c5a16b7e21 | ||
|
|
be14aa0de9 | ||
|
|
8ee7d23cae | ||
|
|
9b38f3a0af | ||
|
|
c541052ca9 | ||
|
|
3910bedae8 | ||
|
|
949347a148 | ||
|
|
da18453d4d | ||
|
|
b196c5ac8f | ||
|
|
ef0bf91de9 | ||
|
|
300682d4ba | ||
|
|
15b42b5825 | ||
|
|
116d722339 | ||
|
|
31a642f73d | ||
|
|
5389b233b8 | ||
|
|
79e686f6a2 | ||
|
|
f3e41f1e0c | ||
|
|
ce40fe6ded | ||
|
|
bb58eeef27 | ||
|
|
7857d2d329 | ||
|
|
667bf164f9 | ||
|
|
daf504b62f | ||
|
|
54dd1a7604 |
@@ -102,12 +102,8 @@ jobs:
|
||||
with:
|
||||
name: rea-dist
|
||||
path: dist
|
||||
- name: Restore generated outputs from downloaded build
|
||||
run: |
|
||||
node scripts/generate-mcp-tool-catalog.mjs
|
||||
node scripts/generate-skill-metadata.mjs
|
||||
node scripts/generate-product-catalog.mjs
|
||||
node scripts/generate-completion-ledger.mjs
|
||||
- name: Generate projections from downloaded build
|
||||
run: npm run build:projections
|
||||
- run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: ${{ !cancelled() }}
|
||||
@@ -321,6 +317,7 @@ jobs:
|
||||
- run: npm run verify:keyed-archive
|
||||
- run: npm run verify:asset-catalog
|
||||
- run: npm run verify:macos-bundle
|
||||
- run: npm run verify:installer-package
|
||||
- name: Native installer and compiled NIB hierarchy regressions
|
||||
run: npm run test:focused -- tests/boundary/filesystem/installScript.test.ts tests/boundary/filesystem/nibNativeHierarchy.test.ts tests/boundary/filesystem/interfaceBuilderBundle.test.ts
|
||||
- name: Real binary metadata and Interface Builder integration
|
||||
|
||||
@@ -398,7 +398,29 @@
|
||||
"calculated_sha256": {
|
||||
"anyOf": [{ "type": "string" }, { "type": "null" }]
|
||||
},
|
||||
"unpacked": { "type": "boolean" }
|
||||
"unpacked": { "type": "boolean" },
|
||||
"checksum_mismatches": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"representation": {
|
||||
"type": "string",
|
||||
"enum": ["stored", "decoded"]
|
||||
},
|
||||
"algorithm": { "type": "string" },
|
||||
"declared": { "type": "string" },
|
||||
"observed": { "type": "string" }
|
||||
},
|
||||
"required": [
|
||||
"representation",
|
||||
"algorithm",
|
||||
"declared",
|
||||
"observed"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"logical_path",
|
||||
|
||||
@@ -280,6 +280,12 @@ Artifact integrity fails closed by default. A request can explicitly select
|
||||
`integrity_policy=record-and-continue` when the investigation needs verified
|
||||
siblings to continue after a mismatch.
|
||||
|
||||
Strict member failures include `details.checksum_mismatches` for conflicting
|
||||
stored or decoded checksums, preserving each algorithm and declared/observed
|
||||
value. An unsupported checksum algorithm leaves that verification facet unknown;
|
||||
readable member bytes are still hashed and can be extracted, with the declared
|
||||
algorithm and value retained in limitations.
|
||||
|
||||
Contradictory bytes are quarantined from nested expansion and recorded with
|
||||
declared and observed hashes, trust, provenance, path, and unpacked state.
|
||||
Verified siblings continue. Comparisons classify the result as a contradiction
|
||||
|
||||
@@ -98,6 +98,14 @@ an XML graph golden. `verify:asset-catalog` compiles source-owned colors with
|
||||
`actool`, invokes real `assetutil`, then checks CLI/MCP results, exact catalog
|
||||
digest, every raw metadata field, pagination and malformed input rejection.
|
||||
Neither artifact workflow requires Hopper or launches it. Both run in macOS CI.
|
||||
`verify:installer-package` needs only macOS's `pkgbuild`, `productbuild` and
|
||||
`pkgutil`. It builds a component package with install scripts, a product package
|
||||
around it, and a pbzx (`--compression latest`) package. It checks that
|
||||
`inspect-artifact` expands Scripts and Payload with exact content digests, and
|
||||
that the expanded Payload members equal Apple's `pkgutil --payload-files`
|
||||
listing. It also checks that the product package nests its component, that the
|
||||
pbzx payload is recorded with a limitation instead of being expanded, and that
|
||||
CLI and stdio MCP results agree.
|
||||
`verify:macos-bundle` needs only macOS with Command Line Tools. It compiles a
|
||||
source-owned app with `clang`: a versioned framework, XPC services, an app
|
||||
extension, a login item, a privileged helper, launchd plists, and a helper tool,
|
||||
|
||||
Generated
+1
@@ -17,6 +17,7 @@
|
||||
"@lydell/node-pty": "1.1.0",
|
||||
"@modelcontextprotocol/client": "2.3.1",
|
||||
"@modelcontextprotocol/server": "2.3.1",
|
||||
"@xmldom/xmldom": "0.9.12",
|
||||
"@xterm/addon-serialize": "0.14.0",
|
||||
"@xterm/headless": "6.0.0",
|
||||
"@zip.js/zip.js": "2.15.0",
|
||||
|
||||
+4
-1
@@ -87,7 +87,8 @@
|
||||
"mcp-catalog:check": "node scripts/generate-mcp-tool-catalog.mjs --check",
|
||||
"prebuild": "npm run deps:check && npm run metadata:check",
|
||||
"build": "node scripts/run-exclusive.mjs build npm run build:unlocked",
|
||||
"build:unlocked": "node scripts/clean-build-output.mjs && tsc -p tsconfig.build.json && node scripts/generate-mcp-tool-catalog.mjs && node scripts/generate-skill-metadata.mjs && node scripts/generate-product-catalog.mjs && node scripts/generate-completion-ledger.mjs",
|
||||
"build:unlocked": "node scripts/clean-build-output.mjs && tsc -p tsconfig.build.json && npm run build:projections",
|
||||
"build:projections": "node scripts/generate-mcp-tool-catalog.mjs && node scripts/generate-skill-metadata.mjs && node scripts/generate-product-catalog.mjs && node scripts/generate-completion-ledger.mjs",
|
||||
"precheck": "npm run deps:check",
|
||||
"check": "turbo run typecheck lint format:check knip metadata:check",
|
||||
"precheck:fast": "npm run deps:check",
|
||||
@@ -141,6 +142,7 @@
|
||||
"verify:apple-dispatch": "npm run build:cached && node scripts/verify/apple/dispatch.mjs",
|
||||
"verify:native-ui": "npm run build:cached && node scripts/verify-native-ui.mjs",
|
||||
"verify:keyed-archive": "npm run build:cached && node scripts/verify/apple/keyed-archive.mjs",
|
||||
"verify:installer-package": "npm run build:cached && node scripts/verify/apple/installer-package.mjs",
|
||||
"verify:macos-bundle": "npm run build:cached && node scripts/verify/apple/macos-bundle.mjs",
|
||||
"verify:ghidra": "npm run build:cached && node scripts/verify-real-ghidra.mjs",
|
||||
"verify:ida": "npm run build:cached && node scripts/verify-real-ida.mjs",
|
||||
@@ -231,6 +233,7 @@
|
||||
"@lydell/node-pty": "1.1.0",
|
||||
"@modelcontextprotocol/client": "2.3.1",
|
||||
"@modelcontextprotocol/server": "2.3.1",
|
||||
"@xmldom/xmldom": "0.9.12",
|
||||
"@xterm/addon-serialize": "0.14.0",
|
||||
"@xterm/headless": "6.0.0",
|
||||
"@zip.js/zip.js": "2.15.0",
|
||||
|
||||
@@ -0,0 +1,174 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import {
|
||||
artifactCli,
|
||||
artifactMcpResult,
|
||||
withArtifactMcp,
|
||||
} from "../../lib/artifact-e2e.mjs";
|
||||
|
||||
const exec = promisify(execFile);
|
||||
|
||||
if (process.platform !== "darwin")
|
||||
throw new Error(
|
||||
"Installer package verification requires macOS pkgbuild, productbuild and pkgutil",
|
||||
);
|
||||
for (const tool of [
|
||||
"/usr/bin/pkgbuild",
|
||||
"/usr/bin/productbuild",
|
||||
"/usr/sbin/pkgutil",
|
||||
])
|
||||
try {
|
||||
await exec(tool, ["--help"]);
|
||||
} catch (cause) {
|
||||
if (cause?.code === "ENOENT")
|
||||
throw new Error(`Installer package verification requires ${tool}`, {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
|
||||
const root = await mkdtemp(join(tmpdir(), "rea-installer-package-"));
|
||||
const sha256 = (text) => createHash("sha256").update(text).digest("hex");
|
||||
|
||||
/** Inventory occurrences keyed by logical path, with each node's content digest. */
|
||||
const inventoryOf = (inspection) => {
|
||||
const inventory = inspection.substeps[0].evidence.normalized_result;
|
||||
const nodes = new Map(
|
||||
inventory.nodes.map((node) => [node.artifact_id, node]),
|
||||
);
|
||||
return {
|
||||
inventory,
|
||||
entries: new Map(
|
||||
inventory.occurrences.map((occurrence) => [
|
||||
occurrence.logical_path,
|
||||
{ ...occurrence, sha256: nodes.get(occurrence.artifact_id)?.sha256 },
|
||||
]),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
/** Apple's Bom listing of a component package's payload, without the root. */
|
||||
const bomPaths = async (path) =>
|
||||
(await exec("/usr/sbin/pkgutil", ["--payload-files", path])).stdout
|
||||
.split("\n")
|
||||
.filter((line) => line.length > 0 && line !== ".")
|
||||
.map((line) => line.replace(/^\.\//u, ""))
|
||||
.sort();
|
||||
|
||||
try {
|
||||
const payload = join(root, "payload");
|
||||
const scripts = join(root, "scripts");
|
||||
await mkdir(join(payload, "usr/local/rea"), { recursive: true });
|
||||
await mkdir(scripts);
|
||||
await writeFile(join(payload, "usr/local/rea/hello.txt"), "hello\n");
|
||||
for (const name of ["preinstall", "postinstall"]) {
|
||||
await writeFile(join(scripts, name), `#!/bin/sh\necho ${name}\n`);
|
||||
await chmod(join(scripts, name), 0o755);
|
||||
}
|
||||
const component = join(root, "Component.pkg");
|
||||
const build = (output, extra) =>
|
||||
exec("/usr/bin/pkgbuild", [
|
||||
"--quiet",
|
||||
"--root",
|
||||
payload,
|
||||
"--identifier",
|
||||
"com.example.rea.installer",
|
||||
"--version",
|
||||
"1.0",
|
||||
"--install-location",
|
||||
"/",
|
||||
...extra,
|
||||
output,
|
||||
]);
|
||||
await build(component, ["--scripts", scripts]);
|
||||
const product = join(root, "Product.pkg");
|
||||
await exec("/usr/bin/productbuild", [
|
||||
"--quiet",
|
||||
"--package",
|
||||
component,
|
||||
product,
|
||||
]);
|
||||
const pbzx = join(root, "Latest.pkg");
|
||||
await build(pbzx, ["--compression", "latest", "--min-os-version", "13.0"]);
|
||||
|
||||
const componentInventory = inventoryOf(
|
||||
await artifactCli("inspect-artifact", component),
|
||||
);
|
||||
assert.equal(componentInventory.inventory.manifest.root_format, "pkg");
|
||||
const payloadPaths = [...componentInventory.entries.keys()]
|
||||
.filter((path) => path.startsWith("Payload/"))
|
||||
.map((path) => path.slice("Payload/".length))
|
||||
.sort();
|
||||
assert.deepEqual(
|
||||
payloadPaths,
|
||||
await bomPaths(component),
|
||||
"Payload members differ from pkgutil",
|
||||
);
|
||||
assert.equal(
|
||||
componentInventory.entries.get("Payload/usr/local/rea/hello.txt")?.sha256,
|
||||
sha256("hello\n"),
|
||||
);
|
||||
for (const name of ["preinstall", "postinstall"]) {
|
||||
const script = componentInventory.entries.get(`Scripts/${name}`);
|
||||
assert.equal(script?.sha256, sha256(`#!/bin/sh\necho ${name}\n`));
|
||||
assert.equal(script?.executable, true);
|
||||
}
|
||||
assert.ok(componentInventory.entries.has("PackageInfo"));
|
||||
assert.ok(componentInventory.entries.has("Bom"));
|
||||
|
||||
const productInventory = inventoryOf(
|
||||
await artifactCli("inspect-artifact", product),
|
||||
);
|
||||
assert.ok(productInventory.entries.has("Distribution"));
|
||||
assert.equal(
|
||||
productInventory.entries.get(
|
||||
"Component.pkg/Payload/usr/local/rea/hello.txt",
|
||||
)?.sha256,
|
||||
sha256("hello\n"),
|
||||
);
|
||||
assert.ok(productInventory.entries.has("Component.pkg/Scripts/postinstall"));
|
||||
|
||||
const pbzxInventory = inventoryOf(
|
||||
await artifactCli("inspect-artifact", pbzx),
|
||||
);
|
||||
const pbzxPayload = pbzxInventory.entries.get("Payload");
|
||||
assert.ok(
|
||||
pbzxPayload?.limitations.some((line) => line.includes("pbzx")),
|
||||
JSON.stringify(pbzxPayload),
|
||||
);
|
||||
assert.ok(
|
||||
![...pbzxInventory.entries.keys()].some((path) =>
|
||||
path.startsWith("Payload/"),
|
||||
),
|
||||
);
|
||||
|
||||
await withArtifactMcp(component, async (client) => {
|
||||
const viaMcp = inventoryOf(
|
||||
await artifactMcpResult(client, "inspect_artifact"),
|
||||
);
|
||||
assert.deepEqual(
|
||||
viaMcp.inventory.manifest,
|
||||
componentInventory.inventory.manifest,
|
||||
);
|
||||
});
|
||||
|
||||
process.stdout.write(
|
||||
`${JSON.stringify({
|
||||
ok: true,
|
||||
mocked: false,
|
||||
cli: true,
|
||||
stdio_mcp: true,
|
||||
payload_members: payloadPaths.length,
|
||||
matches_pkgutil_bom: true,
|
||||
product_nested: true,
|
||||
pbzx_payload: "recorded-not-expanded",
|
||||
})}\n`,
|
||||
);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -87,7 +87,12 @@ header, depending on the binary and analyzer version.
|
||||
## Packages and extraction
|
||||
|
||||
Use `inspect_artifact` for application bundles, archives, ZIP/APK/IPA/MSIX/AppX,
|
||||
ASAR, or DMG inputs when the artifact graph and findings help answer the
|
||||
ASAR, DMG, or PKG inputs. A PKG inventory includes Distribution, PackageInfo,
|
||||
the pre/postinstall scripts, and the gzip payload's files. Pbzx payloads are
|
||||
recorded only. Checksums declared in the PKG's table of contents are verified;
|
||||
with `integrity_policy: "record-and-continue"`, a mismatched member stays in
|
||||
the graph as `mismatched`, with the declared and observed values, and is not
|
||||
expanded. Inspect it when the artifact graph and findings help answer the
|
||||
question. It returns the complete artifact graph inline. Cite graph manifest
|
||||
IDs when using them.
|
||||
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { Transform, type TransformCallback } from "node:stream";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
/** Terminal exhaustion of the budget shared by all readers in one traversal. */
|
||||
export class ArtifactBudgetExceeded extends ArtifactReaderFailure {
|
||||
constructor(message: string) {
|
||||
super("limit", message);
|
||||
}
|
||||
}
|
||||
|
||||
/** One decoded-byte ceiling shared by an archive and all nested readers. */
|
||||
export class ArtifactDecodedBudget {
|
||||
#used = 0;
|
||||
#metadataUsed = 0;
|
||||
|
||||
constructor(
|
||||
private readonly maximum = 512 * 1024 * 1024,
|
||||
private readonly maximumMetadata = 128 * 1024 * 1024,
|
||||
) {}
|
||||
|
||||
/** Charge decoding work, including repeated reads, before exposing a chunk. */
|
||||
consume(bytes: number, path: string): void {
|
||||
if (
|
||||
!Number.isSafeInteger(bytes) ||
|
||||
bytes < 0 ||
|
||||
bytes > this.maximum - this.#used
|
||||
)
|
||||
throw new ArtifactBudgetExceeded(
|
||||
`Decoded archive budget of ${this.maximum} bytes exhausted at ${path}`,
|
||||
);
|
||||
this.#used += bytes;
|
||||
}
|
||||
|
||||
/** Charge retained path text, path-trie segments, and per-entry graph/index capacity. */
|
||||
consumeEntry(path: string): void {
|
||||
const segments = path.split("/").length;
|
||||
const units = 1024 + Buffer.byteLength(path) * 2 + segments * 256;
|
||||
if (units > this.maximumMetadata - this.#metadataUsed)
|
||||
throw new ArtifactBudgetExceeded(
|
||||
`Retained archive metadata budget of ${this.maximumMetadata} capacity units exhausted at ${path}`,
|
||||
);
|
||||
this.#metadataUsed += units;
|
||||
}
|
||||
}
|
||||
|
||||
/** Decoder-stage budget/cancellation guard, independent of member metadata. */
|
||||
export class ArtifactBudgetTransform extends Transform {
|
||||
constructor(
|
||||
private readonly budget: ArtifactDecodedBudget,
|
||||
private readonly path: string,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer,
|
||||
_encoding: BufferEncoding,
|
||||
done: TransformCallback,
|
||||
): void {
|
||||
if (this.signal?.aborted) {
|
||||
done(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`Archive decoding cancelled at ${this.path}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.budget.consume(chunk.length, this.path);
|
||||
} catch (cause: unknown) {
|
||||
done(cause instanceof Error ? cause : new Error(String(cause)));
|
||||
return;
|
||||
}
|
||||
done(null, chunk);
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ export type HashResult = {
|
||||
readonly sha256: string;
|
||||
readonly bytes: number;
|
||||
readonly prefix: Buffer;
|
||||
/** Digest in the additionally requested algorithm, when one was requested. */
|
||||
readonly also?: string;
|
||||
};
|
||||
|
||||
/** Bounded classification evidence; this is not an executable format size limit. */
|
||||
@@ -27,8 +29,11 @@ export const abortIfNeeded = (signal?: AbortSignal): void => {
|
||||
export const hashReadable = async (
|
||||
stream: Readable,
|
||||
signal?: AbortSignal,
|
||||
alsoAlgorithm?: string,
|
||||
): Promise<HashResult> => {
|
||||
const hash = createHash("sha256");
|
||||
const also =
|
||||
alsoAlgorithm === undefined ? undefined : createHash(alsoAlgorithm);
|
||||
const prefixes: Buffer[] = [];
|
||||
let prefixBytes = 0;
|
||||
let bytes = 0;
|
||||
@@ -37,6 +42,7 @@ export const hashReadable = async (
|
||||
const chunk = streamChunkToBuffer(raw);
|
||||
bytes += chunk.length;
|
||||
hash.update(chunk);
|
||||
also?.update(chunk);
|
||||
if (prefixBytes < ARTIFACT_CLASSIFICATION_PREFIX_BYTES) {
|
||||
const selected = chunk.subarray(
|
||||
0,
|
||||
@@ -46,5 +52,10 @@ export const hashReadable = async (
|
||||
prefixBytes += selected.length;
|
||||
}
|
||||
}
|
||||
return { sha256: hash.digest("hex"), bytes, prefix: Buffer.concat(prefixes) };
|
||||
return {
|
||||
sha256: hash.digest("hex"),
|
||||
bytes,
|
||||
prefix: Buffer.concat(prefixes),
|
||||
...(also === undefined ? {} : { also: also.digest("hex") }),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -36,7 +36,7 @@ export const artifactCapabilities = (
|
||||
requiresRoot: false,
|
||||
}),
|
||||
limitations: Object.freeze([
|
||||
"DMG child inventory automatically uses a read-only native macOS mount; PKG remains root-hash-only.",
|
||||
"DMG child inventory automatically uses a read-only native macOS mount. PKG (xar) inventory expands gzip-cpio Scripts and Payload members; pbzx payloads are recorded without expansion.",
|
||||
"ASAR files discovered in filesystem-backed inventories are expanded without bulk extraction; other nested containers remain recorded only.",
|
||||
]),
|
||||
};
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import type { Readable } from "node:stream";
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import type { ZipPackageFormat } from "../domain/zipPackageFormat.js";
|
||||
import type { ArtifactDecodedBudget } from "./ArtifactDecodedBudget.js";
|
||||
import type {
|
||||
ArtifactChecksumObservation,
|
||||
ArtifactIntegrityFailureDetails,
|
||||
} from "../domain/artifactOperationError.js";
|
||||
export type { ArtifactChecksumObservation } from "../domain/artifactOperationError.js";
|
||||
|
||||
/** Archive-neutral entry metadata. Reader adapters never choose output paths. */
|
||||
export interface ArtifactEntry {
|
||||
@@ -15,6 +21,21 @@ export interface ArtifactEntry {
|
||||
readonly unpacked: boolean;
|
||||
readonly limitations: readonly string[];
|
||||
readonly adapterKey: string;
|
||||
/**
|
||||
* Container checksum of the member's bytes in an algorithm other than
|
||||
* SHA-256 (xar declares SHA-1 or MD5). The scan verifies it under the
|
||||
* caller's integrity policy.
|
||||
*/
|
||||
readonly declaredChecksum?: {
|
||||
readonly algorithm: "sha1" | "md5" | "sha512";
|
||||
readonly value: string;
|
||||
};
|
||||
/** The container records this member but does not hold its bytes, such as an unresolved hard link. */
|
||||
readonly contentUnavailable?: boolean;
|
||||
/** A consumed representation contradicted its declared integrity metadata. */
|
||||
readonly integrityMismatched?: boolean;
|
||||
/** Archive format of this member's own bytes, when the scan should expand it. */
|
||||
readonly nestedArchive?: "gzip-cpio";
|
||||
/** Filesystem identity captured during traversal, when supplied by an adapter. */
|
||||
readonly sourceIdentity?: {
|
||||
readonly device: number;
|
||||
@@ -24,9 +45,15 @@ export interface ArtifactEntry {
|
||||
|
||||
/** Read-only adapter over one directory, archive, or virtual container. */
|
||||
export interface ArtifactReader {
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "file";
|
||||
readonly format: "directory" | ZipPackageFormat | "asar" | "pkg" | "file";
|
||||
/** Shared safety budget for compressed representations and nested archives. */
|
||||
readonly decodedBudget?: ArtifactDecodedBudget;
|
||||
entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry>;
|
||||
open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable>;
|
||||
/** Checksums observed after the complete member stream has been consumed. */
|
||||
integrityObservations?(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[];
|
||||
provenance(): readonly ArtifactCommand[];
|
||||
close(): Promise<void>;
|
||||
}
|
||||
@@ -44,12 +71,7 @@ export class ArtifactReaderFailure extends Error {
|
||||
| "unavailable",
|
||||
message: string,
|
||||
options?: ErrorOptions,
|
||||
readonly details?: Readonly<{
|
||||
logicalPath: string;
|
||||
declaredSha256: string | null;
|
||||
calculatedSha256: string | null;
|
||||
unpacked: boolean;
|
||||
}>,
|
||||
readonly details?: ArtifactIntegrityFailureDetails,
|
||||
) {
|
||||
super(message, options);
|
||||
this.name = "ArtifactReaderFailure";
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { Readable, Transform } from "node:stream";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { expect, it } from "vitest";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
import { artifactStreamPipeline } from "./ArtifactStreamPipeline.js";
|
||||
|
||||
it("preserves tagged failures from intermediate verification stages", async () => {
|
||||
const failure = new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"stored checksum disagrees",
|
||||
);
|
||||
const verifier = new Transform({
|
||||
transform(_chunk, _encoding, done) {
|
||||
done(failure);
|
||||
},
|
||||
});
|
||||
const output = artifactStreamPipeline(
|
||||
Readable.from([Buffer.from("encoded")]),
|
||||
[verifier],
|
||||
{ path: "member" },
|
||||
);
|
||||
await expect(buffer(output)).rejects.toBe(failure);
|
||||
});
|
||||
|
||||
it("closes the source when the consumer stops before exhausting member data", async () => {
|
||||
const source = Readable.from(
|
||||
(function* () {
|
||||
for (let index = 0; index < 1000; index++) yield Buffer.alloc(64 * 1024);
|
||||
})(),
|
||||
);
|
||||
const closed = new Promise<void>((resolve) =>
|
||||
source.once("close", () => resolve()),
|
||||
);
|
||||
const output = artifactStreamPipeline(source, [], { path: "member" });
|
||||
for await (const chunk of output) {
|
||||
expect(Buffer.isBuffer(chunk)).toBe(true);
|
||||
break;
|
||||
}
|
||||
await closed;
|
||||
expect(source.destroyed).toBe(true);
|
||||
});
|
||||
|
||||
it("observes abort before a consumer starts reading", async () => {
|
||||
const controller = new AbortController();
|
||||
const source = Readable.from([Buffer.from("data")]);
|
||||
const output = artifactStreamPipeline(source, [], {
|
||||
path: "member",
|
||||
signal: controller.signal,
|
||||
});
|
||||
controller.abort();
|
||||
await expect(buffer(output)).rejects.toMatchObject({ reason: "cancelled" });
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import { PassThrough, Readable, pipeline, type Transform } from "node:stream";
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
/**
|
||||
* Connect a decoder chain as one lifecycle: failures propagate in both
|
||||
* directions, cancellation destroys every stage, and consumer early-return
|
||||
* cannot leave the archive source reading after its handle is closed.
|
||||
*/
|
||||
export const artifactStreamPipeline = (
|
||||
source: Readable,
|
||||
stages: readonly Transform[],
|
||||
context: {
|
||||
readonly path: string;
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
},
|
||||
): Readable => {
|
||||
const output = new PassThrough();
|
||||
const aborted = () =>
|
||||
output.destroy(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`Archive read cancelled: ${context.path}`,
|
||||
),
|
||||
);
|
||||
pipeline([source, ...stages, output], () => {
|
||||
// The readable destination retains any failure for its consumer. Keep
|
||||
// cancellation active until buffered output has also been consumed.
|
||||
});
|
||||
context.signal?.addEventListener("abort", aborted, { once: true });
|
||||
if (context.signal?.aborted) aborted();
|
||||
const dispose = () => {
|
||||
context.signal?.removeEventListener("abort", aborted);
|
||||
output.destroy();
|
||||
};
|
||||
const consumer = Readable.from(
|
||||
(async function* () {
|
||||
try {
|
||||
for await (const chunk of output) yield chunk;
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
if (context.signal?.aborted)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`Archive read cancelled: ${context.path}`,
|
||||
{ cause },
|
||||
);
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Archive member could not be decoded: ${context.path}`,
|
||||
{ cause },
|
||||
);
|
||||
} finally {
|
||||
dispose();
|
||||
}
|
||||
})(),
|
||||
);
|
||||
// Also covers destroy() before the async generator is first advanced.
|
||||
consumer.once("close", dispose);
|
||||
return consumer;
|
||||
};
|
||||
@@ -0,0 +1,94 @@
|
||||
import { isAbsolute } from "node:path";
|
||||
import { normalizeArtifactPath } from "./ArtifactPaths.js";
|
||||
import type { ArtifactEntry, ArtifactReader } from "./ArtifactReader.js";
|
||||
import { AsarArtifactReader } from "./AsarArtifactReader.js";
|
||||
import { CpioArtifactReader } from "./CpioArtifactReader.js";
|
||||
|
||||
/** One reader entry plus its stable logical path and expansion eligibility. */
|
||||
export interface ArtifactTreeEntry {
|
||||
readonly reader: ArtifactReader;
|
||||
readonly entry: ArtifactEntry;
|
||||
readonly path: string;
|
||||
readonly container: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared inventory/extraction traversal. The visitor decides whether verified
|
||||
* content may expand; this boundary owns nested paths, reader creation and
|
||||
* cleanup so the two workflows cannot disagree about container topology.
|
||||
*/
|
||||
export const visitArtifactTree = async (
|
||||
root: ArtifactReader,
|
||||
visit: (item: ArtifactTreeEntry) => Promise<boolean>,
|
||||
options: {
|
||||
readonly signal?: AbortSignal | undefined;
|
||||
readonly integrity?: "fail" | "record-and-continue";
|
||||
readonly expandAsar?: boolean;
|
||||
} = {},
|
||||
): Promise<void> => {
|
||||
const stack = [
|
||||
{
|
||||
reader: root,
|
||||
prefix: "",
|
||||
iterator: root.entries(options.signal)[Symbol.asyncIterator](),
|
||||
owned: false,
|
||||
},
|
||||
];
|
||||
try {
|
||||
while (stack.length > 0) {
|
||||
const frame = stack.at(-1);
|
||||
if (frame === undefined) break;
|
||||
const next = await frame.iterator.next();
|
||||
if (next.done === true) {
|
||||
stack.pop();
|
||||
if (frame.owned) await frame.reader.close();
|
||||
continue;
|
||||
}
|
||||
const entry = next.value;
|
||||
// Charge the full retained namespace, including all enclosing prefixes,
|
||||
// independently from each reader's own raw metadata/index charge.
|
||||
const combined =
|
||||
frame.prefix === "" ? entry.path : `${frame.prefix}/${entry.path}`;
|
||||
root.decodedBudget?.consumeEntry(combined);
|
||||
const path = normalizeArtifactPath(combined);
|
||||
const cpio = entry.nestedArchive === "gzip-cpio";
|
||||
const asar =
|
||||
options.expandAsar !== false &&
|
||||
entry.kind === "file" &&
|
||||
path.toLowerCase().endsWith(".asar") &&
|
||||
isAbsolute(entry.adapterKey);
|
||||
const container =
|
||||
!entry.encrypted && entry.contentUnavailable !== true && (cpio || asar);
|
||||
const expand = await visit({
|
||||
reader: frame.reader,
|
||||
entry,
|
||||
path,
|
||||
container,
|
||||
});
|
||||
if (!container || !expand) continue;
|
||||
const parent = frame.reader;
|
||||
const nested = cpio
|
||||
? new CpioArtifactReader(
|
||||
(signal) => parent.open(entry, signal),
|
||||
options.integrity ?? "fail",
|
||||
parent.decodedBudget,
|
||||
)
|
||||
: new AsarArtifactReader(entry.adapterKey);
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: path,
|
||||
iterator: nested.entries(options.signal)[Symbol.asyncIterator](),
|
||||
owned: true,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
await Promise.allSettled(
|
||||
stack
|
||||
.filter(({ owned }) => owned)
|
||||
.map(async ({ reader, iterator }) => {
|
||||
await iterator.return?.();
|
||||
await reader.close();
|
||||
}),
|
||||
);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,804 @@
|
||||
import { Readable } from "node:stream";
|
||||
import {
|
||||
ArtifactBudgetTransform,
|
||||
ArtifactDecodedBudget,
|
||||
} from "./ArtifactDecodedBudget.js";
|
||||
import { createGunzip } from "node:zlib";
|
||||
import { artifactStreamPipeline } from "./ArtifactStreamPipeline.js";
|
||||
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactChecksumObservation,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
|
||||
/** Longest member name accepted, including its terminating NUL. */
|
||||
const MAX_NAME_BYTES = 64 * 1024;
|
||||
/** Hard-link bytes are buffered to serve every linked path, within these bounds. */
|
||||
const MAX_LINK_BUFFER_BYTES = 16 * 1024 * 1024;
|
||||
const MAX_LINK_BUFFER_TOTAL = 64 * 1024 * 1024;
|
||||
/** Longest symlink target read; PATH_MAX is 1024 on macOS and 4096 on Linux. */
|
||||
const MAX_SYMLINK_TARGET_BYTES = 4096;
|
||||
const TRAILER = "TRAILER!!!";
|
||||
const S_IFMT = 0o170000;
|
||||
const S_IFDIR = 0o040000;
|
||||
const S_IFREG = 0o100000;
|
||||
const S_IFLNK = 0o120000;
|
||||
|
||||
const UNRESOLVED_LINK =
|
||||
"Hard-link bytes are stored with another member of this archive and could not be associated with this path.";
|
||||
|
||||
/** Pull exactly the requested bytes from a stream, holding at most one chunk. */
|
||||
class ByteSource {
|
||||
readonly #iterator: AsyncIterator<unknown>;
|
||||
#buffer: Buffer = Buffer.alloc(0);
|
||||
|
||||
constructor(stream: Readable) {
|
||||
this.#iterator = stream[Symbol.asyncIterator]();
|
||||
}
|
||||
|
||||
async #fill(): Promise<boolean> {
|
||||
let next: IteratorResult<unknown>;
|
||||
try {
|
||||
next = await this.#iterator.next();
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio stream is not valid gzip data",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
if (next.done === true) return false;
|
||||
if (!Buffer.isBuffer(next.value))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio stream yielded non-binary data",
|
||||
);
|
||||
this.#buffer =
|
||||
this.#buffer.length === 0
|
||||
? next.value
|
||||
: Buffer.concat([this.#buffer, next.value]);
|
||||
return true;
|
||||
}
|
||||
|
||||
async read(length: number, label: string): Promise<Buffer> {
|
||||
while (this.#buffer.length < length)
|
||||
if (!(await this.#fill()))
|
||||
throw new ArtifactReaderFailure("format", `cpio ${label} is truncated`);
|
||||
const bytes = this.#buffer.subarray(0, length);
|
||||
this.#buffer = this.#buffer.subarray(length);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Consume the NUL alignment bytes required by newc and crc records. */
|
||||
async padding(length: number, label: string): Promise<void> {
|
||||
if ((await this.read(length, label)).some((byte) => byte !== 0))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`cpio ${label} contains nonzero bytes`,
|
||||
);
|
||||
}
|
||||
|
||||
/** Yield `length` bytes in chunks without buffering them together. */
|
||||
async *take(length: number, label: string): AsyncGenerator<Buffer> {
|
||||
let remaining = length;
|
||||
while (remaining > 0) {
|
||||
if (this.#buffer.length === 0 && !(await this.#fill()))
|
||||
throw new ArtifactReaderFailure("format", `cpio ${label} is truncated`);
|
||||
const size = Math.min(remaining, this.#buffer.length);
|
||||
const chunk = this.#buffer.subarray(0, size);
|
||||
this.#buffer = this.#buffer.subarray(size);
|
||||
remaining -= size;
|
||||
yield chunk;
|
||||
}
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.#iterator.return?.();
|
||||
}
|
||||
|
||||
/** Consume legal zero padding through decoder EOF, including gzip trailer verification. */
|
||||
async finish(): Promise<void> {
|
||||
do {
|
||||
if (this.#buffer.some((byte) => byte !== 0))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio has non-padding bytes after its trailer",
|
||||
);
|
||||
this.#buffer = Buffer.alloc(0);
|
||||
} while (await this.#fill());
|
||||
}
|
||||
}
|
||||
|
||||
interface CpioHeader {
|
||||
readonly format: "odc" | "newc" | "crc";
|
||||
readonly mode: number;
|
||||
readonly fileSize: number;
|
||||
readonly nameSize: number;
|
||||
readonly links: number;
|
||||
/** Device and inode identifying hard links. */
|
||||
readonly identity: string;
|
||||
/** `c_check` of `070702` archives: the unsigned 32-bit sum of the data bytes. */
|
||||
readonly check: number;
|
||||
}
|
||||
|
||||
const field = (
|
||||
bytes: Buffer,
|
||||
start: number,
|
||||
width: number,
|
||||
radix: 8 | 16,
|
||||
): number => {
|
||||
const text = bytes.toString("latin1", start, start + width);
|
||||
const pattern = radix === 8 ? /^[0-7]+$/u : /^[0-9a-fA-F]+$/u;
|
||||
const value = pattern.test(text) ? Number.parseInt(text, radix) : Number.NaN;
|
||||
if (!Number.isSafeInteger(value))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio header has a malformed numeric field",
|
||||
);
|
||||
return value;
|
||||
};
|
||||
|
||||
/** odc (`070707`, octal) and newc/crc (`070701`/`070702`, hexadecimal) headers. */
|
||||
const readHeader = async (source: ByteSource): Promise<CpioHeader> => {
|
||||
const magic = (await source.read(6, "header")).toString("latin1");
|
||||
if (magic === "070707") {
|
||||
const rest = await source.read(70, "header");
|
||||
return {
|
||||
format: "odc",
|
||||
identity: `${field(rest, 0, 6, 8)}:${field(rest, 6, 6, 8)}`,
|
||||
mode: field(rest, 12, 6, 8),
|
||||
links: field(rest, 30, 6, 8),
|
||||
nameSize: field(rest, 53, 6, 8),
|
||||
fileSize: field(rest, 59, 11, 8),
|
||||
check: 0,
|
||||
};
|
||||
}
|
||||
if (magic === "070701" || magic === "070702") {
|
||||
const rest = await source.read(104, "header");
|
||||
return {
|
||||
format: magic === "070702" ? "crc" : "newc",
|
||||
identity: `${field(rest, 56, 8, 16)}:${field(rest, 64, 8, 16)}:${field(rest, 0, 8, 16)}`,
|
||||
mode: field(rest, 8, 8, 16),
|
||||
links: field(rest, 32, 8, 16),
|
||||
fileSize: field(rest, 48, 8, 16),
|
||||
nameSize: field(rest, 88, 8, 16),
|
||||
check: field(rest, 96, 8, 16),
|
||||
};
|
||||
}
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member has an unsupported header; only odc and newc archives are expanded",
|
||||
);
|
||||
};
|
||||
|
||||
const padding = (format: CpioHeader["format"], length: number): number =>
|
||||
format === "odc" ? 0 : (4 - (length % 4)) % 4;
|
||||
|
||||
/** Normalize a member name; reject absolute and traversing names. */
|
||||
const memberPath = (raw: string): string | undefined => {
|
||||
const segments = raw
|
||||
.split("/")
|
||||
.filter((segment) => segment !== "" && segment !== ".");
|
||||
if (raw.startsWith("/") || segments.includes(".."))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`cpio member escapes its archive: ${raw}`,
|
||||
);
|
||||
return segments.length === 0 ? undefined : segments.join("/");
|
||||
};
|
||||
|
||||
/** Project one cpio member as an archive-neutral entry. */
|
||||
const entryOf = (member: {
|
||||
readonly path: string;
|
||||
readonly kind: ArtifactEntry["kind"];
|
||||
readonly key: string;
|
||||
readonly header: CpioHeader;
|
||||
readonly limitations: readonly string[];
|
||||
readonly size?: number | null;
|
||||
readonly contentUnavailable?: boolean;
|
||||
readonly integrityMismatched?: boolean;
|
||||
}): ArtifactEntry => ({
|
||||
path: member.path,
|
||||
kind: member.kind,
|
||||
declaredSize:
|
||||
member.size !== undefined
|
||||
? member.size
|
||||
: member.kind === "file"
|
||||
? member.header.fileSize
|
||||
: null,
|
||||
compressedSize: null,
|
||||
executable: (member.header.mode & 0o111) !== 0,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: member.limitations,
|
||||
adapterKey: member.key,
|
||||
...(member.contentUnavailable === true ? { contentUnavailable: true } : {}),
|
||||
...(member.integrityMismatched === true ? { integrityMismatched: true } : {}),
|
||||
});
|
||||
|
||||
interface PendingLink {
|
||||
readonly path: string;
|
||||
readonly key: string;
|
||||
readonly header: CpioHeader;
|
||||
}
|
||||
|
||||
const drain = async (chunks: AsyncIterable<Buffer>): Promise<void> => {
|
||||
for await (const chunk of chunks) void chunk;
|
||||
};
|
||||
|
||||
const crcObservation = (
|
||||
declared: number,
|
||||
observed: number,
|
||||
): ArtifactChecksumObservation => ({
|
||||
representation: "decoded",
|
||||
algorithm: "cpio-byte-sum",
|
||||
declared: declared.toString(16).padStart(8, "0"),
|
||||
observed: observed.toString(16).padStart(8, "0"),
|
||||
});
|
||||
|
||||
/** Same sentence the scanner uses for a streamed regular-file CRC contradiction. */
|
||||
const crcMismatchDetail = (observation: ArtifactChecksumObservation): string =>
|
||||
`Declared ${observation.representation} ${observation.algorithm} ${observation.declared} disagrees with observed ${observation.observed}.`;
|
||||
|
||||
/** Human-readable limitation for a collected (or skipped) symlink target. */
|
||||
const symlinkTargetLimitation = (
|
||||
header: CpioHeader,
|
||||
target: Buffer | undefined,
|
||||
): string => {
|
||||
if (target === undefined)
|
||||
return `Symlink target of ${header.fileSize} bytes exceeds ${MAX_SYMLINK_TARGET_BYTES} bytes and was not read.`;
|
||||
try {
|
||||
return `Symlink target: ${new TextDecoder("utf-8", { fatal: true }).decode(target)}`;
|
||||
} catch {
|
||||
return "Symlink target is not valid UTF-8 and was not decoded; the archived bytes are preserved only as a byte count.";
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Sequential reader for a gzip-compressed cpio archive, such as an installer
|
||||
* package's Scripts or Payload. Members are decompressed once, in order: open
|
||||
* an entry before advancing to the next one. `070702` data is checked against
|
||||
* its CRC. Hard links whose bytes are stored on another member are served
|
||||
* from that member's bytes; device nodes, FIFOs, and sockets are skipped.
|
||||
*/
|
||||
export class CpioArtifactReader implements ArtifactReader {
|
||||
readonly format = "file" as const;
|
||||
#source: ByteSource | undefined;
|
||||
#signal: AbortSignal | undefined;
|
||||
#current:
|
||||
| {
|
||||
readonly key: string;
|
||||
readonly path: string;
|
||||
readonly header: CpioHeader;
|
||||
}
|
||||
| undefined;
|
||||
#consumed = false;
|
||||
readonly #links = new Map<string, Buffer>();
|
||||
readonly #pending = new Map<string, PendingLink[]>();
|
||||
readonly #aliases = new Map<string, Buffer>();
|
||||
readonly #integrity = new Map<
|
||||
string,
|
||||
readonly ArtifactChecksumObservation[]
|
||||
>();
|
||||
readonly #linkIntegrity = new Map<
|
||||
string,
|
||||
readonly ArtifactChecksumObservation[]
|
||||
>();
|
||||
#buffered = 0;
|
||||
|
||||
constructor(
|
||||
private readonly openCompressed: (
|
||||
signal?: AbortSignal,
|
||||
) => Promise<Readable>,
|
||||
/**
|
||||
* Integrity mode for member CRC failures. Every member representation is
|
||||
* checked before the next header is read. `fail` throws. Under
|
||||
* record-and-continue the disagreement is returned as checksum evidence
|
||||
* so the occurrence and later siblings are still inventoried.
|
||||
*/
|
||||
private readonly integrity: "fail" | "record-and-continue" = "fail",
|
||||
readonly decodedBudget: ArtifactDecodedBudget = new ArtifactDecodedBudget(),
|
||||
) {}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
this.#signal = signal;
|
||||
const compressed = await this.openCompressed(signal);
|
||||
const source = new ByteSource(
|
||||
artifactStreamPipeline(
|
||||
compressed,
|
||||
[
|
||||
createGunzip(),
|
||||
new ArtifactBudgetTransform(this.decodedBudget, "gzip-cpio", signal),
|
||||
],
|
||||
{ path: "gzip-cpio", signal },
|
||||
),
|
||||
);
|
||||
this.#source = source;
|
||||
for (let index = 0; ; index++) {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"cpio expansion was cancelled",
|
||||
);
|
||||
yield* this.#finishCurrent(source);
|
||||
const header = await readHeader(source);
|
||||
if (header.nameSize < 1 || header.nameSize > MAX_NAME_BYTES)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member name size is invalid",
|
||||
);
|
||||
const nameBytes = await source.read(header.nameSize, "member name");
|
||||
await source.padding(
|
||||
padding(header.format, 110 + header.nameSize),
|
||||
"name padding",
|
||||
);
|
||||
// cpio names require exactly one terminal NUL: a missing terminator or
|
||||
// an embedded NUL with trailing bytes is malformed. Decode without
|
||||
// replacement so distinct byte names cannot collapse to one path.
|
||||
if (
|
||||
nameBytes.length === 0 ||
|
||||
nameBytes[nameBytes.length - 1] !== 0 ||
|
||||
nameBytes.indexOf(0) !== nameBytes.length - 1
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member name is not NUL-terminated",
|
||||
);
|
||||
let raw: string;
|
||||
try {
|
||||
raw = new TextDecoder("utf-8", { fatal: true }).decode(
|
||||
nameBytes.subarray(0, nameBytes.length - 1),
|
||||
);
|
||||
} catch {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio member name is not valid UTF-8",
|
||||
);
|
||||
}
|
||||
this.decodedBudget.consumeEntry(raw);
|
||||
if (raw === TRAILER) {
|
||||
if (header.fileSize !== 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"cpio trailer declares member data",
|
||||
);
|
||||
await drain(this.#verified(source, header, raw, undefined, "fail"));
|
||||
await source.finish();
|
||||
yield* this.#unresolvedLinks();
|
||||
return;
|
||||
}
|
||||
yield* this.#member(source, header, raw, `${index}:${raw}`);
|
||||
}
|
||||
}
|
||||
|
||||
async *#member(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
raw: string,
|
||||
key: string,
|
||||
): AsyncGenerator<ArtifactEntry> {
|
||||
const type = header.mode & S_IFMT;
|
||||
const path = memberPath(raw);
|
||||
if (type === S_IFDIR) {
|
||||
const consumed = await this.#consume(source, header, raw, false);
|
||||
if (path !== undefined)
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "directory",
|
||||
key,
|
||||
header,
|
||||
limitations:
|
||||
consumed.mismatch === undefined
|
||||
? []
|
||||
: [crcMismatchDetail(consumed.mismatch)],
|
||||
integrityMismatched: consumed.mismatch !== undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (type === S_IFLNK) {
|
||||
yield* this.#symlink(source, { header, raw, key }, path);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
path !== undefined &&
|
||||
type === S_IFREG &&
|
||||
header.links > 1 &&
|
||||
header.fileSize === 0
|
||||
) {
|
||||
// Zero-size headers carry no bytes, so a CRC archive must declare zero.
|
||||
// Under record-and-continue this is the occurrence's own forgotten
|
||||
// bytes: yield it unavailable so later siblings still inventory.
|
||||
const consumed = await this.#consume(source, header, raw, false);
|
||||
if (consumed.mismatch !== undefined) {
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: null,
|
||||
limitations: [crcMismatchDetail(consumed.mismatch)],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
const stored = this.#links.get(header.identity);
|
||||
if (stored !== undefined) {
|
||||
yield this.#alias(path, key, header, stored);
|
||||
return;
|
||||
}
|
||||
// newc stores a hard link's bytes on its last member; wait for them.
|
||||
// An all-empty group is resolved to empty files at TRAILER!!!.
|
||||
const waiting = this.#pending.get(header.identity) ?? [];
|
||||
waiting.push({ path, key, header });
|
||||
this.#pending.set(header.identity, waiting);
|
||||
return;
|
||||
}
|
||||
this.#current = { key, path: raw, header };
|
||||
this.#consumed = false;
|
||||
if (path === undefined) return;
|
||||
if (type !== S_IFREG) {
|
||||
// FIFOs, device nodes, sockets and other types are not expanded; keep
|
||||
// an explicit unavailable occurrence instead of dropping the path.
|
||||
const consumed = await this.#consume(source, header, raw, false);
|
||||
this.#current = undefined;
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: null,
|
||||
limitations: [
|
||||
`Unsupported cpio member type ${type.toString(8)}; content not expanded.`,
|
||||
...(consumed.mismatch === undefined
|
||||
? []
|
||||
: [crcMismatchDetail(consumed.mismatch)]),
|
||||
],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
limitations: [],
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* One symlink member. A hostile header can declare a multi-gigabyte target,
|
||||
* so oversized targets stream past. The CRC is resolved before the entry is
|
||||
* yielded; under record-and-continue a disagreement stays on the occurrence
|
||||
* so later siblings are still inventoried.
|
||||
*/
|
||||
async *#symlink(
|
||||
source: ByteSource,
|
||||
member: {
|
||||
readonly header: CpioHeader;
|
||||
readonly raw: string;
|
||||
readonly key: string;
|
||||
},
|
||||
path: string | undefined,
|
||||
): AsyncGenerator<ArtifactEntry> {
|
||||
const { header, raw, key } = member;
|
||||
const consumed = await this.#consume(
|
||||
source,
|
||||
header,
|
||||
raw,
|
||||
header.fileSize <= MAX_SYMLINK_TARGET_BYTES,
|
||||
);
|
||||
if (path === undefined) return;
|
||||
if (consumed.mismatch !== undefined) {
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "symlink",
|
||||
key,
|
||||
header,
|
||||
limitations: [
|
||||
crcMismatchDetail(consumed.mismatch),
|
||||
symlinkTargetLimitation(header, consumed.bytes),
|
||||
],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
return;
|
||||
}
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "symlink",
|
||||
key,
|
||||
header,
|
||||
limitations: [symlinkTargetLimitation(header, consumed.bytes)],
|
||||
});
|
||||
}
|
||||
|
||||
#alias(
|
||||
path: string,
|
||||
key: string,
|
||||
header: CpioHeader,
|
||||
bytes: Buffer,
|
||||
): ArtifactEntry {
|
||||
this.#aliases.set(key, bytes);
|
||||
const sourceIntegrity = this.#linkIntegrity.get(header.identity);
|
||||
if (sourceIntegrity !== undefined)
|
||||
this.#integrity.set(key, sourceIntegrity);
|
||||
return entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: bytes.length,
|
||||
limitations: [
|
||||
"Hard link: bytes are stored with another member of this archive.",
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
*#unresolvedLinks(): Generator<ArtifactEntry> {
|
||||
for (const links of this.#pending.values())
|
||||
for (const { path, key, header } of links) {
|
||||
// A complete zero-length group establishes empty content. An orphan
|
||||
// that claims more links than appear is missing its bytes. Pending
|
||||
// groups contain only zero-length members; do not rescan per alias.
|
||||
const complete = links.length >= header.links;
|
||||
if (header.fileSize === 0 && complete) {
|
||||
const empty = Buffer.alloc(0);
|
||||
this.#aliases.set(key, empty);
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: 0,
|
||||
limitations: [
|
||||
"Hard link: every member of this link group is empty.",
|
||||
],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
yield entryOf({
|
||||
path,
|
||||
kind: "file",
|
||||
key,
|
||||
header,
|
||||
size: null,
|
||||
limitations: [UNRESOLVED_LINK],
|
||||
contentUnavailable: true,
|
||||
});
|
||||
}
|
||||
this.#pending.clear();
|
||||
}
|
||||
|
||||
/** Finish the current member, then serve hard links that were waiting for it. */
|
||||
async *#finishCurrent(source: ByteSource): AsyncGenerator<ArtifactEntry> {
|
||||
const current = this.#current;
|
||||
if (current === undefined) return;
|
||||
this.#current = undefined;
|
||||
if (!this.#consumed) {
|
||||
if (this.#shouldKeep(current.header))
|
||||
this.#remember(
|
||||
current.header,
|
||||
await this.#collect(source, current.header, current.path),
|
||||
);
|
||||
else
|
||||
await drain(
|
||||
this.#verified(
|
||||
source,
|
||||
current.header,
|
||||
current.path,
|
||||
undefined,
|
||||
"fail",
|
||||
),
|
||||
);
|
||||
}
|
||||
const waiting = this.#pending.get(current.header.identity);
|
||||
const stored = this.#links.get(current.header.identity);
|
||||
if (waiting === undefined || stored === undefined) return;
|
||||
this.#pending.delete(current.header.identity);
|
||||
for (const { path, key, header } of waiting)
|
||||
yield this.#alias(path, key, header, stored);
|
||||
}
|
||||
|
||||
#shouldKeep(header: CpioHeader): boolean {
|
||||
return (
|
||||
(header.mode & S_IFMT) === S_IFREG &&
|
||||
header.links > 1 &&
|
||||
header.fileSize > 0 &&
|
||||
header.fileSize <= MAX_LINK_BUFFER_BYTES &&
|
||||
this.#buffered + header.fileSize <= MAX_LINK_BUFFER_TOTAL
|
||||
);
|
||||
}
|
||||
|
||||
#remember(
|
||||
header: CpioHeader,
|
||||
bytes: Buffer,
|
||||
observations: readonly ArtifactChecksumObservation[] = [],
|
||||
): void {
|
||||
if (this.#links.has(header.identity)) return;
|
||||
this.#links.set(header.identity, bytes);
|
||||
if (observations.length > 0)
|
||||
this.#linkIntegrity.set(header.identity, observations);
|
||||
this.#buffered += bytes.length;
|
||||
}
|
||||
|
||||
/**
|
||||
* Member bytes and padding. `sum` receives the unsigned 32-bit total used
|
||||
* by `070702`. Padding is consumed before the caller reports a mismatch, so
|
||||
* a recovered archive stays aligned on the next header.
|
||||
*/
|
||||
async *#payload(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
sum: { value: number },
|
||||
): AsyncGenerator<Buffer> {
|
||||
this.#cancelled();
|
||||
for await (const chunk of source.take(header.fileSize, "member data")) {
|
||||
this.#cancelled();
|
||||
if (header.format === "crc")
|
||||
for (const byte of chunk) sum.value = (sum.value + byte) >>> 0;
|
||||
yield chunk;
|
||||
}
|
||||
await source.padding(
|
||||
padding(header.format, header.fileSize),
|
||||
"data padding",
|
||||
);
|
||||
this.#cancelled();
|
||||
}
|
||||
|
||||
/**
|
||||
* One CRC check for every member kind. `fail` throws with the observation
|
||||
* in the message. record-and-continue returns the observation so callers
|
||||
* keep declared and observed values instead of catching a string.
|
||||
*/
|
||||
#crcMismatch(
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
sum: number,
|
||||
policy: "fail" | "record-and-continue" = this.integrity,
|
||||
): ArtifactChecksumObservation | undefined {
|
||||
if (header.format !== "crc" || sum === header.check) return undefined;
|
||||
const observation = crcObservation(header.check, sum);
|
||||
if (policy === "fail")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`cpio CRC disagrees with content: ${path} (${crcMismatchDetail(observation)})`,
|
||||
);
|
||||
return observation;
|
||||
}
|
||||
|
||||
/** Read one member fully. `collect` retains bytes for symlinks and hard links. */
|
||||
async #consume(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
collect: boolean,
|
||||
policy?: "fail" | "record-and-continue",
|
||||
): Promise<{
|
||||
readonly bytes: Buffer | undefined;
|
||||
readonly mismatch: ArtifactChecksumObservation | undefined;
|
||||
}> {
|
||||
const sum = { value: 0 };
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of this.#payload(source, header, sum))
|
||||
if (collect) chunks.push(chunk);
|
||||
return {
|
||||
bytes: collect ? Buffer.concat(chunks) : undefined,
|
||||
mismatch: this.#crcMismatch(header, path, sum.value, policy),
|
||||
};
|
||||
}
|
||||
|
||||
/** Member data and its padding, checked against a `070702` CRC. */
|
||||
async *#verified(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
record?: (observation: ArtifactChecksumObservation) => void,
|
||||
policy?: "fail" | "record-and-continue",
|
||||
): AsyncGenerator<Buffer> {
|
||||
const sum = { value: 0 };
|
||||
yield* this.#payload(source, header, sum);
|
||||
const mismatch = this.#crcMismatch(
|
||||
header,
|
||||
path,
|
||||
sum.value,
|
||||
policy ?? this.integrity,
|
||||
);
|
||||
if (mismatch !== undefined) record?.(mismatch);
|
||||
}
|
||||
|
||||
async #collect(
|
||||
source: ByteSource,
|
||||
header: CpioHeader,
|
||||
path: string,
|
||||
): Promise<Buffer> {
|
||||
// Unopened linked bytes are not a recovered occurrence. A disagreement
|
||||
// still fails the archive instead of being remembered as verified content.
|
||||
const consumed = await this.#consume(source, header, path, true, "fail");
|
||||
return consumed.bytes ?? Buffer.alloc(0);
|
||||
}
|
||||
|
||||
#cancelled(): void {
|
||||
if (this.#signal?.aborted)
|
||||
throw new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
"cpio expansion was cancelled",
|
||||
);
|
||||
}
|
||||
|
||||
open(entry: ArtifactEntry): Promise<Readable> {
|
||||
const alias = this.#aliases.get(entry.adapterKey);
|
||||
if (alias !== undefined) {
|
||||
this.decodedBudget.consume(alias.length, entry.path);
|
||||
return Promise.resolve(Readable.from([alias]));
|
||||
}
|
||||
const source = this.#source;
|
||||
const current = this.#current;
|
||||
if (
|
||||
source === undefined ||
|
||||
current === undefined ||
|
||||
current.key !== entry.adapterKey ||
|
||||
this.#consumed
|
||||
)
|
||||
return Promise.reject(
|
||||
new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
entry.contentUnavailable === true
|
||||
? UNRESOLVED_LINK
|
||||
: "cpio members can be read only once, in archive order",
|
||||
),
|
||||
);
|
||||
this.#consumed = true;
|
||||
const chunks = this.#verified(
|
||||
source,
|
||||
current.header,
|
||||
current.path,
|
||||
this.integrity === "record-and-continue"
|
||||
? (observed) => this.#integrity.set(entry.adapterKey, [observed])
|
||||
: undefined,
|
||||
);
|
||||
if (!this.#shouldKeep(current.header))
|
||||
return Promise.resolve(Readable.from(chunks));
|
||||
// Keep a linked member's bytes for paths that share them.
|
||||
const kept: Buffer[] = [];
|
||||
const remember = (): void => {
|
||||
this.#remember(
|
||||
current.header,
|
||||
Buffer.concat(kept),
|
||||
this.#integrity.get(entry.adapterKey),
|
||||
);
|
||||
};
|
||||
return Promise.resolve(
|
||||
Readable.from(
|
||||
(async function* () {
|
||||
for await (const chunk of chunks) {
|
||||
kept.push(chunk);
|
||||
yield chunk;
|
||||
}
|
||||
remember();
|
||||
})(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
provenance(): readonly ArtifactCommand[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Completed member CRC evidence, including inherited hard-link source contradictions. */
|
||||
integrityObservations(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[] {
|
||||
return this.#integrity.get(entry.adapterKey) ?? [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
await this.#source?.close();
|
||||
this.#source = undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/** xar archives and cpio payloads built field by field for installer-package tests. */
|
||||
import { createHash } from "node:crypto";
|
||||
import { deflateSync, gzipSync, inflateSync } from "node:zlib";
|
||||
|
||||
export interface XarFixtureMember {
|
||||
readonly name: string;
|
||||
readonly type?: "file" | "directory" | "symlink";
|
||||
readonly data?: Uint8Array;
|
||||
readonly encoding?: "raw" | "zlib" | "bzip2";
|
||||
readonly link?: string;
|
||||
readonly mode?: string;
|
||||
/** Replace the declared extracted checksum, to model tampering. */
|
||||
readonly extractedSha1?: string;
|
||||
/** Replace the declared stored checksum while retaining decodable content. */
|
||||
readonly archivedSha1?: string;
|
||||
/** Store these bytes instead of encoding `data`, to model corrupt members. */
|
||||
readonly archived?: Uint8Array;
|
||||
readonly children?: readonly XarFixtureMember[];
|
||||
}
|
||||
|
||||
const sha1 = (bytes: Uint8Array): string =>
|
||||
createHash("sha1").update(bytes).digest("hex");
|
||||
|
||||
const escape = (text: string): string =>
|
||||
text.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
|
||||
|
||||
/** Build a xar archive with a SHA-1 TOC checksum and per-member checksums. */
|
||||
export const xarArchive = (
|
||||
members: readonly XarFixtureMember[],
|
||||
options: {
|
||||
readonly corruptTocChecksum?: boolean;
|
||||
/** Declared TOC checksum size; the real digest is 20 bytes. */
|
||||
readonly tocChecksumSize?: number;
|
||||
/** Override the fixed-header decoded length without altering the TOC. */
|
||||
readonly tocDecodedSize?: number;
|
||||
} = {},
|
||||
): Uint8Array => {
|
||||
const heap: Uint8Array[] = [new Uint8Array(20)];
|
||||
let heapLength = 20;
|
||||
let nextId = 1;
|
||||
const fileXml = (member: XarFixtureMember): string => {
|
||||
const id = nextId++;
|
||||
const type = member.type ?? "file";
|
||||
let data = "";
|
||||
if (type === "file" && member.data !== undefined) {
|
||||
const archived =
|
||||
member.archived ??
|
||||
(member.encoding === "zlib" ? deflateSync(member.data) : member.data);
|
||||
const offset = heapLength;
|
||||
heap.push(archived);
|
||||
heapLength += archived.length;
|
||||
const style =
|
||||
member.encoding === "zlib"
|
||||
? "application/x-gzip"
|
||||
: member.encoding === "bzip2"
|
||||
? "application/x-bzip2"
|
||||
: "application/octet-stream";
|
||||
data = `<data><archived-checksum style="sha1">${member.archivedSha1 ?? sha1(archived)}</archived-checksum><extracted-checksum style="sha1">${member.extractedSha1 ?? sha1(member.data)}</extracted-checksum><encoding style="${style}"/><size>${member.data.length}</size><offset>${offset}</offset><length>${archived.length}</length></data>`;
|
||||
}
|
||||
const link =
|
||||
type === "symlink"
|
||||
? `<link type="file">${escape(member.link ?? "")}</link>`
|
||||
: "";
|
||||
const children = (member.children ?? []).map(fileXml).join("");
|
||||
return `<file id="${id}"><name>${escape(member.name)}</name><type>${type}</type><mode>${member.mode ?? (type === "directory" ? "0755" : "0644")}</mode>${data}${link}${children}</file>`;
|
||||
};
|
||||
const files = members.map(fileXml).join("");
|
||||
const toc = Buffer.from(
|
||||
`<?xml version="1.0" encoding="UTF-8"?><xar><toc><checksum style="sha1"><size>${options.tocChecksumSize ?? 20}</size><offset>0</offset></checksum>${files}</toc></xar>`,
|
||||
);
|
||||
const compressed = deflateSync(toc);
|
||||
const checksum = createHash("sha1").update(compressed).digest();
|
||||
if (options.corruptTocChecksum === true)
|
||||
checksum[0] = (checksum[0] ?? 0) ^ 0xff;
|
||||
heap[0] = checksum;
|
||||
const header = Buffer.alloc(28);
|
||||
header.writeUInt32BE(0x78617221, 0);
|
||||
header.writeUInt16BE(28, 4);
|
||||
header.writeUInt16BE(1, 6);
|
||||
header.writeBigUInt64BE(BigInt(compressed.length), 8);
|
||||
header.writeBigUInt64BE(BigInt(options.tocDecodedSize ?? toc.length), 16);
|
||||
header.writeUInt32BE(1, 24);
|
||||
return Buffer.concat([header, compressed, ...heap]);
|
||||
};
|
||||
|
||||
export interface CpioFixtureMember {
|
||||
readonly name: string;
|
||||
readonly mode: number;
|
||||
readonly data?: Uint8Array | string;
|
||||
readonly ino?: number;
|
||||
readonly links?: number;
|
||||
/** Override a crc archive's c_check, to model corruption. */
|
||||
readonly check?: number;
|
||||
}
|
||||
|
||||
const octal = (value: number, width: number): string =>
|
||||
value.toString(8).padStart(width, "0");
|
||||
const hex = (value: number): string => value.toString(16).padStart(8, "0");
|
||||
|
||||
/** Build a gzip-compressed odc or newc cpio archive ending in its trailer. */
|
||||
export const gzipCpio = (
|
||||
members: readonly CpioFixtureMember[],
|
||||
format: "odc" | "newc" | "crc" = "odc",
|
||||
): Uint8Array => {
|
||||
const parts: Buffer[] = [];
|
||||
const pad = (length: number): void => {
|
||||
if (format !== "odc" && length % 4 !== 0)
|
||||
parts.push(Buffer.alloc(4 - (length % 4)));
|
||||
};
|
||||
for (const member of [...members, { name: "TRAILER!!!", mode: 0 }]) {
|
||||
const data =
|
||||
typeof member.data === "string"
|
||||
? Buffer.from(member.data)
|
||||
: Buffer.from(member.data ?? new Uint8Array());
|
||||
const name = Buffer.from(`${member.name}\0`);
|
||||
const ino = "ino" in member ? (member.ino ?? 1) : 1;
|
||||
const links = "links" in member ? (member.links ?? 1) : 1;
|
||||
const sum = [...data].reduce((total, byte) => (total + byte) >>> 0, 0);
|
||||
const check = "check" in member ? (member.check ?? sum) : sum;
|
||||
if (format === "odc")
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`070707${octal(0, 6)}${octal(ino, 6)}${octal(member.mode, 6)}${octal(0, 6)}${octal(0, 6)}${octal(links, 6)}${octal(0, 6)}${octal(0, 11)}${octal(name.length, 6)}${octal(data.length, 11)}`,
|
||||
),
|
||||
name,
|
||||
data,
|
||||
);
|
||||
else {
|
||||
parts.push(
|
||||
Buffer.from(
|
||||
`${format === "crc" ? "070702" : "070701"}${hex(ino)}${hex(member.mode)}${hex(0)}${hex(0)}${hex(links)}${hex(0)}${hex(data.length)}${hex(0)}${hex(0)}${hex(0)}${hex(0)}${hex(name.length)}${hex(format === "crc" ? check : 0)}`,
|
||||
),
|
||||
name,
|
||||
);
|
||||
pad(110 + name.length);
|
||||
parts.push(data);
|
||||
pad(data.length);
|
||||
}
|
||||
}
|
||||
return gzipSync(Buffer.concat(parts));
|
||||
};
|
||||
|
||||
export const MODE = {
|
||||
directory: 0o040755,
|
||||
file: 0o100644,
|
||||
executable: 0o100755,
|
||||
symlink: 0o120755,
|
||||
fifo: 0o010644,
|
||||
} as const;
|
||||
|
||||
/** Mutate a fixture's XML while keeping its compressed TOC checksum/lengths consistent. */
|
||||
export const rewriteXarToc = (
|
||||
archive: Uint8Array,
|
||||
rewrite: (xml: string) => string,
|
||||
): Uint8Array => {
|
||||
const input = Buffer.from(archive);
|
||||
const oldLength = Number(input.readBigUInt64BE(8));
|
||||
const decoded = inflateSync(input.subarray(28, 28 + oldLength));
|
||||
const xml = Buffer.from(rewrite(decoded.toString("utf8")));
|
||||
const compressed = deflateSync(xml);
|
||||
const header = Buffer.from(input.subarray(0, 28));
|
||||
header.writeBigUInt64BE(BigInt(compressed.length), 8);
|
||||
header.writeBigUInt64BE(BigInt(xml.length), 16);
|
||||
const heap = Buffer.from(input.subarray(28 + oldLength));
|
||||
createHash("sha1").update(compressed).digest().copy(heap, 0);
|
||||
return Buffer.concat([header, compressed, heap]);
|
||||
};
|
||||
@@ -0,0 +1,604 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { Readable } from "node:stream";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { ArtifactDecodedBudget } from "./ArtifactDecodedBudget.js";
|
||||
import { CpioArtifactReader } from "./CpioArtifactReader.js";
|
||||
import {
|
||||
MODE,
|
||||
gzipCpio,
|
||||
xarArchive,
|
||||
rewriteXarToc,
|
||||
type XarFixtureMember,
|
||||
} from "./InstallerPackage.fixture.js";
|
||||
import { XarArtifactReader } from "./XarArtifactReader.js";
|
||||
import { scanReader } from "./inventory/scanReader.js";
|
||||
import { visitArtifactTree } from "./ArtifactTraversal.js";
|
||||
|
||||
const withReader = async (
|
||||
bytes: Uint8Array,
|
||||
run: (reader: XarArtifactReader) => Promise<void>,
|
||||
decodedBudget?: ArtifactDecodedBudget,
|
||||
) => {
|
||||
const root = await createTestTempDirectory("rea-pkg-limits-");
|
||||
const path = join(root, "fixture.pkg");
|
||||
await writeFile(path, bytes);
|
||||
const reader = new XarArtifactReader(
|
||||
path,
|
||||
decodedBudget === undefined ? {} : { decodedBudget },
|
||||
);
|
||||
try {
|
||||
await run(reader);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
};
|
||||
|
||||
it("shares an inflation budget across separate zlib members", async () => {
|
||||
const bytes = xarArchive([
|
||||
{ name: "a", data: Buffer.alloc(40, 65), encoding: "zlib" },
|
||||
{ name: "b", data: Buffer.alloc(40, 66), encoding: "zlib" },
|
||||
]);
|
||||
await withReader(
|
||||
bytes,
|
||||
async (reader) => {
|
||||
let read = 0;
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) {
|
||||
await buffer(await reader.open(entry));
|
||||
read++;
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Decoded archive budget"),
|
||||
});
|
||||
expect(read).toBe(1);
|
||||
},
|
||||
new ArtifactDecodedBudget(64),
|
||||
);
|
||||
});
|
||||
|
||||
it("charges raw member reads to the same cumulative work budget", async () => {
|
||||
await withReader(
|
||||
xarArchive([
|
||||
{ name: "a", data: Buffer.alloc(40, 65) },
|
||||
{ name: "b", data: Buffer.alloc(40, 66) },
|
||||
]),
|
||||
async (reader) => {
|
||||
let read = 0;
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) {
|
||||
await buffer(await reader.open(entry));
|
||||
read++;
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Decoded archive budget"),
|
||||
});
|
||||
expect(read).toBe(1);
|
||||
},
|
||||
new ArtifactDecodedBudget(64),
|
||||
);
|
||||
});
|
||||
|
||||
it("validates unsafe extents even when a member is unsupported and would not be opened", async () => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", encoding: "bzip2", data: Buffer.from("opaque") }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<offset>20<\/offset>/u,
|
||||
`<offset>${Number.MAX_SAFE_INTEGER}</offset>`,
|
||||
),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("heap extent"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["", " ", "1e2", "0x10"])(
|
||||
"rejects nondecimal/empty numeric TOC data: %j",
|
||||
async (value) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("bytes") }]),
|
||||
(xml) =>
|
||||
xml.replace(/<length>\d+<\/length>/u, `<length>${value}</length>`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar TOC has an invalid data length",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects unsafe absolute heap ranges with a tagged format failure", async () => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "Payload", data: Buffer.from("gzip") }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<offset>20<\/offset>/u,
|
||||
`<offset>${Number.MAX_SAFE_INTEGER}</offset>`,
|
||||
),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("heap extent"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds retained full paths from deeply nested TOCs", async () => {
|
||||
let member: XarFixtureMember = { name: "leaf", data: Buffer.from("x") };
|
||||
for (let depth = 0; depth < 300; depth++)
|
||||
member = {
|
||||
name: "directory-segment",
|
||||
type: "directory",
|
||||
children: [member],
|
||||
};
|
||||
await withReader(xarArchive([member]), async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("path text"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("observes cancellation during internal symlink drains", async () => {
|
||||
const controller = new AbortController();
|
||||
const archive = gzipCpio([
|
||||
{
|
||||
name: "huge-link",
|
||||
mode: MODE.symlink,
|
||||
data: "x".repeat(4 * 1024 * 1024),
|
||||
},
|
||||
]);
|
||||
let decoded = 0;
|
||||
const budget = new (class extends ArtifactDecodedBudget {
|
||||
override consume(bytes: number, path: string): void {
|
||||
super.consume(bytes, path);
|
||||
decoded += bytes;
|
||||
if (decoded >= 64 * 1024) controller.abort();
|
||||
}
|
||||
})();
|
||||
const reader = new CpioArtifactReader(
|
||||
async () => Readable.from([archive]),
|
||||
"fail",
|
||||
budget,
|
||||
);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries(controller.signal)) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "cancelled" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("verifies gzip integrity even when the cpio trailer has already been decoded", async () => {
|
||||
const bytes = Buffer.from(
|
||||
gzipCpio([{ name: "a", mode: MODE.file, data: "ok" }]),
|
||||
);
|
||||
bytes[bytes.length - 8] = (bytes[bytes.length - 8] ?? 0) ^ 1;
|
||||
const reader = new CpioArtifactReader(async () => Readable.from([bytes]));
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({ reason: "format" });
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["0755junk", "-1", "", "08"])(
|
||||
"rejects partially parsed or malformed mode %j",
|
||||
async (mode) => {
|
||||
await withReader(
|
||||
xarArchive([{ name: "a", mode, data: Buffer.from("x") }]),
|
||||
async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar TOC has an invalid mode",
|
||||
});
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves CRC contradictions for special nodes and reaches later siblings", async () => {
|
||||
const bytes = gzipCpio(
|
||||
[
|
||||
{ name: "fifo", mode: MODE.fifo, data: "abc", check: 1 },
|
||||
{ name: "good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
);
|
||||
const reader = new CpioArtifactReader(
|
||||
async () => Readable.from([bytes]),
|
||||
"record-and-continue",
|
||||
);
|
||||
try {
|
||||
const paths: string[] = [];
|
||||
for await (const entry of reader.entries()) {
|
||||
paths.push(entry.path);
|
||||
if (entry.path === "fifo")
|
||||
expect(entry.limitations).toContain(
|
||||
"Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000126.",
|
||||
);
|
||||
if (entry.path === "good")
|
||||
expect((await buffer(await reader.open(entry))).toString()).toBe("ok");
|
||||
}
|
||||
expect(paths).toEqual(["fifo", "good"]);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not expose a partial inventory when a previous TOC load failed", async () => {
|
||||
await withReader(
|
||||
xarArchive([{ name: "a", mode: "broken", data: Buffer.from("x") }]),
|
||||
async (reader) => {
|
||||
const scan = async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
};
|
||||
await expect(scan()).rejects.toMatchObject({ reason: "format" });
|
||||
await expect(scan()).rejects.toMatchObject({ reason: "format" });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a TOC whose element count would build an unbounded DOM", async () => {
|
||||
const noise = "<n/>".repeat(100_001);
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
(xml) => xml.replace("<toc>", `<toc>${noise}`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: "xar TOC exceeds 100000 XML nodes",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a TOC checksum when the fixed header declares none", async () => {
|
||||
const bytes = Buffer.from(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
);
|
||||
bytes.writeUInt32BE(0, 24);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("declares no TOC checksum"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["", "nothex", "a".repeat(63)])(
|
||||
"rejects malformed checksum text %j before applying integrity policy",
|
||||
async (digest) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<extracted-checksum style="sha1">[^<]*<\/extracted-checksum>/u,
|
||||
`<extracted-checksum style="sha256">${digest}</extracted-checksum>`,
|
||||
),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar extracted-checksum has malformed sha256 digest text",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects a CRC trailer whose declared checksum is not empty-data zero", async () => {
|
||||
const raw = gunzipSync(gzipCpio([], "crc"));
|
||||
raw.write("00000001", 102, "ascii");
|
||||
const reader = new CpioArtifactReader(async () =>
|
||||
Readable.from([gzipSync(raw)]),
|
||||
);
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
message:
|
||||
"cpio CRC disagrees with content: TRAILER!!! (Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000000.)",
|
||||
});
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("bounds retained metadata independently of small decoded byte counts", async () => {
|
||||
const members = Array.from({ length: 100 }, (_, index) => ({
|
||||
name: `empty-${index}`,
|
||||
mode: MODE.file,
|
||||
}));
|
||||
const reader = new CpioArtifactReader(
|
||||
async () => Readable.from([gzipCpio(members)]),
|
||||
"fail",
|
||||
new ArtifactDecodedBudget(1024 * 1024, 32 * 1024),
|
||||
);
|
||||
let seen = 0;
|
||||
try {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) {
|
||||
void entry;
|
||||
seen++;
|
||||
}
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Retained archive metadata budget"),
|
||||
});
|
||||
expect(seen).toBeGreaterThan(0);
|
||||
expect(seen).toBeLessThan(members.length);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("recovers directory CRC contradictions before yielding and reaches regular siblings", async () => {
|
||||
const reader = new CpioArtifactReader(
|
||||
async () =>
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[
|
||||
{ name: "dir", mode: MODE.directory, check: 1 },
|
||||
{ name: "dir/good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
]),
|
||||
"record-and-continue",
|
||||
);
|
||||
try {
|
||||
const seen: string[] = [];
|
||||
for await (const entry of reader.entries()) {
|
||||
seen.push(entry.path);
|
||||
if (entry.kind === "directory")
|
||||
expect(entry.limitations).toContain(
|
||||
"Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000000.",
|
||||
);
|
||||
else
|
||||
expect((await buffer(await reader.open(entry))).toString()).toBe("ok");
|
||||
}
|
||||
expect(seen).toEqual(["dir", "dir/good"]);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["directory", "symlink"])(
|
||||
"rejects data declarations attached to non-file type %s",
|
||||
async (type) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("data") }]),
|
||||
(xml) => xml.replace("<type>file</type>", `<type>${type}</type>`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(
|
||||
(async () => {
|
||||
for await (const entry of reader.entries()) void entry;
|
||||
})(),
|
||||
).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar non-file member declares data: a",
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("charges long enclosing prefixes for every retained nested entry", async () => {
|
||||
const files = Array.from({ length: 100 }, (_, index) => ({
|
||||
name: `empty-${index}`,
|
||||
mode: MODE.file,
|
||||
}));
|
||||
let member: XarFixtureMember = { name: "Payload", data: gzipCpio(files) };
|
||||
for (let index = 0; index < 18; index++)
|
||||
member = { name: "d".repeat(200), type: "directory", children: [member] };
|
||||
await withReader(
|
||||
xarArchive([member]),
|
||||
async (reader) => {
|
||||
let retained = 0;
|
||||
await expect(
|
||||
visitArtifactTree(reader, async ({ container }) => {
|
||||
if (!container) retained++;
|
||||
return container;
|
||||
}),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Retained archive metadata budget"),
|
||||
});
|
||||
expect(retained).toBeLessThan(files.length + 18);
|
||||
},
|
||||
new ArtifactDecodedBudget(1024 * 1024, 512 * 1024),
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["<!--n-->", "<?n value?>", "<![CDATA[n]]>"])(
|
||||
"bounds DOM nodes produced by %s before parsing",
|
||||
async (node) => {
|
||||
const bytes = rewriteXarToc(xarArchive([]), (xml) =>
|
||||
xml.replace("<toc>", `<toc>${node.repeat(100_001)}`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("XML nodes"),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["data", "extracted-checksum", "offset", "encoding", "type"])(
|
||||
"rejects duplicate singleton <%s> declarations",
|
||||
async (tag) => {
|
||||
const bytes = rewriteXarToc(
|
||||
xarArchive([{ name: "a", data: Buffer.from("ok") }]),
|
||||
(xml) => {
|
||||
const pattern = new RegExp(
|
||||
`<${tag}(?:\\s[^>]*|)>(?:[^]*?)</${tag}>|<${tag}[^>]*/>`,
|
||||
);
|
||||
return xml.replace(pattern, (element) => element + element);
|
||||
},
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining(`duplicate <${tag}>`),
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects unsupported xar versions before reading the TOC", async () => {
|
||||
const bytes = Buffer.from(xarArchive([]));
|
||||
bytes.writeUInt16BE(2, 6);
|
||||
await withReader(bytes, async (reader) => {
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("version 2"),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("ends record-and-continue traversal when its shared decoded budget is exhausted", async () => {
|
||||
const bytes = xarArchive(
|
||||
Array.from({ length: 10 }, (_, index) => ({
|
||||
name: `file-${index}`,
|
||||
data: Buffer.alloc(40, index),
|
||||
})),
|
||||
);
|
||||
await withReader(
|
||||
bytes,
|
||||
async (reader) => {
|
||||
const opened: string[] = [];
|
||||
const observed = {
|
||||
format: reader.format,
|
||||
provenance: reader.provenance.bind(reader),
|
||||
entries: reader.entries.bind(reader),
|
||||
close: reader.close.bind(reader),
|
||||
decodedBudget: reader.decodedBudget,
|
||||
open: async (...args: Parameters<typeof reader.open>) => {
|
||||
opened.push(args[0].path);
|
||||
return reader.open(...args);
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
scanReader(observed, undefined, { mode: "record-and-continue" }),
|
||||
).rejects.toMatchObject({
|
||||
reason: "limit",
|
||||
message: expect.stringContaining("Decoded archive budget"),
|
||||
});
|
||||
expect(opened).toEqual(["file-0", "file-1"]);
|
||||
},
|
||||
new ArtifactDecodedBudget(64),
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts productbuild repeated identical names but rejects contradictory names", async () => {
|
||||
const original = xarArchive([{ name: "a", data: Buffer.from("ok") }]);
|
||||
for (const name of ["a", "different"]) {
|
||||
const bytes = rewriteXarToc(original, (xml) =>
|
||||
xml.replace("<name>a</name>", `<name>a</name><name>${name}</name>`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
if (name === "a")
|
||||
expect((await scanReader(reader)).occurrences).toHaveLength(1);
|
||||
else
|
||||
await expect(scanReader(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringContaining("conflicting <name>"),
|
||||
});
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
it("counts a comment as one node even when its text contains markup openers", async () => {
|
||||
const bytes = rewriteXarToc(xarArchive([]), (xml) =>
|
||||
xml.replace("<toc>", `<toc><!--${"<".repeat(100_001)}-->`),
|
||||
);
|
||||
await withReader(bytes, async (reader) => {
|
||||
expect((await scanReader(reader)).occurrences).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
it("resolves a large complete empty hard-link group without per-alias rescans", async () => {
|
||||
const count = 20_000;
|
||||
const bytes = gzipCpio(
|
||||
Array.from({ length: count }, (_, index) => ({
|
||||
name: `link-${index}`,
|
||||
mode: MODE.file,
|
||||
data: "",
|
||||
ino: 1,
|
||||
links: count,
|
||||
})),
|
||||
"newc",
|
||||
);
|
||||
const reader = new CpioArtifactReader(async () => Readable.from([bytes]));
|
||||
try {
|
||||
let retained = 0;
|
||||
for await (const entry of reader.entries()) {
|
||||
expect(entry.declaredSize).toBe(0);
|
||||
expect(entry.contentUnavailable).not.toBe(true);
|
||||
retained++;
|
||||
}
|
||||
expect(retained).toBe(count);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,444 @@
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { Readable } from "node:stream";
|
||||
import { buffer } from "node:stream/consumers";
|
||||
import { gunzipSync, gzipSync } from "node:zlib";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { ArtifactEntry } from "./ArtifactReader.js";
|
||||
import { CpioArtifactReader } from "./CpioArtifactReader.js";
|
||||
import { MODE, gzipCpio, xarArchive } from "./InstallerPackage.fixture.js";
|
||||
import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js";
|
||||
import { XarArtifactReader } from "./XarArtifactReader.js";
|
||||
|
||||
const writePackage = async (bytes: Uint8Array): Promise<string> => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-reader-");
|
||||
const path = join(directory, "fixture.pkg");
|
||||
await writeFile(path, bytes);
|
||||
return path;
|
||||
};
|
||||
|
||||
const collect = async (reader: {
|
||||
entries(): AsyncIterable<ArtifactEntry>;
|
||||
}): Promise<ArtifactEntry[]> => {
|
||||
const entries: ArtifactEntry[] = [];
|
||||
for await (const entry of reader.entries()) entries.push(entry);
|
||||
return entries;
|
||||
};
|
||||
|
||||
describe("xar installer package reader", () => {
|
||||
it("lists nested members and marks gzip-cpio scripts and payloads", async () => {
|
||||
const scripts = gzipCpio([
|
||||
{ name: "./postinstall", mode: MODE.executable, data: "#!/bin/sh\n" },
|
||||
]);
|
||||
const reader = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{
|
||||
name: "Distribution",
|
||||
data: Buffer.from("<installer-gui-script/>"),
|
||||
encoding: "zlib",
|
||||
},
|
||||
{
|
||||
name: "App.pkg",
|
||||
type: "directory",
|
||||
children: [
|
||||
{
|
||||
name: "PackageInfo",
|
||||
data: Buffer.from("<pkg-info/>"),
|
||||
encoding: "zlib",
|
||||
},
|
||||
{ name: "Scripts", data: scripts },
|
||||
{ name: "Payload", data: Buffer.from("pbzx\0\0\0\0\x01\0\0\0") },
|
||||
{ name: "Current", type: "symlink", link: "App.pkg" },
|
||||
],
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const entries = await collect(reader);
|
||||
expect(entries.map(({ path, kind }) => `${kind}:${path}`)).toEqual([
|
||||
"file:Distribution",
|
||||
"directory:App.pkg",
|
||||
"file:App.pkg/PackageInfo",
|
||||
"file:App.pkg/Scripts",
|
||||
"file:App.pkg/Payload",
|
||||
"symlink:App.pkg/Current",
|
||||
]);
|
||||
const scriptsEntry = entries.find(
|
||||
({ path }) => path === "App.pkg/Scripts",
|
||||
);
|
||||
expect(scriptsEntry?.nestedArchive).toBe("gzip-cpio");
|
||||
expect(
|
||||
entries.find(({ path }) => path === "App.pkg/Payload")?.limitations,
|
||||
).toEqual([expect.stringContaining("pbzx")]);
|
||||
const distribution = entries[0];
|
||||
if (distribution === undefined) throw new Error("missing entry");
|
||||
expect((await buffer(await reader.open(distribution))).toString()).toBe(
|
||||
"<installer-gui-script/>",
|
||||
);
|
||||
if (scriptsEntry === undefined) throw new Error("missing scripts");
|
||||
const nested = new CpioArtifactReader(() => reader.open(scriptsEntry));
|
||||
try {
|
||||
const members: string[] = [];
|
||||
for await (const entry of nested.entries()) {
|
||||
members.push(entry.path);
|
||||
expect((await buffer(await nested.open(entry))).toString()).toBe(
|
||||
"#!/bin/sh\n",
|
||||
);
|
||||
expect(entry.executable).toBe(true);
|
||||
}
|
||||
expect(members).toEqual(["postinstall"]);
|
||||
} finally {
|
||||
await nested.close();
|
||||
}
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects tampered members, TOCs and unsupported encodings", async () => {
|
||||
const tampered = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{
|
||||
name: "Bom",
|
||||
data: Buffer.from("bom"),
|
||||
extractedSha1: "0".repeat(40),
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const [entry] = await collect(tampered);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
await expect(buffer(await tampered.open(entry))).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
} finally {
|
||||
await tampered.close();
|
||||
}
|
||||
const corrupt = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([{ name: "Bom", data: Buffer.from("bom") }], {
|
||||
corruptTocChecksum: true,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await expect(collect(corrupt)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
await corrupt.close();
|
||||
const bzip = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{ name: "Bom", data: Buffer.from("bom"), encoding: "bzip2" },
|
||||
]),
|
||||
),
|
||||
);
|
||||
const [entry] = await collect(bzip);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
expect(entry.contentUnavailable).toBe(true);
|
||||
await expect(bzip.open(entry)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
await bzip.close();
|
||||
const notXar = new XarArtifactReader(
|
||||
await writePackage(Buffer.from("not an archive at all")),
|
||||
);
|
||||
await expect(collect(notXar)).rejects.toMatchObject({ reason: "format" });
|
||||
await notXar.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("gzip cpio reader", () => {
|
||||
for (const format of ["odc", "newc"] as const)
|
||||
it(`reads ${format} members in order and skips unopened data`, async () => {
|
||||
const archive = gzipCpio(
|
||||
[
|
||||
{ name: ".", mode: MODE.directory },
|
||||
{ name: "./usr", mode: MODE.directory },
|
||||
{
|
||||
name: "./usr/bin/tool",
|
||||
mode: MODE.executable,
|
||||
data: "skipped bytes",
|
||||
},
|
||||
{ name: "./usr/lib/link", mode: MODE.symlink, data: "../bin/tool" },
|
||||
{ name: "./usr/fifo", mode: MODE.fifo },
|
||||
{ name: "./usr/share/readme.txt", mode: MODE.file, data: "read me" },
|
||||
],
|
||||
format,
|
||||
);
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(Readable.from([archive])),
|
||||
);
|
||||
const seen: string[] = [];
|
||||
try {
|
||||
for await (const entry of reader.entries()) {
|
||||
seen.push(`${entry.kind}:${entry.path}`);
|
||||
if (entry.path === "usr/share/readme.txt") {
|
||||
expect((await buffer(await reader.open(entry))).toString()).toBe(
|
||||
"read me",
|
||||
);
|
||||
await expect(reader.open(entry)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
}
|
||||
if (entry.kind === "symlink")
|
||||
expect(entry.limitations).toEqual(["Symlink target: ../bin/tool"]);
|
||||
}
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
expect(seen).toEqual([
|
||||
"directory:usr",
|
||||
"file:usr/bin/tool",
|
||||
"symlink:usr/lib/link",
|
||||
"file:usr/fifo",
|
||||
"file:usr/share/readme.txt",
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects traversal and truncated archives", async () => {
|
||||
for (const name of ["../escape", "/absolute"]) {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([gzipCpio([{ name, mode: MODE.file, data: "x" }])]),
|
||||
),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({ reason: "path" });
|
||||
await reader.close();
|
||||
}
|
||||
const whole = gzipCpio([{ name: "a", mode: MODE.file, data: "abc" }]);
|
||||
const { gunzipSync, gzipSync } = await import("node:zlib");
|
||||
const truncated = gzipSync(gunzipSync(whole).subarray(0, 60));
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(Readable.from([truncated])),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({ reason: "format" });
|
||||
await reader.close();
|
||||
});
|
||||
});
|
||||
|
||||
describe("installer package reader hardening", () => {
|
||||
it("refuses a TOC checksum whose declared size is not one digest", async () => {
|
||||
const reader = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([{ name: "Bom", data: Buffer.from("bom") }], {
|
||||
tocChecksumSize: 0x7fffffff,
|
||||
}),
|
||||
),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: "xar TOC declares a 2147483647-byte sha1 checksum",
|
||||
});
|
||||
await reader.close();
|
||||
});
|
||||
|
||||
it("reports corrupt zlib members as malformed and keeps cancellation tagged", async () => {
|
||||
const reader = new XarArtifactReader(
|
||||
await writePackage(
|
||||
xarArchive([
|
||||
{
|
||||
name: "PackageInfo",
|
||||
data: Buffer.from("<pkg-info/>"),
|
||||
encoding: "zlib",
|
||||
archived: Buffer.from("not zlib at all"),
|
||||
},
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const [entry] = await collect(reader);
|
||||
if (entry === undefined) throw new Error("missing entry");
|
||||
await expect(buffer(await reader.open(entry))).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
});
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
await expect(reader.open(entry, controller.signal)).rejects.toMatchObject(
|
||||
{
|
||||
reason: "cancelled",
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("cpio symlink targets", () => {
|
||||
it("streams past an oversized symlink target and keeps reading", async () => {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[
|
||||
{ name: "./huge", mode: MODE.symlink, data: "x".repeat(4097) },
|
||||
{ name: "./after", mode: MODE.file, data: "after" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
try {
|
||||
const entries = await collect(reader);
|
||||
expect(
|
||||
entries.map(({ kind, path, limitations }) => [kind, path, limitations]),
|
||||
).toEqual([
|
||||
[
|
||||
"symlink",
|
||||
"huge",
|
||||
["Symlink target of 4097 bytes exceeds 4096 bytes and was not read."],
|
||||
],
|
||||
["file", "after", []],
|
||||
]);
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("cpio hard links and CRC archives", () => {
|
||||
const readAll = async (archive: Uint8Array) => {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(Readable.from([archive])),
|
||||
);
|
||||
const files: Record<string, string | undefined> = {};
|
||||
try {
|
||||
for await (const entry of reader.entries())
|
||||
if (entry.kind === "file")
|
||||
files[entry.path] =
|
||||
entry.contentUnavailable === true
|
||||
? undefined
|
||||
: (await buffer(await reader.open(entry))).toString();
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
return files;
|
||||
};
|
||||
|
||||
it("serves newc hard links stored on the last member to every path", async () => {
|
||||
expect(
|
||||
await readAll(
|
||||
gzipCpio(
|
||||
[
|
||||
{ name: "./first", mode: MODE.file, ino: 7, links: 2 },
|
||||
{
|
||||
name: "./second",
|
||||
mode: MODE.file,
|
||||
ino: 7,
|
||||
links: 2,
|
||||
data: "shared",
|
||||
},
|
||||
{
|
||||
name: "./third",
|
||||
mode: MODE.file,
|
||||
ino: 9,
|
||||
links: 2,
|
||||
data: "kept",
|
||||
},
|
||||
{ name: "./fourth", mode: MODE.file, ino: 9, links: 2 },
|
||||
],
|
||||
"newc",
|
||||
),
|
||||
),
|
||||
).toEqual({
|
||||
first: "shared",
|
||||
second: "shared",
|
||||
third: "kept",
|
||||
fourth: "kept",
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a hard link whose bytes never appear as unavailable", async () => {
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[{ name: "./orphan", mode: MODE.file, ino: 4, links: 2 }],
|
||||
"newc",
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
const entries = await collect(reader);
|
||||
expect(entries).toEqual([
|
||||
expect.objectContaining({
|
||||
path: "orphan",
|
||||
contentUnavailable: true,
|
||||
declaredSize: null,
|
||||
}),
|
||||
]);
|
||||
const orphan = entries[0];
|
||||
if (orphan === undefined) throw new Error("missing entry");
|
||||
await expect(reader.open(orphan)).rejects.toMatchObject({
|
||||
reason: "unavailable",
|
||||
});
|
||||
await reader.close();
|
||||
});
|
||||
|
||||
it("verifies 070702 member checksums whether or not members are opened", async () => {
|
||||
expect(
|
||||
await readAll(
|
||||
gzipCpio([{ name: "./ok", mode: MODE.file, data: "abc" }], "crc"),
|
||||
),
|
||||
).toEqual({ ok: "abc" });
|
||||
await expect(
|
||||
readAll(
|
||||
gzipCpio(
|
||||
[{ name: "./bad", mode: MODE.file, data: "abc", check: 1 }],
|
||||
"crc",
|
||||
),
|
||||
),
|
||||
).rejects.toMatchObject({ reason: "integrity" });
|
||||
const reader = new CpioArtifactReader(() =>
|
||||
Promise.resolve(
|
||||
Readable.from([
|
||||
gzipCpio(
|
||||
[{ name: "./bad", mode: MODE.file, data: "abc", check: 1 }],
|
||||
"crc",
|
||||
),
|
||||
]),
|
||||
),
|
||||
);
|
||||
await expect(collect(reader)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
await reader.close();
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["newc", "crc"] as const)(
|
||||
"rejects nonzero %s alignment under either integrity policy",
|
||||
async (format) => {
|
||||
for (const policy of ["fail", "record-and-continue"] as const) {
|
||||
for (const [offset, label] of [
|
||||
[113, "name padding"],
|
||||
[117, "data padding"],
|
||||
] as const) {
|
||||
// 110-byte header, a three-byte NUL-terminated name, then one data byte.
|
||||
const decoded = gunzipSync(
|
||||
gzipCpio([{ name: "ab", mode: MODE.file, data: "x" }], format),
|
||||
);
|
||||
decoded[offset] = 1;
|
||||
const reader = new CpioArtifactReader(
|
||||
() => Promise.resolve(Readable.from([gzipSync(decoded)])),
|
||||
policy,
|
||||
);
|
||||
try {
|
||||
await expect(collect(reader)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: `cpio ${label} contains nonzero bytes`,
|
||||
});
|
||||
} finally {
|
||||
await reader.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,875 @@
|
||||
import { createHash, type Hash } from "node:crypto";
|
||||
import { open, type FileHandle } from "node:fs/promises";
|
||||
import { Readable, Transform, type TransformCallback } from "node:stream";
|
||||
import { createInflate, inflateSync } from "node:zlib";
|
||||
|
||||
import { DOMParser, type Element, type Node } from "@xmldom/xmldom";
|
||||
|
||||
import type { ArtifactCommand } from "../domain/artifactGraph.js";
|
||||
import {
|
||||
ArtifactDecodedBudget,
|
||||
ArtifactBudgetTransform,
|
||||
} from "./ArtifactDecodedBudget.js";
|
||||
import { artifactStreamPipeline } from "./ArtifactStreamPipeline.js";
|
||||
import {
|
||||
assertXarTocNodes,
|
||||
xarInteger as integer,
|
||||
xarHeapPosition,
|
||||
XarPathBudget,
|
||||
xarMode,
|
||||
} from "./XarValidation.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactChecksumObservation,
|
||||
type ArtifactReader,
|
||||
} from "./ArtifactReader.js";
|
||||
|
||||
const XAR_MAGIC = 0x78617221;
|
||||
/** A TOC describes members only; real installer TOCs are kilobytes. */
|
||||
const MAX_TOC_BYTES = 64 * 1024 * 1024;
|
||||
const READ_CHUNK_BYTES = 1024 * 1024;
|
||||
|
||||
const CHECKSUM_ALGORITHMS: ReadonlyMap<string, string> = new Map([
|
||||
["sha1", "sha1"],
|
||||
["md5", "md5"],
|
||||
["sha256", "sha256"],
|
||||
["sha512", "sha512"],
|
||||
]);
|
||||
const DIGEST_BYTES: ReadonlyMap<string, number> = new Map([
|
||||
["sha1", 20],
|
||||
["md5", 16],
|
||||
["sha256", 32],
|
||||
["sha512", 64],
|
||||
]);
|
||||
|
||||
const cancelled = (signal?: AbortSignal): void => {
|
||||
if (signal?.aborted === true)
|
||||
throw new ArtifactReaderFailure("cancelled", "PKG traversal was cancelled");
|
||||
};
|
||||
|
||||
/** One TOC `<data>` element: where a member's archived bytes live in the heap. */
|
||||
interface XarData {
|
||||
readonly offset: number;
|
||||
readonly length: number;
|
||||
readonly size: number;
|
||||
readonly encoding: string;
|
||||
readonly extractedChecksum:
|
||||
| { readonly algorithm: string; readonly value: string }
|
||||
| undefined;
|
||||
readonly unsupportedChecksum:
|
||||
| { readonly style: string; readonly value: string }
|
||||
| undefined;
|
||||
readonly archivedChecksum:
|
||||
| { readonly algorithm: string; readonly value: string }
|
||||
| undefined;
|
||||
readonly unsupportedArchivedChecksum:
|
||||
| { readonly style: string; readonly value: string }
|
||||
| undefined;
|
||||
}
|
||||
|
||||
interface XarMember {
|
||||
readonly path: string;
|
||||
readonly kind: "file" | "directory" | "symlink";
|
||||
/** Raw TOC type when it is not file/directory/symlink (FIFO, device, ...). */
|
||||
readonly unsupportedType?: string;
|
||||
readonly mode: number | null;
|
||||
readonly data: XarData | undefined;
|
||||
readonly link: string | undefined;
|
||||
}
|
||||
|
||||
const isElement = (node: Node): node is Element => node.nodeType === 1;
|
||||
|
||||
/** Direct child elements with one tag name, in document order. */
|
||||
const childElements = (element: Element, name: string): Element[] =>
|
||||
Array.from(element.childNodes).filter(
|
||||
(child): child is Element => isElement(child) && child.tagName === name,
|
||||
);
|
||||
|
||||
const childElement = (element: Element, name: string): Element | undefined => {
|
||||
const children = childElements(element, name);
|
||||
if (children.length > 1)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC has duplicate <${name}> declarations in <${element.tagName}>`,
|
||||
);
|
||||
return children[0];
|
||||
};
|
||||
|
||||
// productbuild repeats an identical <name> when flattening a component PKG.
|
||||
// Preserve that observed producer form while rejecting contradictory names.
|
||||
const memberName = (file: Element): string | undefined => {
|
||||
const names = childElements(file, "name").map(
|
||||
(name) => name.textContent ?? "",
|
||||
);
|
||||
if (names.some((name) => name !== names[0]))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar TOC has conflicting <name> declarations",
|
||||
);
|
||||
return names[0];
|
||||
};
|
||||
|
||||
const textOf = (element: Element, name: string): string | undefined =>
|
||||
childElement(element, name)?.textContent ?? undefined;
|
||||
|
||||
const parseChecksum = (
|
||||
parent: Element,
|
||||
tag: string,
|
||||
):
|
||||
| { readonly algorithm: string; readonly value: string }
|
||||
| { readonly unsupported: string; readonly value: string }
|
||||
| undefined => {
|
||||
const element = childElement(parent, tag);
|
||||
if (element === undefined) return undefined;
|
||||
const style = element.getAttribute("style")?.toLowerCase() ?? "";
|
||||
const value = (element.textContent ?? "").trim().toLowerCase();
|
||||
if (CHECKSUM_ALGORITHMS.get(style) === undefined)
|
||||
return { unsupported: style || "(missing style)", value };
|
||||
const expected = DIGEST_BYTES.get(style);
|
||||
if (
|
||||
expected === undefined ||
|
||||
value.length !== expected * 2 ||
|
||||
!/^[0-9a-f]+$/u.test(value)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar ${tag} has malformed ${style} digest text`,
|
||||
);
|
||||
return { algorithm: style, value };
|
||||
};
|
||||
|
||||
const parseData = (file: Element): XarData | undefined => {
|
||||
const data = childElement(file, "data");
|
||||
if (data === undefined) return undefined;
|
||||
const extracted = parseChecksum(data, "extracted-checksum");
|
||||
const archived = parseChecksum(data, "archived-checksum");
|
||||
return {
|
||||
offset: integer(textOf(data, "offset"), "data offset"),
|
||||
length: integer(textOf(data, "length"), "data length"),
|
||||
size: integer(textOf(data, "size"), "data size"),
|
||||
encoding:
|
||||
childElement(data, "encoding")?.getAttribute("style") ??
|
||||
"application/octet-stream",
|
||||
extractedChecksum:
|
||||
extracted === undefined || "unsupported" in extracted
|
||||
? undefined
|
||||
: { algorithm: extracted.algorithm, value: extracted.value },
|
||||
unsupportedChecksum:
|
||||
extracted !== undefined && "unsupported" in extracted
|
||||
? { style: extracted.unsupported, value: extracted.value }
|
||||
: undefined,
|
||||
archivedChecksum:
|
||||
archived === undefined || "unsupported" in archived
|
||||
? undefined
|
||||
: { algorithm: archived.algorithm, value: archived.value },
|
||||
unsupportedArchivedChecksum:
|
||||
archived !== undefined && "unsupported" in archived
|
||||
? { style: archived.unsupported, value: archived.value }
|
||||
: undefined,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Raw members must describe their stored bytes exactly; otherwise the source
|
||||
* range would extend into adjacent heap entries.
|
||||
*/
|
||||
const assertRawSize = (member: XarMember, data: XarData | undefined): void => {
|
||||
if (
|
||||
data !== undefined &&
|
||||
data.encoding === "application/octet-stream" &&
|
||||
data.size !== data.length
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${member.path} declares size ${data.size} with length ${data.length}`,
|
||||
);
|
||||
};
|
||||
|
||||
/** Entry limitations from the TOC record and nested-archive classification. */
|
||||
const memberLimitations = (
|
||||
member: XarMember,
|
||||
unsupportedChecksum: XarData["unsupportedChecksum"],
|
||||
unsupportedArchived: XarData["unsupportedArchivedChecksum"],
|
||||
classified: { readonly limitations: readonly string[] },
|
||||
): string[] => [
|
||||
...(member.kind === "symlink" && member.link !== undefined
|
||||
? [`Symlink target recorded in the TOC: ${member.link}`]
|
||||
: []),
|
||||
...(unsupportedChecksum === undefined
|
||||
? []
|
||||
: [
|
||||
`Member declares unsupported checksum ${unsupportedChecksum.style} ${unsupportedChecksum.value}; integrity not verified.`,
|
||||
]),
|
||||
...(unsupportedArchived === undefined
|
||||
? []
|
||||
: [
|
||||
`Member declares unsupported archived checksum ${unsupportedArchived.style} ${unsupportedArchived.value}; stored-byte integrity not verified.`,
|
||||
]),
|
||||
...(member.data !== undefined &&
|
||||
member.data.encoding !== "application/octet-stream" &&
|
||||
member.data.encoding !== "application/x-gzip"
|
||||
? [
|
||||
`Unsupported xar encoding ${member.data.encoding}; content is unavailable and was not expanded.`,
|
||||
]
|
||||
: []),
|
||||
...classified.limitations,
|
||||
];
|
||||
|
||||
/** File, directory, symlink, or an explicitly unsupported TOC type. */
|
||||
const classifyMemberType = (
|
||||
type: string | undefined,
|
||||
): { readonly kind: XarMember["kind"]; readonly unsupportedType?: string } => {
|
||||
if (type === "directory" || type === "symlink") return { kind: type };
|
||||
if (type === undefined || type === "file") return { kind: "file" };
|
||||
return { kind: "file", unsupportedType: type };
|
||||
};
|
||||
|
||||
/** Walk nested `<file>` elements in document order. */
|
||||
const collectMembers = (
|
||||
toc: Element,
|
||||
budget: ArtifactDecodedBudget,
|
||||
): XarMember[] => {
|
||||
const members: XarMember[] = [];
|
||||
const paths = new XarPathBudget();
|
||||
const pending: Array<{ readonly element: Element; readonly parent: string }> =
|
||||
childElements(toc, "file")
|
||||
.map((element) => ({ element, parent: "" }))
|
||||
.reverse();
|
||||
while (pending.length > 0) {
|
||||
const next = pending.pop();
|
||||
if (next === undefined) break;
|
||||
const name = memberName(next.element);
|
||||
if (
|
||||
name === undefined ||
|
||||
name === "" ||
|
||||
name.includes("/") ||
|
||||
name === "." ||
|
||||
name === ".."
|
||||
)
|
||||
throw new ArtifactReaderFailure("path", "xar member has an unsafe name");
|
||||
const path = paths.join(next.parent, name);
|
||||
budget.consumeEntry(path);
|
||||
const type = textOf(next.element, "type")?.trim();
|
||||
const mode = xarMode(textOf(next.element, "mode"));
|
||||
const { kind, unsupportedType } = classifyMemberType(type);
|
||||
if (kind !== "file" && childElement(next.element, "data") !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar non-file member declares data: ${path}`,
|
||||
);
|
||||
members.push({
|
||||
path,
|
||||
kind,
|
||||
...(unsupportedType === undefined ? {} : { unsupportedType }),
|
||||
mode,
|
||||
data: parseData(next.element),
|
||||
link: kind === "symlink" ? textOf(next.element, "link") : undefined,
|
||||
});
|
||||
const children = childElements(next.element, "file");
|
||||
for (let index = children.length - 1; index >= 0; index--) {
|
||||
const element = children[index];
|
||||
if (element !== undefined) pending.push({ element, parent: path });
|
||||
}
|
||||
}
|
||||
return members;
|
||||
};
|
||||
|
||||
/** Expose a member's extracted checksum so a caller can apply its own policy. */
|
||||
const declaredDigest = (
|
||||
checksum: { readonly algorithm: string; readonly value: string } | undefined,
|
||||
): Pick<ArtifactEntry, "declaredSha256" | "declaredChecksum"> => {
|
||||
if (checksum === undefined) return { declaredSha256: null };
|
||||
if (checksum.algorithm === "sha256")
|
||||
return { declaredSha256: checksum.value };
|
||||
const algorithm = checksum.algorithm;
|
||||
return algorithm === "sha1" || algorithm === "md5" || algorithm === "sha512"
|
||||
? {
|
||||
declaredSha256: null,
|
||||
declaredChecksum: { algorithm, value: checksum.value },
|
||||
}
|
||||
: { declaredSha256: null };
|
||||
};
|
||||
|
||||
/**
|
||||
* Absolute decoded-byte ceiling for one gzip member. The declared size is
|
||||
* attacker-controlled, so it cannot be the only inflation bound.
|
||||
*/
|
||||
const MAX_DECODED_MEMBER_BYTES = 256 * 1024 * 1024;
|
||||
|
||||
/** Bound decoded output to the TOC-declared extracted size. */
|
||||
class SizeBound extends Transform {
|
||||
#seen = 0;
|
||||
#cap: number;
|
||||
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
private readonly expected: number,
|
||||
private readonly budget: ArtifactDecodedBudget,
|
||||
private readonly signal?: AbortSignal,
|
||||
) {
|
||||
super();
|
||||
this.#cap = Math.min(expected, MAX_DECODED_MEMBER_BYTES);
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer,
|
||||
_encoding: BufferEncoding,
|
||||
done: TransformCallback,
|
||||
): void {
|
||||
if (this.signal?.aborted) {
|
||||
done(
|
||||
new ArtifactReaderFailure(
|
||||
"cancelled",
|
||||
`xar member decoding cancelled: ${this.path}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.budget.consume(chunk.length, this.path);
|
||||
} catch (cause: unknown) {
|
||||
done(cause instanceof Error ? cause : new Error(String(cause)));
|
||||
return;
|
||||
}
|
||||
this.#seen += chunk.length;
|
||||
if (this.#seen > this.#cap) {
|
||||
done(
|
||||
new ArtifactReaderFailure(
|
||||
"limit",
|
||||
this.expected > MAX_DECODED_MEMBER_BYTES
|
||||
? `xar member ${this.path} decodes beyond the ${MAX_DECODED_MEMBER_BYTES}-byte member ceiling`
|
||||
: `xar member ${this.path} decodes beyond its declared ${this.expected} bytes`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
done(null, chunk);
|
||||
}
|
||||
|
||||
override _flush(done: TransformCallback): void {
|
||||
done(
|
||||
this.#seen === this.expected
|
||||
? null
|
||||
: new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${this.path} decoded ${this.#seen} bytes, expected ${this.expected}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Verify a member's extracted checksum as its decoded bytes stream past. */
|
||||
class ChecksumVerifier extends Transform {
|
||||
readonly #hash: Hash;
|
||||
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
private readonly expected: {
|
||||
readonly algorithm: string;
|
||||
readonly value: string;
|
||||
},
|
||||
private readonly record?: (
|
||||
observation: ArtifactChecksumObservation,
|
||||
) => void,
|
||||
) {
|
||||
super();
|
||||
this.#hash = createHash(expected.algorithm);
|
||||
}
|
||||
|
||||
override _transform(
|
||||
chunk: Buffer,
|
||||
_encoding: BufferEncoding,
|
||||
done: TransformCallback,
|
||||
): void {
|
||||
this.#hash.update(chunk);
|
||||
done(null, chunk);
|
||||
}
|
||||
|
||||
override _flush(done: TransformCallback): void {
|
||||
const observed = this.#hash.digest("hex");
|
||||
if (this.record !== undefined) {
|
||||
this.record({
|
||||
representation: "stored",
|
||||
algorithm: this.expected.algorithm,
|
||||
declared: this.expected.value,
|
||||
observed,
|
||||
});
|
||||
done();
|
||||
return;
|
||||
}
|
||||
done(
|
||||
observed === this.expected.value
|
||||
? null
|
||||
: new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`xar ${this.expected.algorithm} checksum disagrees with content: ${this.path} (declared ${this.expected.value}, observed ${observed})`,
|
||||
),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** First bytes that identify an installer archive member's own format. */
|
||||
export type XarNestedArchive = "gzip-cpio";
|
||||
|
||||
/**
|
||||
* Read-only reader for xar archives, the container of flat installer
|
||||
* packages. Members are decoded and checksum-verified on demand. Gzip cpio
|
||||
* `Scripts` and `Payload` members are marked for nested expansion; pbzx
|
||||
* (LZMA) payloads are recorded as files with a limitation.
|
||||
*/
|
||||
export class XarArtifactReader implements ArtifactReader {
|
||||
readonly format = "pkg" as const;
|
||||
#handle: FileHandle | undefined;
|
||||
#heap = 0;
|
||||
#size = 0;
|
||||
readonly decodedBudget: ArtifactDecodedBudget;
|
||||
#members: readonly XarMember[] = [];
|
||||
readonly #byPath = new Map<string, XarMember>();
|
||||
#loading: Promise<void> | undefined;
|
||||
#closed = false;
|
||||
readonly #streams = new Set<Readable>();
|
||||
readonly #integrity = new Map<string, ArtifactChecksumObservation[]>();
|
||||
|
||||
readonly #verifyChecksums: boolean;
|
||||
|
||||
/**
|
||||
* @param options.verifyChecksums Fail `open()` streams whose bytes disagree
|
||||
* with the member's extracted checksum (default). Inventory passes false and
|
||||
* verifies the declared checksum itself under the caller's integrity policy.
|
||||
* Archived checksums are always computed. When inventory owns the policy,
|
||||
* integrityObservations exposes their declared/observed values after reading.
|
||||
*/
|
||||
constructor(
|
||||
private readonly path: string,
|
||||
options: {
|
||||
readonly verifyChecksums?: boolean;
|
||||
readonly decodedBudget?: ArtifactDecodedBudget;
|
||||
} = {},
|
||||
) {
|
||||
this.#verifyChecksums = options.verifyChecksums ?? true;
|
||||
this.decodedBudget = options.decodedBudget ?? new ArtifactDecodedBudget();
|
||||
}
|
||||
|
||||
async #load(signal?: AbortSignal): Promise<void> {
|
||||
if (this.#closed)
|
||||
throw new ArtifactReaderFailure("unavailable", "xar archive is closed");
|
||||
this.#loading ??= this.#readToc(signal);
|
||||
await this.#loading;
|
||||
}
|
||||
|
||||
/** Load/index once; a rejected parse remains rejected, never a partial inventory. */
|
||||
async #readToc(signal?: AbortSignal): Promise<void> {
|
||||
cancelled(signal);
|
||||
const handle = await open(this.path, "r");
|
||||
this.#handle = handle;
|
||||
this.#size = (await handle.stat()).size;
|
||||
const header = Buffer.alloc(28);
|
||||
const { bytesRead } = await handle.read(header, 0, 28, 0);
|
||||
if (bytesRead < 28 || header.readUInt32BE(0) !== XAR_MAGIC)
|
||||
throw new ArtifactReaderFailure("format", "File is not a xar archive");
|
||||
const version = header.readUInt16BE(6);
|
||||
if (version !== 1)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`Unsupported xar header version ${version}; expected 1`,
|
||||
);
|
||||
const headerSize = header.readUInt16BE(4);
|
||||
const tocCompressed = header.readBigUInt64BE(8);
|
||||
const tocSize = header.readBigUInt64BE(16);
|
||||
// Fixed-header checksum algorithm at offset 24: 0 is none, 1 is SHA-1,
|
||||
// 2 is MD5. A declared algorithm requires a matching <checksum> element.
|
||||
const headerChecksumAlg = header.readUInt32BE(24);
|
||||
if (headerSize < 28)
|
||||
throw new ArtifactReaderFailure("format", "xar header is too short");
|
||||
if (
|
||||
tocCompressed > BigInt(MAX_TOC_BYTES) ||
|
||||
tocSize > BigInt(MAX_TOC_BYTES)
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`xar TOC exceeds the ${MAX_TOC_BYTES}-byte limit`,
|
||||
);
|
||||
const compressed = Buffer.alloc(Number(tocCompressed));
|
||||
const toc = await handle.read(compressed, 0, compressed.length, headerSize);
|
||||
if (toc.bytesRead < compressed.length)
|
||||
throw new ArtifactReaderFailure("format", "xar TOC is truncated");
|
||||
this.#heap = headerSize + compressed.length;
|
||||
let xml: string;
|
||||
try {
|
||||
const inflated = inflateSync(compressed, {
|
||||
maxOutputLength: MAX_TOC_BYTES,
|
||||
});
|
||||
if (BigInt(inflated.length) !== tocSize)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC decoded ${inflated.length} bytes, expected ${tocSize}`,
|
||||
);
|
||||
try {
|
||||
xml = new TextDecoder("utf-8", { fatal: true }).decode(inflated);
|
||||
} catch (cause: unknown) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar TOC is not valid UTF-8",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
assertXarTocNodes(xml);
|
||||
} catch (cause: unknown) {
|
||||
if (cause instanceof ArtifactReaderFailure) throw cause;
|
||||
throw new ArtifactReaderFailure("format", "xar TOC is not zlib data", {
|
||||
cause,
|
||||
});
|
||||
}
|
||||
let document;
|
||||
try {
|
||||
document = new DOMParser({
|
||||
onError: (level, message) => {
|
||||
if (level !== "warning") throw new Error(message);
|
||||
},
|
||||
}).parseFromString(xml, "text/xml");
|
||||
} catch (cause: unknown) {
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar TOC is not well-formed XML",
|
||||
{ cause },
|
||||
);
|
||||
}
|
||||
const root = document.documentElement;
|
||||
const tocElement = root === null ? undefined : childElement(root, "toc");
|
||||
if (tocElement === undefined)
|
||||
throw new ArtifactReaderFailure("format", "xar TOC has no <toc> element");
|
||||
await this.#verifyToc(tocElement, compressed, headerChecksumAlg);
|
||||
this.#members = collectMembers(tocElement, this.decodedBudget);
|
||||
for (const member of this.#members) {
|
||||
if (member.data !== undefined)
|
||||
xarHeapPosition(
|
||||
this.#heap,
|
||||
member.data.offset,
|
||||
member.data.length,
|
||||
this.#size,
|
||||
);
|
||||
if (this.#byPath.has(member.path))
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`Duplicate xar member path: ${member.path}`,
|
||||
);
|
||||
this.#byPath.set(member.path, member);
|
||||
}
|
||||
}
|
||||
|
||||
/** The TOC checksum in the heap covers the compressed TOC bytes. */
|
||||
async #verifyToc(
|
||||
toc: Element,
|
||||
compressed: Buffer,
|
||||
headerAlg: number,
|
||||
): Promise<void> {
|
||||
// Fixed-header checksum IDs follow libarchive's XAR reader: 0 is none,
|
||||
// 1 is SHA-1, 2 is MD5, 3 is SHA-256, 4 is SHA-512.
|
||||
const expectedStyle =
|
||||
headerAlg === 0
|
||||
? null
|
||||
: headerAlg === 1
|
||||
? "sha1"
|
||||
: headerAlg === 2
|
||||
? "md5"
|
||||
: headerAlg === 3
|
||||
? "sha256"
|
||||
: headerAlg === 4
|
||||
? "sha512"
|
||||
: undefined;
|
||||
if (expectedStyle === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar header declares unsupported TOC checksum algorithm ${headerAlg}`,
|
||||
);
|
||||
const checksum = childElement(toc, "checksum");
|
||||
if (expectedStyle === null && checksum !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar header declares no TOC checksum but the TOC supplies one",
|
||||
);
|
||||
if (checksum === undefined) {
|
||||
if (headerAlg !== 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar header declares a TOC checksum but the TOC has no <checksum> element",
|
||||
);
|
||||
return;
|
||||
}
|
||||
const style = checksum.getAttribute("style")?.toLowerCase() ?? "";
|
||||
if (expectedStyle !== null && style !== expectedStyle)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC checksum style ${style || "(missing)"} does not match header algorithm ${expectedStyle ?? "none"}`,
|
||||
);
|
||||
const algorithm = CHECKSUM_ALGORITHMS.get(style);
|
||||
if (algorithm === undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC declares unsupported checksum style ${style || "(missing)"}`,
|
||||
);
|
||||
const size = integer(textOf(checksum, "size"), "checksum size");
|
||||
// The stored checksum is exactly one digest; never allocate a declared size.
|
||||
if (size !== DIGEST_BYTES.get(style))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC declares a ${size}-byte ${style} checksum`,
|
||||
);
|
||||
const stored = await this.#heapBytes(
|
||||
integer(textOf(checksum, "offset"), "checksum offset"),
|
||||
size,
|
||||
);
|
||||
if (!createHash(algorithm).update(compressed).digest().equals(stored))
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
"xar TOC checksum disagrees with the TOC",
|
||||
);
|
||||
}
|
||||
|
||||
async #heapBytes(offset: number, length: number): Promise<Buffer> {
|
||||
const handle = this.#handle;
|
||||
if (handle === undefined)
|
||||
throw new ArtifactReaderFailure("unavailable", "xar archive is closed");
|
||||
const position = xarHeapPosition(this.#heap, offset, length, this.#size);
|
||||
const bytes = Buffer.alloc(length);
|
||||
let done = 0;
|
||||
while (done < length) {
|
||||
const { bytesRead } = await handle.read(
|
||||
bytes,
|
||||
done,
|
||||
length - done,
|
||||
position + done,
|
||||
);
|
||||
if (bytesRead === 0)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar member extends beyond the archive",
|
||||
);
|
||||
done += bytesRead;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
/** Classify a raw-stored installer member by its leading bytes. */
|
||||
async #nestedArchive(member: XarMember): Promise<{
|
||||
readonly nested: XarNestedArchive | undefined;
|
||||
readonly limitations: readonly string[];
|
||||
}> {
|
||||
const name = member.path.split("/").at(-1);
|
||||
const data = member.data;
|
||||
if (
|
||||
(name !== "Payload" && name !== "Scripts") ||
|
||||
data === undefined ||
|
||||
data.length < 4
|
||||
)
|
||||
return { nested: undefined, limitations: [] };
|
||||
if (data.encoding !== "application/octet-stream")
|
||||
return {
|
||||
nested: undefined,
|
||||
limitations: [
|
||||
`${name} is stored with ${data.encoding}; its archive is not expanded.`,
|
||||
],
|
||||
};
|
||||
const head = await this.#heapBytes(data.offset, 4);
|
||||
if (head[0] === 0x1f && head[1] === 0x8b)
|
||||
return { nested: "gzip-cpio", limitations: [] };
|
||||
if (head.toString("latin1") === "pbzx")
|
||||
return {
|
||||
nested: undefined,
|
||||
limitations: [
|
||||
`${name} is a pbzx (LZMA) archive; its members are not expanded. Expand it with pkgutil --expand-full to inspect them.`,
|
||||
],
|
||||
};
|
||||
return {
|
||||
nested: undefined,
|
||||
limitations: [
|
||||
`${name} has an unrecognized archive format and is not expanded.`,
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async *entries(signal?: AbortSignal): AsyncIterable<ArtifactEntry> {
|
||||
await this.#load(signal);
|
||||
for (const member of this.#members) {
|
||||
cancelled(signal);
|
||||
yield await this.#memberEntry(member);
|
||||
}
|
||||
}
|
||||
|
||||
/** Project one TOC member as an archive-neutral entry. */
|
||||
async #memberEntry(member: XarMember): Promise<ArtifactEntry> {
|
||||
if (member.unsupportedType !== undefined)
|
||||
return {
|
||||
path: member.path,
|
||||
kind: "file",
|
||||
declaredSize: null,
|
||||
compressedSize: null,
|
||||
executable: false,
|
||||
encrypted: false,
|
||||
byteOffset: null,
|
||||
declaredSha256: null,
|
||||
unpacked: false,
|
||||
limitations: [
|
||||
`Unsupported xar member type ${member.unsupportedType}; content not expanded.`,
|
||||
],
|
||||
adapterKey: member.path,
|
||||
contentUnavailable: true,
|
||||
};
|
||||
const data = member.data;
|
||||
assertRawSize(member, data);
|
||||
const unsupportedChecksum = data?.unsupportedChecksum;
|
||||
const unsupportedArchived = data?.unsupportedArchivedChecksum;
|
||||
const unsupportedEncoding =
|
||||
data !== undefined &&
|
||||
data.encoding !== "application/octet-stream" &&
|
||||
data.encoding !== "application/x-gzip";
|
||||
const classified =
|
||||
member.kind === "file" && !unsupportedEncoding
|
||||
? await this.#nestedArchive(member)
|
||||
: {
|
||||
nested: undefined as XarNestedArchive | undefined,
|
||||
limitations: [] as readonly string[],
|
||||
};
|
||||
return {
|
||||
path: member.path,
|
||||
kind: member.kind,
|
||||
declaredSize: member.data?.size ?? (member.kind === "file" ? 0 : null),
|
||||
compressedSize: member.data?.length ?? null,
|
||||
executable: member.mode !== null && (member.mode & 0o111) !== 0,
|
||||
encrypted: false,
|
||||
byteOffset:
|
||||
member.data?.encoding === "application/octet-stream"
|
||||
? this.#heap + member.data.offset
|
||||
: null,
|
||||
...declaredDigest(member.data?.extractedChecksum),
|
||||
unpacked: false,
|
||||
limitations: memberLimitations(
|
||||
member,
|
||||
unsupportedChecksum,
|
||||
unsupportedArchived,
|
||||
classified,
|
||||
),
|
||||
adapterKey: member.path,
|
||||
...(unsupportedEncoding ? { contentUnavailable: true } : {}),
|
||||
...(classified.nested === undefined
|
||||
? {}
|
||||
: { nestedArchive: classified.nested }),
|
||||
};
|
||||
}
|
||||
|
||||
async open(entry: ArtifactEntry, signal?: AbortSignal): Promise<Readable> {
|
||||
cancelled(signal);
|
||||
this.#integrity.delete(entry.adapterKey);
|
||||
await this.#load(signal);
|
||||
const member = this.#byPath.get(entry.adapterKey);
|
||||
if (member === undefined || member.kind !== "file")
|
||||
throw new ArtifactReaderFailure(
|
||||
"path",
|
||||
`xar member is not a file: ${entry.path}`,
|
||||
);
|
||||
if (member.unsupportedType !== undefined)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`xar member ${entry.path} has unsupported type ${member.unsupportedType}`,
|
||||
);
|
||||
const data = member.data;
|
||||
if (data === undefined) return Readable.from([]);
|
||||
if (entry.contentUnavailable === true)
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`xar member ${entry.path} is not expandable`,
|
||||
);
|
||||
// Raw members must describe their stored bytes exactly.
|
||||
if (
|
||||
data.encoding === "application/octet-stream" &&
|
||||
data.size !== data.length
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${entry.path} declares size ${data.size} with length ${data.length}`,
|
||||
);
|
||||
// Validate before creating any streams: otherwise an early throw would
|
||||
// orphan a flowing source that keeps reading after close().
|
||||
if (
|
||||
data.encoding !== "application/octet-stream" &&
|
||||
data.encoding !== "application/x-gzip"
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar member ${entry.path} uses unsupported encoding ${data.encoding}`,
|
||||
);
|
||||
// Archived checksums cover the stored stream, which inventory's scanner
|
||||
// never observes: it hashes decoded bytes. Always verify them here; the
|
||||
// scanner's per-entry recovery still honors record-and-continue.
|
||||
const archived = data.archivedChecksum;
|
||||
const archivedVerifier =
|
||||
archived === undefined
|
||||
? undefined
|
||||
: new ChecksumVerifier(
|
||||
`${entry.path} (archived)`,
|
||||
archived,
|
||||
this.#verifyChecksums
|
||||
? undefined
|
||||
: (observation) => {
|
||||
this.#integrity.set(entry.adapterKey, [observation]);
|
||||
},
|
||||
);
|
||||
const stages: Transform[] = [
|
||||
new ArtifactBudgetTransform(this.decodedBudget, entry.path, signal),
|
||||
];
|
||||
if (archivedVerifier !== undefined) stages.push(archivedVerifier);
|
||||
if (data.encoding === "application/x-gzip")
|
||||
stages.push(
|
||||
createInflate(),
|
||||
new SizeBound(entry.path, data.size, this.decodedBudget, signal),
|
||||
);
|
||||
const checksum = data.extractedChecksum;
|
||||
if (checksum !== undefined && this.#verifyChecksums)
|
||||
stages.push(new ChecksumVerifier(entry.path, checksum));
|
||||
const stream = artifactStreamPipeline(
|
||||
Readable.from(this.#chunks(data.offset, data.length, signal)),
|
||||
stages,
|
||||
{ path: entry.path, signal },
|
||||
);
|
||||
this.#streams.add(stream);
|
||||
stream.once("close", () => this.#streams.delete(stream));
|
||||
return stream;
|
||||
}
|
||||
|
||||
async *#chunks(
|
||||
offset: number,
|
||||
length: number,
|
||||
signal?: AbortSignal,
|
||||
): AsyncGenerator<Buffer> {
|
||||
for (let done = 0; done < length;) {
|
||||
cancelled(signal);
|
||||
const size = Math.min(READ_CHUNK_BYTES, length - done);
|
||||
yield await this.#heapBytes(offset + done, size);
|
||||
done += size;
|
||||
}
|
||||
}
|
||||
|
||||
provenance(): readonly ArtifactCommand[] {
|
||||
return [];
|
||||
}
|
||||
|
||||
/** Stored-byte checksums computed while consuming the member stream. */
|
||||
integrityObservations(
|
||||
entry: ArtifactEntry,
|
||||
): readonly ArtifactChecksumObservation[] {
|
||||
return this.#integrity.get(entry.adapterKey) ?? [];
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closed = true;
|
||||
for (const stream of this.#streams) stream.destroy();
|
||||
await this.#loading?.catch(() => undefined);
|
||||
const handle = this.#handle;
|
||||
this.#handle = undefined;
|
||||
this.#members = [];
|
||||
this.#byPath.clear();
|
||||
this.#integrity.clear();
|
||||
await handle?.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
import { ArtifactReaderFailure } from "./ArtifactReader.js";
|
||||
|
||||
/**
|
||||
* Node ceiling for one TOC. The byte cap still allows millions of tiny
|
||||
* elements, comments, and processing instructions. The DOM is built before
|
||||
* member collection can charge the metadata budget.
|
||||
*/
|
||||
const MAX_TOC_NODES = 100_000;
|
||||
|
||||
/** Bound DOM-producing markup, including comments, CDATA and processing instructions. */
|
||||
export const assertXarTocNodes = (xml: string): void => {
|
||||
let nodes = 0;
|
||||
let index = xml.indexOf("<");
|
||||
while (index !== -1) {
|
||||
if (xml[index + 1] !== "/") {
|
||||
nodes += 1;
|
||||
if (nodes > MAX_TOC_NODES)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`xar TOC exceeds ${MAX_TOC_NODES} XML nodes`,
|
||||
);
|
||||
}
|
||||
const terminator = xml.startsWith("<!--", index)
|
||||
? "-->"
|
||||
: xml.startsWith("<![CDATA[", index)
|
||||
? "]]>"
|
||||
: xml.startsWith("<?", index)
|
||||
? "?>"
|
||||
: ">";
|
||||
const end = xml.indexOf(terminator, index + 1);
|
||||
if (end === -1) return; // The XML parser reports malformed framing.
|
||||
index = xml.indexOf("<", end + terminator.length);
|
||||
}
|
||||
};
|
||||
|
||||
/** Required XAR integer syntax: nonempty unsigned decimal, exactly representable. */
|
||||
export const xarInteger = (
|
||||
value: string | undefined,
|
||||
label: string,
|
||||
): number => {
|
||||
const text = value?.trim() ?? "";
|
||||
const parsed = /^\d+$/u.test(text) ? Number(text) : Number.NaN;
|
||||
if (!Number.isSafeInteger(parsed))
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
`xar TOC has an invalid ${label}`,
|
||||
);
|
||||
return parsed;
|
||||
};
|
||||
|
||||
/** Optional mode metadata is accepted only as a complete unsigned octal field. */
|
||||
export const xarMode = (value: string | undefined): number | null => {
|
||||
if (value === undefined) return null;
|
||||
const text = value.trim();
|
||||
const mode = /^[0-7]+$/u.test(text) ? Number.parseInt(text, 8) : Number.NaN;
|
||||
if (!Number.isSafeInteger(mode))
|
||||
throw new ArtifactReaderFailure("format", "xar TOC has an invalid mode");
|
||||
return mode;
|
||||
};
|
||||
|
||||
/** Validate the entire absolute heap range before allocating or requesting it. */
|
||||
export const xarHeapPosition = (
|
||||
heap: number,
|
||||
offset: number,
|
||||
length: number,
|
||||
size: number,
|
||||
): number => {
|
||||
const position = heap + offset;
|
||||
const end = position + length;
|
||||
if (
|
||||
![position, end, offset, length].every(Number.isSafeInteger) ||
|
||||
offset < 0 ||
|
||||
length < 0 ||
|
||||
end > size
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"format",
|
||||
"xar member has an invalid or out-of-file heap extent",
|
||||
);
|
||||
return position;
|
||||
};
|
||||
|
||||
/** Bound retained path text before constructing paths from deeply nested TOCs. */
|
||||
export class XarPathBudget {
|
||||
#used = 0;
|
||||
private readonly maximumPathBytes = 4096;
|
||||
private readonly maximumTotalBytes = 16 * 1024 * 1024;
|
||||
|
||||
/** Form the next path only when individual and cumulative budgets permit it. */
|
||||
join(parent: string, name: string): string {
|
||||
const bytes =
|
||||
Buffer.byteLength(parent) +
|
||||
Buffer.byteLength(name) +
|
||||
(parent.length > 0 ? 1 : 0);
|
||||
if (
|
||||
bytes > this.maximumPathBytes ||
|
||||
bytes > this.maximumTotalBytes - this.#used
|
||||
)
|
||||
throw new ArtifactReaderFailure(
|
||||
"limit",
|
||||
`xar TOC path text exceeds its ${this.maximumPathBytes}-byte path or ${this.maximumTotalBytes}-byte cumulative budget`,
|
||||
);
|
||||
this.#used += bytes;
|
||||
return parent === "" ? name : `${parent}/${name}`;
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,9 @@ import { lstat, realpath } from "node:fs/promises";
|
||||
|
||||
import { canonicalDigest } from "../../domain/comparisonSemantics.js";
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import {
|
||||
ArtifactPathRegistry,
|
||||
normalizeArtifactPath,
|
||||
} from "../ArtifactPaths.js";
|
||||
import { XarArtifactReader } from "../XarArtifactReader.js";
|
||||
import { ArtifactPathRegistry } from "../ArtifactPaths.js";
|
||||
import { visitArtifactTree } from "../ArtifactTraversal.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
@@ -44,6 +43,7 @@ export const extractArtifact = async (
|
||||
const selectedOccurrences = snapshot.occurrences.filter(
|
||||
(occurrence) =>
|
||||
(occurrence.entry_kind === "file" || occurrence.entry_kind === "slice") &&
|
||||
occurrence.hash_status !== "unavailable" &&
|
||||
occurrence.logical_path !== ".",
|
||||
);
|
||||
const selectedIds = new Set(
|
||||
@@ -63,6 +63,11 @@ export const extractArtifact = async (
|
||||
manifest: snapshot.manifest,
|
||||
occurrences,
|
||||
nodes,
|
||||
unavailablePaths: new Set(
|
||||
snapshot.occurrences
|
||||
.filter(({ hash_status }) => hash_status === "unavailable")
|
||||
.map(({ logical_path }) => logical_path),
|
||||
),
|
||||
};
|
||||
const selected = selectedOccurrences.map((occurrence) => {
|
||||
if (
|
||||
@@ -128,35 +133,52 @@ const materializeSelection = async ({
|
||||
try {
|
||||
const materialized: SelectedOccurrence[] = [];
|
||||
const registry = new ArtifactPathRegistry();
|
||||
for await (const entry of reader.entries(signal)) {
|
||||
const path = normalizeArtifactPath(entry.path);
|
||||
registry.add(path, entry.kind);
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await reader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
stream,
|
||||
selectedItem.node.sha256,
|
||||
signal,
|
||||
);
|
||||
extracted.push({
|
||||
artifact_id: selectedItem.node.artifact_id,
|
||||
relative_path: written.relativePath,
|
||||
sha256: written.sha256,
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
materialized.push(selectedItem);
|
||||
}
|
||||
await visitArtifactTree(
|
||||
reader,
|
||||
async ({ reader: currentReader, entry, path, container: expandable }) => {
|
||||
// Traverse expanded PKG Payload/Scripts like inventory does, so nested
|
||||
// members are materialized instead of silently skipped. An expanded
|
||||
// member acts as a container: register it as a directory and leave its
|
||||
// archive bytes to its children, since a file and a directory cannot
|
||||
// share one output path.
|
||||
registry.add(path, expandable ? "directory" : entry.kind);
|
||||
if (expandable) {
|
||||
return true;
|
||||
}
|
||||
const selectedItem = byPath.get(path);
|
||||
if (selectedItem === undefined) {
|
||||
if (
|
||||
entry.contentUnavailable === true &&
|
||||
inventory.unavailablePaths.has(path)
|
||||
)
|
||||
return false;
|
||||
if (entry.kind === "file" || entry.kind === "slice")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Regular artifact entry is missing from inventory: ${path}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
preflight(entry);
|
||||
const stream = await currentReader.open(entry, signal);
|
||||
const written = await output.write(
|
||||
path,
|
||||
stream,
|
||||
selectedItem.node.sha256,
|
||||
signal,
|
||||
);
|
||||
extracted.push({
|
||||
artifact_id: selectedItem.node.artifact_id,
|
||||
relative_path: written.relativePath,
|
||||
sha256: written.sha256,
|
||||
bytes_written: written.bytesWritten,
|
||||
created: true,
|
||||
});
|
||||
materialized.push(selectedItem);
|
||||
return false;
|
||||
},
|
||||
{ signal, expandAsar: false },
|
||||
);
|
||||
await reader.close();
|
||||
readerClosed = true;
|
||||
extracted.sort((left, right) =>
|
||||
@@ -207,7 +229,11 @@ const createExtractionResult = (
|
||||
cleanup: { attempted: false, verified: true, residual_paths: [] },
|
||||
provenance: [],
|
||||
limitations: [
|
||||
"All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file.",
|
||||
"All available selected regular files were materialized; expanded PKG Payload/Scripts archives are represented by their extracted children.",
|
||||
...[...inventory.unavailablePaths].map(
|
||||
(path) =>
|
||||
`Unavailable member ${path} was not extracted; see its inventory occurrence for the reason.`,
|
||||
),
|
||||
],
|
||||
});
|
||||
};
|
||||
@@ -216,6 +242,7 @@ interface LoadedInventory {
|
||||
readonly manifest: ArtifactGraphManifest;
|
||||
readonly occurrences: ReadonlyMap<string, ArtifactOccurrence>;
|
||||
readonly nodes: ReadonlyMap<string, ArtifactNode>;
|
||||
readonly unavailablePaths: ReadonlySet<string>;
|
||||
}
|
||||
|
||||
const collectOccurrences = (
|
||||
@@ -256,6 +283,7 @@ const createReader = async (
|
||||
)
|
||||
return new ZipArtifactReader(path, format);
|
||||
if (format === "mach-o") return new MachOSliceArtifactReader(path);
|
||||
if (format === "pkg") return new XarArtifactReader(path);
|
||||
throw new ArtifactReaderFailure(
|
||||
"unavailable",
|
||||
`Artifact format has no extraction reader: ${format}`,
|
||||
|
||||
@@ -50,7 +50,12 @@ export const createOccurrence = (
|
||||
compressed_size: entry.compressedSize,
|
||||
executable: entry.executable,
|
||||
encrypted: entry.encrypted,
|
||||
hash_status: entry.encrypted ? "unavailable" : "not-hashed",
|
||||
hash_status:
|
||||
entry.integrityMismatched === true
|
||||
? "mismatched"
|
||||
: entry.encrypted || entry.contentUnavailable === true
|
||||
? "unavailable"
|
||||
: "not-hashed",
|
||||
source_location:
|
||||
entry.byteOffset === null || entry.declaredSize === null
|
||||
? null
|
||||
@@ -104,6 +109,14 @@ export const materializeDirectoryNodes = (
|
||||
format: "directory",
|
||||
executable: false,
|
||||
contentState: "virtual",
|
||||
...(directory.hash_status === "mismatched"
|
||||
? {
|
||||
limitations: [
|
||||
...directory.limitations,
|
||||
"Directory structure identity is derived; its declared member integrity metadata disagreed.",
|
||||
],
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
const existing = nodes.get(node.artifact_id);
|
||||
nodes.set(
|
||||
@@ -111,7 +124,8 @@ export const materializeDirectoryNodes = (
|
||||
existing?.kind === "framework" ? existing : node,
|
||||
);
|
||||
directory.artifact_id = node.artifact_id;
|
||||
directory.hash_status = "verified";
|
||||
if (directory.hash_status !== "mismatched")
|
||||
directory.hash_status = "verified";
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { ArtifactReader } from "../ArtifactReader.js";
|
||||
import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js";
|
||||
import { MachOSliceArtifactReader } from "../MachOSliceArtifactReader.js";
|
||||
import { NativeDmgArtifactReader } from "../NativeDmgArtifactReader.js";
|
||||
import { XarArtifactReader } from "../XarArtifactReader.js";
|
||||
import { ZipArtifactReader } from "../ZipArtifactReader.js";
|
||||
import type { ArtifactNode } from "../../domain/artifactGraph.js";
|
||||
|
||||
@@ -10,6 +11,8 @@ export const createReader = async (
|
||||
path: string,
|
||||
format: ArtifactNode["format"],
|
||||
signal?: AbortSignal,
|
||||
/** The caller verifies declared member checksums itself (inventory scans do). */
|
||||
options: { readonly callerVerifiesChecksums?: boolean } = {},
|
||||
): Promise<ArtifactReader | undefined> => {
|
||||
switch (format) {
|
||||
case "directory":
|
||||
@@ -22,6 +25,10 @@ export const createReader = async (
|
||||
return new ZipArtifactReader(path, format);
|
||||
case "asar":
|
||||
return new AsarArtifactReader(path);
|
||||
case "pkg":
|
||||
return new XarArtifactReader(path, {
|
||||
verifyChecksums: options.callerVerifiesChecksums !== true,
|
||||
});
|
||||
case "mach-o-universal":
|
||||
return process.platform === "darwin"
|
||||
? new MachOSliceArtifactReader(path)
|
||||
@@ -39,9 +46,9 @@ export const inventoryLimitations = (
|
||||
reader: ArtifactReader | undefined,
|
||||
): string[] => {
|
||||
if (reader !== undefined) return [];
|
||||
if (format === "dmg" || format === "pkg")
|
||||
if (format === "dmg")
|
||||
return [
|
||||
`${format.toUpperCase()} root hash is observed; child inventory requires a native macOS adapter.`,
|
||||
"DMG root hash is observed; child inventory requires a native macOS adapter.",
|
||||
];
|
||||
if (format === "mach-o-universal" && process.platform !== "darwin")
|
||||
return ["Universal Mach-O slices require the native macOS lipo adapter."];
|
||||
|
||||
@@ -45,7 +45,10 @@ export const scanCanonicalArtifactInventory = async (
|
||||
const rootDigest = metadata.isDirectory()
|
||||
? null
|
||||
: await hashReadable(createReadStream(path), options.signal);
|
||||
const reader = await createReader(path, rootFormat, options.signal);
|
||||
// The scan applies the integrity policy to declared member checksums.
|
||||
const reader = await createReader(path, rootFormat, options.signal, {
|
||||
callerVerifiesChecksums: true,
|
||||
});
|
||||
|
||||
try {
|
||||
const { nodes, occurrences, pendingContradictions } = await scanReader(
|
||||
@@ -148,7 +151,14 @@ const buildInventorySnapshot = async (
|
||||
edges: orderedEdges,
|
||||
provenance: reader?.provenance() ?? [],
|
||||
integrity_contradictions: integrityContradictions,
|
||||
limitations: buildLimitations(rootFormat, reader, integrityContradictions),
|
||||
limitations: buildLimitations(
|
||||
rootFormat,
|
||||
reader,
|
||||
integrityContradictions,
|
||||
input.occurrences.filter(
|
||||
({ hash_status: status }) => status === "mismatched",
|
||||
).length,
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
@@ -261,11 +271,22 @@ const buildLimitations = (
|
||||
rootFormat: ArtifactNode["format"],
|
||||
reader: ArtifactReader | undefined,
|
||||
integrityContradictions: readonly IntegrityContradiction[],
|
||||
): string[] => [
|
||||
...inventoryLimitations(rootFormat, reader),
|
||||
...(integrityContradictions.length === 0
|
||||
? []
|
||||
: [
|
||||
`${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`,
|
||||
]),
|
||||
];
|
||||
mismatchedOccurrences: number,
|
||||
): string[] => {
|
||||
// integrity_contradictions carry SHA-256 values; other declared checksums
|
||||
// (xar SHA-1/MD5) are recorded on their occurrences.
|
||||
const checksumOnly = mismatchedOccurrences - integrityContradictions.length;
|
||||
return [
|
||||
...inventoryLimitations(rootFormat, reader),
|
||||
...(integrityContradictions.length === 0
|
||||
? []
|
||||
: [
|
||||
`${String(integrityContradictions.length)} integrity contradiction(s) were recorded; mismatched content is observed-untrusted.`,
|
||||
]),
|
||||
...(checksumOnly <= 0
|
||||
? []
|
||||
: [
|
||||
`${String(checksumOnly)} member(s) disagree with a declared non-SHA-256 checksum; their occurrences are mismatched and name the declared and observed values.`,
|
||||
]),
|
||||
];
|
||||
};
|
||||
|
||||
@@ -1,15 +1,12 @@
|
||||
import { isAbsolute } from "node:path";
|
||||
|
||||
import {
|
||||
ArtifactPathRegistry,
|
||||
normalizeArtifactPath,
|
||||
} from "../ArtifactPaths.js";
|
||||
import { ArtifactBudgetExceeded } from "../ArtifactDecodedBudget.js";
|
||||
import { ArtifactPathRegistry } from "../ArtifactPaths.js";
|
||||
import { visitArtifactTree } from "../ArtifactTraversal.js";
|
||||
import {
|
||||
ArtifactReaderFailure,
|
||||
type ArtifactEntry,
|
||||
type ArtifactReader,
|
||||
type ArtifactChecksumObservation,
|
||||
} from "../ArtifactReader.js";
|
||||
import { AsarArtifactReader } from "../AsarArtifactReader.js";
|
||||
import type { ArtifactNode } from "../../domain/artifactGraph.js";
|
||||
import {
|
||||
classifyArtifactContent,
|
||||
@@ -76,7 +73,7 @@ export const scanReader = async (
|
||||
registry: new ArtifactPathRegistry(),
|
||||
expandedContainerIds: new Set<string>(),
|
||||
};
|
||||
await visitArtifactEntries(context, reader, "");
|
||||
await visitArtifactEntries(context, reader);
|
||||
// Archive directories may appear after their children. Resolve containment
|
||||
// against the complete index before directory identities are materialized.
|
||||
for (const occurrence of occurrences)
|
||||
@@ -92,55 +89,42 @@ export const scanReader = async (
|
||||
const visitArtifactEntries = async (
|
||||
context: ScanContext,
|
||||
currentReader: ArtifactReader,
|
||||
prefix: string,
|
||||
): Promise<void> => {
|
||||
const stack: Array<{
|
||||
readonly reader: ArtifactReader;
|
||||
readonly prefix: string;
|
||||
readonly iterator: AsyncIterator<ArtifactEntry>;
|
||||
readonly owned: boolean;
|
||||
}> = [
|
||||
{
|
||||
reader: currentReader,
|
||||
prefix,
|
||||
iterator: currentReader.entries(context.signal)[Symbol.asyncIterator](),
|
||||
owned: false,
|
||||
},
|
||||
];
|
||||
try {
|
||||
while (stack.length > 0) {
|
||||
const frame = stack.at(-1);
|
||||
if (frame === undefined) break;
|
||||
const next = await frame.iterator.next();
|
||||
if (next.done) {
|
||||
stack.pop();
|
||||
if (frame.owned) await frame.reader.close();
|
||||
continue;
|
||||
}
|
||||
const entry = next.value;
|
||||
const logicalPath = normalizeArtifactPath(
|
||||
frame.prefix.length === 0
|
||||
? entry.path
|
||||
: `${frame.prefix}/${entry.path}`,
|
||||
);
|
||||
const expandableAsar = isExpandableAsar(entry, logicalPath);
|
||||
context.registry.add(
|
||||
logicalPath,
|
||||
expandableAsar ? "directory" : entry.kind,
|
||||
);
|
||||
await visitArtifactTree(
|
||||
currentReader,
|
||||
async ({ reader, entry, path: logicalPath, container: expandable }) => {
|
||||
context.registry.add(logicalPath, expandable ? "directory" : entry.kind);
|
||||
const occurrence = createOccurrence(entry, logicalPath, null);
|
||||
let digested:
|
||||
| { readonly node: ArtifactNode; readonly mismatched: boolean }
|
||||
| undefined;
|
||||
let digested: DigestedEntry | undefined;
|
||||
try {
|
||||
digested = await digestArtifactEntry(
|
||||
context,
|
||||
frame.reader,
|
||||
reader,
|
||||
entry,
|
||||
logicalPath,
|
||||
);
|
||||
} catch (cause: unknown) {
|
||||
if (!isUnavailableUnpackedEntry(cause, entry)) throw cause;
|
||||
if (!isUnavailableUnpackedEntry(cause, entry)) {
|
||||
// A streamed CRC, size-bound, or format failure is per-member
|
||||
// evidence loss under record-and-continue: keep the occurrence
|
||||
// unavailable with its diagnostic, without inventing a node.
|
||||
if (
|
||||
context.integrity.mode !== "record-and-continue" ||
|
||||
cause instanceof ArtifactBudgetExceeded ||
|
||||
!(cause instanceof ArtifactReaderFailure) ||
|
||||
(cause.reason !== "integrity" &&
|
||||
cause.reason !== "limit" &&
|
||||
cause.reason !== "format")
|
||||
)
|
||||
throw cause;
|
||||
occurrence.hash_status = "unavailable";
|
||||
occurrence.limitations.push(
|
||||
`Member content could not be verified: ${cause.message}`,
|
||||
);
|
||||
context.occurrences.push(occurrence);
|
||||
context.occurrenceByPath.set(logicalPath, occurrence);
|
||||
return false;
|
||||
}
|
||||
occurrence.hash_status = "unavailable";
|
||||
occurrence.limitations.push(UNAVAILABLE_UNPACKED_LIMITATION);
|
||||
}
|
||||
@@ -153,32 +137,60 @@ const visitArtifactEntries = async (
|
||||
if (digested.mismatched)
|
||||
occurrence.limitations.push(
|
||||
"Declared integrity metadata contradicts observed bytes.",
|
||||
...digested.mismatchDetails,
|
||||
);
|
||||
}
|
||||
context.occurrences.push(occurrence);
|
||||
context.occurrenceByPath.set(logicalPath, occurrence);
|
||||
if (expandableAsar && digested?.mismatched !== true) {
|
||||
const nested = new AsarArtifactReader(entry.adapterKey);
|
||||
// Only traversed containers can own members, not opaque ASAR-named files.
|
||||
if (expandable && digested?.mismatched !== true) {
|
||||
// Only traversed containers can own members, not opaque archive-named files.
|
||||
context.expandedContainerIds.add(occurrence.occurrence_id);
|
||||
stack.push({
|
||||
reader: nested,
|
||||
prefix: logicalPath,
|
||||
iterator: nested.entries(context.signal)[Symbol.asyncIterator](),
|
||||
owned: true,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await Promise.allSettled(
|
||||
stack
|
||||
.filter(({ owned }) => owned)
|
||||
.map(async ({ reader, iterator }) => {
|
||||
await iterator.return?.();
|
||||
await reader.close();
|
||||
}),
|
||||
);
|
||||
}
|
||||
return false;
|
||||
},
|
||||
{ signal: context.signal, integrity: context.integrity.mode },
|
||||
);
|
||||
};
|
||||
|
||||
interface DigestedEntry {
|
||||
readonly node: ArtifactNode;
|
||||
readonly mismatched: boolean;
|
||||
/** Declared and observed values of a non-SHA-256 checksum that disagreed. */
|
||||
readonly mismatchDetails: readonly string[];
|
||||
}
|
||||
|
||||
const checksumDescription = ({
|
||||
representation,
|
||||
algorithm,
|
||||
declared,
|
||||
observed,
|
||||
}: ArtifactChecksumObservation): string =>
|
||||
`Declared ${representation} ${algorithm} ${declared} disagrees with observed ${observed}.`;
|
||||
|
||||
const entryIntegrityFailure = (
|
||||
entry: ArtifactEntry,
|
||||
logicalPath: string,
|
||||
observed: {
|
||||
readonly sha256: string;
|
||||
readonly checksumMismatches: readonly ArtifactChecksumObservation[];
|
||||
},
|
||||
): ArtifactReaderFailure => {
|
||||
const mismatchDetails = observed.checksumMismatches.map(checksumDescription);
|
||||
return new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Artifact integrity metadata disagrees with content: ${logicalPath}${mismatchDetails.length === 0 ? "" : ` (${mismatchDetails.join(" ")})`}`,
|
||||
undefined,
|
||||
{
|
||||
logicalPath,
|
||||
declaredSha256: entry.declaredSha256,
|
||||
calculatedSha256: observed.sha256,
|
||||
unpacked: entry.unpacked,
|
||||
...(observed.checksumMismatches.length === 0
|
||||
? {}
|
||||
: { checksumMismatches: observed.checksumMismatches }),
|
||||
},
|
||||
);
|
||||
};
|
||||
|
||||
const digestArtifactEntry = async (
|
||||
@@ -186,30 +198,47 @@ const digestArtifactEntry = async (
|
||||
currentReader: ArtifactReader,
|
||||
entry: ArtifactEntry,
|
||||
logicalPath: string,
|
||||
): Promise<
|
||||
{ readonly node: ArtifactNode; readonly mismatched: boolean } | undefined
|
||||
> => {
|
||||
if ((entry.kind !== "file" && entry.kind !== "slice") || entry.encrypted)
|
||||
): Promise<DigestedEntry | undefined> => {
|
||||
if (
|
||||
(entry.kind !== "file" && entry.kind !== "slice") ||
|
||||
entry.encrypted ||
|
||||
entry.contentUnavailable === true
|
||||
)
|
||||
return undefined;
|
||||
const checksum = entry.declaredChecksum;
|
||||
const digest = await hashReadable(
|
||||
await currentReader.open(entry, context.signal),
|
||||
context.signal,
|
||||
checksum?.algorithm,
|
||||
);
|
||||
const mismatched =
|
||||
const sha256Mismatch =
|
||||
entry.declaredSha256 !== null && entry.declaredSha256 !== digest.sha256;
|
||||
if (mismatched && entry.declaredSha256 !== null) {
|
||||
if (context.integrity.mode === "fail")
|
||||
throw new ArtifactReaderFailure(
|
||||
"integrity",
|
||||
`Artifact integrity metadata disagrees with content: ${logicalPath}`,
|
||||
undefined,
|
||||
{
|
||||
logicalPath,
|
||||
declaredSha256: entry.declaredSha256,
|
||||
calculatedSha256: digest.sha256,
|
||||
unpacked: entry.unpacked,
|
||||
},
|
||||
);
|
||||
const checksumMismatch =
|
||||
checksum !== undefined && checksum.value !== digest.also;
|
||||
const storedMismatches = (
|
||||
currentReader.integrityObservations?.(entry) ?? []
|
||||
).filter(({ declared, observed }) => declared !== observed);
|
||||
const mismatched =
|
||||
sha256Mismatch || checksumMismatch || storedMismatches.length > 0;
|
||||
const checksumMismatches = [
|
||||
...(checksumMismatch
|
||||
? [
|
||||
{
|
||||
representation: "decoded" as const,
|
||||
algorithm: checksum.algorithm,
|
||||
declared: checksum.value,
|
||||
observed: digest.also ?? "unavailable",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...storedMismatches,
|
||||
];
|
||||
if (mismatched && context.integrity.mode === "fail")
|
||||
throw entryIntegrityFailure(entry, logicalPath, {
|
||||
sha256: digest.sha256,
|
||||
checksumMismatches,
|
||||
});
|
||||
if (sha256Mismatch && entry.declaredSha256 !== null)
|
||||
context.pendingContradictions.push({
|
||||
logicalPath,
|
||||
declaredSha256: entry.declaredSha256,
|
||||
@@ -217,7 +246,6 @@ const digestArtifactEntry = async (
|
||||
entryKind: entry.kind,
|
||||
unpacked: entry.unpacked,
|
||||
});
|
||||
}
|
||||
const classified =
|
||||
entry.kind === "slice"
|
||||
? ({ kind: "universal-slice", format: "mach-o" } as const)
|
||||
@@ -237,14 +265,10 @@ const digestArtifactEntry = async (
|
||||
: [],
|
||||
}),
|
||||
mismatched,
|
||||
mismatchDetails: checksumMismatches.map(checksumDescription),
|
||||
};
|
||||
};
|
||||
|
||||
const isExpandableAsar = (entry: ArtifactEntry, logicalPath: string): boolean =>
|
||||
entry.kind === "file" &&
|
||||
logicalPath.toLowerCase().endsWith(".asar") &&
|
||||
isAbsolute(entry.adapterKey);
|
||||
|
||||
const isUnavailableUnpackedEntry = (
|
||||
cause: unknown,
|
||||
entry: ArtifactEntry,
|
||||
|
||||
@@ -63,7 +63,7 @@ const artifact = <
|
||||
export const ARTIFACT_TOOL_CONTRACTS = [
|
||||
artifact(
|
||||
"inspect_artifact",
|
||||
"Inspect an active archive or application package in one call. Returns the complete content-addressed artifact graph, source Evidence, observations, relationships, integrity contradictions, hypotheses, unexplored branches, limitations, and next probes inline. Read-only DMG mounting is used automatically on supported macOS hosts. Integrity mismatches fail by default; record-and-continue keeps mismatches explicitly untrusted. It does not extract files; use extract_artifact to materialize its regular contents.",
|
||||
"Inspect an active archive or application package in one call. Returns the complete content-addressed artifact graph, source Evidence, observations, relationships, integrity contradictions, hypotheses, unexplored branches, limitations, and next probes inline. Read-only DMG mounting is used automatically on supported macOS hosts. Flat installer packages (PKG) expand their gzip-cpio Scripts and Payload; pbzx payloads are recorded without expansion. Integrity mismatches fail by default; record-and-continue keeps mismatches explicitly untrusted. It does not extract files; use extract_artifact to materialize its regular contents.",
|
||||
artifactInventoryInputSchema,
|
||||
),
|
||||
artifact(
|
||||
|
||||
@@ -53,6 +53,16 @@ export const analysisErrorProjectionSchema = z.discriminatedUnion("code", [
|
||||
declared_sha256: z.string().nullable(),
|
||||
calculated_sha256: z.string().nullable(),
|
||||
unpacked: z.boolean(),
|
||||
checksum_mismatches: z
|
||||
.array(
|
||||
z.strictObject({
|
||||
representation: z.enum(["stored", "decoded"]),
|
||||
algorithm: z.string(),
|
||||
declared: z.string(),
|
||||
observed: z.string(),
|
||||
}),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
.strict(),
|
||||
})
|
||||
|
||||
@@ -249,6 +249,18 @@ const artifactStateErrorDetails = (
|
||||
declared_sha256: error.artifactDetails.declaredSha256,
|
||||
calculated_sha256: error.artifactDetails.calculatedSha256,
|
||||
unpacked: error.artifactDetails.unpacked,
|
||||
...(error.artifactDetails.checksumMismatches === undefined
|
||||
? {}
|
||||
: {
|
||||
checksum_mismatches: error.artifactDetails.checksumMismatches.map(
|
||||
({ representation, algorithm, declared, observed }) => ({
|
||||
representation,
|
||||
algorithm,
|
||||
declared,
|
||||
observed,
|
||||
}),
|
||||
),
|
||||
}),
|
||||
};
|
||||
if (error instanceof ArtifactOperationError)
|
||||
return {
|
||||
|
||||
@@ -1,5 +1,22 @@
|
||||
import { AnalysisError } from "./analysisErrorBase.js";
|
||||
|
||||
/** Declared and observed checksums of one consumed member representation. */
|
||||
export interface ArtifactChecksumObservation {
|
||||
readonly representation: "stored" | "decoded";
|
||||
readonly algorithm: string;
|
||||
readonly declared: string;
|
||||
readonly observed: string;
|
||||
}
|
||||
|
||||
/** Evidence identifying an artifact member's failed integrity constraints. */
|
||||
export interface ArtifactIntegrityFailureDetails {
|
||||
readonly logicalPath: string;
|
||||
readonly declaredSha256: string | null;
|
||||
readonly calculatedSha256: string | null;
|
||||
readonly unpacked: boolean;
|
||||
readonly checksumMismatches?: readonly ArtifactChecksumObservation[];
|
||||
}
|
||||
|
||||
/** Artifact inventory or extraction failed a typed safety boundary. */
|
||||
export class ArtifactOperationError extends AnalysisError {
|
||||
readonly _tag = "ArtifactOperationError";
|
||||
@@ -25,12 +42,7 @@ export class ArtifactOperationError extends AnalysisError {
|
||||
| "path"
|
||||
| "unavailable"
|
||||
| "io",
|
||||
readonly artifactDetails?: Readonly<{
|
||||
logicalPath: string;
|
||||
declaredSha256: string | null;
|
||||
calculatedSha256: string | null;
|
||||
unpacked: boolean;
|
||||
}>,
|
||||
readonly artifactDetails?: ArtifactIntegrityFailureDetails,
|
||||
/** The specific constraint that failed, such as the colliding path. */
|
||||
readonly detail?: string,
|
||||
) {
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
MODE,
|
||||
gzipCpio,
|
||||
xarArchive,
|
||||
rewriteXarToc,
|
||||
} from "../../../src/artifacts/InstallerPackage.fixture.js";
|
||||
import { runProviderAnalysis } from "../../../src/composition/directAnalysis.js";
|
||||
import {
|
||||
artifactInventoryResultSchema,
|
||||
artifactExtractionResultSchema,
|
||||
} from "../../../src/domain/artifactGraph.js";
|
||||
import { analysisErrorProjectionSchema } from "../../../src/contracts/errorSchemas.js";
|
||||
import { parseEvidence } from "../../../src/domain/evidence.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const sha256 = (text: string): string =>
|
||||
createHash("sha256").update(text).digest("hex");
|
||||
|
||||
const productPackage = (payloadText: string, extractedSha1?: string) =>
|
||||
xarArchive([
|
||||
{
|
||||
name: "Distribution",
|
||||
data: Buffer.from("<installer-gui-script/>"),
|
||||
encoding: "zlib",
|
||||
},
|
||||
{
|
||||
name: "App.pkg",
|
||||
type: "directory",
|
||||
children: [
|
||||
{
|
||||
name: "PackageInfo",
|
||||
data: Buffer.from('<pkg-info identifier="com.example.app"/>'),
|
||||
encoding: "zlib",
|
||||
...(extractedSha1 === undefined ? {} : { extractedSha1 }),
|
||||
},
|
||||
{
|
||||
name: "Scripts",
|
||||
data: gzipCpio([
|
||||
{ name: ".", mode: MODE.directory },
|
||||
{
|
||||
name: "./postinstall",
|
||||
mode: MODE.executable,
|
||||
data: "#!/bin/sh\n",
|
||||
},
|
||||
]),
|
||||
},
|
||||
{
|
||||
name: "Payload",
|
||||
data: gzipCpio([
|
||||
{ name: ".", mode: MODE.directory },
|
||||
{ name: "./Applications", mode: MODE.directory },
|
||||
{
|
||||
name: "./Applications/App.app/Contents/Info.plist",
|
||||
mode: MODE.file,
|
||||
data: payloadText,
|
||||
},
|
||||
]),
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
describe("installer package checksum evidence", () => {
|
||||
it("reports a stored SHA-1 mismatch independently of a valid decoded MD5", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-checksum-reason-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
const data = Buffer.from("readable content");
|
||||
const declared = "0".repeat(40);
|
||||
await writeFile(
|
||||
path,
|
||||
rewriteXarToc(
|
||||
xarArchive([{ name: "member", data, archivedSha1: declared }]),
|
||||
(xml) =>
|
||||
xml.replace(
|
||||
/<extracted-checksum style="sha1">[^<]+<\/extracted-checksum>/u,
|
||||
`<extracted-checksum style="md5">${createHash("md5").update(data).digest("hex")}</extracted-checksum>`,
|
||||
),
|
||||
),
|
||||
);
|
||||
const { error, ...projection } = z
|
||||
.object({ error: z.string() })
|
||||
.passthrough()
|
||||
.parse(await runProviderAnalysis(path, "inventory_artifact", {}));
|
||||
expect(error).toBe("Analysis failed");
|
||||
const failed = analysisErrorProjectionSchema.parse(projection);
|
||||
expect(failed).toMatchObject({
|
||||
code: "artifact_integrity_mismatch",
|
||||
details: {
|
||||
logical_path: "member",
|
||||
checksum_mismatches: [
|
||||
{
|
||||
representation: "stored",
|
||||
algorithm: "sha1",
|
||||
declared,
|
||||
observed: createHash("sha1").update(data).digest("hex"),
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["raw", "zlib"] as const)(
|
||||
"hashes and extracts %s content with unsupported stored and decoded checksum algorithms",
|
||||
async (encoding) => {
|
||||
const directory = await createTestTempDirectory(
|
||||
"rea-pkg-unknown-checksum-",
|
||||
);
|
||||
const path = join(directory, "Installer.pkg");
|
||||
const data = Buffer.from("readable content");
|
||||
await writeFile(
|
||||
path,
|
||||
rewriteXarToc(xarArchive([{ name: "member", data, encoding }]), (xml) =>
|
||||
xml
|
||||
.replaceAll(
|
||||
'<archived-checksum style="sha1">',
|
||||
'<archived-checksum style="unknown-stored">',
|
||||
)
|
||||
.replaceAll(
|
||||
'<extracted-checksum style="sha1">',
|
||||
'<extracted-checksum style="unknown-decoded">',
|
||||
),
|
||||
),
|
||||
);
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
parseEvidence(await runProviderAnalysis(path, "inventory_artifact", {}))
|
||||
.normalized_result,
|
||||
);
|
||||
const member = inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "member",
|
||||
);
|
||||
expect(member).toMatchObject({
|
||||
hash_status: "verified",
|
||||
limitations: [
|
||||
expect.stringContaining("unsupported checksum unknown-decoded"),
|
||||
expect.stringContaining(
|
||||
"unsupported archived checksum unknown-stored",
|
||||
),
|
||||
],
|
||||
});
|
||||
expect(
|
||||
inventory.nodes.find(
|
||||
({ artifact_id }) => artifact_id === member?.artifact_id,
|
||||
),
|
||||
).toMatchObject({ sha256: sha256(data.toString()), size: data.length });
|
||||
const output = join(directory, "extracted");
|
||||
const extraction = artifactExtractionResultSchema.parse(
|
||||
parseEvidence(
|
||||
await runProviderAnalysis(path, "extract_artifact", {
|
||||
output_root: output,
|
||||
}),
|
||||
).normalized_result,
|
||||
);
|
||||
expect(extraction.artifacts).toHaveLength(1);
|
||||
expect(await readFile(join(output, "member"))).toEqual(data);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("installer package inventory", () => {
|
||||
it("inventories xar members and expands gzip-cpio scripts and payloads", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-inventory-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(path, productPackage("<plist/>"));
|
||||
const evidence = parseEvidence(
|
||||
await runProviderAnalysis(path, "inventory_artifact", {}),
|
||||
);
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
evidence.normalized_result,
|
||||
);
|
||||
expect(inventory.manifest.root_format).toBe("pkg");
|
||||
expect(inventory.limitations).toEqual([]);
|
||||
const occurrences = new Map(
|
||||
inventory.occurrences.map((occurrence) => [
|
||||
occurrence.logical_path,
|
||||
occurrence,
|
||||
]),
|
||||
);
|
||||
expect([...occurrences.keys()]).toEqual(
|
||||
expect.arrayContaining([
|
||||
"Distribution",
|
||||
"App.pkg/PackageInfo",
|
||||
"App.pkg/Scripts",
|
||||
"App.pkg/Scripts/postinstall",
|
||||
"App.pkg/Payload",
|
||||
"App.pkg/Payload/Applications/App.app/Contents/Info.plist",
|
||||
]),
|
||||
);
|
||||
const plist = occurrences.get(
|
||||
"App.pkg/Payload/Applications/App.app/Contents/Info.plist",
|
||||
);
|
||||
const node = inventory.nodes.find(
|
||||
({ artifact_id: id }) => id === plist?.artifact_id,
|
||||
);
|
||||
expect(node?.sha256).toBe(sha256("<plist/>"));
|
||||
expect(occurrences.get("App.pkg/Scripts/postinstall")?.executable).toBe(
|
||||
true,
|
||||
);
|
||||
const scripts = occurrences.get("App.pkg/Scripts");
|
||||
expect(
|
||||
occurrences.get("App.pkg/Scripts/postinstall")?.parent_occurrence_id,
|
||||
).toBe(scripts?.occurrence_id);
|
||||
});
|
||||
|
||||
it("fails on a member whose xar checksum disagrees with its bytes", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-tampered-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(path, productPackage("<plist/>", "0".repeat(40)));
|
||||
expect(
|
||||
await runProviderAnalysis(path, "inventory_artifact", {}),
|
||||
).toMatchObject({
|
||||
error: "Analysis failed",
|
||||
code: "artifact_integrity_mismatch",
|
||||
details: { logical_path: "App.pkg/PackageInfo", declared_sha256: null },
|
||||
});
|
||||
});
|
||||
|
||||
it("records a xar checksum mismatch under record-and-continue", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-recorded-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(path, productPackage("<plist/>", "0".repeat(40)));
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
parseEvidence(
|
||||
await runProviderAnalysis(path, "inventory_artifact", {
|
||||
integrity_policy: "record-and-continue",
|
||||
}),
|
||||
).normalized_result,
|
||||
);
|
||||
const info = inventory.occurrences.find(
|
||||
({ logical_path: logical }) => logical === "App.pkg/PackageInfo",
|
||||
);
|
||||
const observedSha1 = createHash("sha1")
|
||||
.update('<pkg-info identifier="com.example.app"/>')
|
||||
.digest("hex");
|
||||
expect(info).toMatchObject({ hash_status: "mismatched" });
|
||||
expect(info?.limitations).toContain(
|
||||
`Declared decoded sha1 ${"0".repeat(40)} disagrees with observed ${observedSha1}.`,
|
||||
);
|
||||
expect(inventory.limitations).toContainEqual(
|
||||
expect.stringContaining(
|
||||
"1 member(s) disagree with a declared non-SHA-256 checksum",
|
||||
),
|
||||
);
|
||||
// Untampered members are still inventoried and expanded.
|
||||
expect(
|
||||
inventory.occurrences.some(
|
||||
({ logical_path: logical }) =>
|
||||
logical === "App.pkg/Scripts/postinstall",
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("records an unresolved hard link as unavailable instead of empty", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-hardlink-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{
|
||||
name: "Payload",
|
||||
data: gzipCpio(
|
||||
[{ name: "./orphan", mode: MODE.file, ino: 3, links: 2 }],
|
||||
"newc",
|
||||
),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
parseEvidence(await runProviderAnalysis(path, "inventory_artifact", {}))
|
||||
.normalized_result,
|
||||
);
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path: logical }) => logical === "Payload/orphan",
|
||||
),
|
||||
).toMatchObject({
|
||||
artifact_id: null,
|
||||
hash_status: "unavailable",
|
||||
limitations: [expect.stringContaining("Hard-link bytes")],
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("installer package CRC recovery", () => {
|
||||
it("recovers later siblings after a misaligned CRC mismatch", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-crc-align-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{
|
||||
name: "Payload",
|
||||
// "abc" is 3 bytes (1 padding byte): a CRC failure here must not
|
||||
// desynchronize the following member.
|
||||
data: gzipCpio(
|
||||
[
|
||||
{ name: "./bad", mode: MODE.file, data: "abc", check: 1 },
|
||||
{ name: "./good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
parseEvidence(
|
||||
await runProviderAnalysis(path, "inventory_artifact", {
|
||||
integrity_policy: "record-and-continue",
|
||||
}),
|
||||
).normalized_result,
|
||||
);
|
||||
const bad = inventory.occurrences.find(
|
||||
({ logical_path: logical }) => logical === "Payload/bad",
|
||||
);
|
||||
expect(bad).toMatchObject({ hash_status: "mismatched" });
|
||||
expect(
|
||||
inventory.nodes.find(
|
||||
({ artifact_id }) => artifact_id === bad?.artifact_id,
|
||||
),
|
||||
).toMatchObject({ sha256: sha256("abc"), size: 3 });
|
||||
expect(bad?.limitations).toContain(
|
||||
"Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000126.",
|
||||
);
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path: logical }) => logical === "Payload/good",
|
||||
),
|
||||
).toMatchObject({ hash_status: "verified" });
|
||||
});
|
||||
|
||||
it("keeps later siblings after a zero-size hard-link CRC mismatch", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-link-crc-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{
|
||||
name: "Payload",
|
||||
data: gzipCpio(
|
||||
[
|
||||
{ name: "./bad", mode: MODE.file, ino: 4, links: 2, check: 1 },
|
||||
{ name: "./good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const inventory = artifactInventoryResultSchema.parse(
|
||||
parseEvidence(
|
||||
await runProviderAnalysis(path, "inventory_artifact", {
|
||||
integrity_policy: "record-and-continue",
|
||||
}),
|
||||
).normalized_result,
|
||||
);
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path: logical }) => logical === "Payload/bad",
|
||||
),
|
||||
).toMatchObject({ hash_status: "unavailable" });
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path: logical }) => logical === "Payload/good",
|
||||
),
|
||||
).toMatchObject({ hash_status: "verified" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { deflateSync } from "node:zlib";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { extractArtifact } from "../../../src/artifacts/extraction/ArtifactExtraction.js";
|
||||
import {
|
||||
MODE,
|
||||
gzipCpio,
|
||||
xarArchive,
|
||||
} from "../../../src/artifacts/InstallerPackage.fixture.js";
|
||||
import { scanArtifactInventory } from "../../../src/artifacts/inventory/ArtifactInventory.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
it("extracts regular nested siblings while retaining skipped special-node evidence", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-extract-special-");
|
||||
const inputPath = join(directory, "Installer.pkg");
|
||||
const outputRoot = join(directory, "extracted");
|
||||
await writeFile(
|
||||
inputPath,
|
||||
xarArchive([
|
||||
{
|
||||
name: "Payload",
|
||||
data: gzipCpio([
|
||||
{ name: "./fifo", mode: MODE.fifo },
|
||||
{ name: "./usr/bin/tool", mode: MODE.executable, data: "tool bytes" },
|
||||
]),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const extracted = await extractArtifact({
|
||||
inputPath,
|
||||
outputRoot,
|
||||
inputFormat: "pkg",
|
||||
});
|
||||
expect(await readFile(join(outputRoot, "Payload/usr/bin/tool"), "utf8")).toBe(
|
||||
"tool bytes",
|
||||
);
|
||||
expect(extracted.artifacts.map(({ relative_path }) => relative_path)).toEqual(
|
||||
["Payload/usr/bin/tool"],
|
||||
);
|
||||
expect(extracted.extraction_manifest.selected_occurrence_ids).toHaveLength(1);
|
||||
expect(extracted.limitations).toContainEqual(
|
||||
expect.stringContaining("Payload/fifo was not extracted"),
|
||||
);
|
||||
});
|
||||
|
||||
it("retains unsupported-encoding occurrences under the default inventory policy", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-unsupported-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{ name: "encoded", encoding: "bzip2", data: Buffer.from("opaque") },
|
||||
{ name: "plain", data: Buffer.from("available") },
|
||||
]),
|
||||
);
|
||||
const inventory = await scanArtifactInventory(path);
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "encoded",
|
||||
),
|
||||
).toMatchObject({
|
||||
hash_status: "unavailable",
|
||||
artifact_id: null,
|
||||
limitations: [
|
||||
expect.stringContaining("Unsupported xar encoding application/x-bzip2"),
|
||||
],
|
||||
});
|
||||
expect(
|
||||
inventory.occurrences.find(({ logical_path }) => logical_path === "plain"),
|
||||
).toMatchObject({ hash_status: "verified" });
|
||||
});
|
||||
|
||||
it("rejects a header whose decoded TOC length contradicts the actual data", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-toc-size-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([{ name: "plain", data: Buffer.from("ok") }], {
|
||||
tocDecodedSize: 1,
|
||||
}),
|
||||
);
|
||||
await expect(scanArtifactInventory(path)).rejects.toMatchObject({
|
||||
reason: "format",
|
||||
message: expect.stringMatching(/xar TOC decoded \d+ bytes, expected 1/u),
|
||||
});
|
||||
});
|
||||
|
||||
it("records stored-checksum contradictions without losing decoded content identity", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-stored-integrity-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
const decoded = Buffer.from("same decoded bytes");
|
||||
const archivedSha1 = "0".repeat(40);
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{ name: "encoded", encoding: "zlib", data: decoded, archivedSha1 },
|
||||
{ name: "plain", data: Buffer.from("ok") },
|
||||
]),
|
||||
);
|
||||
const inventory = await scanArtifactInventory(path, {
|
||||
integrity: { mode: "record-and-continue" },
|
||||
});
|
||||
const entry = inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "encoded",
|
||||
);
|
||||
expect(entry).toMatchObject({ hash_status: "mismatched" });
|
||||
expect(
|
||||
inventory.nodes.find(
|
||||
({ artifact_id }) => artifact_id === entry?.artifact_id,
|
||||
),
|
||||
).toMatchObject({
|
||||
size: decoded.length,
|
||||
sha256: createHash("sha256").update(decoded).digest("hex"),
|
||||
});
|
||||
const observed = createHash("sha1")
|
||||
.update(deflateSync(decoded))
|
||||
.digest("hex");
|
||||
expect(entry?.limitations).toContain(
|
||||
`Declared stored sha1 ${archivedSha1} disagrees with observed ${observed}.`,
|
||||
);
|
||||
await expect(scanArtifactInventory(path)).rejects.toMatchObject({
|
||||
reason: "integrity",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves recovered directory CRC mismatches through graph materialization", async () => {
|
||||
const directory = await createTestTempDirectory("rea-pkg-directory-crc-");
|
||||
const path = join(directory, "Installer.pkg");
|
||||
await writeFile(
|
||||
path,
|
||||
xarArchive([
|
||||
{
|
||||
name: "Payload",
|
||||
data: gzipCpio(
|
||||
[
|
||||
{ name: "dir", mode: MODE.directory, check: 1 },
|
||||
{ name: "dir/good", mode: MODE.file, data: "ok" },
|
||||
],
|
||||
"crc",
|
||||
),
|
||||
},
|
||||
]),
|
||||
);
|
||||
const inventory = await scanArtifactInventory(path, {
|
||||
integrity: { mode: "record-and-continue" },
|
||||
});
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "Payload/dir",
|
||||
),
|
||||
).toMatchObject({
|
||||
hash_status: "mismatched",
|
||||
limitations: [
|
||||
"Declared decoded cpio-byte-sum 00000001 disagrees with observed 00000000.",
|
||||
],
|
||||
});
|
||||
expect(
|
||||
inventory.occurrences.find(
|
||||
({ logical_path }) => logical_path === "Payload/dir/good",
|
||||
),
|
||||
).toMatchObject({ hash_status: "verified" });
|
||||
});
|
||||
Reference in New Issue
Block a user