Classify coverage once at inventory production and require it in current outputs. Normalize older retained Evidence at the inventory parser, reject inconsistent page statuses, and remove comparison heuristics over limitation prose. Validation: public CLI-to-MCP plist comparison, changed-behavior and unknowns workflow; boundary checks for legacy Evidence, partial coverage, page disagreement, content and executable changes; generated docs checks. Fixes #2173.
445 lines
15 KiB
TypeScript
445 lines
15 KiB
TypeScript
import { chmod, mkdir, writeFile } from "node:fs/promises";
|
|
import { join } from "node:path";
|
|
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
import { fragmentInventoryEvidence } from "../../fixtures/artifactEvidence.js";
|
|
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
|
|
|
import { inventoryArtifact } from "../../fixtures/artifactInventory.js";
|
|
import { artifactInventoryResultSchema } from "../../../src/domain/artifactGraph.js";
|
|
import {
|
|
artifactComparisonResultSchema,
|
|
compareArtifacts,
|
|
} from "../../../src/domain/artifactComparison.js";
|
|
import { parseArtifactInventoryEvidence } from "../../../src/domain/artifactInventoryEvidence.js";
|
|
import { createEvidence } from "../../../src/domain/evidence.js";
|
|
import { jsonValueSchema } from "../../../src/domain/jsonValue.js";
|
|
|
|
const PROVIDER = {
|
|
id: "rea-artifact-graph",
|
|
name: "REA artifact graph",
|
|
version: "1",
|
|
} as const;
|
|
|
|
const observe = async (path: string) => {
|
|
const inventory = await inventoryArtifact(path);
|
|
return createEvidence(
|
|
{
|
|
path,
|
|
sha256: inventory.manifest.root_sha256,
|
|
format: inventory.manifest.root_format,
|
|
},
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: jsonValueSchema.parse(inventory),
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
);
|
|
};
|
|
|
|
const changedArtifactComparison = async () => {
|
|
const parent = await createTestTempDirectory("rea-artifact-compare-");
|
|
const leftPath = join(parent, "left.app");
|
|
const rightPath = join(parent, "right.app");
|
|
await Promise.all([mkdir(leftPath), mkdir(rightPath)]);
|
|
await Promise.all([
|
|
writeFile(join(leftPath, "main.js"), "old();"),
|
|
writeFile(join(leftPath, "same.txt"), "same"),
|
|
writeFile(join(rightPath, "main.js"), "newer();"),
|
|
writeFile(join(rightPath, "same.txt"), "same"),
|
|
writeFile(join(rightPath, "added.txt"), "added"),
|
|
]);
|
|
const left = await observe(leftPath);
|
|
const right = await observe(rightPath);
|
|
return {
|
|
left,
|
|
right,
|
|
result: compareArtifacts(left, right),
|
|
};
|
|
};
|
|
|
|
describe("leaf artifact comparison", () => {
|
|
it("accepts persisted complete leaf inventory Evidence without coverage_status", async () => {
|
|
const root = await createTestTempDirectory("rea-artifact-plist-compare-");
|
|
const leftPath = join(root, "left.plist");
|
|
const rightPath = join(root, "right.plist");
|
|
const xml =
|
|
'<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>hello</string></dict></plist>';
|
|
await Promise.all([writeFile(leftPath, xml), writeFile(rightPath, xml)]);
|
|
const [left, right] = await Promise.all([
|
|
observe(leftPath),
|
|
observe(rightPath),
|
|
]);
|
|
// Persisted inventory Evidence predates coverage_status. The known leaf
|
|
// notice remains complete when normalized at the inventory boundary.
|
|
const parsed = artifactInventoryResultSchema.parse(left.normalized_result);
|
|
const { coverage_status: _coverageStatus, ...legacyInventory } = parsed;
|
|
const legacy = createEvidence(
|
|
{
|
|
path: left.subject!.local_path,
|
|
sha256: left.subject!.digest.sha256,
|
|
format: left.subject!.format,
|
|
},
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: jsonValueSchema.parse(legacyInventory),
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
);
|
|
expect(compareArtifacts(legacy, right)).toMatchObject({
|
|
status: "unchanged",
|
|
summary: { unchanged: 1, unknown: 0 },
|
|
changes: [],
|
|
});
|
|
});
|
|
|
|
it("keeps content and executable changes visible for complete plist roots", async () => {
|
|
const root = await createTestTempDirectory("rea-artifact-plist-deltas-");
|
|
const leftPath = join(root, "left.plist");
|
|
const rightPath = join(root, "right.plist");
|
|
const plist = (label: string) =>
|
|
`<?xml version="1.0"?><plist version="1.0"><dict><key>label</key><string>${label}</string></dict></plist>`;
|
|
await Promise.all([
|
|
writeFile(leftPath, plist("hello"), { mode: 0o644 }),
|
|
writeFile(rightPath, plist("world"), { mode: 0o644 }),
|
|
]);
|
|
const [left, changed] = await Promise.all([
|
|
observe(leftPath),
|
|
observe(rightPath),
|
|
]);
|
|
expect(compareArtifacts(left, changed)).toMatchObject({
|
|
status: "changed",
|
|
summary: { changed: 1, unknown: 0 },
|
|
changes: [
|
|
expect.objectContaining({
|
|
logical_path: ".",
|
|
classification: "changed",
|
|
dimensions: expect.arrayContaining(["content"]),
|
|
}),
|
|
],
|
|
});
|
|
|
|
if (process.platform !== "win32") {
|
|
await writeFile(rightPath, plist("hello"), { mode: 0o644 });
|
|
await chmod(rightPath, 0o755);
|
|
const executable = await observe(rightPath);
|
|
expect(compareArtifacts(left, executable)).toMatchObject({
|
|
status: "changed",
|
|
summary: { changed: 1, unknown: 0 },
|
|
changes: [
|
|
expect.objectContaining({
|
|
logical_path: ".",
|
|
classification: "changed",
|
|
dimensions: expect.arrayContaining(["executable"]),
|
|
}),
|
|
],
|
|
});
|
|
}
|
|
});
|
|
});
|
|
|
|
describe("artifact inventory coverage", () => {
|
|
it("keeps explicit and legacy unknown coverage fail-closed", async () => {
|
|
const root = await createTestTempDirectory("rea-artifact-coverage-");
|
|
const path = join(root, "complete.plist");
|
|
await writeFile(
|
|
path,
|
|
'<?xml version="1.0"?><plist version="1.0"><dict/></plist>',
|
|
);
|
|
const complete = await observe(path);
|
|
const graph = artifactInventoryResultSchema.parse(
|
|
complete.normalized_result,
|
|
);
|
|
const partial = createEvidence(
|
|
{
|
|
path: complete.subject!.local_path,
|
|
sha256: complete.subject!.digest.sha256,
|
|
format: complete.subject!.format,
|
|
},
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: jsonValueSchema.parse({ ...graph, coverage_status: "partial" }),
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
);
|
|
expect(compareArtifacts(partial, partial)).toMatchObject({
|
|
status: "unknown",
|
|
summary: { unchanged: 0, unknown: 1 },
|
|
});
|
|
|
|
const { coverage_status: _coverageStatus, ...legacyGraph } = graph;
|
|
const unknownLegacy = createEvidence(
|
|
{
|
|
path: complete.subject!.local_path,
|
|
sha256: complete.subject!.digest.sha256,
|
|
format: complete.subject!.format,
|
|
},
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: jsonValueSchema.parse({
|
|
...legacyGraph,
|
|
limitations: ["Producer omitted unexamined child entries."],
|
|
}),
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
},
|
|
);
|
|
expect(compareArtifacts(unknownLegacy, unknownLegacy)).toMatchObject({
|
|
status: "unknown",
|
|
summary: { unchanged: 0, unknown: 1 },
|
|
});
|
|
});
|
|
|
|
it("rejects inventory fragments that disagree on producer coverage", async () => {
|
|
const parent = await createTestTempDirectory("rea-artifact-page-coverage-");
|
|
const path = join(parent, "fixture.app");
|
|
await mkdir(path);
|
|
await writeFile(join(path, "main.js"), "main();");
|
|
const source = await observe(path);
|
|
const fragments = fragmentInventoryEvidence(source, 2);
|
|
const firstFragment = fragments[0];
|
|
const fragment = fragments[1];
|
|
if (firstFragment === undefined || fragment === undefined)
|
|
throw new Error("Missing test fragment");
|
|
const fragmentGraph = artifactInventoryResultSchema.parse(
|
|
fragment.normalized_result,
|
|
);
|
|
const mixed = createEvidence(
|
|
{
|
|
path: fragment.subject!.local_path,
|
|
sha256: fragment.subject!.digest.sha256,
|
|
format: fragment.subject!.format,
|
|
},
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: fragment.parameters,
|
|
result: jsonValueSchema.parse({
|
|
...fragmentGraph,
|
|
coverage_status: "partial",
|
|
}),
|
|
confidence: "derived",
|
|
authority: "shipped-artifact",
|
|
},
|
|
);
|
|
expect(() =>
|
|
parseArtifactInventoryEvidence([firstFragment, mixed]),
|
|
).toThrow("Artifact inventory pages disagree on coverage status");
|
|
});
|
|
});
|
|
|
|
describe("artifact comparison", () => {
|
|
it("classifies deterministic path changes and cites both inventories", async () => {
|
|
const firstObservation = await changedArtifactComparison();
|
|
const secondObservation = await changedArtifactComparison();
|
|
const first = firstObservation.result;
|
|
const second = secondObservation.result;
|
|
expect(first).toEqual(second);
|
|
expect(artifactComparisonResultSchema.parse(first)).toMatchObject({
|
|
status: "changed",
|
|
summary: { added: 1, changed: 2, unknown: 0 },
|
|
changes: expect.arrayContaining([
|
|
expect.objectContaining({ logical_path: "added.txt" }),
|
|
]),
|
|
});
|
|
expect(first.changes[0]?.evidence_links).toEqual([
|
|
firstObservation.left.evidence_id,
|
|
firstObservation.right.evidence_id,
|
|
]);
|
|
expect(first.changes).toEqual(
|
|
expect.arrayContaining([
|
|
expect.objectContaining({
|
|
logical_path: "added.txt",
|
|
classification: "added",
|
|
}),
|
|
expect.objectContaining({
|
|
logical_path: "main.js",
|
|
classification: "changed",
|
|
dimensions: expect.arrayContaining(["content", "size"]),
|
|
}),
|
|
]),
|
|
);
|
|
});
|
|
|
|
it("reports incomplete inventory as truncated, never unchanged", async () => {
|
|
const root = await createTestTempDirectory("rea-artifact-truncated-");
|
|
await writeFile(join(root, "one.txt"), "one");
|
|
const complete = await observe(root);
|
|
const completeInventory = artifactInventoryResultSchema.parse(
|
|
complete.normalized_result,
|
|
);
|
|
const incomplete = createEvidence(
|
|
{
|
|
path: root,
|
|
sha256: complete.subject?.digest.sha256 ?? "0".repeat(64),
|
|
format: "directory",
|
|
},
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: jsonValueSchema.parse({
|
|
...completeInventory,
|
|
nodes: [],
|
|
}),
|
|
confidence: "observed",
|
|
authority: "shipped-artifact",
|
|
limitations: ["Inventory node observations are unavailable."],
|
|
},
|
|
);
|
|
const comparison = compareArtifacts(incomplete, incomplete);
|
|
expect(comparison).toMatchObject({
|
|
status: "truncated",
|
|
summary: { unchanged: 0, unknown: 2 },
|
|
});
|
|
expect(comparison.limitations).toContain(
|
|
"Left artifact inventory is incomplete.",
|
|
);
|
|
});
|
|
});
|
|
|
|
describe("artifact comparison completeness", () => {
|
|
it("compares every graph member for inventories larger than 500 entries", async () => {
|
|
const root = await createTestTempDirectory("rea-artifact-pages-");
|
|
const entryCount = 501;
|
|
await Promise.all(
|
|
Array.from({ length: entryCount }, async (_, index) =>
|
|
writeFile(
|
|
join(root, `file-${String(index).padStart(3, "0")}.txt`),
|
|
String(index),
|
|
),
|
|
),
|
|
);
|
|
const complete = await observe(root);
|
|
const graph = artifactInventoryResultSchema.parse(
|
|
complete.normalized_result,
|
|
);
|
|
// The real risk above 500 entries is silent truncation, so assert the
|
|
// manifest counts reconcile with the arrays actually returned and that
|
|
// every written file is present. Deriving from `entryCount` keeps this
|
|
// valid at any size instead of freezing a page boundary.
|
|
expect(graph.manifest.node_count).toBe(graph.nodes.length);
|
|
expect(graph.manifest.occurrence_count).toBe(graph.occurrences.length);
|
|
expect(graph.manifest.edge_count).toBe(graph.edges.length);
|
|
const written = new Set(
|
|
Array.from(
|
|
{ length: entryCount },
|
|
(_, index) => `file-${String(index).padStart(3, "0")}.txt`,
|
|
),
|
|
);
|
|
// Occurrences carry the inventory paths, so completeness is checked there.
|
|
const paths = new Set(
|
|
graph.occurrences.map((occurrence) => occurrence.logical_path),
|
|
);
|
|
expect([...written].filter((path) => !paths.has(path))).toEqual([]);
|
|
expect(
|
|
graph.occurrences.filter(
|
|
(occurrence) => occurrence.entry_kind === "file",
|
|
),
|
|
).toHaveLength(entryCount);
|
|
expect(compareArtifacts(complete, complete)).toMatchObject({
|
|
status: "unchanged",
|
|
summary: { unchanged: 502, unknown: 0 },
|
|
changes: [],
|
|
});
|
|
}, 15_000);
|
|
|
|
it("assembles real inventory fragments and retains every citation", async () => {
|
|
const parent = await createTestTempDirectory("rea-artifact-fragments-");
|
|
const leftPath = join(parent, "left.app");
|
|
const rightPath = join(parent, "right.app");
|
|
await Promise.all([mkdir(leftPath), mkdir(rightPath)]);
|
|
// Cross the former 100-citation limit with distinct, real graph members.
|
|
const fragmentCount = 101;
|
|
await Promise.all(
|
|
Array.from({ length: fragmentCount }, async (_, index) => {
|
|
await Promise.all([
|
|
writeFile(join(leftPath, `file-${index}.js`), `left(${index});`),
|
|
writeFile(join(rightPath, `file-${index}.js`), `right(${index});`),
|
|
]);
|
|
}),
|
|
);
|
|
const left = await observe(leftPath);
|
|
const right = await observe(rightPath);
|
|
const leftFragments = fragmentInventoryEvidence(left, fragmentCount);
|
|
const rightFragments = fragmentInventoryEvidence(right, fragmentCount);
|
|
const citations = [...leftFragments, ...rightFragments]
|
|
.map(({ evidence_id }) => evidence_id)
|
|
.sort();
|
|
expect(new Set(citations).size).toBe(fragmentCount * 2);
|
|
const comparison = compareArtifacts(leftFragments, rightFragments);
|
|
const complete = compareArtifacts(left, right);
|
|
expect(
|
|
comparison.changes
|
|
.filter(({ logical_path }) => logical_path !== ".")
|
|
.map(({ logical_path, classification }) => ({
|
|
logical_path,
|
|
classification,
|
|
}))
|
|
.sort((left, right) =>
|
|
left.logical_path.localeCompare(right.logical_path),
|
|
),
|
|
).toEqual(
|
|
Array.from({ length: fragmentCount }, (_, index) => ({
|
|
logical_path: `file-${index}.js`,
|
|
classification: "changed",
|
|
})).sort((left, right) =>
|
|
left.logical_path.localeCompare(right.logical_path),
|
|
),
|
|
);
|
|
expect({
|
|
...comparison,
|
|
changes: comparison.changes.map((change) => ({
|
|
...change,
|
|
evidence_links: [...change.evidence_links].sort(),
|
|
})),
|
|
}).toEqual({
|
|
...complete,
|
|
changes: complete.changes.map((change) => ({
|
|
...change,
|
|
evidence_links: citations,
|
|
})),
|
|
});
|
|
}, 15_000);
|
|
|
|
it("rejects non-inventory and tampered Evidence", async () => {
|
|
const root = await createTestTempDirectory("rea-artifact-invalid-");
|
|
const evidence = await observe(root);
|
|
expect(() =>
|
|
compareArtifacts({ ...evidence, operation: "binary_overview" }, evidence),
|
|
).toThrow(/identifier/u);
|
|
const wrongOperation = createEvidence(undefined, PROVIDER, {
|
|
operation: "binary_overview",
|
|
parameters: {},
|
|
result: evidence.normalized_result,
|
|
});
|
|
expect(() => compareArtifacts(wrongOperation, evidence)).toThrow(
|
|
/inspect_artifact/u,
|
|
);
|
|
const mismatchedSubject = createEvidence(
|
|
{ path: root, sha256: "f".repeat(64), format: "directory" },
|
|
PROVIDER,
|
|
{
|
|
operation: "inventory_artifact",
|
|
parameters: {},
|
|
result: evidence.normalized_result,
|
|
},
|
|
);
|
|
expect(() => compareArtifacts(mismatchedSubject, evidence)).toThrow(
|
|
/root digest/u,
|
|
);
|
|
});
|
|
});
|