perf(evidence): scale imports and dependency graphs (#941)

* perf(evidence): avoid rebuilding ledger on record imports

* perf(evidence): scale unknown graph validation

* fix(evidence): preserve import and projection semantics

* fix(evidence): invalidate cache on metadata replacement

* test(evidence): isolate snapshot import regression

* fix(evidence): preserve snapshots on additive imports
This commit is contained in:
morluto
2026-10-07 23:47:47 +08:00
committed by GitHub
parent 93aa9eb5c2
commit 7dc40d3d9e
8 changed files with 613 additions and 47 deletions
@@ -68,7 +68,8 @@ export abstract class BinarySessionRecords {
): Result<number, EvidenceIntegrityError> {
const imported = this.#records.mergeEvidenceBundle(bundle);
if (!imported.ok) return imported;
if (imported.value.changed) this.#emitSnapshotChanged();
if (imported.value.metadataChanged) this.invalidateSnapshot();
else if (imported.value.changed) this.#emitSnapshotChanged();
return ok(imported.value.recordsAdded);
}
@@ -77,6 +77,78 @@ const update = (
});
describe("residual unknown registry reads and identity", () => {
it("merges record-only bundles without changing retained unknown history", () => {
const store = ledger();
const created = store.recordUnknown(
input("Does a later record preserve this unknown?"),
mutation("record-only-import"),
);
expect(created.ok).toBe(true);
const originalUnknowns = store.export().unknowns;
const addition = evidence("record-only-addition");
const bundle = createEvidenceBundle([addition]);
expect(store.import(bundle)).toEqual({
ok: true,
value: {
recordsAdded: 1,
unknownsAdded: 0,
changed: true,
metadataChanged: false,
},
});
expect(store.export().unknowns).toEqual(originalUnknowns);
expect(store.import(bundle)).toEqual({
ok: true,
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: false,
metadataChanged: false,
},
});
});
});
describe("residual unknown dependency histories", () => {
it("imports a valid dependency chain deeper than the JavaScript call stack", () => {
const records = [];
const unknowns: ResidualUnknown[] = [];
let previousId: string | undefined;
for (let index = 0; index < 10_000; index += 1) {
const mutationEvidence = mutation(`deep-chain-${index}`);
records.push(mutationEvidence);
const relationships =
previousId === undefined
? []
: [{ type: "depends-on" as const, unknown_id: previousId }];
const unknown = createResidualUnknown(
input(`Does chain item ${index} meet the expected state?`, {
domain: "chain-profile",
required_authority: null,
relationships,
}),
mutationEvidence.evidence_id,
null,
);
unknowns.push(unknown);
previousId = unknown.unknown_id;
}
const bundle = createEvidenceBundle(records, unknowns);
const store = ledger();
expect(store.import(bundle)).toEqual({
ok: true,
value: {
recordsAdded: 10_000,
unknownsAdded: 10_000,
changed: true,
metadataChanged: false,
},
});
expect(store.export()).toEqual(bundle);
});
it("returns detached unknowns and evidence bundles from every read surface", () => {
const store = ledger();
expect(store.record(evidence("detached-record")).ok).toBe(true);
@@ -38,9 +38,55 @@ describe("evidence ledger recording", () => {
});
expect(ledger.import(createEvidenceBundle([evidence]))).toEqual({
ok: true,
value: { recordsAdded: 0, unknownsAdded: 0, changed: false },
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: false,
metadataChanged: false,
},
});
expect(ledger.export().records).toEqual([evidence]);
const relocatedBundleRecord = createEvidence(
{ ...TARGET, path: "/relocated/fixture-renamed" },
PROVIDER,
{ operation: "health", parameters: {}, result: true },
);
expect(
ledger.import(createEvidenceBundle([relocatedBundleRecord])),
).toEqual({
ok: true,
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: true,
metadataChanged: true,
},
});
expect(ledger.get(evidence.evidence_id)).toEqual(relocatedBundleRecord);
expect(ledger.export().records).toEqual([relocatedBundleRecord]);
expect(
ledger.import(createEvidenceBundle([relocatedBundleRecord])),
).toEqual({
ok: true,
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: false,
metadataChanged: false,
},
});
const conflictingBundle = createEvidenceBundle([
evidence,
relocatedBundleRecord,
]);
expect(ledger.import(conflictingBundle).ok).toBe(false);
expect(ledger.get(evidence.evidence_id)).toEqual(relocatedBundleRecord);
ledger.clear();
expect(ledger.export().records).toEqual([]);
});
it("ignores JSON object key order during imported identity checks", () => {
const ledger = new EvidenceLedger();
const ordered = createEvidence(TARGET, PROVIDER, {
operation: "health",
parameters: { alpha: 1, beta: 2 },
@@ -60,11 +106,78 @@ describe("evidence ledger recording", () => {
);
expect(ledger.import(reorderedBundle)).toEqual({
ok: true,
value: { recordsAdded: 0, unknownsAdded: 0, changed: false },
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: false,
metadataChanged: false,
},
});
ledger.clear();
expect(ledger.export().records).toEqual([]);
});
});
describe("evidence ledger metadata imports", () => {
it("reports and commits metadata replacement when importing unknown history", () => {
const ledger = new EvidenceLedger();
const mutation = createEvidence(TARGET, PROVIDER, {
predicateType: "rea.residual-unknown-mutation",
operation: "record_unknown",
parameters: {},
result: { action: "record", question: "Which path is current?" },
});
const created = ledger.recordUnknown(
recordUnknownInputSchema.parse({
approved: true,
question: "Which path is current?",
severity: "high",
domain: "fixture",
required_authority: null,
required_confidence: "derived",
required_environment: null,
recommended_probes: [],
relationships: [],
}),
mutation,
);
expect(created.ok).toBe(true);
const relocatedMutation = createEvidence(
{ ...TARGET, path: "/relocated/mutation" },
PROVIDER,
{
predicateType: "rea.residual-unknown-mutation",
operation: "record_unknown",
parameters: {},
result: { action: "record", question: "Which path is current?" },
},
);
expect(relocatedMutation.evidence_id).toBe(mutation.evidence_id);
const bundle = createEvidenceBundle(
[relocatedMutation],
ledger.export().unknowns,
);
expect(ledger.import(bundle)).toEqual({
ok: true,
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: true,
metadataChanged: true,
},
});
expect(ledger.get(mutation.evidence_id)).toEqual(relocatedMutation);
expect(ledger.import(bundle)).toEqual({
ok: true,
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: false,
metadataChanged: false,
},
});
});
it("retains more than ten thousand inline evidence records", () => {
const ledger = new EvidenceLedger();
@@ -27,6 +27,7 @@ export interface EvidenceImportDelta {
readonly recordsAdded: number;
readonly unknownsAdded: number;
readonly changed: boolean;
readonly metadataChanged: boolean;
}
type ImportResult = Result<EvidenceImportDelta, EvidenceLedgerFailure>;
@@ -70,11 +71,50 @@ export class EvidenceLedger {
}),
);
}
// A record-only bundle cannot change the already-valid unknown graph. The
// bundle parser has validated every incoming record and its manifests, so
// checking ID conflicts and committing only the additions preserves the
// atomic merge without rebuilding and reparsing the entire ledger.
if (bundle.unknowns.length === 0) {
const additions = new Map<string, Evidence>();
const replacements = new Map<string, Evidence>();
let metadataChanged = false;
for (const evidence of bundle.records) {
const existing = this.#records.get(evidence.evidence_id);
if (existing !== undefined) {
if (!recordsAgree(existing, evidence))
return err(
new EvidenceIntegrityError("Conflicting evidence record"),
);
if (canonicalize(existing) !== canonicalize(evidence)) {
replacements.set(evidence.evidence_id, evidence);
metadataChanged = true;
}
continue;
}
additions.set(evidence.evidence_id, evidence);
}
for (const [id, evidence] of replacements)
this.#records.set(id, evidence);
for (const [id, evidence] of additions) this.#records.set(id, evidence);
return ok({
recordsAdded: additions.size,
unknownsAdded: 0,
changed: additions.size > 0 || metadataChanged,
metadataChanged,
});
}
const pending = new Map(this.#records);
let metadataChanged = false;
for (const evidence of bundle.records) {
const existing = pending.get(evidence.evidence_id);
if (existing !== undefined && !recordsAgree(existing, evidence))
return err(new EvidenceIntegrityError("Conflicting evidence record"));
if (
existing !== undefined &&
canonicalize(existing) !== canonicalize(evidence)
)
metadataChanged = true;
pending.set(evidence.evidence_id, evidence);
}
const pendingUnknowns = new Map(this.#unknownRevisions);
@@ -108,7 +148,8 @@ export class EvidenceLedger {
return ok({
recordsAdded: added,
unknownsAdded,
changed: added > 0 || unknownsAdded > 0,
changed: added > 0 || unknownsAdded > 0 || metadataChanged,
metadataChanged,
});
}
@@ -133,11 +133,21 @@ describe("investigation record ownership", () => {
).toBe(true);
expect(records.mergeEvidenceBundle(bundle)).toEqual({
ok: true,
value: { recordsAdded: 0, unknownsAdded: 1, changed: true },
value: {
recordsAdded: 0,
unknownsAdded: 1,
changed: true,
metadataChanged: false,
},
});
expect(records.mergeEvidenceBundle(bundle)).toEqual({
ok: true,
value: { recordsAdded: 0, unknownsAdded: 0, changed: false },
value: {
recordsAdded: 0,
unknownsAdded: 0,
changed: false,
metadataChanged: false,
},
});
records.clear();
expect(records.exportEvidenceBundle()).toEqual(createEvidenceBundle([]));
+122
View File
@@ -0,0 +1,122 @@
import { describe, expect, it } from "vitest";
import { createEvidence } from "./evidence.js";
import {
createEvidenceBundle,
evidenceBundleForTarget,
parseEvidenceBundle,
} from "./evidenceBundle.js";
import {
createResidualUnknown,
recordUnknownInputSchema,
} from "./residualUnknown.js";
const provider = { id: "fixture", name: "Fixture", version: "1" };
const targetDigest = "a".repeat(64);
const foreignDigest = "b".repeat(64);
const makeUnknown = (
question: string,
scopeDigest: string,
dependency?: string,
) => {
const input = recordUnknownInputSchema.parse({
approved: true,
question,
severity: "medium",
domain: "bundle-projection",
required_authority: null,
required_confidence: "derived",
required_environment: null,
recommended_probes: [],
relationships:
dependency === undefined
? []
: [{ type: "depends-on", unknown_id: dependency }],
});
const mutation = createEvidence(undefined, provider, {
predicateType: "rea.residual-unknown-mutation",
operation: "record_unknown",
parameters: { domain: input.domain, severity: input.severity },
result: {
action: "record",
question: input.question,
required_authority: input.required_authority,
required_confidence: input.required_confidence,
},
});
return {
evidence: mutation,
unknown: createResidualUnknown(input, mutation.evidence_id, scopeDigest),
};
};
describe("evidenceBundleForTarget", () => {
it("removes dependent unknowns when their foreign-scope root is excluded", () => {
const foreignRoot = makeUnknown("Foreign root", foreignDigest);
const child = makeUnknown(
"Target child",
targetDigest,
foreignRoot.unknown.unknown_id,
);
const grandchild = makeUnknown(
"Target grandchild",
targetDigest,
child.unknown.unknown_id,
);
const retained = makeUnknown("Independent target", targetDigest);
const bundle = createEvidenceBundle(
[
foreignRoot.evidence,
child.evidence,
grandchild.evidence,
retained.evidence,
],
[
foreignRoot.unknown,
child.unknown,
grandchild.unknown,
retained.unknown,
],
);
expect(evidenceBundleForTarget(bundle, targetDigest)).toEqual(
createEvidenceBundle([retained.evidence], [retained.unknown]),
);
});
it("projects a valid graph with more than 125,000 dependents", () => {
const foreignRoot = makeUnknown("Foreign high-fan-out root", foreignDigest);
const middle = makeUnknown(
"Target middle depending on foreign root",
targetDigest,
foreignRoot.unknown.unknown_id,
);
const unknowns = [foreignRoot.unknown, middle.unknown];
for (let index = 0; index < 125_000; index += 1) {
const input = recordUnknownInputSchema.parse({
approved: true,
question: `Target child ${index}`,
severity: "medium",
domain: "bundle-projection",
required_authority: null,
required_confidence: "derived",
required_environment: null,
recommended_probes: [],
relationships: [
{ type: "depends-on", unknown_id: middle.unknown.unknown_id },
],
});
unknowns.push(
createResidualUnknown(input, middle.evidence.evidence_id, targetDigest),
);
}
const bundle = parseEvidenceBundle(
createEvidenceBundle([foreignRoot.evidence, middle.evidence], unknowns),
);
expect(evidenceBundleForTarget(bundle, targetDigest)).toEqual(
createEvidenceBundle([], []),
);
});
});
+108 -41
View File
@@ -111,28 +111,57 @@ export const evidenceBundleForTarget = (
const recordsById = new Map(
bundle.records.map((record) => [record.evidence_id, record]),
);
let unknowns = bundle.unknowns.filter(
const candidates = bundle.unknowns.filter(
({ scope_digest: scopeDigest }) => scopeDigest === sha256,
);
let changed = true;
while (changed) {
const ids = new Set(unknowns.map(({ unknown_id: id }) => id));
const retained = unknowns.filter(
(unknown) =>
unknown.relationships.every(({ unknown_id: id }) => ids.has(id)) &&
referencedEvidenceIds(unknown).every((id) => {
const record = recordsById.get(id);
return (
record?.subject?.digest.sha256 === sha256 ||
(unknown.mutation_evidence_ids.includes(id) &&
record?.subject === null &&
record.predicate_type === "rea.residual-unknown-mutation")
);
}),
const retained = new Set<ResidualUnknown>();
const revisionsById = new Map<string, number>();
const dependents = new Map<string, Set<ResidualUnknown>>();
for (const unknown of candidates) {
const mutationEvidenceIds = new Set(unknown.mutation_evidence_ids);
const evidenceAvailable = referencedEvidenceIds(unknown).every((id) => {
const record = recordsById.get(id);
return (
record?.subject?.digest.sha256 === sha256 ||
(mutationEvidenceIds.has(id) &&
record?.subject === null &&
record.predicate_type === "rea.residual-unknown-mutation")
);
});
if (!evidenceAvailable) continue;
retained.add(unknown);
revisionsById.set(
unknown.unknown_id,
(revisionsById.get(unknown.unknown_id) ?? 0) + 1,
);
changed = retained.length !== unknowns.length;
unknowns = retained;
for (const relatedId of new Set(
unknown.relationships.map(({ unknown_id: id }) => id),
)) {
const references =
dependents.get(relatedId) ?? new Set<ResidualUnknown>();
references.add(unknown);
dependents.set(relatedId, references);
}
}
const pending: ResidualUnknown[] = [];
for (const unknown of retained)
if (
unknown.relationships.some(({ unknown_id: id }) => !revisionsById.has(id))
)
pending.push(unknown);
for (let index = 0; index < pending.length; index += 1) {
const unknown = pending[index];
if (unknown === undefined || !retained.delete(unknown)) continue;
const remaining = (revisionsById.get(unknown.unknown_id) ?? 1) - 1;
if (remaining > 0) {
revisionsById.set(unknown.unknown_id, remaining);
continue;
}
revisionsById.delete(unknown.unknown_id);
for (const dependent of dependents.get(unknown.unknown_id) ?? [])
pending.push(dependent);
}
const unknowns = candidates.filter((unknown) => retained.has(unknown));
const mutationIds = new Set(
unknowns.flatMap(({ mutation_evidence_ids: ids }) => ids),
);
@@ -285,26 +314,28 @@ export const validateResidualUnknownAddition = (
);
if (unknown.resolution?.disposition === "verified")
validateVerifiedResolution(unknown, evidenceById);
for (const relationship of unknown.relationships)
// In a valid graph, no existing node can point to an absent ID. Since the
// schema rejects self-relations, a new node's outgoing edges cannot close a
// cycle; keep graph traversal for revisions of existing nodes only.
if (current !== undefined) {
const dependencies = unknown.relationships
.filter((relationship) => relationship.type === "depends-on")
.map(({ unknown_id }) => unknown_id);
if (
relationship.type === "depends-on" &&
reachesUnknown(
relationship.unknown_id,
unknown.unknown_id,
currentHeads,
unknown,
)
dependencies.length > 0 &&
reachesUnknown(dependencies, unknown.unknown_id, currentHeads, unknown)
)
throw new TypeError("Residual unknown dependency graph contains a cycle");
}
};
const reachesUnknown = (
start: string,
starts: readonly string[],
goal: string,
currentHeads: ReadonlyMap<string, ResidualUnknown>,
candidate: ResidualUnknown,
): boolean => {
const pending = [start];
const pending = [...starts];
const visited = new Set<string>();
while (pending.length > 0) {
const id = pending.pop();
@@ -356,20 +387,56 @@ const rejectDependencyCycles = (
): void => {
const visited = new Set<string>();
const active = new Set<string>();
const visit = (id: string): void => {
if (active.has(id))
throw new TypeError("Residual unknown dependency graph contains a cycle");
if (visited.has(id)) return;
active.add(id);
const unknown = unknownById.get(id);
if (unknown === undefined)
const pending: {
readonly id: string;
readonly relationships: ResidualUnknown["relationships"];
nextRelationship: number;
}[] = [];
for (const root of unknownById.keys()) {
if (visited.has(root)) continue;
const rootUnknown = unknownById.get(root);
if (rootUnknown === undefined)
throw new TypeError("Residual unknown dependency graph is inconsistent");
for (const relationship of unknown.relationships)
if (relationship.type === "depends-on") visit(relationship.unknown_id);
active.delete(id);
visited.add(id);
};
for (const id of unknownById.keys()) visit(id);
active.add(root);
pending.push({
id: root,
relationships: rootUnknown.relationships,
nextRelationship: 0,
});
while (pending.length > 0) {
const frame = pending[pending.length - 1];
if (frame === undefined) break;
let descended = false;
while (frame.nextRelationship < frame.relationships.length) {
const relationship = frame.relationships[frame.nextRelationship];
frame.nextRelationship += 1;
if (relationship?.type !== "depends-on") continue;
const dependency = relationship.unknown_id;
if (active.has(dependency))
throw new TypeError(
"Residual unknown dependency graph contains a cycle",
);
if (visited.has(dependency)) continue;
const unknown = unknownById.get(dependency);
if (unknown === undefined)
throw new TypeError(
"Residual unknown dependency graph is inconsistent",
);
active.add(dependency);
pending.push({
id: dependency,
relationships: unknown.relationships,
nextRelationship: 0,
});
descended = true;
break;
}
if (descended) continue;
pending.pop();
active.delete(frame.id);
visited.add(frame.id);
}
}
};
/** Encode a validated bundle as byte-stable RFC 8785 canonical JSON. */
@@ -10,6 +10,8 @@ import { HopperStartError } from "../../../src/domain/hopperErrors.js";
import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js";
import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js";
import { err, ok as resultOk } from "../../../src/domain/result.js";
import { createEvidenceBundle } from "../../../src/domain/evidenceBundle.js";
import { createEvidence } from "../../../src/domain/evidence.js";
import { observed as ok } from "../../fixtures/analysisExecution.js";
import {
ControllableAnalysisClient,
@@ -267,6 +269,144 @@ describe("fresh run identity", () => {
});
});
describe("evidence metadata imports and snapshot cache", () => {
it("retains cached queries and exports after additive evidence imports", async () => {
const [target] = await createBinarySessionTargets();
const calls: string[] = [];
const session = createTestBinarySession(createCacheProvider(calls));
expect((await session.open(target)).ok).toBe(true);
expect(
(
await session.execute("address_name", {
address: "0x1000",
document: "first",
})
).ok,
).toBe(true);
const before = session.exportAnalysisSnapshot();
expect(before.ok).toBe(true);
if (!before.ok) {
await session.close();
return;
}
const existing = session.exportEvidenceBundle().records[0];
expect(existing?.subject).not.toBeNull();
if (existing?.subject === null || existing === undefined) {
await session.close();
return;
}
const addition = createEvidence(
{
path: existing.subject.local_path,
sha256: existing.subject.digest.sha256,
format: existing.subject.format,
...(existing.subject.architecture === null
? {}
: { architecture: existing.subject.architecture }),
},
{ id: "fixture", name: "Fixture", version: "1" },
{
predicateType: "rea.analysis",
operation: "health",
parameters: { source: "additive-import" },
result: true,
},
);
expect(
session.importEvidenceBundle(createEvidenceBundle([addition])),
).toEqual({ ok: true, value: 1 });
expect(session.exportAnalysisSnapshot()).toMatchObject({
ok: true,
value: {
entries: before.value.entries,
evidence_bundle: { records: expect.arrayContaining([addition]) },
},
});
const callsAfterInitialRead = [...calls];
expect(
(
await session.execute("address_name", {
address: "0x1000",
document: "first",
})
).ok,
).toBe(true);
expect(calls).toEqual(callsAfterInitialRead);
expect(session.exportAnalysisSnapshot().ok).toBe(true);
await session.close();
});
it("invalidates cached analysis when an import changes evidence path metadata", async () => {
const [target] = await createBinarySessionTargets();
const calls: string[] = [];
const session = createTestBinarySession(createCacheProvider(calls));
expect((await session.open(target)).ok).toBe(true);
expect(
(
await session.execute("address_name", {
address: "0x1000",
document: "first",
})
).ok,
).toBe(true);
expect(session.exportAnalysisSnapshot().ok).toBe(true);
const evidence = session
.exportEvidenceBundle()
.records.find(({ operation }) => operation === "address_name");
expect(evidence?.subject).not.toBeNull();
if (evidence?.subject === null || evidence === undefined) {
await session.close();
return;
}
expect(
session.importEvidenceBundle(createEvidenceBundle([evidence])),
).toEqual({ ok: true, value: 0 });
expect(session.exportAnalysisSnapshot().ok).toBe(true);
const callsAfterInitialRead = [...calls];
expect(
(
await session.execute("address_name", {
address: "0x1000",
document: "first",
})
).ok,
).toBe(true);
expect(calls).toEqual(callsAfterInitialRead);
const relocated = {
...evidence,
subject: {
...evidence.subject,
local_path: `${evidence.subject.local_path}.moved`,
name: `${evidence.subject.name}.moved`,
},
};
expect(
session.importEvidenceBundle(createEvidenceBundle([relocated])),
).toEqual({ ok: true, value: 0 });
expect(session.exportEvidenceBundle().records).toContainEqual(relocated);
expect(
(
await session.execute("address_name", {
address: "0x1000",
document: "first",
})
).ok,
).toBe(true);
expect(calls).toHaveLength(callsAfterInitialRead.length + 1);
expect(session.exportAnalysisSnapshot()).toMatchObject({
ok: false,
error: {
message: expect.stringContaining(
"Analysis snapshots are unavailable after analysis metadata mutations",
),
},
});
await session.close();
});
});
describe("replay of exact immutable calls", () => {
it("replays exact immutable calls from a matching provider-neutral snapshot", async () => {
const [first, second] = await createBinarySessionTargets();