Treat missing, partial, unknown, and unresolved call coverage as insufficient for call-set equality. Combine IDA caller/callee coverage without losing either explanation and recheck source coverage when verifying historical comparisons. Scope generated function-comparison unknowns with optional typed comparison/dimension facts, preserving legacy IDs and revision digests when absent. Keep unrelated dimensions conclusive, explicit dependencies applicable, and ambiguous function identity unknown. A confirmed change for matched functions remains a failure even if calls are incomplete. Validation: function/reconstruction/residual-unknown and bundle tests, recorded IDA protocol regression, public MCP comparison/verification, and advertised SDK JSON Schema checks. Real IDA provider execution was not verified. Fixes #2222
383 lines
13 KiB
TypeScript
383 lines
13 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
|
|
import canonicalize from "canonicalize";
|
|
import { z } from "zod";
|
|
|
|
import { artifactComparisonResultSchema } from "./artifactComparison.js";
|
|
import { artifactInventoryResultSchema } from "./artifactGraph.js";
|
|
import { uniqueSorted } from "./canonicalOrdering.js";
|
|
import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js";
|
|
import { comparisonSourceEvidenceSides } from "./comparisonSourceEvidence.js";
|
|
import { parseEvidence, type Evidence } from "./evidence.js";
|
|
import { parseEvidenceBundle } from "./evidenceBundle.js";
|
|
import { functionComparisonResultSchema } from "./functionComparisonSchemas.js";
|
|
import {
|
|
PROCESS_PROVIDER,
|
|
isProcessEvidenceProvider,
|
|
} from "./process/processEvidenceProvider.js";
|
|
import { functionDossierSchema } from "./hopperValues.js";
|
|
import { functionCallCoverageLimitations } from "./functionCallCoverage.js";
|
|
import {
|
|
deriveProcessComparisonStatus,
|
|
PROCESS_COMPARISON_DIMENSIONS,
|
|
processCaptureComparisonSchema,
|
|
} from "./process/processComparison.js";
|
|
import { parseProcessCapture } from "./process/processCaptureParsing.js";
|
|
import {
|
|
reconstructionClaimResultSchema,
|
|
reconstructionSpecificationSchema,
|
|
reconstructionVerificationResultSchema,
|
|
type ReconstructionClaim as Claim,
|
|
type ReconstructionClaimResult as ClaimResult,
|
|
type ReconstructionObservedStatus as ObservedStatus,
|
|
type ReconstructionVerificationResult,
|
|
} from "./reconstructionVerificationSchemas.js";
|
|
import {
|
|
reconstructionClaimUnknowns,
|
|
reconstructionProbes,
|
|
reconstructionUnknownHeads,
|
|
} from "./reconstructionUnknowns.js";
|
|
import type { ResidualUnknown } from "./residualUnknown.js";
|
|
|
|
const providers = {
|
|
behavioral: {
|
|
operation: "compare_process_captures",
|
|
predicates: ["rea.process-comparison"],
|
|
...PROCESS_PROVIDER,
|
|
},
|
|
"structural-function": {
|
|
operation: "compare_functions",
|
|
predicates: ["rea.function-comparison"],
|
|
id: "rea-function-comparison",
|
|
name: "REA function comparison",
|
|
},
|
|
"structural-artifact": {
|
|
operation: "compare_artifacts",
|
|
predicates: ["rea.artifact-comparison"],
|
|
id: "rea-artifact-comparison",
|
|
name: "REA artifact comparison",
|
|
},
|
|
} as const;
|
|
|
|
interface FunctionSourceFacts {
|
|
readonly evidence_id: string;
|
|
readonly callCoverageLimitations: readonly string[];
|
|
}
|
|
|
|
const acceptsPredicate = (
|
|
expected: readonly string[],
|
|
actual: string,
|
|
): boolean => expected.some((predicate) => predicate === actual);
|
|
|
|
/** Verify only explicitly declared claims; no global equivalence is inferred. */
|
|
export const verifyReconstruction = (
|
|
specificationInput: unknown,
|
|
bundleInput: unknown,
|
|
): ReconstructionVerificationResult => {
|
|
const specification =
|
|
reconstructionSpecificationSchema.parse(specificationInput);
|
|
const bundle = parseEvidenceBundle(bundleInput);
|
|
const records = new Map(
|
|
bundle.records.map((item) => [item.evidence_id, item]),
|
|
);
|
|
const heads = reconstructionUnknownHeads(bundle.unknowns);
|
|
const results = specification.claims
|
|
.map((claim) => evaluateClaim(claim, records, heads))
|
|
.sort((left, right) =>
|
|
compareUnicodeCodePoints(left.claim_id, right.claim_id),
|
|
);
|
|
const failed = count(results, "fail");
|
|
const unresolved = count(results, "unknown");
|
|
const probes = reconstructionProbes(results, heads);
|
|
const evidenceLinks = uniqueSorted(
|
|
results.flatMap(({ evidence_links: links }) => links),
|
|
);
|
|
const output = {
|
|
status: failed > 0 ? "fail" : unresolved > 0 ? "unknown" : "pass",
|
|
specification_sha256: digest({
|
|
...specification,
|
|
claims: [...specification.claims].sort((left, right) =>
|
|
compareUnicodeCodePoints(left.claim_id, right.claim_id),
|
|
),
|
|
}),
|
|
summary: {
|
|
total: results.length,
|
|
passed: count(results, "pass"),
|
|
failed,
|
|
unknown: unresolved,
|
|
behavioral: results.filter(({ kind }) => kind === "behavioral").length,
|
|
structural: results.filter(({ kind }) => kind !== "behavioral").length,
|
|
},
|
|
claims: {
|
|
items: results,
|
|
},
|
|
recommended_probes: probes,
|
|
evidence_links: evidenceLinks,
|
|
limitations: [
|
|
"Pass means every declared claim passed; it does not establish global implementation equivalence.",
|
|
],
|
|
};
|
|
return reconstructionVerificationResultSchema.parse(output);
|
|
};
|
|
|
|
const evaluateClaim = (
|
|
claim: Claim,
|
|
records: ReadonlyMap<string, Evidence>,
|
|
heads: ReadonlyMap<string, ResidualUnknown>,
|
|
): ClaimResult => {
|
|
const comparison = records.get(claim.comparison_evidence_id);
|
|
if (comparison === undefined)
|
|
throw new TypeError(
|
|
`Missing comparison Evidence ${claim.comparison_evidence_id}`,
|
|
);
|
|
validateComparisonIdentity(claim, comparison);
|
|
const sides = comparisonSourceEvidenceSides(comparison);
|
|
const allSourceIds = uniqueSorted([...sides.left, ...sides.right]);
|
|
if (!sameSet(comparison.evidence_links, allSourceIds))
|
|
throw new TypeError(
|
|
"Comparison Evidence closure disagrees with its parameters",
|
|
);
|
|
const sources = allSourceIds.map((id) => {
|
|
const source = records.get(id);
|
|
if (source === undefined)
|
|
throw new TypeError(`Missing source Evidence ${id}`);
|
|
return source;
|
|
});
|
|
const functionSourceFacts = readValidatedSourceFacts(claim, sources);
|
|
const observed = observedStatus(claim, comparison);
|
|
const limitations = [
|
|
...authorityLimitations(claim, sources, sides),
|
|
...sourceCallCoverageLimitations(claim, functionSourceFacts, observed),
|
|
];
|
|
const relatedUnknowns = reconstructionClaimUnknowns(
|
|
heads,
|
|
comparison.evidence_id,
|
|
allSourceIds,
|
|
claim.dimension,
|
|
);
|
|
const relevantUnknowns = relatedUnknowns;
|
|
const status = classify(observed, limitations, relevantUnknowns);
|
|
const displayedUnknowns = relevantUnknowns;
|
|
return reconstructionClaimResultSchema.parse({
|
|
claim_id: claim.claim_id,
|
|
kind: claim.kind,
|
|
dimension: claim.dimension,
|
|
status,
|
|
observed_status: observed,
|
|
comparison_evidence_id: comparison.evidence_id,
|
|
left_evidence_ids: sides.left,
|
|
right_evidence_ids: sides.right,
|
|
evidence_links: uniqueSorted([comparison.evidence_id, ...allSourceIds]),
|
|
unknown_ids: displayedUnknowns.map(({ unknown_id: id }) => id),
|
|
limitations: uniqueSorted([
|
|
...comparison.limitations,
|
|
...limitations,
|
|
...displayedUnknowns.map(
|
|
({ question }) => `Residual unknown: ${question}`,
|
|
),
|
|
]),
|
|
});
|
|
};
|
|
|
|
const readValidatedSourceFacts = (
|
|
claim: Claim,
|
|
sources: readonly Evidence[],
|
|
): readonly FunctionSourceFacts[] => {
|
|
const expected =
|
|
claim.kind === "behavioral"
|
|
? ["capture_process_scenario", "rea.process-capture"]
|
|
: claim.kind === "structural-function"
|
|
? ["analyze_function", "rea.analysis"]
|
|
: ["inventory_artifact", "rea.analysis"];
|
|
if (
|
|
sources.some(
|
|
(source) =>
|
|
source.operation !== expected[0] ||
|
|
source.predicate_type !== expected[1],
|
|
)
|
|
)
|
|
throw new TypeError(
|
|
"Comparison source Evidence has an unexpected observation type",
|
|
);
|
|
const functionSources: FunctionSourceFacts[] = [];
|
|
for (const source of sources) {
|
|
if (claim.kind === "behavioral")
|
|
parseProcessCapture(source.normalized_result);
|
|
else if (claim.kind === "structural-function") {
|
|
const dossier = functionDossierSchema.parse(source.normalized_result);
|
|
functionSources.push({
|
|
evidence_id: source.evidence_id,
|
|
callCoverageLimitations: functionCallCoverageLimitations(
|
|
source.relationship_coverage,
|
|
dossier,
|
|
),
|
|
});
|
|
} else artifactInventoryResultSchema.parse(source.normalized_result);
|
|
}
|
|
return functionSources;
|
|
};
|
|
|
|
const validateComparisonIdentity = (claim: Claim, evidence: Evidence): void => {
|
|
parseEvidence(evidence);
|
|
const expected = providers[claim.kind];
|
|
if (
|
|
evidence.operation !== expected.operation ||
|
|
!acceptsPredicate(expected.predicates, evidence.predicate_type) ||
|
|
(claim.kind === "behavioral"
|
|
? !isProcessEvidenceProvider(evidence.provider)
|
|
: evidence.provider.id !== expected.id ||
|
|
evidence.provider.name !== expected.name ||
|
|
evidence.provider.version !== "1") ||
|
|
evidence.confidence !== "derived" ||
|
|
evidence.authority !== "analyst-inference" ||
|
|
evidence.subject !== null
|
|
)
|
|
throw new TypeError("Comparison Evidence identity or authority disagrees");
|
|
};
|
|
|
|
const observedStatus = (claim: Claim, evidence: Evidence): ObservedStatus => {
|
|
if (claim.kind === "behavioral") {
|
|
const result = processCaptureComparisonSchema.parse(
|
|
evidence.normalized_result,
|
|
);
|
|
validateProcessStatus(result);
|
|
return claim.dimension === "overall"
|
|
? result.status
|
|
: result[claim.dimension];
|
|
}
|
|
if (claim.kind === "structural-function") {
|
|
const result = functionComparisonResultSchema.parse(
|
|
evidence.normalized_result,
|
|
);
|
|
if (claim.dimension === "overall") return result.status;
|
|
const dimension = result.dimensions.find(
|
|
(item) => item.dimension === claim.dimension,
|
|
);
|
|
if (dimension === undefined)
|
|
throw new TypeError("Function comparison dimension missing");
|
|
if (
|
|
dimension.evidence_links.some(
|
|
(id) => !evidence.evidence_links.includes(id),
|
|
)
|
|
)
|
|
throw new TypeError(
|
|
"Function dimension cites Evidence outside comparison closure",
|
|
);
|
|
return dimension.status;
|
|
}
|
|
const result = artifactComparisonResultSchema.parse(
|
|
evidence.normalized_result,
|
|
);
|
|
if (
|
|
result.changes.some((item) =>
|
|
item.evidence_links.some((id) => !evidence.evidence_links.includes(id)),
|
|
)
|
|
)
|
|
throw new TypeError(
|
|
"Artifact change cites Evidence outside comparison closure",
|
|
);
|
|
return result.status;
|
|
};
|
|
|
|
const validateProcessStatus = (
|
|
result: z.infer<typeof processCaptureComparisonSchema>,
|
|
): void => {
|
|
const expected = deriveProcessComparisonStatus(
|
|
PROCESS_COMPARISON_DIMENSIONS.map((dimension) => result[dimension]),
|
|
);
|
|
if (result.status !== expected)
|
|
throw new TypeError("Process comparison status contradicts its dimensions");
|
|
};
|
|
|
|
const authorityLimitations = (
|
|
claim: Claim,
|
|
sources: readonly Evidence[],
|
|
sides: {
|
|
readonly left: readonly string[];
|
|
readonly right: readonly string[];
|
|
},
|
|
): string[] => {
|
|
const requiredAuthority =
|
|
claim.kind === "behavioral" ? "controlled-replay" : "shipped-artifact";
|
|
const limitations: string[] = [];
|
|
if (sides.left.some((id) => sides.right.includes(id)))
|
|
limitations.push(
|
|
"A source observation appears on both comparison sides; independent original and reconstruction observations are unavailable.",
|
|
);
|
|
if (
|
|
sources.some(
|
|
(source) =>
|
|
source.authority !== requiredAuthority ||
|
|
(claim.kind === "structural-function"
|
|
? source.confidence === "inferred"
|
|
: source.confidence !== "observed"),
|
|
)
|
|
)
|
|
limitations.push(
|
|
`Source Evidence lacks qualifying ${requiredAuthority} authority.`,
|
|
);
|
|
if (
|
|
claim.kind !== "behavioral" &&
|
|
sources.some(({ subject }) => subject === null)
|
|
)
|
|
limitations.push("Structural source artifact identity is unavailable.");
|
|
if (claim.kind === "behavioral") {
|
|
const byId = new Map(sources.map((source) => [source.evidence_id, source]));
|
|
const left = byId.get(sides.left[0] ?? "");
|
|
const right = byId.get(sides.right[0] ?? "");
|
|
if (
|
|
left?.environment === null ||
|
|
right?.environment === null ||
|
|
JSON.stringify(left?.environment) !== JSON.stringify(right?.environment)
|
|
)
|
|
limitations.push(
|
|
"Controlled replay environments are unavailable or incompatible.",
|
|
);
|
|
}
|
|
return limitations;
|
|
};
|
|
|
|
const sourceCallCoverageLimitations = (
|
|
claim: Claim,
|
|
functionSources: readonly FunctionSourceFacts[],
|
|
observed: ObservedStatus,
|
|
): string[] => {
|
|
if (
|
|
claim.kind !== "structural-function" ||
|
|
(claim.dimension !== "calls" &&
|
|
!(claim.dimension === "overall" && observed !== "changed"))
|
|
)
|
|
return [];
|
|
return functionSources.flatMap(({ evidence_id, callCoverageLimitations }) =>
|
|
callCoverageLimitations.map((reason) => `Source ${evidence_id}: ${reason}`),
|
|
);
|
|
};
|
|
|
|
const classify = (
|
|
observed: ObservedStatus,
|
|
limitations: readonly string[],
|
|
unknowns: readonly ResidualUnknown[],
|
|
): ClaimResult["status"] => {
|
|
if (limitations.length > 0) return "unknown";
|
|
if (["changed", "added", "removed", "contradiction"].includes(observed))
|
|
return "fail";
|
|
if (observed === "unknown" || observed === "truncated" || unknowns.length > 0)
|
|
return "unknown";
|
|
return "pass";
|
|
};
|
|
|
|
const digest = (value: unknown): string => {
|
|
const encoded = canonicalize(value);
|
|
if (encoded === undefined)
|
|
throw new TypeError("Specification canonicalization failed");
|
|
return createHash("sha256").update(encoded).digest("hex");
|
|
};
|
|
|
|
const count = (
|
|
items: readonly ClaimResult[],
|
|
status: ClaimResult["status"],
|
|
): number => items.filter((item) => item.status === status).length;
|
|
const sameSet = (left: readonly string[], right: readonly string[]): boolean =>
|
|
uniqueSorted(left).length === uniqueSorted(right).length &&
|
|
uniqueSorted(left).every((id, index) => id === uniqueSorted(right)[index]);
|