fix(evidence): preserve call coverage in function comparisons
Treat missing, partial, unknown, and unresolved call coverage as insufficient for call-set equality. Combine IDA caller/callee coverage without losing either explanation and recheck source coverage when verifying historical comparisons. Scope generated function-comparison unknowns with optional typed comparison/dimension facts, preserving legacy IDs and revision digests when absent. Keep unrelated dimensions conclusive, explicit dependencies applicable, and ambiguous function identity unknown. A confirmed change for matched functions remains a failure even if calls are incomplete. Validation: function/reconstruction/residual-unknown and bundle tests, recorded IDA protocol regression, public MCP comparison/verification, and advertised SDK JSON Schema checks. Real IDA provider execution was not verified. Fixes #2222
This commit is contained in:
@@ -69,6 +69,7 @@ const source = (
|
||||
result: dossier,
|
||||
confidence: "derived",
|
||||
authority: "shipped-artifact",
|
||||
relationshipCoverage: { status: "complete" },
|
||||
...observation,
|
||||
},
|
||||
);
|
||||
@@ -99,14 +100,15 @@ const comparison = (
|
||||
provider?: EvidenceProvider;
|
||||
target?: EvidenceSubjectTarget;
|
||||
} = {},
|
||||
resultInput = baselineResult,
|
||||
): Evidence =>
|
||||
createEvidence(identity.target, identity.provider ?? provider, {
|
||||
predicateType: "rea.function-comparison",
|
||||
operation: "compare_functions",
|
||||
parameters,
|
||||
result: jsonValueSchema.parse({
|
||||
...baselineResult,
|
||||
dimensions: baselineResult.dimensions.map((dimension) => ({
|
||||
...resultInput,
|
||||
dimensions: resultInput.dimensions.map((dimension) => ({
|
||||
...dimension,
|
||||
evidence_links: sources.map(({ evidence_id: id }) => id),
|
||||
})),
|
||||
@@ -195,6 +197,100 @@ describe("function reconstruction source parameters", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("function reconstruction call coverage", () => {
|
||||
it("rechecks old unchanged comparisons without affecting unrelated finite dimensions", () => {
|
||||
const incomplete = source("7", {
|
||||
relationshipCoverage: {
|
||||
status: "unknown",
|
||||
reason: "Call graph unavailable.",
|
||||
},
|
||||
});
|
||||
const incompleteSides = {
|
||||
left_evidence_id: left.evidence_id,
|
||||
right_evidence_id: incomplete.evidence_id,
|
||||
};
|
||||
const oldUnchanged = comparison(incompleteSides, {}, [left, incomplete]);
|
||||
for (const dimension of ["calls", "overall"] as const) {
|
||||
const result = admittedVerification(
|
||||
oldUnchanged,
|
||||
[left, incomplete],
|
||||
dimension,
|
||||
)();
|
||||
expect(result).toMatchObject({
|
||||
status: "unknown",
|
||||
summary: { total: 1, passed: 0, failed: 0, unknown: 1 },
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
dimension,
|
||||
status: "unknown",
|
||||
observed_status: "unchanged",
|
||||
limitations: expect.arrayContaining([
|
||||
`Source ${incomplete.evidence_id}: Call graph unavailable.`,
|
||||
]),
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
expect(
|
||||
admittedVerification(oldUnchanged, [left, incomplete], "pseudocode")(),
|
||||
).toMatchObject({
|
||||
status: "pass",
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
dimension: "pseudocode",
|
||||
status: "pass",
|
||||
observed_status: "unchanged",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps a confirmed historical overall change as a failure despite unknown calls", () => {
|
||||
const changedDossier = jsonValueSchema.parse({
|
||||
...functionDossierSchema.parse(dossier),
|
||||
pseudocode: "return 1;",
|
||||
});
|
||||
const changedComplete = source("8", { result: changedDossier });
|
||||
const incomplete = source("9", {
|
||||
result: changedDossier,
|
||||
relationshipCoverage: {
|
||||
status: "unknown",
|
||||
reason: "Call graph unavailable.",
|
||||
},
|
||||
});
|
||||
const changedResult = compareFunctions(left, changedComplete);
|
||||
const oldChanged = comparison(
|
||||
{
|
||||
left_evidence_id: left.evidence_id,
|
||||
right_evidence_id: incomplete.evidence_id,
|
||||
},
|
||||
{},
|
||||
[left, incomplete],
|
||||
{},
|
||||
changedResult,
|
||||
);
|
||||
expect(oldChanged.normalized_result).toMatchObject({ status: "changed" });
|
||||
expect(
|
||||
admittedVerification(oldChanged, [left, incomplete], "overall")(),
|
||||
).toMatchObject({
|
||||
status: "fail",
|
||||
summary: { total: 1, passed: 0, failed: 1, unknown: 0 },
|
||||
claims: {
|
||||
items: [
|
||||
expect.objectContaining({
|
||||
status: "fail",
|
||||
observed_status: "changed",
|
||||
}),
|
||||
],
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("function reconstruction source aliases", () => {
|
||||
it.each([
|
||||
["no source declarations", {}],
|
||||
|
||||
Reference in New Issue
Block a user