diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 5d9c566c2..b8fe74853 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -122,9 +122,11 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Nine context/grading regressions pass. Four real-answer +Source bytes stay unchanged. Nine context/grading regressions pass. Five real-answer mutations—omitted null, false page exhaustion, contradictory source scope, -unsupported absence claim—fail correctly. JS syntax, Python compilation and +unsupported absence claim and non-string receipt identity—fail correctly. +Malformed identities produce failed facts rather than crashing the grader. +JS syntax, Python compilation and relevant standard lint pass. **Decision: keep existing complete defaults; add no production optimization.** diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index a13656e21..23fe9a21f 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -181,7 +181,8 @@ def main(): expected_observations = [] if isinstance(canonical_observations, list): for observation in canonical_observations: - record = records.get(observation.get('evidence_id')) if isinstance(observation, dict) else None + identity = observation.get('evidence_id') if isinstance(observation, dict) else None + record = records.get(identity) if isinstance(identity, str) else None if record is None: expected_observations.append(None) continue @@ -201,7 +202,8 @@ def main(): if isinstance(canonical_observations, list) and all(isinstance(x, dict) for x in canonical_observations): check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations], [x.get('kind', 'complete') for x in reference['delivery_observations']]) - checks['pcap_observations_have_reported_ids'] = {x.get('evidence_id') for x in canonical_observations} <= reported_ids + identities = [x.get('evidence_id') for x in canonical_observations] + checks['pcap_observations_have_reported_ids'] = all(isinstance(value, str) for value in identities) and set(identities) <= reported_ids else: checks['pcap_requested_delivery_chain'] = False checks['pcap_observations_have_reported_ids'] = False