From 56a90afdded70c7b79e1fae4aa778f0153c80514 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 06:38:10 +0000 Subject: [PATCH] fix(experiments): reject malformed PCAP receipt identities as failed facts Parse model-supplied identity values before dictionary/set lookup so malformed JSON remains a factual failure. Validate the actual stored-answer mutation and related grading regressions without rerunning models. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/pcap-small-captures.md | 6 ++++-- experiments/analysis-efficiency/review-model.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 5d9c566c2..b8fe74853 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -122,9 +122,11 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Nine context/grading regressions pass. Four real-answer +Source bytes stay unchanged. Nine context/grading regressions pass. Five real-answer mutations—omitted null, false page exhaustion, contradictory source scope, -unsupported absence claim—fail correctly. JS syntax, Python compilation and +unsupported absence claim and non-string receipt identity—fail correctly. +Malformed identities produce failed facts rather than crashing the grader. +JS syntax, Python compilation and relevant standard lint pass. **Decision: keep existing complete defaults; add no production optimization.** diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index a13656e21..23fe9a21f 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -181,7 +181,8 @@ def main(): expected_observations = [] if isinstance(canonical_observations, list): for observation in canonical_observations: - record = records.get(observation.get('evidence_id')) if isinstance(observation, dict) else None + identity = observation.get('evidence_id') if isinstance(observation, dict) else None + record = records.get(identity) if isinstance(identity, str) else None if record is None: expected_observations.append(None) continue @@ -201,7 +202,8 @@ def main(): if isinstance(canonical_observations, list) and all(isinstance(x, dict) for x in canonical_observations): check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations], [x.get('kind', 'complete') for x in reference['delivery_observations']]) - checks['pcap_observations_have_reported_ids'] = {x.get('evidence_id') for x in canonical_observations} <= reported_ids + identities = [x.get('evidence_id') for x in canonical_observations] + checks['pcap_observations_have_reported_ids'] = all(isinstance(value, str) for value in identities) and set(identities) <= reported_ids else: checks['pcap_requested_delivery_chain'] = False checks['pcap_observations_have_reported_ids'] = False