From 0f24c61196d5299a771fe92dc4432e5d937404fc Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:05:31 +0000 Subject: [PATCH] fix(perf): report empty PCAP observation lists as failed facts Reject an empty observation list at citation admission before reading its first receipt, and keep writing failed factual reports. Add the full-grader regression reproducing the prior IndexError. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/pcap-small-captures.md | 7 ++++--- experiments/analysis-efficiency/review-model.py | 2 +- experiments/analysis-efficiency/test_codex_context.py | 10 ++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 59a8148a1..fcf7021b4 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Twenty-four context/grading regressions pass, including +Source bytes stay unchanged. Twenty-five context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Twenty-four grading/context regressions pass, +reference page's verified coverage. Twenty-five grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. @@ -184,7 +184,8 @@ to make the tool name and delegated Evidence operation identical. The receipt index parses the required envelope/nested-view containers before allowing lookup, so missing or malformed normalized results/limitations become -failed facts with a written report. Coverage uses type-sensitive JSON comparison +failed facts with a written report. Empty observation lists also remain failed +facts and never enter citation checks that require a first receipt. Coverage uses type-sensitive JSON comparison to distinguish booleans, integers and floats. Receipt-identity conflict checks use the same type-sensitive rule, so a scalar type change cannot hide in a repeated ID. Existing PCAP scope-alias checks already preserve JSON types. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index ad2d216ba..43a06c996 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -246,7 +246,7 @@ def main(): checks['pcap_delivery_observations_nonempty'] = isinstance(observations, list) and bool(observations) check('pcap_original_delivery_observations', json.dumps(canonical_observations, sort_keys=True), json.dumps(expected_observations, sort_keys=True)) - if isinstance(canonical_observations, list) and all(isinstance(x, dict) for x in canonical_observations): + if isinstance(canonical_observations, list) and canonical_observations and all(isinstance(x, dict) for x in canonical_observations): check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations], [x.get('kind', 'complete') for x in reference['delivery_observations']]) check('pcap_requested_views', json.dumps([x.get('view') for x in canonical_observations], sort_keys=True), diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index b286f2629..46839078f 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -298,6 +298,16 @@ class PcapGraderWorkflowTests(unittest.TestCase): self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) self.assertFalse(report['all_factual_checks_pass']) + def test_empty_delivery_observations_write_failed_facts(self): + answer, records, reference = self.fixture() + answer['delivery_observations'] = [] + answer['evidence_ids'] = [] + report = self.grade(answer, records, reference) + self.assertFalse(report['factual_checks']['pcap_delivery_observations_nonempty']) + self.assertFalse(report['factual_checks']['pcap_cited_call_arguments']) + self.assertFalse(report['factual_checks']['pcap_cited_parameters']) + self.assertFalse(report['all_factual_checks_pass']) + def test_cited_view_parameters_and_call_arguments_bind_actual_requests(self): for index in [0, 1]: for parameters in [None, {}, {'parent_evidence_id': 'ev_' + 'f' * 64,