diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index a31c17e2b..fb3d4b4be 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Thirty-two context/grading regressions pass, including +Source bytes stay unchanged. Thirty-three context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Thirty-two grading/context regressions pass, +reference page's verified coverage. Thirty-three grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. @@ -273,3 +273,8 @@ response ID. An unreadable planned reply cannot independently pass workflow checks merely because tool names, arguments and terminal status match. The regression reproduces the previous workflow-only false success; original model factual/workflow grades remain unchanged. + +Workflow citation cardinality must equal the planned call count before binding +per-call IDs. A shorter or longer answer list cannot silently shorten Python +zip’s expected plan; missing URI calls and extra citations are rejected. +Original complete/summary model workflows continue to pass unchanged. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index c9c58acf1..e74b43ccf 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -297,7 +297,8 @@ def main(): check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True), json.dumps(expected_arguments, sort_keys=True)) workflow_checks['pcap_exact_call_sequence'] = ( - json.dumps(pcap_attempts, sort_keys=True) == json.dumps([ + len(identities) == len(expected_tools) + and json.dumps(pcap_attempts, sort_keys=True) == json.dumps([ {'tool': tool, 'arguments': arguments, 'status': 'completed', 'evidence_id': identity, 'readable_evidence': True} for tool, arguments, identity in zip(expected_tools, expected_arguments, identities) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 0c7fdcab7..574a58774 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -463,6 +463,22 @@ class PcapGraderWorkflowTests(unittest.TestCase): answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture' self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass']) + def test_workflow_citation_count_must_match_the_planned_calls(self): + for count, made_page_call in [(1, False), (1, True), (3, True)]: + with self.subTest(count=count, made_page_call=made_page_call): + answer, records, reference = self.fixture() + if count == 1: + answer['delivery_observations'] = answer['delivery_observations'][:1] + answer['evidence_ids'] = answer['evidence_ids'][:1] + else: + extra = copy.deepcopy(answer['delivery_observations'][-1]) + extra['evidence_id'] = 'ev_' + 'e' * 64 + answer['delivery_observations'].append(extra) + answer['evidence_ids'].append(extra['evidence_id']) + report = self.grade(answer, records if made_page_call else records[:1], reference) + self.assertFalse(report['all_factual_checks_pass']) + self.assertFalse(report['all_workflow_checks_pass']) + def test_unreadable_planned_reply_does_not_pass_workflow_acceptance(self): answer, records, reference = self.fixture() report = self.grade(answer, records, reference,