diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 7b10fd515..eb97e2b7e 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -572,6 +572,15 @@ const inspectPcapTarget = async () => { view, }); const page = normalized(selected); + const total = capture.retained_counts.http_request_uris; + const examined = Math.min(view.limit, total); + const expectedCoverage = { + status: "page", + examined, + total, + next_offset: examined < total ? examined : null, + exhausted: examined >= total, + }; if ( page.parent_evidence_id !== n.parent_evidence_id || page.parent_operation !== "inspect_pcap_capture" || @@ -579,8 +588,7 @@ const inspectPcapTarget = async () => { !isDeepStrictEqual(page.view, view) || page.page.facet !== "http-uris" || page.source_coverage !== capture.coverage || - page.coverage.total !== capture.retained_counts.http_request_uris || - page.coverage.examined !== page.page.items.length || + !isDeepStrictEqual(page.coverage, expectedCoverage) || page.artifact.path !== capture.target.path || page.artifact.sha256 !== capture.target.sha256 ) diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index a7d90febd..9739a0910 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -153,6 +153,17 @@ without writing an answer, and an unmodified real reply still passes. This checks receipt uniqueness before recording observations; it does not establish that a provider actually returned duplicate IDs. +The page guard checks the complete expected coverage object, including status, +continuation and exhaustion. Private actual-reply injections demonstrate that +the preceding driver accepts inconsistent coverage, while the corrected driver +rejects wrong status, next offset or exhaustion. Unmodified exhausted and +continuable URI pages still pass. The grader also requires each cited receipt's +subject digest and local path to identify the reference capture, and uses the +reference page's verified coverage. Seventeen grading/context regressions pass, +including missing or malformed subjects and faithful but inconsistent coverage; +all four original model answers still pass factual regrading. Strict resource +grades remain false where terminal whole-scope counters are unknown. + **Decision: keep existing complete defaults; add no production optimization.** For these compact recordings and this question, summary/page costs more total input in both real-model pairs. No stable speed or CPU/memory benefit justifies diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 92d847626..a64199a11 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -238,6 +238,14 @@ def main(): and len(set(identities)) == len(identities) ) cited = [records.get(identity) if isinstance(identity, str) else None for identity in identities] + checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all( + isinstance(record, dict) + and isinstance(record.get('subject'), dict) + and isinstance(record['subject'].get('digest'), dict) + and record['subject']['digest'].get('sha256') == reference['artifact_sha256'] + and record['subject'].get('local_path') == reference['capture']['target']['path'] + for record in cited + ) checks['pcap_cited_source_supports_uri_prefix'] = False checks['pcap_cited_capture'] = False if all(isinstance(record, dict) for record in cited): @@ -267,6 +275,7 @@ def main(): and selected_page.get('facet') == 'http-uris' and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True) and page.get('source_coverage') == capture.get('coverage') + and page.get('coverage') == reference['delivery_observations'][-1].get('coverage') and page.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']} ) else: @@ -275,6 +284,7 @@ def main(): checks['pcap_requested_views'] = False checks['pcap_cited_source_supports_uri_prefix'] = False checks['pcap_cited_capture'] = False + checks['pcap_cited_subjects_match_artifact'] = False elif domain == 'flutter-aot': for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']: check(key, answer.get(key), reference[key]) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 60cef7874..b510839b9 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -81,12 +81,13 @@ class PcapGraderWorkflowTests(unittest.TestCase): 'retained_counts': {'http_request_uris': 3}} prefix = ['http://example.test/one', 'http://example.test/two'] coverage = {'status': 'page', 'examined': 2, 'total': 3, 'next_offset': 2, 'exhausted': False} - summary = {'evidence_id': summary_id, 'limitations': ['source', 'view'], + subject = {'digest': {'sha256': capture['target']['sha256']}, 'local_path': capture['target']['path']} + summary = {'evidence_id': summary_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'], 'normalized_result': {'kind': 'summary', 'view': {'kind': 'summary'}, 'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent, 'summary': capture, 'coverage': {'status': 'complete-within-view', 'examined': 1, 'total': 1, 'next_offset': None, 'exhausted': True}, 'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}} - page = {'evidence_id': page_id, 'limitations': ['source', 'view'], + page = {'evidence_id': page_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'], 'normalized_result': {'kind': 'pcap', 'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, 'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent, @@ -184,11 +185,18 @@ class PcapGraderWorkflowTests(unittest.TestCase): source.update({key: [] for key in capture['retained_counts']}) source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three'] source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'} - receipt = {'evidence_id': 'ev_' + 'a' * 64, 'limitations': ['source'], 'normalized_result': source} + receipt = {'evidence_id': 'ev_' + 'a' * 64, 'subject': copy.deepcopy(records[0]['subject']), + 'limitations': ['source'], 'normalized_result': source} reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'], 'limitations': receipt['limitations'], 'coverage': source['coverage']}] answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]}) self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass']) + for subject in [None, {'digest': {'sha256': 'e' * 64}, 'local_path': source['target']['path']}]: + changed = copy.deepcopy(receipt);changed['subject'] = subject + report = self.grade(answer, [changed], reference) + self.assertTrue(report['factual_checks']['pcap_cited_capture']) + self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact']) + self.assertFalse(report['all_factual_checks_pass']) mutations = [lambda x: x.__setitem__('packet_count', 4), lambda x: x['client'].__setitem__('version', '2'), lambda x: x.__setitem__('capture_start', 'changed'), @@ -206,6 +214,34 @@ class PcapGraderWorkflowTests(unittest.TestCase): answer['capture']['diagnostics_code_units'] = None self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass']) + def test_cited_receipt_subjects_must_identify_the_reference_capture(self): + for index in [0, 1]: + for subject in [None, [], 'artifact', {}, {'digest': None}, + {'digest': {'sha256': 'e' * 64}, 'local_path': '/fixture/http.pcapng'}, + {'digest': {'sha256': 'd' * 64}, 'local_path': '/fixture/other.pcapng'}]: + with self.subTest(index=index, subject=subject): + answer, records, reference = self.fixture() + records[index]['subject'] = subject + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact']) + self.assertFalse(report['all_factual_checks_pass']) + answer, records, reference = self.fixture();records[index].pop('subject') + self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass']) + + def test_faithful_inconsistent_page_coverage_does_not_support_the_prefix(self): + for field, wrong in [('status', 'complete-within-view'), ('next_offset', None), ('exhausted', True)]: + with self.subTest(field=field): + answer, records, reference = self.fixture() + records[-1]['normalized_result']['coverage'][field] = wrong + answer['delivery_observations'][-1] = self.observation(records[-1]) + answer['http_request_uris_first2']['coverage'][field] = wrong + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed']) + self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) + self.assertFalse(report['all_factual_checks_pass']) + def test_reported_ids_are_only_the_ordered_unique_response_receipts(self): answer, records, reference = self.fixture() original = answer['evidence_ids']