From d6885075632ced8b3a25a418aca805ad94cd8fe2 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 11:45:02 +0000 Subject: [PATCH] fix(experiments): validate PCAP coverage and cited artifact subjects Verify complete page coverage at the SDK measurement boundary. Bind every cited Evidence subject digest and local path to the reference capture in the model grader, and reuse the reference page coverage. Add failing-then-passing regressions, real SDK consumer fault/control checks, and factual regrading of all four stored model answers without model reruns. Original measurements and resource uncertainty stay intact. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/measure.mjs | 12 +++++- .../pcap-small-captures.md | 11 +++++ .../analysis-efficiency/review-model.py | 10 +++++ .../analysis-efficiency/test_codex_context.py | 42 +++++++++++++++++-- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 7b10fd515..eb97e2b7e 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -572,6 +572,15 @@ const inspectPcapTarget = async () => { view, }); const page = normalized(selected); + const total = capture.retained_counts.http_request_uris; + const examined = Math.min(view.limit, total); + const expectedCoverage = { + status: "page", + examined, + total, + next_offset: examined < total ? examined : null, + exhausted: examined >= total, + }; if ( page.parent_evidence_id !== n.parent_evidence_id || page.parent_operation !== "inspect_pcap_capture" || @@ -579,8 +588,7 @@ const inspectPcapTarget = async () => { !isDeepStrictEqual(page.view, view) || page.page.facet !== "http-uris" || page.source_coverage !== capture.coverage || - page.coverage.total !== capture.retained_counts.http_request_uris || - page.coverage.examined !== page.page.items.length || + !isDeepStrictEqual(page.coverage, expectedCoverage) || page.artifact.path !== capture.target.path || page.artifact.sha256 !== capture.target.sha256 ) diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index a7d90febd..9739a0910 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -153,6 +153,17 @@ without writing an answer, and an unmodified real reply still passes. This checks receipt uniqueness before recording observations; it does not establish that a provider actually returned duplicate IDs. +The page guard checks the complete expected coverage object, including status, +continuation and exhaustion. Private actual-reply injections demonstrate that +the preceding driver accepts inconsistent coverage, while the corrected driver +rejects wrong status, next offset or exhaustion. Unmodified exhausted and +continuable URI pages still pass. The grader also requires each cited receipt's +subject digest and local path to identify the reference capture, and uses the +reference page's verified coverage. Seventeen grading/context regressions pass, +including missing or malformed subjects and faithful but inconsistent coverage; +all four original model answers still pass factual regrading. Strict resource +grades remain false where terminal whole-scope counters are unknown. + **Decision: keep existing complete defaults; add no production optimization.** For these compact recordings and this question, summary/page costs more total input in both real-model pairs. No stable speed or CPU/memory benefit justifies diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 92d847626..a64199a11 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -238,6 +238,14 @@ def main(): and len(set(identities)) == len(identities) ) cited = [records.get(identity) if isinstance(identity, str) else None for identity in identities] + checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all( + isinstance(record, dict) + and isinstance(record.get('subject'), dict) + and isinstance(record['subject'].get('digest'), dict) + and record['subject']['digest'].get('sha256') == reference['artifact_sha256'] + and record['subject'].get('local_path') == reference['capture']['target']['path'] + for record in cited + ) checks['pcap_cited_source_supports_uri_prefix'] = False checks['pcap_cited_capture'] = False if all(isinstance(record, dict) for record in cited): @@ -267,6 +275,7 @@ def main(): and selected_page.get('facet') == 'http-uris' and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True) and page.get('source_coverage') == capture.get('coverage') + and page.get('coverage') == reference['delivery_observations'][-1].get('coverage') and page.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']} ) else: @@ -275,6 +284,7 @@ def main(): checks['pcap_requested_views'] = False checks['pcap_cited_source_supports_uri_prefix'] = False checks['pcap_cited_capture'] = False + checks['pcap_cited_subjects_match_artifact'] = False elif domain == 'flutter-aot': for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']: check(key, answer.get(key), reference[key]) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 60cef7874..b510839b9 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -81,12 +81,13 @@ class PcapGraderWorkflowTests(unittest.TestCase): 'retained_counts': {'http_request_uris': 3}} prefix = ['http://example.test/one', 'http://example.test/two'] coverage = {'status': 'page', 'examined': 2, 'total': 3, 'next_offset': 2, 'exhausted': False} - summary = {'evidence_id': summary_id, 'limitations': ['source', 'view'], + subject = {'digest': {'sha256': capture['target']['sha256']}, 'local_path': capture['target']['path']} + summary = {'evidence_id': summary_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'], 'normalized_result': {'kind': 'summary', 'view': {'kind': 'summary'}, 'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent, 'summary': capture, 'coverage': {'status': 'complete-within-view', 'examined': 1, 'total': 1, 'next_offset': None, 'exhausted': True}, 'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}} - page = {'evidence_id': page_id, 'limitations': ['source', 'view'], + page = {'evidence_id': page_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'], 'normalized_result': {'kind': 'pcap', 'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, 'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent, @@ -184,11 +185,18 @@ class PcapGraderWorkflowTests(unittest.TestCase): source.update({key: [] for key in capture['retained_counts']}) source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three'] source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'} - receipt = {'evidence_id': 'ev_' + 'a' * 64, 'limitations': ['source'], 'normalized_result': source} + receipt = {'evidence_id': 'ev_' + 'a' * 64, 'subject': copy.deepcopy(records[0]['subject']), + 'limitations': ['source'], 'normalized_result': source} reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'], 'limitations': receipt['limitations'], 'coverage': source['coverage']}] answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]}) self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass']) + for subject in [None, {'digest': {'sha256': 'e' * 64}, 'local_path': source['target']['path']}]: + changed = copy.deepcopy(receipt);changed['subject'] = subject + report = self.grade(answer, [changed], reference) + self.assertTrue(report['factual_checks']['pcap_cited_capture']) + self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact']) + self.assertFalse(report['all_factual_checks_pass']) mutations = [lambda x: x.__setitem__('packet_count', 4), lambda x: x['client'].__setitem__('version', '2'), lambda x: x.__setitem__('capture_start', 'changed'), @@ -206,6 +214,34 @@ class PcapGraderWorkflowTests(unittest.TestCase): answer['capture']['diagnostics_code_units'] = None self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass']) + def test_cited_receipt_subjects_must_identify_the_reference_capture(self): + for index in [0, 1]: + for subject in [None, [], 'artifact', {}, {'digest': None}, + {'digest': {'sha256': 'e' * 64}, 'local_path': '/fixture/http.pcapng'}, + {'digest': {'sha256': 'd' * 64}, 'local_path': '/fixture/other.pcapng'}]: + with self.subTest(index=index, subject=subject): + answer, records, reference = self.fixture() + records[index]['subject'] = subject + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact']) + self.assertFalse(report['all_factual_checks_pass']) + answer, records, reference = self.fixture();records[index].pop('subject') + self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass']) + + def test_faithful_inconsistent_page_coverage_does_not_support_the_prefix(self): + for field, wrong in [('status', 'complete-within-view'), ('next_offset', None), ('exhausted', True)]: + with self.subTest(field=field): + answer, records, reference = self.fixture() + records[-1]['normalized_result']['coverage'][field] = wrong + answer['delivery_observations'][-1] = self.observation(records[-1]) + answer['http_request_uris_first2']['coverage'][field] = wrong + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed']) + self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) + self.assertFalse(report['all_factual_checks_pass']) + def test_reported_ids_are_only_the_ordered_unique_response_receipts(self): answer, records, reference = self.fixture() original = answer['evidence_ids']