From 073f6c07e996aec6148a4782a56b7861cbdea1a7 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:00:21 +0000 Subject: [PATCH] fix(experiments): validate PCAP receipt ownership before observation Validate subjects and normalized targets against selected bytes/path at the common PCAP observation boundary. Require every retained view parent to be linked and distinct from the response identity; enforce the same relationship for model citations. Remove redundant page target checks. Actual SDK before/after faults and normal complete/summary controls pass, as do 18 grading/context and six comparator regressions plus factual regrading of all four stored model answers. No model rerun or production/default change. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/measure.mjs | 25 +++++++++++++++-- .../pcap-small-captures.md | 11 ++++++-- .../analysis-efficiency/review-model.py | 11 ++++++++ .../analysis-efficiency/test_codex_context.py | 28 +++++++++++++++++-- 4 files changed, 67 insertions(+), 8 deletions(-) diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index eb97e2b7e..2c90dca43 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -525,6 +525,27 @@ const inspectEvmTarget = async () => { const pcapObservation = (evidence) => { const value = normalized(evidence); + const target = + value.kind === "summary" + ? value.summary?.target + : value.kind === "pcap" + ? value.artifact + : value.target; + if ( + evidence.subject?.digest?.sha256 !== inputSha || + evidence.subject?.local_path !== values.path || + target?.sha256 !== inputSha || + target?.path !== values.path + ) + throw new Error("PCAP Evidence does not identify the selected capture"); + if ( + values.delivery === "summary" && + (typeof value.parent_evidence_id !== "string" || + value.parent_evidence_id === evidence.evidence_id || + !Array.isArray(evidence.evidence_links) || + !evidence.evidence_links.includes(value.parent_evidence_id)) + ) + throw new Error("PCAP retained parent lacks a distinct Evidence link"); return { evidence_id: evidence.evidence_id, limitations: evidence.limitations, @@ -588,9 +609,7 @@ const inspectPcapTarget = async () => { !isDeepStrictEqual(page.view, view) || page.page.facet !== "http-uris" || page.source_coverage !== capture.coverage || - !isDeepStrictEqual(page.coverage, expectedCoverage) || - page.artifact.path !== capture.target.path || - page.artifact.sha256 !== capture.target.sha256 + !isDeepStrictEqual(page.coverage, expectedCoverage) ) throw new Error( "PCAP page identity, scope or coverage differs from its parent", diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 9739a0910..e2f99c6b2 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Fifteen context/grading regressions pass, including +Source bytes stay unchanged. Eighteen context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,11 +159,18 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Seventeen grading/context regressions pass, +reference page's verified coverage. Eighteen grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. +PCAP observation production now validates both the Evidence subject and +normalized target against the selected path/digest for complete, summary and +page replies. Retained parents must be declared in each view receipt's Evidence +links and differ from that response's own ID. The grader enforces the same link +relationship. Private actual-reply injections and normal complete/summary +controls exercise this boundary; these do not establish a provider defect. + **Decision: keep existing complete defaults; add no production optimization.** For these compact recordings and this question, summary/page costs more total input in both real-model pairs. No stable speed or CPU/memory benefit justifies diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index a64199a11..768b6f17e 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -246,6 +246,16 @@ def main(): and record['subject'].get('local_path') == reference['capture']['target']['path'] for record in cited ) + checks['pcap_cited_parents_linked'] = bool(cited) and all(isinstance(record, dict) for record in cited) and ( + len(cited) == 1 or (len(cited) == 2 and all( + isinstance(record['normalized_result'].get('parent_evidence_id'), str) + and re.fullmatch(r'ev_[a-f0-9]{64}', record['normalized_result']['parent_evidence_id']) + and record['normalized_result']['parent_evidence_id'] != record['evidence_id'] + and isinstance(record.get('evidence_links'), list) + and record['normalized_result']['parent_evidence_id'] in record['evidence_links'] + for record in cited + )) + ) checks['pcap_cited_source_supports_uri_prefix'] = False checks['pcap_cited_capture'] = False if all(isinstance(record, dict) for record in cited): @@ -285,6 +295,7 @@ def main(): checks['pcap_cited_source_supports_uri_prefix'] = False checks['pcap_cited_capture'] = False checks['pcap_cited_subjects_match_artifact'] = False + checks['pcap_cited_parents_linked'] = False elif domain == 'flutter-aot': for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']: check(key, answer.get(key), reference[key]) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index b510839b9..c21a7ab78 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -82,12 +82,14 @@ class PcapGraderWorkflowTests(unittest.TestCase): prefix = ['http://example.test/one', 'http://example.test/two'] coverage = {'status': 'page', 'examined': 2, 'total': 3, 'next_offset': 2, 'exhausted': False} subject = {'digest': {'sha256': capture['target']['sha256']}, 'local_path': capture['target']['path']} - summary = {'evidence_id': summary_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'], + summary = {'evidence_id': summary_id, 'subject': copy.deepcopy(subject), 'evidence_links': [parent], + 'limitations': ['source', 'view'], 'normalized_result': {'kind': 'summary', 'view': {'kind': 'summary'}, 'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent, 'summary': capture, 'coverage': {'status': 'complete-within-view', 'examined': 1, 'total': 1, 'next_offset': None, 'exhausted': True}, 'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}} - page = {'evidence_id': page_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'], + page = {'evidence_id': page_id, 'subject': copy.deepcopy(subject), 'evidence_links': [parent], + 'limitations': ['source', 'view'], 'normalized_result': {'kind': 'pcap', 'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, 'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent, @@ -185,7 +187,7 @@ class PcapGraderWorkflowTests(unittest.TestCase): source.update({key: [] for key in capture['retained_counts']}) source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three'] source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'} - receipt = {'evidence_id': 'ev_' + 'a' * 64, 'subject': copy.deepcopy(records[0]['subject']), + receipt = {'evidence_id': 'ev_' + 'a' * 64, 'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [], 'limitations': ['source'], 'normalized_result': source} reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'], 'limitations': receipt['limitations'], 'coverage': source['coverage']}] @@ -242,6 +244,26 @@ class PcapGraderWorkflowTests(unittest.TestCase): self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) self.assertFalse(report['all_factual_checks_pass']) + def test_retained_parents_require_declared_distinct_evidence_links(self): + for index in [0, 1]: + for links in [None, [], ['ev_' + 'f' * 64], {'parent': 'ev_' + 'a' * 64}]: + with self.subTest(index=index, links=links): + answer, records, reference = self.fixture();records[index]['evidence_links'] = links + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertFalse(report['factual_checks']['pcap_cited_parents_linked']) + self.assertFalse(report['all_factual_checks_pass']) + answer, records, reference = self.fixture();records[index].pop('evidence_links') + self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass']) + answer, records, reference = self.fixture() + parent = records[index]['evidence_id'] + for record in records: + record['normalized_result']['parent_evidence_id'] = parent;record['evidence_links'] = [parent] + answer['delivery_observations'] = [self.observation(record) for record in records] + report = self.grade(answer, records, reference) + self.assertFalse(report['factual_checks']['pcap_cited_parents_linked']) + self.assertFalse(report['all_factual_checks_pass']) + def test_reported_ids_are_only_the_ordered_unique_response_receipts(self): answer, records, reference = self.fixture() original = answer['evidence_ids']