From 6b167f98123f964d9d5ebf5e0088e458bed52f61 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:09:31 +0000 Subject: [PATCH] fix(experiments): bind PCAP prefix coverage and requested view Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/measure.mjs | 4 +++- .../pcap-small-captures.md | 15 ++++++++++++-- .../analysis-efficiency/review-model.py | 20 ++++++++++++------- .../analysis-efficiency/test_codex_context.py | 13 ++++++++++++ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 8327d74f4..1130bc6dc 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -566,15 +566,17 @@ const inspectPcapTarget = async () => { if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture") throw new Error("Runtime returned no requested PCAP summary"); capture = n.summary; + const view = { kind: "pcap", facet: "http-uris", offset: 0, limit: 2 }; const selected = await call("inspect_analysis_view", { source: { kind: "retained-evidence", evidence_id: n.parent_evidence_id }, - view: { kind: "pcap", facet: "http-uris", offset: 0, limit: 2 }, + view, }); const page = normalized(selected); if ( page.parent_evidence_id !== n.parent_evidence_id || page.parent_operation !== "inspect_pcap_capture" || page.kind !== "pcap" || + !isDeepStrictEqual(page.view, view) || page.page.facet !== "http-uris" || page.source_coverage !== capture.coverage || page.coverage.total !== capture.retained_counts.http_request_uris || diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 8fa3c7411..c2c891d24 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,15 +126,26 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Fourteen context/grading regressions pass, including +Source bytes stay unchanged. Fifteen context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID -lists that reject parent lookups, duplicates and nested representations. Five real-answer +lists that reject parent lookups, duplicates and nested representations. URI +prefix coverage must come from its cited page, even if an uncited page has the +same items. The SDK driver also checks the returned view against the exact +requested kind, facet, offset and limit. Five real-answer mutations—omitted null, false page exhaustion, contradictory source scope, unsupported absence claim and non-string receipt identity—fail correctly. Malformed identities produce failed facts rather than crashing the grader. JS syntax, Python compilation and relevant standard lint pass. +A fresh real SDK summary/URI smoke matches the original selected facts. Three +private consumer fault injections alter the returned view's offset, limit or +presence after a real SDK reply; the driver rejects each. These establish the +experiment consumer's guard, not provider malformed-reply behavior. The initial +injection setup loaded the separate CommonJS client and did not inject a fault; +that discarded setup attempt remains private alongside the corrected ESM runs. +Original measured driver sources, token totals and timing data remain unchanged. + **Decision: keep existing complete defaults; add no production optimization.** For these compact recordings and this question, summary/page costs more total input in both real-model pairs. No stable speed or CPU/memory benefit justifies diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index be4312477..92d847626 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -183,21 +183,27 @@ def main(): prefix = answer.get('http_request_uris_first2') records = {value['evidence_id']: value for values in producer_results.values() for value in values if isinstance(value, dict) and 'evidence_id' in value} + observations = answer.get('delivery_observations') + canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations if isinstance(prefix, dict): checks['pcap_prefix_fields_supported'] = set(prefix) <= {'items', 'coverage', 'source_coverage'} and 'items' in prefix if 'source_coverage' in prefix: check('pcap_prefix_source_coverage', prefix['source_coverage'], reference['capture']['coverage']) if 'coverage' in prefix: - page_coverages = [value['normalized_result']['coverage'] for value in records.values() - if value['normalized_result'].get('kind') == 'pcap' - and value['normalized_result']['page'].get('facet') == 'http-uris' - and value['normalized_result']['page'].get('items') == prefix.get('items')] - checks['pcap_prefix_coverage_observed'] = any(json.dumps(prefix['coverage'], sort_keys=True) == json.dumps(value, sort_keys=True) for value in page_coverages) + last = canonical_observations[-1] if isinstance(canonical_observations, list) and canonical_observations else None + identity = last.get('evidence_id') if isinstance(last, dict) else None + record = records.get(identity) if isinstance(identity, str) else None + page = record['normalized_result'] if record is not None else {} + checks['pcap_prefix_coverage_observed'] = ( + page.get('kind') == 'pcap' + and page.get('page', {}).get('facet') == 'http-uris' + and json.dumps(page['page'].get('items'), sort_keys=True) == json.dumps(prefix.get('items'), sort_keys=True) + and 'coverage' in page + and json.dumps(prefix['coverage'], sort_keys=True) == json.dumps(page['coverage'], sort_keys=True) + ) prefix = prefix.get('items') check('http_request_uris_first2', ('http_request_uris_first2' in answer, json.dumps(prefix, sort_keys=True)), (True, json.dumps(reference['http_request_uris_first2'], sort_keys=True))) - observations = answer.get('delivery_observations') - canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations expected_observations = [] if isinstance(canonical_observations, list): for observation in canonical_observations: diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index c06f4228b..60cef7874 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -219,6 +219,19 @@ class PcapGraderWorkflowTests(unittest.TestCase): self.assertFalse(report['factual_checks']['pcap_observations_have_reported_ids']) self.assertFalse(report['all_factual_checks_pass']) + def test_prefix_coverage_must_come_from_the_cited_uri_receipt(self): + answer, records, reference = self.fixture() + extra = copy.deepcopy(records[-1]);extra['evidence_id'] = 'ev_' + 'e' * 64 + extra['normalized_result']['coverage']['next_offset'] = None + extra['normalized_result']['coverage']['exhausted'] = True + records.append(extra) + answer['http_request_uris_first2']['coverage'] = extra['normalized_result']['coverage'] + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertTrue(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) + self.assertFalse(report['factual_checks']['pcap_prefix_coverage_observed']) + self.assertFalse(report['all_factual_checks_pass']) + class ContextCaptureTests(unittest.TestCase): def test_filters_ambient_account_and_nested_usage_state(self):