diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 523d49e1d..43c71b424 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -524,6 +524,11 @@ const inspectEvmTarget = async () => { }; const pcapObservation = (evidence) => { + if ( + typeof evidence.evidence_id !== "string" || + !/^ev_[a-f0-9]{64}$/.test(evidence.evidence_id) + ) + throw new Error("PCAP Evidence lacks a valid receipt identity"); const value = normalized(evidence); const target = value.kind === "summary" diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 76bb794a4..590270ec8 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Twenty-two context/grading regressions pass, including +Source bytes stay unchanged. Twenty-three context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Twenty-two grading/context regressions pass, +reference page's verified coverage. Twenty-three grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. @@ -188,6 +188,12 @@ failed facts with a written report. Coverage uses type-sensitive JSON comparison to distinguish booleans, integers and floats. Receipt-identity conflict checks use the same type-sensitive rule, so a scalar type change cannot hide in a repeated ID. Existing PCAP scope-alias checks already preserve JSON types. +Requested and cited URI views use the same type-sensitive JSON rule, rejecting +boolean or floating-point offsets/limits even when an answer faithfully copies +them. The observation producer rejects missing, null or malformed response IDs +before recording either capture or page receipts. Private real-reply injections +exercise these ID failures alongside normal complete/summary controls; they do +not establish malformed provider behavior. Negative regressions reproduce prior crashes and false passes; all original real-model factual grades still pass without model reruns. The SDK workflow is unchanged by these grader fixes. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 30216b3dc..e1ba69e25 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -246,8 +246,8 @@ def main(): if isinstance(canonical_observations, list) and all(isinstance(x, dict) for x in canonical_observations): check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations], [x.get('kind', 'complete') for x in reference['delivery_observations']]) - check('pcap_requested_views', [x.get('view') for x in canonical_observations], - [x.get('view') for x in reference['delivery_observations']]) + check('pcap_requested_views', json.dumps([x.get('view') for x in canonical_observations], sort_keys=True), + json.dumps([x.get('view') for x in reference['delivery_observations']], sort_keys=True)) identities = [x.get('evidence_id') for x in canonical_observations] checks['pcap_observations_have_reported_ids'] = ( isinstance(answer.get('evidence_ids'), list) @@ -306,7 +306,7 @@ def main(): and isinstance(source.get('parent_evidence_id'), str) and page.get('parent_evidence_id') == source['parent_evidence_id'] and page.get('parent_operation') == 'inspect_pcap_capture' - and page.get('view') == {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2} + and json.dumps(page.get('view'), sort_keys=True) == json.dumps({'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, sort_keys=True) and selected_page.get('facet') == 'http-uris' and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True) and page.get('source_coverage') == capture.get('coverage') diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index f3334aa0d..18e6e7b8b 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -263,6 +263,18 @@ class PcapGraderWorkflowTests(unittest.TestCase): self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed']) self.assertFalse(report['all_factual_checks_pass']) + def test_cited_views_preserve_exact_json_scalar_types(self): + for field, wrong in [('offset', False), ('offset', 0.0), ('limit', 2.0), ('limit', True)]: + with self.subTest(field=field, wrong=wrong): + answer, records, reference = self.fixture() + records[-1]['normalized_result']['view'][field] = wrong + answer['delivery_observations'][-1] = self.observation(records[-1]) + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertFalse(report['factual_checks']['pcap_requested_views']) + self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) + self.assertFalse(report['all_factual_checks_pass']) + def test_malformed_receipt_envelopes_write_failed_facts_instead_of_crashing(self): tools = ['inspect_pcap_capture', 'inspect_analysis_view'] for index in [0, 1]: