fix(perf): validate summary coverage and normalized provenance

Reject PCAP summaries that do not describe their single complete view and
reject present normalized limitations/unknowns outside string arrays at the
shared observation boundary and factual grader. Preserve omission and empty
arrays. Add regressions that reproduce faithful malformed receipts and keep
original model measurements unchanged.

Validation: 28 context/grader tests, seven actual SDK consumer fault/control
probes, and factual regrading of all four original model answers.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>

experiments/analysis-efficiency/measure.mjs

@@ -534,6 +534,13 @@ const pcapObservation = (evidence, requestedView) => {
   if (typeof evidenceId !== "string" || !PCAP_EVIDENCE_ID.test(evidenceId))
     throw new Error("PCAP Evidence receipt identity is missing or malformed");
   const value = normalized(evidence);
+  for (const key of ["limitations", "unknowns"])
+    if (
+      Object.hasOwn(value, key) &&
+      (!Array.isArray(value[key]) ||
+        !value[key].every((item) => typeof item === "string"))
+    )
+      throw new Error("PCAP normalized provenance lacks a valid string list");
   const target =
     value.kind === "summary"
       ? value.summary?.target
@@ -618,6 +625,16 @@ const inspectPcapTarget = async () => {
   if (values.delivery === "summary") {
     if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture")
       throw new Error("Runtime returned no requested PCAP summary");
+    if (
+      !isDeepStrictEqual(n.coverage, {
+        status: "complete-within-view",
+        examined: 1,
+        total: 1,
+        next_offset: null,
+        exhausted: true,
+      })
+    )
+      throw new Error("PCAP summary coverage differs from its complete view");
     capture = n.summary;
     const parentEvidenceId = observations[0].parent_evidence_id;
     const view = { kind: "pcap", facet: "http-uris", offset: 0, limit: 2 };

experiments/analysis-efficiency/pcap-small-captures.md

@@ -126,7 +126,7 @@ validated.
 Both real CLI complete results match full MCP normalized results, subjects,
 parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
 normalized facts and subjects; source forms/caller parameters remain distinct.
-Source bytes stay unchanged. Twenty-six context/grading regressions pass, including
+Source bytes stay unchanged. Twenty-eight context/grading regressions pass, including
 cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
 lists that reject parent lookups, duplicates and nested representations. URI
 prefix coverage must come from its cited page, even if an uncited page has the
@@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver
 rejects wrong status, next offset or exhaustion. Unmodified exhausted and
 continuable URI pages still pass. The grader also requires each cited receipt's
 subject digest and local path to identify the reference capture, and uses the
-reference page's verified coverage. Twenty-six grading/context regressions pass,
+reference page's verified coverage. Twenty-eight grading/context regressions pass,
 including missing or malformed subjects and faithful but inconsistent coverage;
 all four original model answers still pass factual regrading. Strict resource
 grades remain false where terminal whole-scope counters are unknown.
@@ -228,3 +228,12 @@ parent citations, missing nulls, false scope and malformed identities. These are
 grader regressions, not real-provider evidence. All four stored actual model
 answers still pass; no model reruns are required. Transcript readers close their
 owned file handles after grading.
+
+The SDK driver and factual grader require summary coverage to describe its one
+complete view and require each present normalized limitations/unknowns field to
+be a string array. Missing fields and valid empty arrays remain distinct.
+Controlled consumer mutations over actual SDK replies reproduce the previously
+accepted inconsistent summary coverage and verify rejection after the guard;
+complete and summary controls still pass. These mutations do not establish a
+malformed-response defect in the provider. The four stored model answers still
+pass factual regrading without another model run.

experiments/analysis-efficiency/review-model.py

@@ -193,6 +193,10 @@ def main():
             normalized = value.get('normalized_result')
             supported = (isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
                          and all(isinstance(item, str) for item in value['limitations']))
+            if supported:
+                supported = all(key not in normalized or (isinstance(normalized[key], list)
+                                and all(isinstance(item, str) for item in normalized[key]))
+                                for key in ['limitations', 'unknowns'])
             if supported and ('summary' in normalized or normalized.get('kind') == 'summary'):
                 supported = isinstance(normalized.get('summary'), dict)
             if supported and ('page' in normalized or normalized.get('kind') == 'pcap'):
@@ -252,6 +256,10 @@ def main():
                   [x.get('kind', 'complete') for x in reference['delivery_observations']])
             check('pcap_requested_views', json.dumps([x.get('view') for x in canonical_observations], sort_keys=True),
                   json.dumps([x.get('view') for x in reference['delivery_observations']], sort_keys=True))
+            if len(reference['delivery_observations']) == 2:
+                check('pcap_summary_coverage', json.dumps(canonical_observations[0].get('coverage'), sort_keys=True),
+                      json.dumps({'status': 'complete-within-view', 'examined': 1, 'total': 1,
+                                  'next_offset': None, 'exhausted': True}, sort_keys=True))
             identities = [x.get('evidence_id') for x in canonical_observations]
             checks['pcap_observations_have_reported_ids'] = (
                 isinstance(answer.get('evidence_ids'), list)
@@ -332,6 +340,8 @@ def main():
                     )
         else:
             checks['pcap_requested_delivery_chain'] = False
+            if len(reference['delivery_observations']) == 2:
+                checks['pcap_summary_coverage'] = False
             checks['pcap_observations_have_reported_ids'] = False
             checks['pcap_requested_views'] = False
             checks['pcap_cited_source_supports_uri_prefix'] = False

experiments/analysis-efficiency/test_codex_context.py

@@ -354,6 +354,37 @@ class PcapGraderWorkflowTests(unittest.TestCase):
                     answer, records, reference = self.fixture();records[index]['normalized_result'][field] = wrong
                     self.assertFalse(self.grade(answer, records, reference, tools)['all_factual_checks_pass'])
 
+    def test_faithful_summary_coverage_must_describe_the_single_complete_view(self):
+        for field, wrong in [('status', 'page'), ('examined', 0), ('total', 2),
+                             ('next_offset', 0), ('exhausted', False), ('examined', 1.0)]:
+            with self.subTest(field=field, wrong=wrong):
+                answer, records, reference = self.fixture()
+                records[0]['normalized_result']['coverage'][field] = wrong
+                answer['delivery_observations'][0] = self.observation(records[0])
+                report = self.grade(answer, records, reference)
+                self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
+                self.assertFalse(report['all_factual_checks_pass'])
+
+    def test_present_normalized_provenance_requires_string_arrays(self):
+        for index in [0, 1]:
+            for field in ['limitations', 'unknowns']:
+                for wrong in [None, {}, 'scope', [None], [1]]:
+                    with self.subTest(index=index, field=field, wrong=wrong):
+                        answer, records, reference = self.fixture()
+                        records[index]['normalized_result'][field] = wrong
+                        answer['delivery_observations'][index] = self.observation(records[index])
+                        self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
+                for empty in [True, False]:
+                    answer, records, reference = self.fixture()
+                    key = 'normalized_limitations' if field == 'limitations' else field
+                    if empty:
+                        records[index]['normalized_result'][field] = []
+                        answer['delivery_observations'][index][key] = []
+                    else:
+                        records[index]['normalized_result'].pop(field)
+                        answer['delivery_observations'][index].pop(key)
+                    self.assertTrue(self.grade(answer, records, reference)['all_factual_checks_pass'])
+
     def test_missing_or_wrong_kind_does_not_hide_malformed_view_containers(self):
         for index, container in [(0, 'summary'), (1, 'page')]:
             for kind in [None, 'unexpected']: