From 620ac0f5a7c3bbed636d1e5f8b15cbedb83e4826 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:22:54 +0000 Subject: [PATCH] fix(perf): reject malformed PCAP provenance before grading Parse present summary/page containers independently of the claimed kind so malformed nested values write failed facts rather than crashing. Validate outer limitations as arrays of strings before producing observations and admitting citations. Preserve empty valid lists and original measurements. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/measure.mjs | 5 +++++ .../pcap-small-captures.md | 4 ++-- .../analysis-efficiency/review-model.py | 7 ++++--- .../analysis-efficiency/test_codex_context.py | 20 ++++++++++++++++++- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 698ccb23c..709bd79b6 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -525,6 +525,11 @@ const inspectEvmTarget = async () => { const PCAP_EVIDENCE_ID = /^ev_[a-f0-9]{64}$/u; const pcapObservation = (evidence, requestedView) => { + if ( + !Array.isArray(evidence.limitations) || + !evidence.limitations.every((item) => typeof item === "string") + ) + throw new Error("PCAP Evidence lacks a valid limitations list"); const evidenceId = evidence.evidence_id; if (typeof evidenceId !== "string" || !PCAP_EVIDENCE_ID.test(evidenceId)) throw new Error("PCAP Evidence receipt identity is missing or malformed"); diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index fcf7021b4..63a898362 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Twenty-five context/grading regressions pass, including +Source bytes stay unchanged. Twenty-six context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Twenty-five grading/context regressions pass, +reference page's verified coverage. Twenty-six grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 43a06c996..1737af34c 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -191,10 +191,11 @@ def main(): continue identity = value['evidence_id'] normalized = value.get('normalized_result') - supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list) - if supported and normalized.get('kind') == 'summary': + supported = (isinstance(normalized, dict) and isinstance(value.get('limitations'), list) + and all(isinstance(item, str) for item in value['limitations'])) + if supported and ('summary' in normalized or normalized.get('kind') == 'summary'): supported = isinstance(normalized.get('summary'), dict) - elif supported and normalized.get('kind') == 'pcap': + if supported and ('page' in normalized or normalized.get('kind') == 'pcap'): supported = isinstance(normalized.get('page'), dict) entry = (tool, arguments, value) if supported else None if identity not in records: diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 46839078f..9c5470601 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -335,9 +335,12 @@ class PcapGraderWorkflowTests(unittest.TestCase): for index in [0, 1]: for field, wrong in [('normalized_result', None), ('normalized_result', []), ('normalized_result', 'capture'), ('limitations', None), - ('limitations', {})]: + ('limitations', {}), ('limitations', [None]), + ('limitations', ['source', 1])]: with self.subTest(index=index, field=field, wrong=wrong): answer, records, reference = self.fixture();records[index][field] = wrong + if field == 'limitations': + answer['delivery_observations'][index]['limitations'] = copy.deepcopy(wrong) report = self.grade(answer, records, reference, tools) self.assertTrue(report['factual_checks']['capture']) self.assertFalse(report['all_factual_checks_pass']) @@ -351,6 +354,21 @@ class PcapGraderWorkflowTests(unittest.TestCase): answer, records, reference = self.fixture();records[index]['normalized_result'][field] = wrong self.assertFalse(self.grade(answer, records, reference, tools)['all_factual_checks_pass']) + def test_missing_or_wrong_kind_does_not_hide_malformed_view_containers(self): + for index, container in [(0, 'summary'), (1, 'page')]: + for kind in [None, 'unexpected']: + for wrong in [None, [], 'view']: + with self.subTest(index=index, kind=kind, container=container): + answer, records, reference = self.fixture() + if kind is None: + records[index]['normalized_result'].pop('kind') + else: + records[index]['normalized_result']['kind'] = kind + records[index]['normalized_result'][container] = wrong + report = self.grade(answer, records, reference, + producer_tools=['inspect_pcap_capture', 'inspect_analysis_view']) + self.assertFalse(report['all_factual_checks_pass']) + def test_retained_parents_require_declared_distinct_evidence_links(self): for index in [0, 1]: for links in [None, [], ['ev_' + 'f' * 64], {'parent': 'ev_' + 'a' * 64}]: