diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 262779a71..918dd80c0 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Thirty-three context/grading regressions pass, including +Source bytes stay unchanged. Thirty-four context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Thirty-three grading/context regressions pass, +reference page's verified coverage. Thirty-four grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. @@ -283,3 +283,9 @@ The six comparator CLI regressions now live in the normally discovered Vitest evaluation suite, so routine local/PR/CI gates execute them. The original six scenarios and CLI assertions are retained; the former manual Node suite is removed. They remain synthetic consumer checks, not provider evidence. + +Failed calls with scalar/list replies or malformed nested resource-error +containers are parsed defensively. The grader writes its report, preserves the +original failure and any supported cited facts, and rejects workflow acceptance +instead of crashing. Genuine structured resource Evidence references remain +observable; no malformed producer behavior is inferred from synthetic cases. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index e74b43ccf..782c320e1 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -152,9 +152,12 @@ def main(): producer_results.setdefault(item['tool'], []).append(parsed) else: tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result}) - if parsed is not None and parsed.get('error', {}).get('code') == 'resource_constraint': - reference_value = parsed['error'].get('details', {}).get('reported_limits', {}).get('evidence_reference') - observed_ids.update(evidence_ids(reference_value)) + error = parsed.get('error') if isinstance(parsed, dict) else None + if isinstance(error, dict) and error.get('code') == 'resource_constraint': + details = error.get('details') + limits = details.get('reported_limits') if isinstance(details, dict) else None + if isinstance(limits, dict): + observed_ids.update(evidence_ids(limits.get('evidence_reference'))) reported_ids = evidence_ids(answer.get('evidence_ids')) checks = { 'reported_evidence_ids_observed_in_responses': bool(reported_ids) and reported_ids <= observed_ids, diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 574a58774..1b828aaf6 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -463,6 +463,37 @@ class PcapGraderWorkflowTests(unittest.TestCase): answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture' self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass']) + def test_failed_reply_shapes_preserve_a_written_report_and_partial_facts(self): + payloads = ['malformed', [1], None, False, 7, {'error': 'malformed'}, + {'error': []}, {'error': None}] + payloads += [{'error': {'code': 'resource_constraint', 'details': value}} + for value in [None, [], 'details']] + payloads += [{'error': {'code': 'resource_constraint', 'details': {'reported_limits': value}}} + for value in [None, [], 'limits']] + for payload in payloads: + with self.subTest(payload=payload): + answer, records, reference = self.fixture() + result = {'structuredContent': payload} + event = {'type': 'item.completed', 'item': { + 'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': 'failed', + 'arguments': {'view': {'kind': 'pcap', 'facet': 'endpoints'}}, 'result': result}} + report = self.grade(answer, records, reference, extra_events=[event]) + self.assertTrue(report['all_factual_checks_pass']) + self.assertFalse(report['all_workflow_checks_pass']) + self.assertEqual(report['tool_errors'][-1]['response'], payload if payload is not None else result) + answer, records, reference = self.fixture() + partial_id = 'ev_' + 'e' * 64 + payload = {'error': {'code': 'resource_constraint', 'details': { + 'reported_limits': {'evidence_reference': {'evidence_id': partial_id}}}}} + event = {'type': 'item.completed', 'item': { + 'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': 'failed', + 'arguments': {}, 'result': {'structuredContent': payload}}} + answer['evidence_ids'].append(partial_id) + report = self.grade(answer, records, reference, extra_events=[event]) + self.assertTrue(report['factual_checks']['reported_evidence_ids_observed_in_responses']) + self.assertFalse(report['all_factual_checks_pass']) + self.assertFalse(report['all_workflow_checks_pass']) + def test_workflow_citation_count_must_match_the_planned_calls(self): for count, made_page_call in [(1, False), (1, True), (3, True)]: with self.subTest(count=count, made_page_call=made_page_call):