diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 8f072b932..a31c17e2b 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Thirty-one context/grading regressions pass, including +Source bytes stay unchanged. Thirty-two context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Thirty-one grading/context regressions pass, +reference page's verified coverage. Thirty-two grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. @@ -267,3 +267,9 @@ completed status. Extra, reversed, failed or undecodable attempts fail workflow acceptance even when the final cited facts are correct; their actual cost and errors remain recorded. All four original model trials pass both factual and workflow checks without model reruns. Strict resource grades remain unchanged. + +Each planned call must also decode as an Evidence object with its own cited +response ID. An unreadable planned reply cannot independently pass workflow +checks merely because tool names, arguments and terminal status match. The +regression reproduces the previous workflow-only false success; original model +factual/workflow grades remain unchanged. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 0592f4ea6..c9c58acf1 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -131,11 +131,16 @@ def main(): item = event.get('item', {}) if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call': continue - if args.domain == 'pcap': - pcap_attempts.append({'tool': item.get('tool'), 'arguments': item.get('arguments'), - 'status': item.get('status')}) result = item.get('result') or {} parsed = decode(result) + if args.domain == 'pcap': + pcap_attempts.append({'tool': item.get('tool'), 'arguments': item.get('arguments'), + 'status': item.get('status'), + 'evidence_id': parsed.get('evidence_id') if isinstance(parsed, dict) else None, + 'readable_evidence': isinstance(parsed, dict) + and isinstance(parsed.get('normalized_result'), dict) + and isinstance(parsed.get('evidence_id'), str) + and re.fullmatch(r'ev_[a-f0-9]{64}', parsed['evidence_id']) is not None}) if item['status'] == 'completed': # Only response text contributes observed IDs, never model arguments. for text in strings(result): @@ -293,8 +298,9 @@ def main(): json.dumps(expected_arguments, sort_keys=True)) workflow_checks['pcap_exact_call_sequence'] = ( json.dumps(pcap_attempts, sort_keys=True) == json.dumps([ - {'tool': tool, 'arguments': arguments, 'status': 'completed'} - for tool, arguments in zip(expected_tools, expected_arguments) + {'tool': tool, 'arguments': arguments, 'status': 'completed', + 'evidence_id': identity, 'readable_evidence': True} + for tool, arguments, identity in zip(expected_tools, expected_arguments, identities) ], sort_keys=True) ) check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True), diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index c1468801e..0c7fdcab7 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -117,7 +117,7 @@ class PcapGraderWorkflowTests(unittest.TestCase): 'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'], 'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']} - def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None): + def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None, producer_results=None): review = runpy.run_path(str(Path(__file__).with_name('review-model.py'))) with tempfile.TemporaryDirectory() as directory: root = Path(directory);trial = root / 'trial';trial.mkdir() @@ -141,7 +141,7 @@ class PcapGraderWorkflowTests(unittest.TestCase): arguments = producer_arguments[index] events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call', 'tool': tool, 'arguments': arguments, 'status': 'completed', - 'result': {'structuredContent': value}}}) + 'result': {'structuredContent': value} if producer_results is None else producer_results[index]}}) events.extend(extra_events or []) (trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events)) source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference)) @@ -463,6 +463,13 @@ class PcapGraderWorkflowTests(unittest.TestCase): answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture' self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass']) + def test_unreadable_planned_reply_does_not_pass_workflow_acceptance(self): + answer, records, reference = self.fixture() + report = self.grade(answer, records, reference, + producer_results=[{'structuredContent': records[0]}, {}]) + self.assertFalse(report['all_factual_checks_pass']) + self.assertFalse(report['all_workflow_checks_pass']) + def test_extra_calls_are_not_an_equivalent_workflow_even_with_correct_facts(self): for status in ['completed', 'failed']: for with_receipt in [True, False]: