fix(perf): require readable planned PCAP replies

Bind each terminal call to its own decoded Evidence ID and require a readable
Evidence result before passing the separate nominal workflow grade. A planned
URI call with empty content must not pass merely because its name, arguments
and status match. Preserve factual and resource grades independently.

Validation: 32 targeted context/grading tests, including the reproduced
workflow-only false success, and all four original factual/workflow regrades.
No unchanged SDK/comparator or model rerun.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-13 19:03:41 +00:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent 8ee5ca17ee
commit 6ef33162b8
3 changed files with 28 additions and 9 deletions
@@ -117,7 +117,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None):
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None, producer_results=None):
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory);trial = root / 'trial';trial.mkdir()
@@ -141,7 +141,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
arguments = producer_arguments[index]
events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
'tool': tool, 'arguments': arguments, 'status': 'completed',
'result': {'structuredContent': value}}})
'result': {'structuredContent': value} if producer_results is None else producer_results[index]}})
events.extend(extra_events or [])
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
@@ -463,6 +463,13 @@ class PcapGraderWorkflowTests(unittest.TestCase):
answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture'
self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
def test_unreadable_planned_reply_does_not_pass_workflow_acceptance(self):
answer, records, reference = self.fixture()
report = self.grade(answer, records, reference,
producer_results=[{'structuredContent': records[0]}, {}])
self.assertFalse(report['all_factual_checks_pass'])
self.assertFalse(report['all_workflow_checks_pass'])
def test_extra_calls_are_not_an_equivalent_workflow_even_with_correct_facts(self):
for status in ['completed', 'failed']:
for with_receipt in [True, False]: