fix(perf): grade exact PCAP workflow call sequences

Keep selected factual correctness separate from nominal workflow equivalence.
Record all completed MCP attempts, including failed/undecodable replies, and
require the planned ordered calls, arguments and completion status. Extra
inventory, repeated or reordered attempts cannot pass workflow acceptance;
retain their actual cost/errors and supported cited facts.

Validation: 31 targeted context/grading regressions and factual plus workflow
regrading of all four original model trials without reruns. SDK and comparator
code are unchanged and are not retested.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-13 18:50:58 +00:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent e6cf2e354e
commit 8ee5ca17ee
3 changed files with 49 additions and 4 deletions
@@ -117,7 +117,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None):
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None):
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory);trial = root / 'trial';trial.mkdir()
@@ -142,6 +142,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
'tool': tool, 'arguments': arguments, 'status': 'completed',
'result': {'structuredContent': value}}})
events.extend(extra_events or [])
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
resource = root / 'resource.log'
@@ -462,9 +463,31 @@ class PcapGraderWorkflowTests(unittest.TestCase):
answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture'
self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
def test_extra_calls_are_not_an_equivalent_workflow_even_with_correct_facts(self):
for status in ['completed', 'failed']:
for with_receipt in [True, False]:
with self.subTest(status=status, with_receipt=with_receipt):
answer, records, reference = self.fixture()
extra = copy.deepcopy(records[-1]);extra['evidence_id'] = 'ev_' + 'e' * 64
extra['normalized_result']['view']['facet'] = 'endpoints'
event = {'type': 'item.completed', 'item': {
'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': status,
'arguments': {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
'view': extra['normalized_result']['view']},
'result': {'structuredContent': extra} if with_receipt else {}}}
report = self.grade(answer, records, reference, extra_events=[event])
self.assertTrue(report['all_factual_checks_pass'])
self.assertFalse(report.get('all_workflow_checks_pass', True))
answer, records, reference = self.fixture()
report = self.grade(answer, records[::-1], reference)
self.assertTrue(report['all_factual_checks_pass'])
self.assertFalse(report.get('all_workflow_checks_pass', True))
def test_repeated_identical_receipts_pass_but_conflicting_id_ownership_fails(self):
answer, records, reference = self.fixture();duplicate = copy.deepcopy(records[0])
self.assertTrue(self.grade(answer, [duplicate, *records], reference)['all_factual_checks_pass'])
report = self.grade(answer, [duplicate, *records], reference)
self.assertTrue(report['all_factual_checks_pass'])
self.assertFalse(report['all_workflow_checks_pass'])
for duplicate_first in [True, False]:
with self.subTest(duplicate_first=duplicate_first):
items = [duplicate, *records] if duplicate_first else [*records, duplicate]