fix(perf): grade exact PCAP workflow call sequences

Keep selected factual correctness separate from nominal workflow equivalence.
Record all completed MCP attempts, including failed/undecodable replies, and
require the planned ordered calls, arguments and completion status. Extra
inventory, repeated or reordered attempts cannot pass workflow acceptance;
retain their actual cost/errors and supported cited facts.

Validation: 31 targeted context/grading regressions and factual plus workflow
regrading of all four original model trials without reruns. SDK and comparator
code are unchanged and are not retested.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-13 18:50:58 +00:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent e6cf2e354e
commit 8ee5ca17ee
3 changed files with 49 additions and 4 deletions
@@ -126,7 +126,7 @@ validated.
Both real CLI complete results match full MCP normalized results, subjects,
parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
normalized facts and subjects; source forms/caller parameters remain distinct.
Source bytes stay unchanged. Thirty context/grading regressions pass, including
Source bytes stay unchanged. Thirty-one context/grading regressions pass, including
cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
lists that reject parent lookups, duplicates and nested representations. URI
prefix coverage must come from its cited page, even if an uncited page has the
@@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver
rejects wrong status, next offset or exhaustion. Unmodified exhausted and
continuable URI pages still pass. The grader also requires each cited receipt's
subject digest and local path to identify the reference capture, and uses the
reference page's verified coverage. Thirty grading/context regressions pass,
reference page's verified coverage. Thirty-one grading/context regressions pass,
including missing or malformed subjects and faithful but inconsistent coverage;
all four original model answers still pass factual regrading. Strict resource
grades remain false where terminal whole-scope counters are unknown.
@@ -259,3 +259,11 @@ guard and fail afterward; unmodified complete/summary controls pass. The model
grader tests selected facts against cited transcript receipts; its synthetic
fixtures do not establish Evidence authentication. The original real-model
receipts are independently authenticated against the pinned runtime.
The grader separates truthful cited facts from equivalence of the measured
workflow. PCAP workflow checks require the planned ordered capture call (complete)
or summary capture followed by the URI-page call, with exact arguments and
completed status. Extra, reversed, failed or undecodable attempts fail workflow
acceptance even when the final cited facts are correct; their actual cost and
errors remain recorded. All four original model trials pass both factual and
workflow checks without model reruns. Strict resource grades remain unchanged.
@@ -122,6 +122,8 @@ def main():
observed_ids = set()
producer_results = {}
pcap_calls = []
pcap_attempts = []
workflow_checks = {'pcap_exact_call_sequence': False} if args.domain == 'pcap' else {}
tool_errors = []
with (args.trial / 'events.jsonl').open() as event_lines:
for line in event_lines:
@@ -129,6 +131,9 @@ def main():
item = event.get('item', {})
if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
continue
if args.domain == 'pcap':
pcap_attempts.append({'tool': item.get('tool'), 'arguments': item.get('arguments'),
'status': item.get('status')})
result = item.get('result') or {}
parsed = decode(result)
if item['status'] == 'completed':
@@ -286,6 +291,12 @@ def main():
expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views]
check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True),
json.dumps(expected_arguments, sort_keys=True))
workflow_checks['pcap_exact_call_sequence'] = (
json.dumps(pcap_attempts, sort_keys=True) == json.dumps([
{'tool': tool, 'arguments': arguments, 'status': 'completed'}
for tool, arguments in zip(expected_tools, expected_arguments)
], sort_keys=True)
)
check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True),
json.dumps(expected_parameters, sort_keys=True))
checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
@@ -539,6 +550,7 @@ def main():
statuses[status] = statuses.get(status, 0) + 1
check('network_response_statuses', network['observed_response_statuses'], statuses)
factual_checks = dict(checks)
checks.update(workflow_checks)
scopes = [json.loads(line) for line in args.resource_log.read_text().splitlines() if line.startswith('{')]
scope = next(value for value in reversed(scopes) if 'resource_scope' in value)
checks['no_resource_abort'] = scope['abort'] is None
@@ -558,6 +570,8 @@ def main():
'review_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'domain': domain, 'checks': checks, 'all_selected_checks_pass': all(checks.values()),
'factual_checks': factual_checks, 'all_factual_checks_pass': all(factual_checks.values()),
**({'workflow_checks': workflow_checks, 'all_workflow_checks_pass': all(workflow_checks.values())}
if domain == 'pcap' else {}),
'differences': differences, 'tool_errors': tool_errors, 'summary': summary,
'fingerprints': {key: value for key, value in settings.items() if 'sha256' in key or key in ['runtime_revision', 'enabled_tools', 'authority']},
'answer_file_sha256': hashlib.sha256((args.trial / 'answer.txt').read_bytes()).hexdigest(),
@@ -117,7 +117,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None):
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None):
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory);trial = root / 'trial';trial.mkdir()
@@ -142,6 +142,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
'tool': tool, 'arguments': arguments, 'status': 'completed',
'result': {'structuredContent': value}}})
events.extend(extra_events or [])
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
resource = root / 'resource.log'
@@ -462,9 +463,31 @@ class PcapGraderWorkflowTests(unittest.TestCase):
answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture'
self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
def test_extra_calls_are_not_an_equivalent_workflow_even_with_correct_facts(self):
for status in ['completed', 'failed']:
for with_receipt in [True, False]:
with self.subTest(status=status, with_receipt=with_receipt):
answer, records, reference = self.fixture()
extra = copy.deepcopy(records[-1]);extra['evidence_id'] = 'ev_' + 'e' * 64
extra['normalized_result']['view']['facet'] = 'endpoints'
event = {'type': 'item.completed', 'item': {
'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': status,
'arguments': {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
'view': extra['normalized_result']['view']},
'result': {'structuredContent': extra} if with_receipt else {}}}
report = self.grade(answer, records, reference, extra_events=[event])
self.assertTrue(report['all_factual_checks_pass'])
self.assertFalse(report.get('all_workflow_checks_pass', True))
answer, records, reference = self.fixture()
report = self.grade(answer, records[::-1], reference)
self.assertTrue(report['all_factual_checks_pass'])
self.assertFalse(report.get('all_workflow_checks_pass', True))
def test_repeated_identical_receipts_pass_but_conflicting_id_ownership_fails(self):
answer, records, reference = self.fixture();duplicate = copy.deepcopy(records[0])
self.assertTrue(self.grade(answer, [duplicate, *records], reference)['all_factual_checks_pass'])
report = self.grade(answer, [duplicate, *records], reference)
self.assertTrue(report['all_factual_checks_pass'])
self.assertFalse(report['all_workflow_checks_pass'])
for duplicate_first in [True, False]:
with self.subTest(duplicate_first=duplicate_first):
items = [duplicate, *records] if duplicate_first else [*records, duplicate]