fix(perf): grade exact PCAP workflow call sequences
Keep selected factual correctness separate from nominal workflow equivalence. Record all completed MCP attempts, including failed/undecodable replies, and require the planned ordered calls, arguments and completion status. Extra inventory, repeated or reordered attempts cannot pass workflow acceptance; retain their actual cost/errors and supported cited facts. Validation: 31 targeted context/grading regressions and factual plus workflow regrading of all four original model trials without reruns. SDK and comparator code are unchanged and are not retested. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
parent
e6cf2e354e
commit
8ee5ca17ee
@@ -126,7 +126,7 @@ validated.
|
||||
Both real CLI complete results match full MCP normalized results, subjects,
|
||||
parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
|
||||
normalized facts and subjects; source forms/caller parameters remain distinct.
|
||||
Source bytes stay unchanged. Thirty context/grading regressions pass, including
|
||||
Source bytes stay unchanged. Thirty-one context/grading regressions pass, including
|
||||
cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
|
||||
lists that reject parent lookups, duplicates and nested representations. URI
|
||||
prefix coverage must come from its cited page, even if an uncited page has the
|
||||
@@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver
|
||||
rejects wrong status, next offset or exhaustion. Unmodified exhausted and
|
||||
continuable URI pages still pass. The grader also requires each cited receipt's
|
||||
subject digest and local path to identify the reference capture, and uses the
|
||||
reference page's verified coverage. Thirty grading/context regressions pass,
|
||||
reference page's verified coverage. Thirty-one grading/context regressions pass,
|
||||
including missing or malformed subjects and faithful but inconsistent coverage;
|
||||
all four original model answers still pass factual regrading. Strict resource
|
||||
grades remain false where terminal whole-scope counters are unknown.
|
||||
@@ -259,3 +259,11 @@ guard and fail afterward; unmodified complete/summary controls pass. The model
|
||||
grader tests selected facts against cited transcript receipts; its synthetic
|
||||
fixtures do not establish Evidence authentication. The original real-model
|
||||
receipts are independently authenticated against the pinned runtime.
|
||||
|
||||
The grader separates truthful cited facts from equivalence of the measured
|
||||
workflow. PCAP workflow checks require the planned ordered capture call (complete)
|
||||
or summary capture followed by the URI-page call, with exact arguments and
|
||||
completed status. Extra, reversed, failed or undecodable attempts fail workflow
|
||||
acceptance even when the final cited facts are correct; their actual cost and
|
||||
errors remain recorded. All four original model trials pass both factual and
|
||||
workflow checks without model reruns. Strict resource grades remain unchanged.
|
||||
|
||||
@@ -122,6 +122,8 @@ def main():
|
||||
observed_ids = set()
|
||||
producer_results = {}
|
||||
pcap_calls = []
|
||||
pcap_attempts = []
|
||||
workflow_checks = {'pcap_exact_call_sequence': False} if args.domain == 'pcap' else {}
|
||||
tool_errors = []
|
||||
with (args.trial / 'events.jsonl').open() as event_lines:
|
||||
for line in event_lines:
|
||||
@@ -129,6 +131,9 @@ def main():
|
||||
item = event.get('item', {})
|
||||
if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
|
||||
continue
|
||||
if args.domain == 'pcap':
|
||||
pcap_attempts.append({'tool': item.get('tool'), 'arguments': item.get('arguments'),
|
||||
'status': item.get('status')})
|
||||
result = item.get('result') or {}
|
||||
parsed = decode(result)
|
||||
if item['status'] == 'completed':
|
||||
@@ -286,6 +291,12 @@ def main():
|
||||
expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views]
|
||||
check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True),
|
||||
json.dumps(expected_arguments, sort_keys=True))
|
||||
workflow_checks['pcap_exact_call_sequence'] = (
|
||||
json.dumps(pcap_attempts, sort_keys=True) == json.dumps([
|
||||
{'tool': tool, 'arguments': arguments, 'status': 'completed'}
|
||||
for tool, arguments in zip(expected_tools, expected_arguments)
|
||||
], sort_keys=True)
|
||||
)
|
||||
check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True),
|
||||
json.dumps(expected_parameters, sort_keys=True))
|
||||
checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
|
||||
@@ -539,6 +550,7 @@ def main():
|
||||
statuses[status] = statuses.get(status, 0) + 1
|
||||
check('network_response_statuses', network['observed_response_statuses'], statuses)
|
||||
factual_checks = dict(checks)
|
||||
checks.update(workflow_checks)
|
||||
scopes = [json.loads(line) for line in args.resource_log.read_text().splitlines() if line.startswith('{')]
|
||||
scope = next(value for value in reversed(scopes) if 'resource_scope' in value)
|
||||
checks['no_resource_abort'] = scope['abort'] is None
|
||||
@@ -558,6 +570,8 @@ def main():
|
||||
'review_source_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
||||
'domain': domain, 'checks': checks, 'all_selected_checks_pass': all(checks.values()),
|
||||
'factual_checks': factual_checks, 'all_factual_checks_pass': all(factual_checks.values()),
|
||||
**({'workflow_checks': workflow_checks, 'all_workflow_checks_pass': all(workflow_checks.values())}
|
||||
if domain == 'pcap' else {}),
|
||||
'differences': differences, 'tool_errors': tool_errors, 'summary': summary,
|
||||
'fingerprints': {key: value for key, value in settings.items() if 'sha256' in key or key in ['runtime_revision', 'enabled_tools', 'authority']},
|
||||
'answer_file_sha256': hashlib.sha256((args.trial / 'answer.txt').read_bytes()).hexdigest(),
|
||||
|
||||
@@ -117,7 +117,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
|
||||
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
|
||||
|
||||
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None):
|
||||
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None, extra_events=None):
|
||||
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory);trial = root / 'trial';trial.mkdir()
|
||||
@@ -142,6 +142,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
|
||||
'tool': tool, 'arguments': arguments, 'status': 'completed',
|
||||
'result': {'structuredContent': value}}})
|
||||
events.extend(extra_events or [])
|
||||
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
|
||||
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
|
||||
resource = root / 'resource.log'
|
||||
@@ -462,9 +463,31 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture'
|
||||
self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
|
||||
|
||||
def test_extra_calls_are_not_an_equivalent_workflow_even_with_correct_facts(self):
|
||||
for status in ['completed', 'failed']:
|
||||
for with_receipt in [True, False]:
|
||||
with self.subTest(status=status, with_receipt=with_receipt):
|
||||
answer, records, reference = self.fixture()
|
||||
extra = copy.deepcopy(records[-1]);extra['evidence_id'] = 'ev_' + 'e' * 64
|
||||
extra['normalized_result']['view']['facet'] = 'endpoints'
|
||||
event = {'type': 'item.completed', 'item': {
|
||||
'type': 'mcp_tool_call', 'tool': 'inspect_analysis_view', 'status': status,
|
||||
'arguments': {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
|
||||
'view': extra['normalized_result']['view']},
|
||||
'result': {'structuredContent': extra} if with_receipt else {}}}
|
||||
report = self.grade(answer, records, reference, extra_events=[event])
|
||||
self.assertTrue(report['all_factual_checks_pass'])
|
||||
self.assertFalse(report.get('all_workflow_checks_pass', True))
|
||||
answer, records, reference = self.fixture()
|
||||
report = self.grade(answer, records[::-1], reference)
|
||||
self.assertTrue(report['all_factual_checks_pass'])
|
||||
self.assertFalse(report.get('all_workflow_checks_pass', True))
|
||||
|
||||
def test_repeated_identical_receipts_pass_but_conflicting_id_ownership_fails(self):
|
||||
answer, records, reference = self.fixture();duplicate = copy.deepcopy(records[0])
|
||||
self.assertTrue(self.grade(answer, [duplicate, *records], reference)['all_factual_checks_pass'])
|
||||
report = self.grade(answer, [duplicate, *records], reference)
|
||||
self.assertTrue(report['all_factual_checks_pass'])
|
||||
self.assertFalse(report['all_workflow_checks_pass'])
|
||||
for duplicate_first in [True, False]:
|
||||
with self.subTest(duplicate_first=duplicate_first):
|
||||
items = [duplicate, *records] if duplicate_first else [*records, duplicate]
|
||||
|
||||
Reference in New Issue
Block a user