fix(experiments): preserve PCAP receipt producer provenance

Keep tool and Evidence together in the receipt index, reject conflicting ID ownership, and validate requested tool and delegated operation roles. Preserve legitimate summary delegation to inspect_analysis_view and identical repeated receipts. Validate the same operation contract at the SDK observation boundary. Twenty grading/context tests and six comparator tests pass after reproducing the gap; real SDK fault/control runs and all four stored actual-model factual grades pass without model reruns.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-14 02:40:31 +08:00
committed by morluto
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent 073f6c07e9
commit 3e88ebf735
4 changed files with 85 additions and 13 deletions
@@ -82,13 +82,15 @@ class PcapGraderWorkflowTests(unittest.TestCase):
prefix = ['http://example.test/one', 'http://example.test/two']
coverage = {'status': 'page', 'examined': 2, 'total': 3, 'next_offset': 2, 'exhausted': False}
subject = {'digest': {'sha256': capture['target']['sha256']}, 'local_path': capture['target']['path']}
summary = {'evidence_id': summary_id, 'subject': copy.deepcopy(subject), 'evidence_links': [parent],
summary = {'evidence_id': summary_id, 'operation': 'inspect_analysis_view',
'subject': copy.deepcopy(subject), 'evidence_links': [parent],
'limitations': ['source', 'view'],
'normalized_result': {'kind': 'summary', 'view': {'kind': 'summary'},
'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent,
'summary': capture, 'coverage': {'status': 'complete-within-view', 'examined': 1, 'total': 1, 'next_offset': None, 'exhausted': True},
'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}}
page = {'evidence_id': page_id, 'subject': copy.deepcopy(subject), 'evidence_links': [parent],
page = {'evidence_id': page_id, 'operation': 'inspect_analysis_view',
'subject': copy.deepcopy(subject), 'evidence_links': [parent],
'limitations': ['source', 'view'],
'normalized_result': {'kind': 'pcap',
'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2},
@@ -111,7 +113,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
def grade(self, answer, records, reference):
def grade(self, answer, records, reference, producer_tools=None):
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory);trial = root / 'trial';trial.mkdir()
@@ -120,8 +122,9 @@ class PcapGraderWorkflowTests(unittest.TestCase):
(trial / name).write_text(json.dumps(value))
(trial / 'answer.txt').write_text(json.dumps(answer))
events = [{'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
'tool': 'inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture',
'status': 'completed', 'result': {'structuredContent': value}}} for value in records]
'tool': producer_tools[index] if producer_tools is not None else
('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture'),
'status': 'completed', 'result': {'structuredContent': value}}} for index, value in enumerate(records)]
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
resource = root / 'resource.log'
@@ -187,12 +190,16 @@ class PcapGraderWorkflowTests(unittest.TestCase):
source.update({key: [] for key in capture['retained_counts']})
source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three']
source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'}
receipt = {'evidence_id': 'ev_' + 'a' * 64, 'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [],
receipt = {'evidence_id': 'ev_' + 'a' * 64, 'operation': 'inspect_pcap_capture',
'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [],
'limitations': ['source'], 'normalized_result': source}
reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'],
'limitations': receipt['limitations'], 'coverage': source['coverage']}]
answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]})
self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
self.assertFalse(self.grade(answer, [receipt], reference, ['inspect_analysis_view'])['all_factual_checks_pass'])
wrong_operation = copy.deepcopy(receipt);wrong_operation['operation'] = 'inspect_analysis_view'
self.assertFalse(self.grade(answer, [wrong_operation], reference)['all_factual_checks_pass'])
for subject in [None, {'digest': {'sha256': 'e' * 64}, 'local_path': source['target']['path']}]:
changed = copy.deepcopy(receipt);changed['subject'] = subject
report = self.grade(answer, [changed], reference)
@@ -264,6 +271,29 @@ class PcapGraderWorkflowTests(unittest.TestCase):
self.assertFalse(report['factual_checks']['pcap_cited_parents_linked'])
self.assertFalse(report['all_factual_checks_pass'])
def test_cited_receipts_preserve_tool_and_delegated_operation_provenance(self):
for tools in [['inspect_analysis_view', 'inspect_analysis_view'],
['inspect_pcap_capture', 'inspect_pcap_capture'],
['unrelated_tool', 'inspect_analysis_view']]:
with self.subTest(tools=tools):
answer, records, reference = self.fixture()
report = self.grade(answer, records, reference, tools)
self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
self.assertFalse(report['all_factual_checks_pass'])
for index in [0, 1]:
answer, records, reference = self.fixture();records[index]['operation'] = 'inspect_pcap_capture'
self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
def test_repeated_identical_receipts_pass_but_conflicting_id_ownership_fails(self):
answer, records, reference = self.fixture();duplicate = copy.deepcopy(records[0])
self.assertTrue(self.grade(answer, [duplicate, *records], reference)['all_factual_checks_pass'])
for duplicate_first in [True, False]:
with self.subTest(duplicate_first=duplicate_first):
items = [duplicate, *records] if duplicate_first else [*records, duplicate]
tools = ['unrelated_tool', 'inspect_pcap_capture', 'inspect_analysis_view'] if duplicate_first else \
['inspect_pcap_capture', 'inspect_analysis_view', 'unrelated_tool']
self.assertFalse(self.grade(answer, items, reference, tools)['all_factual_checks_pass'])
def test_reported_ids_are_only_the_ordered_unique_response_receipts(self):
answer, records, reference = self.fixture()
original = answer['evidence_ids']