From 8924a3a06c8dd90823cf0ef219dd44d941b8206b Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:55:57 +0000 Subject: [PATCH] fix(perf): bind PCAP receipts to requested parameters and views Preserve completed call arguments with receipt ownership in the grader, verify actual call and envelope parameters, and validate delegated summary/URI views once at the common observation boundary. Preserve existing delegation and original measurements. Add complete-grader regressions for request mismatch. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- experiments/analysis-efficiency/measure.mjs | 29 ++++++--- .../pcap-small-captures.md | 11 +++- .../analysis-efficiency/review-model.py | 60 ++++++++++++------- .../analysis-efficiency/test_codex_context.py | 55 +++++++++++++++-- 4 files changed, 118 insertions(+), 37 deletions(-) diff --git a/experiments/analysis-efficiency/measure.mjs b/experiments/analysis-efficiency/measure.mjs index 506531277..698ccb23c 100644 --- a/experiments/analysis-efficiency/measure.mjs +++ b/experiments/analysis-efficiency/measure.mjs @@ -524,7 +524,7 @@ const inspectEvmTarget = async () => { }; const PCAP_EVIDENCE_ID = /^ev_[a-f0-9]{64}$/u; -const pcapObservation = (evidence) => { +const pcapObservation = (evidence, requestedView) => { const evidenceId = evidence.evidence_id; if (typeof evidenceId !== "string" || !PCAP_EVIDENCE_ID.test(evidenceId)) throw new Error("PCAP Evidence receipt identity is missing or malformed"); @@ -544,16 +544,15 @@ const pcapObservation = (evidence) => { throw new Error("PCAP Evidence does not identify the selected capture"); // Summary delivery delegates to a retained view, whose Evidence operation is // inspect_analysis_view even when inspect_pcap_capture was requested. - const operation = - values.delivery === "summary" - ? "inspect_analysis_view" - : "inspect_pcap_capture"; + const operation = requestedView + ? "inspect_analysis_view" + : "inspect_pcap_capture"; if (evidence.operation !== operation) throw new Error( "PCAP Evidence operation differs from the requested delivery", ); if ( - values.delivery === "summary" && + requestedView && (typeof value.parent_evidence_id !== "string" || !PCAP_EVIDENCE_ID.test(value.parent_evidence_id) || value.parent_evidence_id === evidenceId || @@ -561,6 +560,14 @@ const pcapObservation = (evidence) => { !evidence.evidence_links.includes(value.parent_evidence_id)) ) throw new Error("PCAP retained parent lacks a distinct Evidence link"); + const parameters = requestedView + ? { parent_evidence_id: value.parent_evidence_id, view: requestedView } + : { path: values.path }; + if ( + !isDeepStrictEqual(evidence.parameters, parameters) || + (requestedView && !isDeepStrictEqual(value.view, requestedView)) + ) + throw new Error("PCAP Evidence parameters or view differ from the request"); return { evidence_id: evidenceId, limitations: evidence.limitations, @@ -596,7 +603,12 @@ const inspectPcapTarget = async () => { throw new Error("Selected PCAP request is not advertised by this runtime"); const inspected = await call("inspect_pcap_capture", input); const n = normalized(inspected); - const observations = [pcapObservation(inspected)]; + const observations = [ + pcapObservation( + inspected, + values.delivery === "summary" ? { kind: "summary" } : undefined, + ), + ]; let capture, uris; if (values.delivery === "summary") { if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture") @@ -609,6 +621,7 @@ const inspectPcapTarget = async () => { view, }); const page = normalized(selected); + const observation = pcapObservation(selected, view); const total = capture.retained_counts.http_request_uris; const examined = Math.min(view.limit, total); const expectedCoverage = { @@ -622,7 +635,6 @@ const inspectPcapTarget = async () => { page.parent_evidence_id !== parentEvidenceId || page.parent_operation !== "inspect_pcap_capture" || page.kind !== "pcap" || - !isDeepStrictEqual(page.view, view) || page.page.facet !== "http-uris" || page.source_coverage !== capture.coverage || !isDeepStrictEqual(page.coverage, expectedCoverage) @@ -631,7 +643,6 @@ const inspectPcapTarget = async () => { "PCAP page identity, scope or coverage differs from its parent", ); uris = page.page.items; - const observation = pcapObservation(selected); if ( observations.some((item) => item.evidence_id === observation.evidence_id) ) diff --git a/experiments/analysis-efficiency/pcap-small-captures.md b/experiments/analysis-efficiency/pcap-small-captures.md index 590270ec8..59a8148a1 100644 --- a/experiments/analysis-efficiency/pcap-small-captures.md +++ b/experiments/analysis-efficiency/pcap-small-captures.md @@ -126,7 +126,7 @@ validated. Both real CLI complete results match full MCP normalized results, subjects, parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained normalized facts and subjects; source forms/caller parameters remain distinct. -Source bytes stay unchanged. Twenty-three context/grading regressions pass, including +Source bytes stay unchanged. Twenty-four context/grading regressions pass, including cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID lists that reject parent lookups, duplicates and nested representations. URI prefix coverage must come from its cited page, even if an uncited page has the @@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver rejects wrong status, next offset or exhaustion. Unmodified exhausted and continuable URI pages still pass. The grader also requires each cited receipt's subject digest and local path to identify the reference capture, and uses the -reference page's verified coverage. Twenty-three grading/context regressions pass, +reference page's verified coverage. Twenty-four grading/context regressions pass, including missing or malformed subjects and faithful but inconsistent coverage; all four original model answers still pass factual regrading. Strict resource grades remain false where terminal whole-scope counters are unknown. @@ -194,6 +194,13 @@ them. The observation producer rejects missing, null or malformed response IDs before recording either capture or page receipts. Private real-reply injections exercise these ID failures alongside normal complete/summary controls; they do not establish malformed provider behavior. +Completed-call arguments stay with their producing tool and receipt in the +grader's identity index. Cited envelope parameters and actual call arguments +must match the selected path or retained parent/view, including delegated +summary parameters. The observation producer validates both summary and URI +views and envelope parameters at their common boundary. Private real-reply +faults reproduce missing-parameter/summary-view acceptance before the fix and +are rejected afterward; normal complete/summary controls still pass. Negative regressions reproduce prior crashes and false passes; all original real-model factual grades still pass without model reruns. The SDK workflow is unchanged by these grader fixes. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index e1ba69e25..ad2d216ba 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -121,6 +121,7 @@ def main(): reference = read(args.reference / 'answer.json') if args.reference else None observed_ids = set() producer_results = {} + pcap_calls = [] tool_errors = [] with (args.trial / 'events.jsonl').open() as event_lines: for line in event_lines: @@ -135,7 +136,10 @@ def main(): for text in strings(result): observed_ids.update(re.findall(r'ev_[a-f0-9]{64}', text)) if parsed is not None: - producer_results.setdefault(item['tool'], []).append(parsed) + if args.domain == 'pcap': + pcap_calls.append((item['tool'], item.get('arguments'), parsed)) + else: + producer_results.setdefault(item['tool'], []).append(parsed) else: tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result}) if parsed is not None and parsed.get('error', {}).get('code') == 'resource_constraint': @@ -182,23 +186,22 @@ def main(): (True, json.dumps(reference['capture'], sort_keys=True))) prefix = answer.get('http_request_uris_first2') records = {} - for tool, values in producer_results.items(): - for value in values: - if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str): - continue - identity = value['evidence_id'] - normalized = value.get('normalized_result') - supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list) - if supported and normalized.get('kind') == 'summary': - supported = isinstance(normalized.get('summary'), dict) - elif supported and normalized.get('kind') == 'pcap': - supported = isinstance(normalized.get('page'), dict) - entry = (tool, value) if supported else None - if identity not in records: - records[identity] = entry - elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True): - # An ID with conflicting producers or envelopes is ambiguous. - records[identity] = None + for tool, arguments, value in pcap_calls: + if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str): + continue + identity = value['evidence_id'] + normalized = value.get('normalized_result') + supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list) + if supported and normalized.get('kind') == 'summary': + supported = isinstance(normalized.get('summary'), dict) + elif supported and normalized.get('kind') == 'pcap': + supported = isinstance(normalized.get('page'), dict) + entry = (tool, arguments, value) if supported else None + if identity not in records: + records[identity] = entry + elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True): + # An ID with conflicting producers or envelopes is ambiguous. + records[identity] = None observations = answer.get('delivery_observations') canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations if isinstance(prefix, dict): @@ -209,7 +212,7 @@ def main(): last = canonical_observations[-1] if isinstance(canonical_observations, list) and canonical_observations else None identity = last.get('evidence_id') if isinstance(last, dict) else None entry = records.get(identity) if isinstance(identity, str) else None - record = entry[1] if entry is not None else None + record = entry[2] if entry is not None else None page = record['normalized_result'] if record is not None else {} checks['pcap_prefix_coverage_observed'] = ( page.get('kind') == 'pcap' @@ -226,7 +229,7 @@ def main(): for observation in canonical_observations: identity = observation.get('evidence_id') if isinstance(observation, dict) else None entry = records.get(identity) if isinstance(identity, str) else None - record = entry[1] if entry is not None else None + record = entry[2] if entry is not None else None if record is None: expected_observations.append(None) continue @@ -256,13 +259,26 @@ def main(): and len(set(identities)) == len(identities) ) entries = [records.get(identity) if isinstance(identity, str) else None for identity in identities] - cited = [entry[1] if entry is not None else None for entry in entries] + cited = [entry[2] if entry is not None else None for entry in entries] expected_tools = ['inspect_pcap_capture'] + (['inspect_analysis_view'] if len(reference['delivery_observations']) == 2 else []) check('pcap_cited_producer_tools', [entry[0] if entry is not None else None for entry in entries], expected_tools) # Summary delivery delegates to a retained view; its envelope operation # is inspect_analysis_view even though the requested tool is inspect_pcap_capture. expected_operations = ['inspect_pcap_capture'] if len(expected_tools) == 1 else ['inspect_analysis_view'] * 2 check('pcap_cited_operations', [record.get('operation') if isinstance(record, dict) else None for record in cited], expected_operations) + path = reference['capture']['target']['path'] + expected_arguments = [{'path': path}] + expected_parameters = [{'path': path}] + if len(expected_tools) == 2: + parent = cited[0]['normalized_result'].get('parent_evidence_id') if isinstance(cited[0], dict) else None + views = [observation.get('view') for observation in reference['delivery_observations']] + expected_arguments = [{'path': path, 'detail': 'summary'}, + {'source': {'kind': 'retained-evidence', 'evidence_id': parent}, 'view': views[-1]}] + expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views] + check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True), + json.dumps(expected_arguments, sort_keys=True)) + check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True), + json.dumps(expected_parameters, sort_keys=True)) checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all( isinstance(record, dict) and isinstance(record.get('subject'), dict) @@ -323,6 +339,8 @@ def main(): checks['pcap_cited_parents_linked'] = False checks['pcap_cited_producer_tools'] = False checks['pcap_cited_operations'] = False + checks['pcap_cited_call_arguments'] = False + checks['pcap_cited_parameters'] = False elif domain == 'flutter-aot': for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']: check(key, answer.get(key), reference[key]) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 18e6e7b8b..b286f2629 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -98,6 +98,8 @@ class PcapGraderWorkflowTests(unittest.TestCase): 'page': {'facet': 'http-uris', 'items': prefix}, 'coverage': coverage, 'source_coverage': 'complete', 'artifact': capture['target'], 'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}} + for record in [summary, page]: + record['parameters'] = {'parent_evidence_id': parent, 'view': copy.deepcopy(record['normalized_result']['view'])} observations = [self.observation(value) for value in [summary, page]] reference = {'artifact_sha256': 'd' * 64, 'capture': capture, 'http_request_uris_first2': prefix, 'delivery_observations': observations} @@ -113,7 +115,7 @@ class PcapGraderWorkflowTests(unittest.TestCase): 'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'], 'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']} - def grade(self, answer, records, reference, producer_tools=None): + def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None): review = runpy.run_path(str(Path(__file__).with_name('review-model.py'))) with tempfile.TemporaryDirectory() as directory: root = Path(directory);trial = root / 'trial';trial.mkdir() @@ -121,10 +123,23 @@ class PcapGraderWorkflowTests(unittest.TestCase): ('trial-settings.json', {}), ('cleanup.json', {'profile_removed': True})]: (trial / name).write_text(json.dumps(value)) (trial / 'answer.txt').write_text(json.dumps(answer)) - events = [{'type': 'item.completed', 'item': {'type': 'mcp_tool_call', - 'tool': producer_tools[index] if producer_tools is not None else - ('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture'), - 'status': 'completed', 'result': {'structuredContent': value}}} for index, value in enumerate(records)] + events = [] + for index, value in enumerate(records): + tool = producer_tools[index] if producer_tools is not None else \ + ('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture') + arguments = {'path': reference['capture']['target']['path']} + if tool == 'inspect_analysis_view': + normalized = value.get('normalized_result') + parent = normalized.get('parent_evidence_id') if isinstance(normalized, dict) else None + arguments = {'source': {'kind': 'retained-evidence', 'evidence_id': parent}, + 'view': reference['delivery_observations'][-1].get('view')} + elif len(reference['delivery_observations']) == 2: + arguments['detail'] = 'summary' + if producer_arguments is not None: + arguments = producer_arguments[index] + events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call', + 'tool': tool, 'arguments': arguments, 'status': 'completed', + 'result': {'structuredContent': value}}}) (trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events)) source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference)) resource = root / 'resource.log' @@ -191,12 +206,20 @@ class PcapGraderWorkflowTests(unittest.TestCase): source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three'] source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'} receipt = {'evidence_id': 'ev_' + 'a' * 64, 'operation': 'inspect_pcap_capture', + 'parameters': {'path': source['target']['path']}, 'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [], 'limitations': ['source'], 'normalized_result': source} reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'], 'limitations': receipt['limitations'], 'coverage': source['coverage']}] answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]}) self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass']) + for parameters in [None, {}, {'path': '/fixture/other.pcapng'}]: + changed = copy.deepcopy(receipt);changed['parameters'] = parameters + report = self.grade(answer, [changed], reference) + self.assertFalse(report['factual_checks']['pcap_cited_parameters']) + self.assertFalse(report['all_factual_checks_pass']) + self.assertFalse(self.grade(answer, [receipt], reference, + producer_arguments=[{'path': '/fixture/other.pcapng'}])['all_factual_checks_pass']) self.assertFalse(self.grade(answer, [receipt], reference, ['inspect_analysis_view'])['all_factual_checks_pass']) wrong_operation = copy.deepcopy(receipt);wrong_operation['operation'] = 'inspect_analysis_view' self.assertFalse(self.grade(answer, [wrong_operation], reference)['all_factual_checks_pass']) @@ -275,6 +298,28 @@ class PcapGraderWorkflowTests(unittest.TestCase): self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix']) self.assertFalse(report['all_factual_checks_pass']) + def test_cited_view_parameters_and_call_arguments_bind_actual_requests(self): + for index in [0, 1]: + for parameters in [None, {}, {'parent_evidence_id': 'ev_' + 'f' * 64, + 'view': {'kind': 'summary'}}]: + with self.subTest(index=index, parameters=parameters): + answer, records, reference = self.fixture() + records[index]['parameters'] = parameters + report = self.grade(answer, records, reference) + self.assertTrue(report['factual_checks']['pcap_original_delivery_observations']) + self.assertFalse(report['factual_checks']['pcap_cited_parameters']) + self.assertFalse(report['all_factual_checks_pass']) + for wrong in [None, {}, {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'f' * 64}, + 'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}}, + {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64}, + 'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': False, 'limit': 2}}]: + with self.subTest(arguments=wrong): + answer, records, reference = self.fixture() + report = self.grade(answer, records, reference, + producer_arguments=[{'path': reference['capture']['target']['path'], 'detail': 'summary'}, wrong]) + self.assertFalse(report['factual_checks']['pcap_cited_call_arguments']) + self.assertFalse(report['all_factual_checks_pass']) + def test_malformed_receipt_envelopes_write_failed_facts_instead_of_crashing(self): tools = ['inspect_pcap_capture', 'inspect_analysis_view'] for index in [0, 1]: