fix(perf): bind PCAP receipts to requested parameters and views

Preserve completed call arguments with receipt ownership in the grader, verify
actual call and envelope parameters, and validate delegated summary/URI views
once at the common observation boundary. Preserve existing delegation and
original measurements. Add complete-grader regressions for request mismatch.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-14 02:40:31 +08:00
committed by morluto
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent 5f46da90f9
commit 8924a3a06c
4 changed files with 118 additions and 37 deletions
+20 -9
View File
@@ -524,7 +524,7 @@ const inspectEvmTarget = async () => {
};
const PCAP_EVIDENCE_ID = /^ev_[a-f0-9]{64}$/u;
const pcapObservation = (evidence) => {
const pcapObservation = (evidence, requestedView) => {
const evidenceId = evidence.evidence_id;
if (typeof evidenceId !== "string" || !PCAP_EVIDENCE_ID.test(evidenceId))
throw new Error("PCAP Evidence receipt identity is missing or malformed");
@@ -544,16 +544,15 @@ const pcapObservation = (evidence) => {
throw new Error("PCAP Evidence does not identify the selected capture");
// Summary delivery delegates to a retained view, whose Evidence operation is
// inspect_analysis_view even when inspect_pcap_capture was requested.
const operation =
values.delivery === "summary"
? "inspect_analysis_view"
: "inspect_pcap_capture";
const operation = requestedView
? "inspect_analysis_view"
: "inspect_pcap_capture";
if (evidence.operation !== operation)
throw new Error(
"PCAP Evidence operation differs from the requested delivery",
);
if (
values.delivery === "summary" &&
requestedView &&
(typeof value.parent_evidence_id !== "string" ||
!PCAP_EVIDENCE_ID.test(value.parent_evidence_id) ||
value.parent_evidence_id === evidenceId ||
@@ -561,6 +560,14 @@ const pcapObservation = (evidence) => {
!evidence.evidence_links.includes(value.parent_evidence_id))
)
throw new Error("PCAP retained parent lacks a distinct Evidence link");
const parameters = requestedView
? { parent_evidence_id: value.parent_evidence_id, view: requestedView }
: { path: values.path };
if (
!isDeepStrictEqual(evidence.parameters, parameters) ||
(requestedView && !isDeepStrictEqual(value.view, requestedView))
)
throw new Error("PCAP Evidence parameters or view differ from the request");
return {
evidence_id: evidenceId,
limitations: evidence.limitations,
@@ -596,7 +603,12 @@ const inspectPcapTarget = async () => {
throw new Error("Selected PCAP request is not advertised by this runtime");
const inspected = await call("inspect_pcap_capture", input);
const n = normalized(inspected);
const observations = [pcapObservation(inspected)];
const observations = [
pcapObservation(
inspected,
values.delivery === "summary" ? { kind: "summary" } : undefined,
),
];
let capture, uris;
if (values.delivery === "summary") {
if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture")
@@ -609,6 +621,7 @@ const inspectPcapTarget = async () => {
view,
});
const page = normalized(selected);
const observation = pcapObservation(selected, view);
const total = capture.retained_counts.http_request_uris;
const examined = Math.min(view.limit, total);
const expectedCoverage = {
@@ -622,7 +635,6 @@ const inspectPcapTarget = async () => {
page.parent_evidence_id !== parentEvidenceId ||
page.parent_operation !== "inspect_pcap_capture" ||
page.kind !== "pcap" ||
!isDeepStrictEqual(page.view, view) ||
page.page.facet !== "http-uris" ||
page.source_coverage !== capture.coverage ||
!isDeepStrictEqual(page.coverage, expectedCoverage)
@@ -631,7 +643,6 @@ const inspectPcapTarget = async () => {
"PCAP page identity, scope or coverage differs from its parent",
);
uris = page.page.items;
const observation = pcapObservation(selected);
if (
observations.some((item) => item.evidence_id === observation.evidence_id)
)
@@ -126,7 +126,7 @@ validated.
Both real CLI complete results match full MCP normalized results, subjects,
parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
normalized facts and subjects; source forms/caller parameters remain distinct.
Source bytes stay unchanged. Twenty-three context/grading regressions pass, including
Source bytes stay unchanged. Twenty-four context/grading regressions pass, including
cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
lists that reject parent lookups, duplicates and nested representations. URI
prefix coverage must come from its cited page, even if an uncited page has the
@@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver
rejects wrong status, next offset or exhaustion. Unmodified exhausted and
continuable URI pages still pass. The grader also requires each cited receipt's
subject digest and local path to identify the reference capture, and uses the
reference page's verified coverage. Twenty-three grading/context regressions pass,
reference page's verified coverage. Twenty-four grading/context regressions pass,
including missing or malformed subjects and faithful but inconsistent coverage;
all four original model answers still pass factual regrading. Strict resource
grades remain false where terminal whole-scope counters are unknown.
@@ -194,6 +194,13 @@ them. The observation producer rejects missing, null or malformed response IDs
before recording either capture or page receipts. Private real-reply injections
exercise these ID failures alongside normal complete/summary controls; they do
not establish malformed provider behavior.
Completed-call arguments stay with their producing tool and receipt in the
grader's identity index. Cited envelope parameters and actual call arguments
must match the selected path or retained parent/view, including delegated
summary parameters. The observation producer validates both summary and URI
views and envelope parameters at their common boundary. Private real-reply
faults reproduce missing-parameter/summary-view acceptance before the fix and
are rejected afterward; normal complete/summary controls still pass.
Negative regressions reproduce
prior crashes and false passes; all original real-model factual grades still
pass without model reruns. The SDK workflow is unchanged by these grader fixes.
+39 -21
View File
@@ -121,6 +121,7 @@ def main():
reference = read(args.reference / 'answer.json') if args.reference else None
observed_ids = set()
producer_results = {}
pcap_calls = []
tool_errors = []
with (args.trial / 'events.jsonl').open() as event_lines:
for line in event_lines:
@@ -135,7 +136,10 @@ def main():
for text in strings(result):
observed_ids.update(re.findall(r'ev_[a-f0-9]{64}', text))
if parsed is not None:
producer_results.setdefault(item['tool'], []).append(parsed)
if args.domain == 'pcap':
pcap_calls.append((item['tool'], item.get('arguments'), parsed))
else:
producer_results.setdefault(item['tool'], []).append(parsed)
else:
tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result})
if parsed is not None and parsed.get('error', {}).get('code') == 'resource_constraint':
@@ -182,23 +186,22 @@ def main():
(True, json.dumps(reference['capture'], sort_keys=True)))
prefix = answer.get('http_request_uris_first2')
records = {}
for tool, values in producer_results.items():
for value in values:
if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str):
continue
identity = value['evidence_id']
normalized = value.get('normalized_result')
supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
if supported and normalized.get('kind') == 'summary':
supported = isinstance(normalized.get('summary'), dict)
elif supported and normalized.get('kind') == 'pcap':
supported = isinstance(normalized.get('page'), dict)
entry = (tool, value) if supported else None
if identity not in records:
records[identity] = entry
elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True):
# An ID with conflicting producers or envelopes is ambiguous.
records[identity] = None
for tool, arguments, value in pcap_calls:
if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str):
continue
identity = value['evidence_id']
normalized = value.get('normalized_result')
supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
if supported and normalized.get('kind') == 'summary':
supported = isinstance(normalized.get('summary'), dict)
elif supported and normalized.get('kind') == 'pcap':
supported = isinstance(normalized.get('page'), dict)
entry = (tool, arguments, value) if supported else None
if identity not in records:
records[identity] = entry
elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True):
# An ID with conflicting producers or envelopes is ambiguous.
records[identity] = None
observations = answer.get('delivery_observations')
canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations
if isinstance(prefix, dict):
@@ -209,7 +212,7 @@ def main():
last = canonical_observations[-1] if isinstance(canonical_observations, list) and canonical_observations else None
identity = last.get('evidence_id') if isinstance(last, dict) else None
entry = records.get(identity) if isinstance(identity, str) else None
record = entry[1] if entry is not None else None
record = entry[2] if entry is not None else None
page = record['normalized_result'] if record is not None else {}
checks['pcap_prefix_coverage_observed'] = (
page.get('kind') == 'pcap'
@@ -226,7 +229,7 @@ def main():
for observation in canonical_observations:
identity = observation.get('evidence_id') if isinstance(observation, dict) else None
entry = records.get(identity) if isinstance(identity, str) else None
record = entry[1] if entry is not None else None
record = entry[2] if entry is not None else None
if record is None:
expected_observations.append(None)
continue
@@ -256,13 +259,26 @@ def main():
and len(set(identities)) == len(identities)
)
entries = [records.get(identity) if isinstance(identity, str) else None for identity in identities]
cited = [entry[1] if entry is not None else None for entry in entries]
cited = [entry[2] if entry is not None else None for entry in entries]
expected_tools = ['inspect_pcap_capture'] + (['inspect_analysis_view'] if len(reference['delivery_observations']) == 2 else [])
check('pcap_cited_producer_tools', [entry[0] if entry is not None else None for entry in entries], expected_tools)
# Summary delivery delegates to a retained view; its envelope operation
# is inspect_analysis_view even though the requested tool is inspect_pcap_capture.
expected_operations = ['inspect_pcap_capture'] if len(expected_tools) == 1 else ['inspect_analysis_view'] * 2
check('pcap_cited_operations', [record.get('operation') if isinstance(record, dict) else None for record in cited], expected_operations)
path = reference['capture']['target']['path']
expected_arguments = [{'path': path}]
expected_parameters = [{'path': path}]
if len(expected_tools) == 2:
parent = cited[0]['normalized_result'].get('parent_evidence_id') if isinstance(cited[0], dict) else None
views = [observation.get('view') for observation in reference['delivery_observations']]
expected_arguments = [{'path': path, 'detail': 'summary'},
{'source': {'kind': 'retained-evidence', 'evidence_id': parent}, 'view': views[-1]}]
expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views]
check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True),
json.dumps(expected_arguments, sort_keys=True))
check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True),
json.dumps(expected_parameters, sort_keys=True))
checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
isinstance(record, dict)
and isinstance(record.get('subject'), dict)
@@ -323,6 +339,8 @@ def main():
checks['pcap_cited_parents_linked'] = False
checks['pcap_cited_producer_tools'] = False
checks['pcap_cited_operations'] = False
checks['pcap_cited_call_arguments'] = False
checks['pcap_cited_parameters'] = False
elif domain == 'flutter-aot':
for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']:
check(key, answer.get(key), reference[key])
@@ -98,6 +98,8 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'page': {'facet': 'http-uris', 'items': prefix}, 'coverage': coverage,
'source_coverage': 'complete', 'artifact': capture['target'],
'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}}
for record in [summary, page]:
record['parameters'] = {'parent_evidence_id': parent, 'view': copy.deepcopy(record['normalized_result']['view'])}
observations = [self.observation(value) for value in [summary, page]]
reference = {'artifact_sha256': 'd' * 64, 'capture': capture,
'http_request_uris_first2': prefix, 'delivery_observations': observations}
@@ -113,7 +115,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
def grade(self, answer, records, reference, producer_tools=None):
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None):
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
with tempfile.TemporaryDirectory() as directory:
root = Path(directory);trial = root / 'trial';trial.mkdir()
@@ -121,10 +123,23 @@ class PcapGraderWorkflowTests(unittest.TestCase):
('trial-settings.json', {}), ('cleanup.json', {'profile_removed': True})]:
(trial / name).write_text(json.dumps(value))
(trial / 'answer.txt').write_text(json.dumps(answer))
events = [{'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
'tool': producer_tools[index] if producer_tools is not None else
('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture'),
'status': 'completed', 'result': {'structuredContent': value}}} for index, value in enumerate(records)]
events = []
for index, value in enumerate(records):
tool = producer_tools[index] if producer_tools is not None else \
('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture')
arguments = {'path': reference['capture']['target']['path']}
if tool == 'inspect_analysis_view':
normalized = value.get('normalized_result')
parent = normalized.get('parent_evidence_id') if isinstance(normalized, dict) else None
arguments = {'source': {'kind': 'retained-evidence', 'evidence_id': parent},
'view': reference['delivery_observations'][-1].get('view')}
elif len(reference['delivery_observations']) == 2:
arguments['detail'] = 'summary'
if producer_arguments is not None:
arguments = producer_arguments[index]
events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
'tool': tool, 'arguments': arguments, 'status': 'completed',
'result': {'structuredContent': value}}})
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
resource = root / 'resource.log'
@@ -191,12 +206,20 @@ class PcapGraderWorkflowTests(unittest.TestCase):
source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three']
source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'}
receipt = {'evidence_id': 'ev_' + 'a' * 64, 'operation': 'inspect_pcap_capture',
'parameters': {'path': source['target']['path']},
'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [],
'limitations': ['source'], 'normalized_result': source}
reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'],
'limitations': receipt['limitations'], 'coverage': source['coverage']}]
answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]})
self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
for parameters in [None, {}, {'path': '/fixture/other.pcapng'}]:
changed = copy.deepcopy(receipt);changed['parameters'] = parameters
report = self.grade(answer, [changed], reference)
self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
self.assertFalse(report['all_factual_checks_pass'])
self.assertFalse(self.grade(answer, [receipt], reference,
producer_arguments=[{'path': '/fixture/other.pcapng'}])['all_factual_checks_pass'])
self.assertFalse(self.grade(answer, [receipt], reference, ['inspect_analysis_view'])['all_factual_checks_pass'])
wrong_operation = copy.deepcopy(receipt);wrong_operation['operation'] = 'inspect_analysis_view'
self.assertFalse(self.grade(answer, [wrong_operation], reference)['all_factual_checks_pass'])
@@ -275,6 +298,28 @@ class PcapGraderWorkflowTests(unittest.TestCase):
self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
self.assertFalse(report['all_factual_checks_pass'])
def test_cited_view_parameters_and_call_arguments_bind_actual_requests(self):
for index in [0, 1]:
for parameters in [None, {}, {'parent_evidence_id': 'ev_' + 'f' * 64,
'view': {'kind': 'summary'}}]:
with self.subTest(index=index, parameters=parameters):
answer, records, reference = self.fixture()
records[index]['parameters'] = parameters
report = self.grade(answer, records, reference)
self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
self.assertFalse(report['all_factual_checks_pass'])
for wrong in [None, {}, {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'f' * 64},
'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}},
{'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': False, 'limit': 2}}]:
with self.subTest(arguments=wrong):
answer, records, reference = self.fixture()
report = self.grade(answer, records, reference,
producer_arguments=[{'path': reference['capture']['target']['path'], 'detail': 'summary'}, wrong])
self.assertFalse(report['factual_checks']['pcap_cited_call_arguments'])
self.assertFalse(report['all_factual_checks_pass'])
def test_malformed_receipt_envelopes_write_failed_facts_instead_of_crashing(self):
tools = ['inspect_pcap_capture', 'inspect_analysis_view']
for index in [0, 1]: