fix(perf): bind PCAP receipts to requested parameters and views
Preserve completed call arguments with receipt ownership in the grader, verify actual call and envelope parameters, and validate delegated summary/URI views once at the common observation boundary. Preserve existing delegation and original measurements. Add complete-grader regressions for request mismatch. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
committed by
morluto
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
parent
5f46da90f9
commit
8924a3a06c
@@ -524,7 +524,7 @@ const inspectEvmTarget = async () => {
|
||||
};
|
||||
|
||||
const PCAP_EVIDENCE_ID = /^ev_[a-f0-9]{64}$/u;
|
||||
const pcapObservation = (evidence) => {
|
||||
const pcapObservation = (evidence, requestedView) => {
|
||||
const evidenceId = evidence.evidence_id;
|
||||
if (typeof evidenceId !== "string" || !PCAP_EVIDENCE_ID.test(evidenceId))
|
||||
throw new Error("PCAP Evidence receipt identity is missing or malformed");
|
||||
@@ -544,16 +544,15 @@ const pcapObservation = (evidence) => {
|
||||
throw new Error("PCAP Evidence does not identify the selected capture");
|
||||
// Summary delivery delegates to a retained view, whose Evidence operation is
|
||||
// inspect_analysis_view even when inspect_pcap_capture was requested.
|
||||
const operation =
|
||||
values.delivery === "summary"
|
||||
? "inspect_analysis_view"
|
||||
: "inspect_pcap_capture";
|
||||
const operation = requestedView
|
||||
? "inspect_analysis_view"
|
||||
: "inspect_pcap_capture";
|
||||
if (evidence.operation !== operation)
|
||||
throw new Error(
|
||||
"PCAP Evidence operation differs from the requested delivery",
|
||||
);
|
||||
if (
|
||||
values.delivery === "summary" &&
|
||||
requestedView &&
|
||||
(typeof value.parent_evidence_id !== "string" ||
|
||||
!PCAP_EVIDENCE_ID.test(value.parent_evidence_id) ||
|
||||
value.parent_evidence_id === evidenceId ||
|
||||
@@ -561,6 +560,14 @@ const pcapObservation = (evidence) => {
|
||||
!evidence.evidence_links.includes(value.parent_evidence_id))
|
||||
)
|
||||
throw new Error("PCAP retained parent lacks a distinct Evidence link");
|
||||
const parameters = requestedView
|
||||
? { parent_evidence_id: value.parent_evidence_id, view: requestedView }
|
||||
: { path: values.path };
|
||||
if (
|
||||
!isDeepStrictEqual(evidence.parameters, parameters) ||
|
||||
(requestedView && !isDeepStrictEqual(value.view, requestedView))
|
||||
)
|
||||
throw new Error("PCAP Evidence parameters or view differ from the request");
|
||||
return {
|
||||
evidence_id: evidenceId,
|
||||
limitations: evidence.limitations,
|
||||
@@ -596,7 +603,12 @@ const inspectPcapTarget = async () => {
|
||||
throw new Error("Selected PCAP request is not advertised by this runtime");
|
||||
const inspected = await call("inspect_pcap_capture", input);
|
||||
const n = normalized(inspected);
|
||||
const observations = [pcapObservation(inspected)];
|
||||
const observations = [
|
||||
pcapObservation(
|
||||
inspected,
|
||||
values.delivery === "summary" ? { kind: "summary" } : undefined,
|
||||
),
|
||||
];
|
||||
let capture, uris;
|
||||
if (values.delivery === "summary") {
|
||||
if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture")
|
||||
@@ -609,6 +621,7 @@ const inspectPcapTarget = async () => {
|
||||
view,
|
||||
});
|
||||
const page = normalized(selected);
|
||||
const observation = pcapObservation(selected, view);
|
||||
const total = capture.retained_counts.http_request_uris;
|
||||
const examined = Math.min(view.limit, total);
|
||||
const expectedCoverage = {
|
||||
@@ -622,7 +635,6 @@ const inspectPcapTarget = async () => {
|
||||
page.parent_evidence_id !== parentEvidenceId ||
|
||||
page.parent_operation !== "inspect_pcap_capture" ||
|
||||
page.kind !== "pcap" ||
|
||||
!isDeepStrictEqual(page.view, view) ||
|
||||
page.page.facet !== "http-uris" ||
|
||||
page.source_coverage !== capture.coverage ||
|
||||
!isDeepStrictEqual(page.coverage, expectedCoverage)
|
||||
@@ -631,7 +643,6 @@ const inspectPcapTarget = async () => {
|
||||
"PCAP page identity, scope or coverage differs from its parent",
|
||||
);
|
||||
uris = page.page.items;
|
||||
const observation = pcapObservation(selected);
|
||||
if (
|
||||
observations.some((item) => item.evidence_id === observation.evidence_id)
|
||||
)
|
||||
|
||||
@@ -126,7 +126,7 @@ validated.
|
||||
Both real CLI complete results match full MCP normalized results, subjects,
|
||||
parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
|
||||
normalized facts and subjects; source forms/caller parameters remain distinct.
|
||||
Source bytes stay unchanged. Twenty-three context/grading regressions pass, including
|
||||
Source bytes stay unchanged. Twenty-four context/grading regressions pass, including
|
||||
cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
|
||||
lists that reject parent lookups, duplicates and nested representations. URI
|
||||
prefix coverage must come from its cited page, even if an uncited page has the
|
||||
@@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver
|
||||
rejects wrong status, next offset or exhaustion. Unmodified exhausted and
|
||||
continuable URI pages still pass. The grader also requires each cited receipt's
|
||||
subject digest and local path to identify the reference capture, and uses the
|
||||
reference page's verified coverage. Twenty-three grading/context regressions pass,
|
||||
reference page's verified coverage. Twenty-four grading/context regressions pass,
|
||||
including missing or malformed subjects and faithful but inconsistent coverage;
|
||||
all four original model answers still pass factual regrading. Strict resource
|
||||
grades remain false where terminal whole-scope counters are unknown.
|
||||
@@ -194,6 +194,13 @@ them. The observation producer rejects missing, null or malformed response IDs
|
||||
before recording either capture or page receipts. Private real-reply injections
|
||||
exercise these ID failures alongside normal complete/summary controls; they do
|
||||
not establish malformed provider behavior.
|
||||
Completed-call arguments stay with their producing tool and receipt in the
|
||||
grader's identity index. Cited envelope parameters and actual call arguments
|
||||
must match the selected path or retained parent/view, including delegated
|
||||
summary parameters. The observation producer validates both summary and URI
|
||||
views and envelope parameters at their common boundary. Private real-reply
|
||||
faults reproduce missing-parameter/summary-view acceptance before the fix and
|
||||
are rejected afterward; normal complete/summary controls still pass.
|
||||
Negative regressions reproduce
|
||||
prior crashes and false passes; all original real-model factual grades still
|
||||
pass without model reruns. The SDK workflow is unchanged by these grader fixes.
|
||||
|
||||
@@ -121,6 +121,7 @@ def main():
|
||||
reference = read(args.reference / 'answer.json') if args.reference else None
|
||||
observed_ids = set()
|
||||
producer_results = {}
|
||||
pcap_calls = []
|
||||
tool_errors = []
|
||||
with (args.trial / 'events.jsonl').open() as event_lines:
|
||||
for line in event_lines:
|
||||
@@ -135,7 +136,10 @@ def main():
|
||||
for text in strings(result):
|
||||
observed_ids.update(re.findall(r'ev_[a-f0-9]{64}', text))
|
||||
if parsed is not None:
|
||||
producer_results.setdefault(item['tool'], []).append(parsed)
|
||||
if args.domain == 'pcap':
|
||||
pcap_calls.append((item['tool'], item.get('arguments'), parsed))
|
||||
else:
|
||||
producer_results.setdefault(item['tool'], []).append(parsed)
|
||||
else:
|
||||
tool_errors.append({'tool': item['tool'], 'arguments': item['arguments'], 'response': parsed if parsed is not None else result})
|
||||
if parsed is not None and parsed.get('error', {}).get('code') == 'resource_constraint':
|
||||
@@ -182,23 +186,22 @@ def main():
|
||||
(True, json.dumps(reference['capture'], sort_keys=True)))
|
||||
prefix = answer.get('http_request_uris_first2')
|
||||
records = {}
|
||||
for tool, values in producer_results.items():
|
||||
for value in values:
|
||||
if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str):
|
||||
continue
|
||||
identity = value['evidence_id']
|
||||
normalized = value.get('normalized_result')
|
||||
supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
|
||||
if supported and normalized.get('kind') == 'summary':
|
||||
supported = isinstance(normalized.get('summary'), dict)
|
||||
elif supported and normalized.get('kind') == 'pcap':
|
||||
supported = isinstance(normalized.get('page'), dict)
|
||||
entry = (tool, value) if supported else None
|
||||
if identity not in records:
|
||||
records[identity] = entry
|
||||
elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True):
|
||||
# An ID with conflicting producers or envelopes is ambiguous.
|
||||
records[identity] = None
|
||||
for tool, arguments, value in pcap_calls:
|
||||
if not isinstance(value, dict) or not isinstance(value.get('evidence_id'), str):
|
||||
continue
|
||||
identity = value['evidence_id']
|
||||
normalized = value.get('normalized_result')
|
||||
supported = isinstance(normalized, dict) and isinstance(value.get('limitations'), list)
|
||||
if supported and normalized.get('kind') == 'summary':
|
||||
supported = isinstance(normalized.get('summary'), dict)
|
||||
elif supported and normalized.get('kind') == 'pcap':
|
||||
supported = isinstance(normalized.get('page'), dict)
|
||||
entry = (tool, arguments, value) if supported else None
|
||||
if identity not in records:
|
||||
records[identity] = entry
|
||||
elif json.dumps(records[identity], sort_keys=True) != json.dumps(entry, sort_keys=True):
|
||||
# An ID with conflicting producers or envelopes is ambiguous.
|
||||
records[identity] = None
|
||||
observations = answer.get('delivery_observations')
|
||||
canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations
|
||||
if isinstance(prefix, dict):
|
||||
@@ -209,7 +212,7 @@ def main():
|
||||
last = canonical_observations[-1] if isinstance(canonical_observations, list) and canonical_observations else None
|
||||
identity = last.get('evidence_id') if isinstance(last, dict) else None
|
||||
entry = records.get(identity) if isinstance(identity, str) else None
|
||||
record = entry[1] if entry is not None else None
|
||||
record = entry[2] if entry is not None else None
|
||||
page = record['normalized_result'] if record is not None else {}
|
||||
checks['pcap_prefix_coverage_observed'] = (
|
||||
page.get('kind') == 'pcap'
|
||||
@@ -226,7 +229,7 @@ def main():
|
||||
for observation in canonical_observations:
|
||||
identity = observation.get('evidence_id') if isinstance(observation, dict) else None
|
||||
entry = records.get(identity) if isinstance(identity, str) else None
|
||||
record = entry[1] if entry is not None else None
|
||||
record = entry[2] if entry is not None else None
|
||||
if record is None:
|
||||
expected_observations.append(None)
|
||||
continue
|
||||
@@ -256,13 +259,26 @@ def main():
|
||||
and len(set(identities)) == len(identities)
|
||||
)
|
||||
entries = [records.get(identity) if isinstance(identity, str) else None for identity in identities]
|
||||
cited = [entry[1] if entry is not None else None for entry in entries]
|
||||
cited = [entry[2] if entry is not None else None for entry in entries]
|
||||
expected_tools = ['inspect_pcap_capture'] + (['inspect_analysis_view'] if len(reference['delivery_observations']) == 2 else [])
|
||||
check('pcap_cited_producer_tools', [entry[0] if entry is not None else None for entry in entries], expected_tools)
|
||||
# Summary delivery delegates to a retained view; its envelope operation
|
||||
# is inspect_analysis_view even though the requested tool is inspect_pcap_capture.
|
||||
expected_operations = ['inspect_pcap_capture'] if len(expected_tools) == 1 else ['inspect_analysis_view'] * 2
|
||||
check('pcap_cited_operations', [record.get('operation') if isinstance(record, dict) else None for record in cited], expected_operations)
|
||||
path = reference['capture']['target']['path']
|
||||
expected_arguments = [{'path': path}]
|
||||
expected_parameters = [{'path': path}]
|
||||
if len(expected_tools) == 2:
|
||||
parent = cited[0]['normalized_result'].get('parent_evidence_id') if isinstance(cited[0], dict) else None
|
||||
views = [observation.get('view') for observation in reference['delivery_observations']]
|
||||
expected_arguments = [{'path': path, 'detail': 'summary'},
|
||||
{'source': {'kind': 'retained-evidence', 'evidence_id': parent}, 'view': views[-1]}]
|
||||
expected_parameters = [{'parent_evidence_id': parent, 'view': view} for view in views]
|
||||
check('pcap_cited_call_arguments', json.dumps([entry[1] if entry is not None else None for entry in entries], sort_keys=True),
|
||||
json.dumps(expected_arguments, sort_keys=True))
|
||||
check('pcap_cited_parameters', json.dumps([record.get('parameters') if isinstance(record, dict) else None for record in cited], sort_keys=True),
|
||||
json.dumps(expected_parameters, sort_keys=True))
|
||||
checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
|
||||
isinstance(record, dict)
|
||||
and isinstance(record.get('subject'), dict)
|
||||
@@ -323,6 +339,8 @@ def main():
|
||||
checks['pcap_cited_parents_linked'] = False
|
||||
checks['pcap_cited_producer_tools'] = False
|
||||
checks['pcap_cited_operations'] = False
|
||||
checks['pcap_cited_call_arguments'] = False
|
||||
checks['pcap_cited_parameters'] = False
|
||||
elif domain == 'flutter-aot':
|
||||
for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']:
|
||||
check(key, answer.get(key), reference[key])
|
||||
|
||||
@@ -98,6 +98,8 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
'page': {'facet': 'http-uris', 'items': prefix}, 'coverage': coverage,
|
||||
'source_coverage': 'complete', 'artifact': capture['target'],
|
||||
'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}}
|
||||
for record in [summary, page]:
|
||||
record['parameters'] = {'parent_evidence_id': parent, 'view': copy.deepcopy(record['normalized_result']['view'])}
|
||||
observations = [self.observation(value) for value in [summary, page]]
|
||||
reference = {'artifact_sha256': 'd' * 64, 'capture': capture,
|
||||
'http_request_uris_first2': prefix, 'delivery_observations': observations}
|
||||
@@ -113,7 +115,7 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
'parent_evidence_id': value['parent_evidence_id'], 'coverage': value['coverage'],
|
||||
'unknowns': value['unknowns'], 'normalized_limitations': value['limitations']}
|
||||
|
||||
def grade(self, answer, records, reference, producer_tools=None):
|
||||
def grade(self, answer, records, reference, producer_tools=None, producer_arguments=None):
|
||||
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory);trial = root / 'trial';trial.mkdir()
|
||||
@@ -121,10 +123,23 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
('trial-settings.json', {}), ('cleanup.json', {'profile_removed': True})]:
|
||||
(trial / name).write_text(json.dumps(value))
|
||||
(trial / 'answer.txt').write_text(json.dumps(answer))
|
||||
events = [{'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
|
||||
'tool': producer_tools[index] if producer_tools is not None else
|
||||
('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture'),
|
||||
'status': 'completed', 'result': {'structuredContent': value}}} for index, value in enumerate(records)]
|
||||
events = []
|
||||
for index, value in enumerate(records):
|
||||
tool = producer_tools[index] if producer_tools is not None else \
|
||||
('inspect_analysis_view' if value['normalized_result'].get('kind') == 'pcap' else 'inspect_pcap_capture')
|
||||
arguments = {'path': reference['capture']['target']['path']}
|
||||
if tool == 'inspect_analysis_view':
|
||||
normalized = value.get('normalized_result')
|
||||
parent = normalized.get('parent_evidence_id') if isinstance(normalized, dict) else None
|
||||
arguments = {'source': {'kind': 'retained-evidence', 'evidence_id': parent},
|
||||
'view': reference['delivery_observations'][-1].get('view')}
|
||||
elif len(reference['delivery_observations']) == 2:
|
||||
arguments['detail'] = 'summary'
|
||||
if producer_arguments is not None:
|
||||
arguments = producer_arguments[index]
|
||||
events.append({'type': 'item.completed', 'item': {'type': 'mcp_tool_call',
|
||||
'tool': tool, 'arguments': arguments, 'status': 'completed',
|
||||
'result': {'structuredContent': value}}})
|
||||
(trial / 'events.jsonl').write_text(''.join(json.dumps(value) + '\n' for value in events))
|
||||
source = root / 'reference';source.mkdir();(source / 'answer.json').write_text(json.dumps(reference))
|
||||
resource = root / 'resource.log'
|
||||
@@ -191,12 +206,20 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three']
|
||||
source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'}
|
||||
receipt = {'evidence_id': 'ev_' + 'a' * 64, 'operation': 'inspect_pcap_capture',
|
||||
'parameters': {'path': source['target']['path']},
|
||||
'subject': copy.deepcopy(records[0]['subject']), 'evidence_links': [],
|
||||
'limitations': ['source'], 'normalized_result': source}
|
||||
reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'],
|
||||
'limitations': receipt['limitations'], 'coverage': source['coverage']}]
|
||||
answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]})
|
||||
self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
|
||||
for parameters in [None, {}, {'path': '/fixture/other.pcapng'}]:
|
||||
changed = copy.deepcopy(receipt);changed['parameters'] = parameters
|
||||
report = self.grade(answer, [changed], reference)
|
||||
self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
|
||||
self.assertFalse(report['all_factual_checks_pass'])
|
||||
self.assertFalse(self.grade(answer, [receipt], reference,
|
||||
producer_arguments=[{'path': '/fixture/other.pcapng'}])['all_factual_checks_pass'])
|
||||
self.assertFalse(self.grade(answer, [receipt], reference, ['inspect_analysis_view'])['all_factual_checks_pass'])
|
||||
wrong_operation = copy.deepcopy(receipt);wrong_operation['operation'] = 'inspect_analysis_view'
|
||||
self.assertFalse(self.grade(answer, [wrong_operation], reference)['all_factual_checks_pass'])
|
||||
@@ -275,6 +298,28 @@ class PcapGraderWorkflowTests(unittest.TestCase):
|
||||
self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
|
||||
self.assertFalse(report['all_factual_checks_pass'])
|
||||
|
||||
def test_cited_view_parameters_and_call_arguments_bind_actual_requests(self):
|
||||
for index in [0, 1]:
|
||||
for parameters in [None, {}, {'parent_evidence_id': 'ev_' + 'f' * 64,
|
||||
'view': {'kind': 'summary'}}]:
|
||||
with self.subTest(index=index, parameters=parameters):
|
||||
answer, records, reference = self.fixture()
|
||||
records[index]['parameters'] = parameters
|
||||
report = self.grade(answer, records, reference)
|
||||
self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
|
||||
self.assertFalse(report['factual_checks']['pcap_cited_parameters'])
|
||||
self.assertFalse(report['all_factual_checks_pass'])
|
||||
for wrong in [None, {}, {'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'f' * 64},
|
||||
'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}},
|
||||
{'source': {'kind': 'retained-evidence', 'evidence_id': 'ev_' + 'a' * 64},
|
||||
'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': False, 'limit': 2}}]:
|
||||
with self.subTest(arguments=wrong):
|
||||
answer, records, reference = self.fixture()
|
||||
report = self.grade(answer, records, reference,
|
||||
producer_arguments=[{'path': reference['capture']['target']['path'], 'detail': 'summary'}, wrong])
|
||||
self.assertFalse(report['factual_checks']['pcap_cited_call_arguments'])
|
||||
self.assertFalse(report['all_factual_checks_pass'])
|
||||
|
||||
def test_malformed_receipt_envelopes_write_failed_facts_instead_of_crashing(self):
|
||||
tools = ['inspect_pcap_capture', 'inspect_analysis_view']
|
||||
for index in [0, 1]:
|
||||
|
||||
Reference in New Issue
Block a user