fix(experiments): validate PCAP coverage and cited artifact subjects

Verify complete page coverage at the SDK measurement boundary. Bind every cited Evidence subject digest and local path to the reference capture in the model grader, and reuse the reference page coverage. Add failing-then-passing regressions, real SDK consumer fault/control checks, and factual regrading of all four stored model answers without model reruns. Original measurements and resource uncertainty stay intact.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-14 02:40:31 +08:00
committed by morluto
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent e5abf4acf4
commit d688507563
4 changed files with 70 additions and 5 deletions
+10 -2
View File
@@ -572,6 +572,15 @@ const inspectPcapTarget = async () => {
view,
});
const page = normalized(selected);
const total = capture.retained_counts.http_request_uris;
const examined = Math.min(view.limit, total);
const expectedCoverage = {
status: "page",
examined,
total,
next_offset: examined < total ? examined : null,
exhausted: examined >= total,
};
if (
page.parent_evidence_id !== n.parent_evidence_id ||
page.parent_operation !== "inspect_pcap_capture" ||
@@ -579,8 +588,7 @@ const inspectPcapTarget = async () => {
!isDeepStrictEqual(page.view, view) ||
page.page.facet !== "http-uris" ||
page.source_coverage !== capture.coverage ||
page.coverage.total !== capture.retained_counts.http_request_uris ||
page.coverage.examined !== page.page.items.length ||
!isDeepStrictEqual(page.coverage, expectedCoverage) ||
page.artifact.path !== capture.target.path ||
page.artifact.sha256 !== capture.target.sha256
)
@@ -153,6 +153,17 @@ without writing an answer, and an unmodified real reply still passes. This
checks receipt uniqueness before recording observations; it does not establish
that a provider actually returned duplicate IDs.
The page guard checks the complete expected coverage object, including status,
continuation and exhaustion. Private actual-reply injections demonstrate that
the preceding driver accepts inconsistent coverage, while the corrected driver
rejects wrong status, next offset or exhaustion. Unmodified exhausted and
continuable URI pages still pass. The grader also requires each cited receipt's
subject digest and local path to identify the reference capture, and uses the
reference page's verified coverage. Seventeen grading/context regressions pass,
including missing or malformed subjects and faithful but inconsistent coverage;
all four original model answers still pass factual regrading. Strict resource
grades remain false where terminal whole-scope counters are unknown.
**Decision: keep existing complete defaults; add no production optimization.**
For these compact recordings and this question, summary/page costs more total
input in both real-model pairs. No stable speed or CPU/memory benefit justifies
@@ -238,6 +238,14 @@ def main():
and len(set(identities)) == len(identities)
)
cited = [records.get(identity) if isinstance(identity, str) else None for identity in identities]
checks['pcap_cited_subjects_match_artifact'] = bool(cited) and all(
isinstance(record, dict)
and isinstance(record.get('subject'), dict)
and isinstance(record['subject'].get('digest'), dict)
and record['subject']['digest'].get('sha256') == reference['artifact_sha256']
and record['subject'].get('local_path') == reference['capture']['target']['path']
for record in cited
)
checks['pcap_cited_source_supports_uri_prefix'] = False
checks['pcap_cited_capture'] = False
if all(isinstance(record, dict) for record in cited):
@@ -267,6 +275,7 @@ def main():
and selected_page.get('facet') == 'http-uris'
and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True)
and page.get('source_coverage') == capture.get('coverage')
and page.get('coverage') == reference['delivery_observations'][-1].get('coverage')
and page.get('artifact') == {key: capture['target'][key] for key in ['path', 'sha256']}
)
else:
@@ -275,6 +284,7 @@ def main():
checks['pcap_requested_views'] = False
checks['pcap_cited_source_supports_uri_prefix'] = False
checks['pcap_cited_capture'] = False
checks['pcap_cited_subjects_match_artifact'] = False
elif domain == 'flutter-aot':
for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']:
check(key, answer.get(key), reference[key])
@@ -81,12 +81,13 @@ class PcapGraderWorkflowTests(unittest.TestCase):
'retained_counts': {'http_request_uris': 3}}
prefix = ['http://example.test/one', 'http://example.test/two']
coverage = {'status': 'page', 'examined': 2, 'total': 3, 'next_offset': 2, 'exhausted': False}
summary = {'evidence_id': summary_id, 'limitations': ['source', 'view'],
subject = {'digest': {'sha256': capture['target']['sha256']}, 'local_path': capture['target']['path']}
summary = {'evidence_id': summary_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'],
'normalized_result': {'kind': 'summary', 'view': {'kind': 'summary'},
'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent,
'summary': capture, 'coverage': {'status': 'complete-within-view', 'examined': 1, 'total': 1, 'next_offset': None, 'exhausted': True},
'limitations': ['source', 'view'], 'unknowns': ['unrecorded traffic']}}
page = {'evidence_id': page_id, 'limitations': ['source', 'view'],
page = {'evidence_id': page_id, 'subject': copy.deepcopy(subject), 'limitations': ['source', 'view'],
'normalized_result': {'kind': 'pcap',
'view': {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2},
'parent_operation': 'inspect_pcap_capture', 'parent_evidence_id': parent,
@@ -184,11 +185,18 @@ class PcapGraderWorkflowTests(unittest.TestCase):
source.update({key: [] for key in capture['retained_counts']})
source['http_request_uris'] = reference['http_request_uris_first2'] + ['http://example.test/three']
source['diagnostics'] = {'tshark_stderr': '', 'capinfos_stderr': '\U0001f4a5'}
receipt = {'evidence_id': 'ev_' + 'a' * 64, 'limitations': ['source'], 'normalized_result': source}
receipt = {'evidence_id': 'ev_' + 'a' * 64, 'subject': copy.deepcopy(records[0]['subject']),
'limitations': ['source'], 'normalized_result': source}
reference['delivery_observations'] = [{'evidence_id': receipt['evidence_id'],
'limitations': receipt['limitations'], 'coverage': source['coverage']}]
answer = copy.deepcopy({**reference, 'evidence_ids': [receipt['evidence_id']]})
self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
for subject in [None, {'digest': {'sha256': 'e' * 64}, 'local_path': source['target']['path']}]:
changed = copy.deepcopy(receipt);changed['subject'] = subject
report = self.grade(answer, [changed], reference)
self.assertTrue(report['factual_checks']['pcap_cited_capture'])
self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact'])
self.assertFalse(report['all_factual_checks_pass'])
mutations = [lambda x: x.__setitem__('packet_count', 4),
lambda x: x['client'].__setitem__('version', '2'),
lambda x: x.__setitem__('capture_start', 'changed'),
@@ -206,6 +214,34 @@ class PcapGraderWorkflowTests(unittest.TestCase):
answer['capture']['diagnostics_code_units'] = None
self.assertTrue(self.grade(answer, [receipt], reference)['all_factual_checks_pass'])
def test_cited_receipt_subjects_must_identify_the_reference_capture(self):
for index in [0, 1]:
for subject in [None, [], 'artifact', {}, {'digest': None},
{'digest': {'sha256': 'e' * 64}, 'local_path': '/fixture/http.pcapng'},
{'digest': {'sha256': 'd' * 64}, 'local_path': '/fixture/other.pcapng'}]:
with self.subTest(index=index, subject=subject):
answer, records, reference = self.fixture()
records[index]['subject'] = subject
report = self.grade(answer, records, reference)
self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
self.assertFalse(report['factual_checks']['pcap_cited_subjects_match_artifact'])
self.assertFalse(report['all_factual_checks_pass'])
answer, records, reference = self.fixture();records[index].pop('subject')
self.assertFalse(self.grade(answer, records, reference)['all_factual_checks_pass'])
def test_faithful_inconsistent_page_coverage_does_not_support_the_prefix(self):
for field, wrong in [('status', 'complete-within-view'), ('next_offset', None), ('exhausted', True)]:
with self.subTest(field=field):
answer, records, reference = self.fixture()
records[-1]['normalized_result']['coverage'][field] = wrong
answer['delivery_observations'][-1] = self.observation(records[-1])
answer['http_request_uris_first2']['coverage'][field] = wrong
report = self.grade(answer, records, reference)
self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed'])
self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
self.assertFalse(report['all_factual_checks_pass'])
def test_reported_ids_are_only_the_ordered_unique_response_receipts(self):
answer, records, reference = self.fixture()
original = answer['evidence_ids']