test(perf): validate PCAP receipt IDs and exact view types

Reject malformed response identities at observation production and preserve
JSON scalar types when checking requested and cited PCAP views. Keep normal
complete and delegated summary workflows unchanged.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
N0zoM1z0
2026-10-14 02:40:31 +08:00
committed by morluto
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Kaoru0822
parent 6f0237af0c
commit f07446681c
4 changed files with 28 additions and 5 deletions
@@ -524,6 +524,11 @@ const inspectEvmTarget = async () => {
};
const pcapObservation = (evidence) => {
if (
typeof evidence.evidence_id !== "string" ||
!/^ev_[a-f0-9]{64}$/.test(evidence.evidence_id)
)
throw new Error("PCAP Evidence lacks a valid receipt identity");
const value = normalized(evidence);
const target =
value.kind === "summary"
@@ -126,7 +126,7 @@ validated.
Both real CLI complete results match full MCP normalized results, subjects,
parameters, IDs and limitations. Portable CLI summary/URI views match MCP retained
normalized facts and subjects; source forms/caller parameters remain distinct.
Source bytes stay unchanged. Twenty-two context/grading regressions pass, including
Source bytes stay unchanged. Twenty-three context/grading regressions pass, including
cited complete metadata/counts/UTF-16 diagnostics and exact ordered response ID
lists that reject parent lookups, duplicates and nested representations. URI
prefix coverage must come from its cited page, even if an uncited page has the
@@ -159,7 +159,7 @@ the preceding driver accepts inconsistent coverage, while the corrected driver
rejects wrong status, next offset or exhaustion. Unmodified exhausted and
continuable URI pages still pass. The grader also requires each cited receipt's
subject digest and local path to identify the reference capture, and uses the
reference page's verified coverage. Twenty-two grading/context regressions pass,
reference page's verified coverage. Twenty-three grading/context regressions pass,
including missing or malformed subjects and faithful but inconsistent coverage;
all four original model answers still pass factual regrading. Strict resource
grades remain false where terminal whole-scope counters are unknown.
@@ -188,6 +188,12 @@ failed facts with a written report. Coverage uses type-sensitive JSON comparison
to distinguish booleans, integers and floats. Receipt-identity conflict checks
use the same type-sensitive rule, so a scalar type change cannot hide in a
repeated ID. Existing PCAP scope-alias checks already preserve JSON types.
Requested and cited URI views use the same type-sensitive JSON rule, rejecting
boolean or floating-point offsets/limits even when an answer faithfully copies
them. The observation producer rejects missing, null or malformed response IDs
before recording either capture or page receipts. Private real-reply injections
exercise these ID failures alongside normal complete/summary controls; they do
not establish malformed provider behavior.
Negative regressions reproduce
prior crashes and false passes; all original real-model factual grades still
pass without model reruns. The SDK workflow is unchanged by these grader fixes.
@@ -246,8 +246,8 @@ def main():
if isinstance(canonical_observations, list) and all(isinstance(x, dict) for x in canonical_observations):
check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations],
[x.get('kind', 'complete') for x in reference['delivery_observations']])
check('pcap_requested_views', [x.get('view') for x in canonical_observations],
[x.get('view') for x in reference['delivery_observations']])
check('pcap_requested_views', json.dumps([x.get('view') for x in canonical_observations], sort_keys=True),
json.dumps([x.get('view') for x in reference['delivery_observations']], sort_keys=True))
identities = [x.get('evidence_id') for x in canonical_observations]
checks['pcap_observations_have_reported_ids'] = (
isinstance(answer.get('evidence_ids'), list)
@@ -306,7 +306,7 @@ def main():
and isinstance(source.get('parent_evidence_id'), str)
and page.get('parent_evidence_id') == source['parent_evidence_id']
and page.get('parent_operation') == 'inspect_pcap_capture'
and page.get('view') == {'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}
and json.dumps(page.get('view'), sort_keys=True) == json.dumps({'kind': 'pcap', 'facet': 'http-uris', 'offset': 0, 'limit': 2}, sort_keys=True)
and selected_page.get('facet') == 'http-uris'
and json.dumps(selected_page.get('items'), sort_keys=True) == json.dumps(prefix, sort_keys=True)
and page.get('source_coverage') == capture.get('coverage')
@@ -263,6 +263,18 @@ class PcapGraderWorkflowTests(unittest.TestCase):
self.assertTrue(report['factual_checks']['pcap_prefix_coverage_observed'])
self.assertFalse(report['all_factual_checks_pass'])
def test_cited_views_preserve_exact_json_scalar_types(self):
for field, wrong in [('offset', False), ('offset', 0.0), ('limit', 2.0), ('limit', True)]:
with self.subTest(field=field, wrong=wrong):
answer, records, reference = self.fixture()
records[-1]['normalized_result']['view'][field] = wrong
answer['delivery_observations'][-1] = self.observation(records[-1])
report = self.grade(answer, records, reference)
self.assertTrue(report['factual_checks']['pcap_original_delivery_observations'])
self.assertFalse(report['factual_checks']['pcap_requested_views'])
self.assertFalse(report['factual_checks']['pcap_cited_source_supports_uri_prefix'])
self.assertFalse(report['all_factual_checks_pass'])
def test_malformed_receipt_envelopes_write_failed_facts_instead_of_crashing(self):
tools = ['inspect_pcap_capture', 'inspect_analysis_view']
for index in [0, 1]: