test(perf): compare complete and retained-page PCAP workflows
Pin official public HTTP recordings and a fixed metadata/URI question. Preserve each delivery scope, accept equivalent explicit model labels without losing contradictions, and verify related grading regressions. This adds experiments only; production and complete defaults stay unchanged. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
committed by
morluto
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
parent
0bc422d7be
commit
fb10bdc546
@@ -180,7 +180,7 @@ def main():
|
||||
'inspect_android_package', 'search_android_classes',
|
||||
'inspect_android_class', 'inspect_android_method',
|
||||
'inspect_firmware_regions', 'inspect_managed_artifact',
|
||||
'inspect_sqlite_database',
|
||||
'inspect_sqlite_database', 'inspect_pcap_capture',
|
||||
'inspect_managed_members', 'inspect_managed_native_boundaries',
|
||||
'inspect_artifact', 'inspect_wasm_artifact',
|
||||
'inspect_evm_interface', 'get_evidence_bundle', 'export_evidence_bundle',
|
||||
|
||||
@@ -70,6 +70,32 @@ for (const [side, answer] of [
|
||||
value: answer.evidence_ids,
|
||||
});
|
||||
}
|
||||
// PCAP delivery observations preserve each actual receipt's identity, limitations
|
||||
// and view coverage. Summary/page scopes differ from complete source scope;
|
||||
// retain them as run-specific facts rather than equating page exhaustion with
|
||||
// capture coverage. The common capture facts and URI prefix still must match.
|
||||
if (
|
||||
["wireshark-http.cap", "wireshark-http_redirects.pcapng"].includes(
|
||||
left.artifact,
|
||||
)
|
||||
) {
|
||||
for (const [side, answer] of [
|
||||
["left", a],
|
||||
["right", b],
|
||||
]) {
|
||||
if (
|
||||
!Array.isArray(answer.delivery_observations) ||
|
||||
!answer.delivery_observations.length
|
||||
)
|
||||
throw new Error("Missing original PCAP delivery observations");
|
||||
runSpecific.push({
|
||||
side,
|
||||
path: ["delivery_observations"],
|
||||
value: answer.delivery_observations,
|
||||
});
|
||||
delete answer.delivery_observations;
|
||||
}
|
||||
}
|
||||
// Binwalk assigns a new opaque region UUID in each invocation. Preserve both
|
||||
// original lookup values in this report; compare the observed region facts
|
||||
// separately. The product and original replies retain those IDs unchanged.
|
||||
|
||||
@@ -263,6 +263,34 @@
|
||||
"url": "https://raw.githubusercontent.com/qgis/QGIS/e1223b8e9e71fc50ee995ed556f711c54ccc101b/resources/data/world_map.gpkg",
|
||||
"derivation": "unmodified official resource file; streaming Git blob and SHA-256 verification"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wireshark-http.cap",
|
||||
"file": "http.cap",
|
||||
"bytes": 25803,
|
||||
"sha256": "25a72bdf10339f2c29916920c8b9501d294923108de8f29b19aba7cc001ab60d",
|
||||
"provenance": {
|
||||
"project": "Wireshark public sample captures",
|
||||
"source_page": "https://wiki.wireshark.org/SampleCaptures",
|
||||
"url": "https://wiki.wireshark.org/uploads/27707187aeb30df68e70c8fb9d614981/http.cap",
|
||||
"resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/27707187aeb30df68e70c8fb9d614981/http.cap",
|
||||
"upstream_expected_digest": null,
|
||||
"derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "wireshark-http_redirects.pcapng",
|
||||
"file": "http_redirects.pcapng",
|
||||
"bytes": 47804,
|
||||
"sha256": "ff432a6b71d06cd20720f7763e584014edd0a43c3b55981c682d1b7640207abc",
|
||||
"provenance": {
|
||||
"project": "Wireshark public sample captures",
|
||||
"source_page": "https://wiki.wireshark.org/SampleCaptures",
|
||||
"url": "https://wiki.wireshark.org/uploads/__moin_import__/attachments/SampleCaptures/http_redirects.pcapng",
|
||||
"resolved_url": "https://gitlab.com/wireshark/wireshark/-/wikis/uploads/__moin_import__/attachments/SampleCaptures/http_redirects.pcapng",
|
||||
"upstream_expected_digest": null,
|
||||
"derivation": "Unmodified capture published by official Wireshark wiki. Local digest pins downloaded bytes; no independently published upstream digest is claimed."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -523,7 +523,115 @@ const inspectEvmTarget = async () => {
|
||||
};
|
||||
};
|
||||
|
||||
const pcapObservation = (evidence) => {
|
||||
const value = normalized(evidence);
|
||||
return {
|
||||
evidence_id: evidence.evidence_id,
|
||||
limitations: evidence.limitations,
|
||||
...Object.fromEntries(
|
||||
[
|
||||
"kind",
|
||||
"view",
|
||||
"parent_evidence_id",
|
||||
"coverage",
|
||||
"limitations",
|
||||
"unknowns",
|
||||
]
|
||||
.filter((key) => Object.hasOwn(value, key) && key !== "limitations")
|
||||
.map((key) => [key, value[key]]),
|
||||
),
|
||||
...(Object.hasOwn(value, "limitations")
|
||||
? { normalized_limitations: value.limitations }
|
||||
: {}),
|
||||
};
|
||||
};
|
||||
const inspectPcapTarget = async () => {
|
||||
const input = {
|
||||
path: values.path,
|
||||
...(values.delivery === "summary" ? { detail: "summary" } : {}),
|
||||
};
|
||||
const definition = catalogDefinitions.get("inspect_pcap_capture");
|
||||
const ajv = new Ajv2020({ strict: false, validateFormats: false });
|
||||
if (
|
||||
!definition ||
|
||||
!ajv.validateSchema(definition.inputSchema) ||
|
||||
!ajv.validate(definition.inputSchema, input)
|
||||
)
|
||||
throw new Error("Selected PCAP request is not advertised by this runtime");
|
||||
const inspected = await call("inspect_pcap_capture", input);
|
||||
const n = normalized(inspected);
|
||||
const observations = [pcapObservation(inspected)];
|
||||
let capture, uris;
|
||||
if (values.delivery === "summary") {
|
||||
if (n.kind !== "summary" || n.parent_operation !== "inspect_pcap_capture")
|
||||
throw new Error("Runtime returned no requested PCAP summary");
|
||||
capture = n.summary;
|
||||
const selected = await call("inspect_analysis_view", {
|
||||
source: { kind: "retained-evidence", evidence_id: n.parent_evidence_id },
|
||||
view: { kind: "pcap", facet: "http-uris", offset: 0, limit: 2 },
|
||||
});
|
||||
const page = normalized(selected);
|
||||
if (
|
||||
page.parent_evidence_id !== n.parent_evidence_id ||
|
||||
page.parent_operation !== "inspect_pcap_capture" ||
|
||||
page.kind !== "pcap" ||
|
||||
page.page.facet !== "http-uris" ||
|
||||
page.source_coverage !== capture.coverage ||
|
||||
page.coverage.total !== capture.retained_counts.http_request_uris ||
|
||||
page.coverage.examined !== page.page.items.length ||
|
||||
page.artifact.path !== capture.target.path ||
|
||||
page.artifact.sha256 !== capture.target.sha256
|
||||
)
|
||||
throw new Error(
|
||||
"PCAP page identity, scope or coverage differs from its parent",
|
||||
);
|
||||
uris = page.page.items;
|
||||
observations.push(pcapObservation(selected));
|
||||
} else {
|
||||
const {
|
||||
endpoints,
|
||||
dns_queries,
|
||||
tls_sni_values,
|
||||
protocol_distribution,
|
||||
http_request_uris,
|
||||
unattributed_dns_addresses,
|
||||
diagnostics,
|
||||
...metadata
|
||||
} = n;
|
||||
capture = {
|
||||
...metadata,
|
||||
retained_counts: {
|
||||
endpoints: endpoints.length,
|
||||
dns_queries: dns_queries.length,
|
||||
tls_sni_values: tls_sni_values.length,
|
||||
protocol_distribution: protocol_distribution.length,
|
||||
http_request_uris: http_request_uris.length,
|
||||
unattributed_dns_addresses: unattributed_dns_addresses.length,
|
||||
},
|
||||
diagnostics_code_units:
|
||||
diagnostics === undefined
|
||||
? null
|
||||
: {
|
||||
tshark: diagnostics.tshark_stderr.length,
|
||||
capinfos: diagnostics.capinfos_stderr.length,
|
||||
},
|
||||
};
|
||||
uris = http_request_uris.slice(0, 2);
|
||||
}
|
||||
if (uris.length !== Math.min(2, capture.retained_counts.http_request_uris))
|
||||
throw new Error("Runtime did not return the requested retained URI prefix");
|
||||
return {
|
||||
artifact_sha256: targetSha(inspected),
|
||||
capture,
|
||||
http_request_uris_first2: uris,
|
||||
delivery_observations: observations,
|
||||
evidence_ids: observations.map((item) => item.evidence_id),
|
||||
};
|
||||
};
|
||||
|
||||
const workflows = {
|
||||
"wireshark-http.cap": inspectPcapTarget,
|
||||
"wireshark-http_redirects.pcapng": inspectPcapTarget,
|
||||
"qgis-world-map.gpkg": async () => {
|
||||
const columns = ["fid", "NAME", "ISO_A3"];
|
||||
const input = {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Statically inspect the supplied saved Wireshark capture at {{artifact_path}} using only registered REA inspect_pcap_capture and inspect_analysis_view. Report its original client/target identity and every returned scalar metadata/count/coverage field exactly, preserving explicit nulls. Include retained_counts for endpoints, dns_queries, tls_sni_values, protocol_distribution, http_request_uris and unattributed_dns_addresses; derive array lengths only when reading complete Evidence. Include diagnostics_code_units as exact UTF-16 lengths of retained capinfos/tshark stderr, or null if diagnostics are unrecorded. In capture, omit inventory arrays and diagnostic text; do not omit their observed counts or change source coverage. Report the first two retained HTTP URIs in original order, or all if fewer; do not request other inventory pages. Report delivery_observations in dependency order for the Evidence used: evidence_id, outer limitations, and each present normalized kind, view, parent_evidence_id, coverage, unknowns and normalized_limitations. Preserve presence, explicit nulls, scalar types and original limitation order. Distinguish source coverage from URI-page exhaustion. Final JSON fields: artifact_sha256, capture, http_request_uris_first2, delivery_observations, evidence_ids. Include only actual response IDs. Saved packet/URI observations do not establish current deployment, process identity, missing traffic, decrypted payloads or current DNS resolution. Do not use shell, host file tools, network, other agents, traffic acquisition or execute recorded targets. Stop once these selected facts are supported.
|
||||
@@ -67,9 +67,28 @@ def scope_with_verified_counts(value, expected_scope, returned_counts):
|
||||
return scope, checks
|
||||
|
||||
|
||||
def pcap_observation_fields(value):
|
||||
"""Equivalent explicit labels are accepted; conflicting duplicates remain failures."""
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
result = dict(value)
|
||||
for label, canonical in {
|
||||
'outer_limitations': 'limitations', 'normalized_kind': 'kind',
|
||||
'normalized_view': 'view', 'normalized_parent_evidence_id': 'parent_evidence_id',
|
||||
'normalized_coverage': 'coverage', 'normalized_unknowns': 'unknowns',
|
||||
'normalized_source_coverage': 'source_coverage',
|
||||
}.items():
|
||||
if label not in result:
|
||||
continue
|
||||
if canonical in result and json.dumps(result[label], sort_keys=True) != json.dumps(result[canonical], sort_keys=True):
|
||||
return {**result, 'conflicting_fields': [label, canonical]}
|
||||
result[canonical] = result.pop(label)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'flutter-aot', 'sqlite', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
|
||||
parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'flutter-aot', 'sqlite', 'pcap', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
|
||||
parser.add_argument('--trial', type=Path, required=True)
|
||||
parser.add_argument('--reference', type=Path)
|
||||
parser.add_argument('--resource-log', type=Path, required=True)
|
||||
@@ -138,6 +157,54 @@ def main():
|
||||
# Missing is not SQL/JSON null; booleans are not numeric counters.
|
||||
check(key, (key in answer, json.dumps(answer.get(key), sort_keys=True)),
|
||||
(True, json.dumps(reference[key], sort_keys=True)))
|
||||
elif domain == 'pcap':
|
||||
check('capture', ('capture' in answer, json.dumps(answer.get('capture'), sort_keys=True)),
|
||||
(True, json.dumps(reference['capture'], sort_keys=True)))
|
||||
prefix = answer.get('http_request_uris_first2')
|
||||
records = {value['evidence_id']: value for values in producer_results.values()
|
||||
for value in values if isinstance(value, dict) and 'evidence_id' in value}
|
||||
if isinstance(prefix, dict):
|
||||
checks['pcap_prefix_fields_supported'] = set(prefix) <= {'items', 'coverage', 'source_coverage'} and 'items' in prefix
|
||||
if 'source_coverage' in prefix:
|
||||
check('pcap_prefix_source_coverage', prefix['source_coverage'], reference['capture']['coverage'])
|
||||
if 'coverage' in prefix:
|
||||
page_coverages = [value['normalized_result']['coverage'] for value in records.values()
|
||||
if value['normalized_result'].get('kind') == 'pcap'
|
||||
and value['normalized_result']['page'].get('facet') == 'http-uris'
|
||||
and value['normalized_result']['page'].get('items') == prefix.get('items')]
|
||||
checks['pcap_prefix_coverage_observed'] = any(json.dumps(prefix['coverage'], sort_keys=True) == json.dumps(value, sort_keys=True) for value in page_coverages)
|
||||
prefix = prefix.get('items')
|
||||
check('http_request_uris_first2', ('http_request_uris_first2' in answer, json.dumps(prefix, sort_keys=True)),
|
||||
(True, json.dumps(reference['http_request_uris_first2'], sort_keys=True)))
|
||||
observations = answer.get('delivery_observations')
|
||||
canonical_observations = [pcap_observation_fields(value) for value in observations] if isinstance(observations, list) else observations
|
||||
expected_observations = []
|
||||
if isinstance(canonical_observations, list):
|
||||
for observation in canonical_observations:
|
||||
record = records.get(observation.get('evidence_id')) if isinstance(observation, dict) else None
|
||||
if record is None:
|
||||
expected_observations.append(None)
|
||||
continue
|
||||
value = record['normalized_result']
|
||||
expected = {'evidence_id': record['evidence_id'], 'limitations': record['limitations']}
|
||||
for key in ['kind', 'view', 'parent_evidence_id', 'coverage', 'unknowns']:
|
||||
if key in value:
|
||||
expected[key] = value[key]
|
||||
if 'limitations' in value:
|
||||
expected['normalized_limitations'] = value['limitations']
|
||||
if 'source_coverage' in observation and 'source_coverage' in value:
|
||||
expected['source_coverage'] = value['source_coverage']
|
||||
expected_observations.append(expected)
|
||||
checks['pcap_delivery_observations_nonempty'] = isinstance(observations, list) and bool(observations)
|
||||
check('pcap_original_delivery_observations', json.dumps(canonical_observations, sort_keys=True),
|
||||
json.dumps(expected_observations, sort_keys=True))
|
||||
if isinstance(canonical_observations, list) and all(isinstance(x, dict) for x in canonical_observations):
|
||||
check('pcap_requested_delivery_chain', [x.get('kind', 'complete') for x in canonical_observations],
|
||||
[x.get('kind', 'complete') for x in reference['delivery_observations']])
|
||||
checks['pcap_observations_have_reported_ids'] = {x.get('evidence_id') for x in canonical_observations} <= reported_ids
|
||||
else:
|
||||
checks['pcap_requested_delivery_chain'] = False
|
||||
checks['pcap_observations_have_reported_ids'] = False
|
||||
elif domain == 'flutter-aot':
|
||||
for key in ['target', 'abi', 'libapp', 'string_pool', 'coverage', 'limitations']:
|
||||
check(key, answer.get(key), reference[key])
|
||||
|
||||
@@ -38,6 +38,38 @@ class ManagedCoverageTests(unittest.TestCase):
|
||||
self.assertNotEqual(scope, expected)
|
||||
|
||||
|
||||
class PcapCoverageTests(unittest.TestCase):
|
||||
def test_explicit_labels_preserve_original_nulls_and_source_scope(self):
|
||||
canonical = runpy.run_path(str(Path(__file__).with_name('review-model.py')))['pcap_observation_fields']
|
||||
original = {'outer_limitations': [], 'normalized_kind': 'pcap',
|
||||
'normalized_coverage': {'examined': 2, 'total': 23, 'next_offset': 2, 'exhausted': False},
|
||||
'normalized_source_coverage': 'partial', 'normalized_parent_evidence_id': None}
|
||||
result = canonical(original)
|
||||
self.assertEqual(result['limitations'], [])
|
||||
self.assertEqual(result['source_coverage'], 'partial')
|
||||
self.assertEqual(result['coverage']['total'], 23)
|
||||
self.assertIsNone(result['parent_evidence_id'])
|
||||
self.assertIn('outer_limitations', original)
|
||||
self.assertNotIn('parent_evidence_id', canonical({}))
|
||||
|
||||
def test_conflicting_duplicates_and_json_type_substitutions_stay_failures(self):
|
||||
canonical = runpy.run_path(str(Path(__file__).with_name('review-model.py')))['pcap_observation_fields']
|
||||
for wrong in ['partial', None]:
|
||||
result = canonical({'normalized_source_coverage': 'complete', 'source_coverage': wrong})
|
||||
self.assertIn('conflicting_fields', result)
|
||||
self.assertEqual(result['normalized_source_coverage'], 'complete')
|
||||
self.assertEqual(result['source_coverage'], wrong)
|
||||
result = canonical({'normalized_coverage': {'examined': 1}, 'coverage': {'examined': True}})
|
||||
self.assertIn('conflicting_fields', result)
|
||||
self.assertEqual(canonical({'outer_limitations': [], 'limitations': []}), {'limitations': []})
|
||||
|
||||
def test_unsupported_claims_are_retained_for_exact_producer_checking(self):
|
||||
canonical = runpy.run_path(str(Path(__file__).with_name('review-model.py')))['pcap_observation_fields']
|
||||
original = {'normalized_kind': 'pcap', 'unrecorded_traffic_absent': True}
|
||||
self.assertIn('unrecorded_traffic_absent', canonical(original))
|
||||
self.assertIsNone(canonical(None))
|
||||
|
||||
|
||||
class ContextCaptureTests(unittest.TestCase):
|
||||
def test_filters_ambient_account_and_nested_usage_state(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
Reference in New Issue
Block a user