test(experiments): compare source facts across equivalent report forms

This commit is contained in:
N0zoM1z0
2026-10-11 18:33:39 +00:00
parent 22326e8199
commit c11a75cbe9
3 changed files with 111 additions and 7 deletions
+7
View File
@@ -137,6 +137,13 @@ signature with that field omitted fails this stronger assertion.
Firmware's run-local region UUIDs remain in answers but are excluded from parity
against an independently acquired report. Classification evidence, explicit
unknowns and coverage remain part of the requested fact checks.
The question comparison accepts grouped original path records, extra known
coverage counts and explicitly named firmware length/coverage report fields.
Derived extent ends must agree with offset plus reported length. It verifies
engine version and executable digest rather than display capitalization; a manual
review is still needed for limitations and claims beyond these assertions.
Grades record the grader source digest, and earlier strict grades are retained
when the comparison changes. These are report parsers, not REA API aliases.
The supplied feedback is a research lead. Its Ghidra timing claim also changed
auto-analysis options, so it does not isolate Code Mode's effect. Relevant primary
+61 -7
View File
@@ -1,6 +1,7 @@
"""Check requested JSON facts and real Evidence references against a replay oracle."""
import argparse
import hashlib
import json
from pathlib import Path
@@ -36,27 +37,75 @@ def requested_facts(value, task):
'artifact_sha256', 'entry_address', 'containing_procedure',
'procedure', 'pseudocode_prefix', 'caller_count', 'callee_count']}
if task == 'electron-inventory':
return {field: value.get(field) for field in [
facts = {field: value.get(field) for field in [
'artifact_sha256', 'manifest', 'entry_kind_counts', 'requested_paths',
'integrity_contradictions', 'coverage', 'unexplored_branches']}
paths = facts['requested_paths']
if isinstance(paths, dict) and set(paths) == {'package.json', 'main.js', 'preload.js'}:
if all(isinstance(rows, list) and all(isinstance(row, dict) and
row.get('logical_path') == name for row in rows)
for name, rows in paths.items()):
paths = [row for rows in paths.values() for row in rows]
if isinstance(paths, list) and all(isinstance(row, dict) and
isinstance(row.get('logical_path'), str) for row in paths):
paths = sorted(paths, key=lambda row: row['logical_path'])
facts['requested_paths'] = paths
contradictions = facts['integrity_contradictions']
if isinstance(contradictions, dict) and set(contradictions) == {'inspection', 'inventory'}:
if all(rows == [] for rows in contradictions.values()):
facts['integrity_contradictions'] = []
branches = facts['unexplored_branches']
if isinstance(branches, list) and all(isinstance(row, dict) for row in branches):
# Branch identity includes the source Evidence identity. Compare its
# reported meaning here and verify its actual source reference below.
facts['unexplored_branches'] = [{key: row.get(key) for key in [
'reason', 'detail', 'next_probe']} for row in branches]
return facts
if task == 'firmware':
result = value.get('result')
if not isinstance(result, dict):
result = {}
engine = result.get('engine', result.get('provider'))
if isinstance(engine, dict):
engine = {'version': engine.get('version'),
'executable_sha256': engine.get('executable_sha256',
engine.get('engine_artifact_sha256'))}
regions = result.get('regions')
if isinstance(regions, list) and all(isinstance(row, dict) for row in regions):
# Binwalk's UUID is run-local; preserve it in the answer but do not
# compare it with the independent replay's different occurrence ID.
regions = [{key: row.get(key) for key in [
'offset', 'reported_size', 'signature', 'description',
'reported_confidence', 'size_basis']} for row in regions]
selected_regions = []
for row in regions:
selected = {key: row.get(key) for key in [
'offset', 'signature', 'description', 'reported_confidence']}
if 'reported_size' in row:
size = row['reported_size']
elif 'reported_extent_bytes' in row:
size = row['reported_extent_bytes']
else:
extent = row.get('reported_extent')
size = None
if isinstance(extent, dict) and extent.get('offset') == row.get('offset'):
length = extent.get('length')
offset = row.get('offset')
if isinstance(length, int) and isinstance(offset, int) and \
extent.get('end_exclusive') == offset + length:
size = length
selected['reported_size'] = size
selected_regions.append(selected)
regions = selected_regions
return {'artifact_sha256': value.get('artifact_sha256'),
'engine': result.get('engine'), 'input_size': result.get('input_size'),
'regions': regions, 'coverage': result.get('coverage')}
'engine': engine, 'input_size': result.get('input_size', result.get('input_size_bytes')),
'regions': regions, 'coverage': result.get('coverage', result.get('examined_coverage'))}
if task == 'managed':
def selected(field, keys):
row = value.get(field)
return {key: row.get(key) for key in keys} if isinstance(row, dict) else row
coverage = value.get('coverage')
if isinstance(coverage, dict):
coverage = {key: {field: row.get(field) for field in ['state', 'issues']}
if isinstance(row, dict) else row
for key, row in coverage.items() if key in ['managed_artifact', 'native_boundaries']}
return {
'artifact_sha256': value.get('artifact_sha256'),
'assembly': selected('assembly', ['name', 'version']),
@@ -64,7 +113,7 @@ def requested_facts(value, task):
'classification': value.get('classification'),
'target_frameworks': value.get('target_frameworks'),
'native_boundaries': selected('native_boundaries', ['module_refs', 'pinvoke_imports']),
'coverage': value.get('coverage'),
'coverage': coverage,
}
facts = {field: value.get(field) for field in [
'artifact_sha256', 'package_name', 'version_name', 'version_code', 'matches', 'onCreate']}
@@ -150,11 +199,16 @@ def main():
observed |= ids
unreadable += skipped
cited = evidence_ids(answer.get('evidence_ids'))
if args.task == 'electron-inventory':
for branch in answer.get('unexplored_branches', []):
if isinstance(branch, dict):
cited |= evidence_ids(branch.get('evidence_id'))
if not cited:
failures.append({'kind': 'missing_evidence_references'})
if cited - observed:
failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)})
result = {'task': args.task,
'grader_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy',
'passed': not failures, 'fact_assertions': len(expected),
'answer_format': answer_format,
+43
View File
@@ -252,3 +252,46 @@ sampled peak of 1399996416 bytes and zero observed limit events. Declarations ar
cached per exact tool name for this immutable live-catalog connection; repeated
discovery does not regenerate them. No provider implementation or published
REA API has changed.
## Typed-discovery serial agent checkpoint
All rows below use real Codex CLI 0.161.0, Terra high, the same rendered question
within each pair, and the same provider settings. The bridge/runtime/runner source
fingerprints match commit `22326e819`; model-written original arguments can differ.
Comparisons include required mode-specific instructions and are not a controlled
causal decomposition of inference, caching and provider time.
| Case | Profile | Wall seconds | Input | Cached | Uncached | Output | MCP calls | Fact/reference check |
| ------------------ | ------- | -----------: | -----: | -----: | -------: | -----: | --------: | --------------------------------- |
| android | direct | 67.329 | 107007 | 83712 | 23295 | 1670 | 3 | pass |
| android | bridge | 48.231 | 113256 | 91648 | 21608 | 1456 | 3 | pass |
| managed | bridge | 73.145 | 116974 | 92672 | 24302 | 2065 | 3 | classification field omitted |
| managed | direct | 33.15 | 108096 | 75776 | 32320 | 2315 | 2 | facts match; reference unverified |
| electron-inventory | direct | 89.494 | 266121 | 238848 | 27273 | 3215 | 3 | pass |
| electron-inventory | bridge | 145.577 | 462809 | 407040 | 55769 | 5624 | 8 | pass |
| firmware | bridge | 81.944 | 152886 | 126976 | 25910 | 2192 | 3 | pass |
| firmware | direct | 30.161 | 103224 | 74752 | 28472 | 1761 | 1 | pass |
The managed bridge omits the producer classification container field (`pe`) while
preserving status/runtime/implementation/architecture and all classification
evidence. Its strict complete-classification assertion fails; this is an omission,
not an incorrect architecture claim. Normal MCP matches requested facts but its
large-result JSONL truncation again prevents automated verification of one ID.
The initial grader treated grouped path records, extra known coverage counts and
an empty per-source contradiction object as mismatches. Task-specific comparison
now accepts those lossless report forms. Firmware compares observed offset/length
facts, checks any supplied derived end arithmetically, and verifies engine version
and executable digest. Run-local UUIDs and differently named display fields are
not compared as if they were stable source identity. Original strict grades and
pre-equivalence grades remain outside Git, and new grades include a grader digest.
These report-format comparisons do not add aliases to any REA tool contract.
Obsidian bridge discovery takes six searches and two executions. The first returns
the entire nested inventory; the second repeats it and computes a focused answer.
The live nested Evidence schema describes normalized_result as generic JSON, so
derived types alone cannot teach its concrete manifest/occurrences layout. Its
recursive record walk happens to match this corpus, but format-specific field
selection is the stronger recipe. Both model profiles choose record-and-continue
for integrity; deterministic inventory replays used fail. No contradictions were
observed, and the exact caller-selected arguments are retained.