test(experiments): compare source facts across equivalent report forms
This commit is contained in:
@@ -137,6 +137,13 @@ signature with that field omitted fails this stronger assertion.
|
||||
Firmware's run-local region UUIDs remain in answers but are excluded from parity
|
||||
against an independently acquired report. Classification evidence, explicit
|
||||
unknowns and coverage remain part of the requested fact checks.
|
||||
The question comparison accepts grouped original path records, extra known
|
||||
coverage counts and explicitly named firmware length/coverage report fields.
|
||||
Derived extent ends must agree with offset plus reported length. It verifies
|
||||
engine version and executable digest rather than display capitalization; a manual
|
||||
review is still needed for limitations and claims beyond these assertions.
|
||||
Grades record the grader source digest, and earlier strict grades are retained
|
||||
when the comparison changes. These are report parsers, not REA API aliases.
|
||||
|
||||
The supplied feedback is a research lead. Its Ghidra timing claim also changed
|
||||
auto-analysis options, so it does not isolate Code Mode's effect. Relevant primary
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Check requested JSON facts and real Evidence references against a replay oracle."""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
@@ -36,27 +37,75 @@ def requested_facts(value, task):
|
||||
'artifact_sha256', 'entry_address', 'containing_procedure',
|
||||
'procedure', 'pseudocode_prefix', 'caller_count', 'callee_count']}
|
||||
if task == 'electron-inventory':
|
||||
return {field: value.get(field) for field in [
|
||||
facts = {field: value.get(field) for field in [
|
||||
'artifact_sha256', 'manifest', 'entry_kind_counts', 'requested_paths',
|
||||
'integrity_contradictions', 'coverage', 'unexplored_branches']}
|
||||
paths = facts['requested_paths']
|
||||
if isinstance(paths, dict) and set(paths) == {'package.json', 'main.js', 'preload.js'}:
|
||||
if all(isinstance(rows, list) and all(isinstance(row, dict) and
|
||||
row.get('logical_path') == name for row in rows)
|
||||
for name, rows in paths.items()):
|
||||
paths = [row for rows in paths.values() for row in rows]
|
||||
if isinstance(paths, list) and all(isinstance(row, dict) and
|
||||
isinstance(row.get('logical_path'), str) for row in paths):
|
||||
paths = sorted(paths, key=lambda row: row['logical_path'])
|
||||
facts['requested_paths'] = paths
|
||||
contradictions = facts['integrity_contradictions']
|
||||
if isinstance(contradictions, dict) and set(contradictions) == {'inspection', 'inventory'}:
|
||||
if all(rows == [] for rows in contradictions.values()):
|
||||
facts['integrity_contradictions'] = []
|
||||
branches = facts['unexplored_branches']
|
||||
if isinstance(branches, list) and all(isinstance(row, dict) for row in branches):
|
||||
# Branch identity includes the source Evidence identity. Compare its
|
||||
# reported meaning here and verify its actual source reference below.
|
||||
facts['unexplored_branches'] = [{key: row.get(key) for key in [
|
||||
'reason', 'detail', 'next_probe']} for row in branches]
|
||||
return facts
|
||||
if task == 'firmware':
|
||||
result = value.get('result')
|
||||
if not isinstance(result, dict):
|
||||
result = {}
|
||||
engine = result.get('engine', result.get('provider'))
|
||||
if isinstance(engine, dict):
|
||||
engine = {'version': engine.get('version'),
|
||||
'executable_sha256': engine.get('executable_sha256',
|
||||
engine.get('engine_artifact_sha256'))}
|
||||
regions = result.get('regions')
|
||||
if isinstance(regions, list) and all(isinstance(row, dict) for row in regions):
|
||||
# Binwalk's UUID is run-local; preserve it in the answer but do not
|
||||
# compare it with the independent replay's different occurrence ID.
|
||||
regions = [{key: row.get(key) for key in [
|
||||
'offset', 'reported_size', 'signature', 'description',
|
||||
'reported_confidence', 'size_basis']} for row in regions]
|
||||
selected_regions = []
|
||||
for row in regions:
|
||||
selected = {key: row.get(key) for key in [
|
||||
'offset', 'signature', 'description', 'reported_confidence']}
|
||||
if 'reported_size' in row:
|
||||
size = row['reported_size']
|
||||
elif 'reported_extent_bytes' in row:
|
||||
size = row['reported_extent_bytes']
|
||||
else:
|
||||
extent = row.get('reported_extent')
|
||||
size = None
|
||||
if isinstance(extent, dict) and extent.get('offset') == row.get('offset'):
|
||||
length = extent.get('length')
|
||||
offset = row.get('offset')
|
||||
if isinstance(length, int) and isinstance(offset, int) and \
|
||||
extent.get('end_exclusive') == offset + length:
|
||||
size = length
|
||||
selected['reported_size'] = size
|
||||
selected_regions.append(selected)
|
||||
regions = selected_regions
|
||||
return {'artifact_sha256': value.get('artifact_sha256'),
|
||||
'engine': result.get('engine'), 'input_size': result.get('input_size'),
|
||||
'regions': regions, 'coverage': result.get('coverage')}
|
||||
'engine': engine, 'input_size': result.get('input_size', result.get('input_size_bytes')),
|
||||
'regions': regions, 'coverage': result.get('coverage', result.get('examined_coverage'))}
|
||||
if task == 'managed':
|
||||
def selected(field, keys):
|
||||
row = value.get(field)
|
||||
return {key: row.get(key) for key in keys} if isinstance(row, dict) else row
|
||||
coverage = value.get('coverage')
|
||||
if isinstance(coverage, dict):
|
||||
coverage = {key: {field: row.get(field) for field in ['state', 'issues']}
|
||||
if isinstance(row, dict) else row
|
||||
for key, row in coverage.items() if key in ['managed_artifact', 'native_boundaries']}
|
||||
return {
|
||||
'artifact_sha256': value.get('artifact_sha256'),
|
||||
'assembly': selected('assembly', ['name', 'version']),
|
||||
@@ -64,7 +113,7 @@ def requested_facts(value, task):
|
||||
'classification': value.get('classification'),
|
||||
'target_frameworks': value.get('target_frameworks'),
|
||||
'native_boundaries': selected('native_boundaries', ['module_refs', 'pinvoke_imports']),
|
||||
'coverage': value.get('coverage'),
|
||||
'coverage': coverage,
|
||||
}
|
||||
facts = {field: value.get(field) for field in [
|
||||
'artifact_sha256', 'package_name', 'version_name', 'version_code', 'matches', 'onCreate']}
|
||||
@@ -150,11 +199,16 @@ def main():
|
||||
observed |= ids
|
||||
unreadable += skipped
|
||||
cited = evidence_ids(answer.get('evidence_ids'))
|
||||
if args.task == 'electron-inventory':
|
||||
for branch in answer.get('unexplored_branches', []):
|
||||
if isinstance(branch, dict):
|
||||
cited |= evidence_ids(branch.get('evidence_id'))
|
||||
if not cited:
|
||||
failures.append({'kind': 'missing_evidence_references'})
|
||||
if cited - observed:
|
||||
failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)})
|
||||
result = {'task': args.task,
|
||||
'grader_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
|
||||
'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy',
|
||||
'passed': not failures, 'fact_assertions': len(expected),
|
||||
'answer_format': answer_format,
|
||||
|
||||
@@ -252,3 +252,46 @@ sampled peak of 1399996416 bytes and zero observed limit events. Declarations ar
|
||||
cached per exact tool name for this immutable live-catalog connection; repeated
|
||||
discovery does not regenerate them. No provider implementation or published
|
||||
REA API has changed.
|
||||
|
||||
## Typed-discovery serial agent checkpoint
|
||||
|
||||
All rows below use real Codex CLI 0.161.0, Terra high, the same rendered question
|
||||
within each pair, and the same provider settings. The bridge/runtime/runner source
|
||||
fingerprints match commit `22326e819`; model-written original arguments can differ.
|
||||
Comparisons include required mode-specific instructions and are not a controlled
|
||||
causal decomposition of inference, caching and provider time.
|
||||
|
||||
| Case | Profile | Wall seconds | Input | Cached | Uncached | Output | MCP calls | Fact/reference check |
|
||||
| ------------------ | ------- | -----------: | -----: | -----: | -------: | -----: | --------: | --------------------------------- |
|
||||
| android | direct | 67.329 | 107007 | 83712 | 23295 | 1670 | 3 | pass |
|
||||
| android | bridge | 48.231 | 113256 | 91648 | 21608 | 1456 | 3 | pass |
|
||||
| managed | bridge | 73.145 | 116974 | 92672 | 24302 | 2065 | 3 | classification field omitted |
|
||||
| managed | direct | 33.15 | 108096 | 75776 | 32320 | 2315 | 2 | facts match; reference unverified |
|
||||
| electron-inventory | direct | 89.494 | 266121 | 238848 | 27273 | 3215 | 3 | pass |
|
||||
| electron-inventory | bridge | 145.577 | 462809 | 407040 | 55769 | 5624 | 8 | pass |
|
||||
| firmware | bridge | 81.944 | 152886 | 126976 | 25910 | 2192 | 3 | pass |
|
||||
| firmware | direct | 30.161 | 103224 | 74752 | 28472 | 1761 | 1 | pass |
|
||||
|
||||
The managed bridge omits the producer classification container field (`pe`) while
|
||||
preserving status/runtime/implementation/architecture and all classification
|
||||
evidence. Its strict complete-classification assertion fails; this is an omission,
|
||||
not an incorrect architecture claim. Normal MCP matches requested facts but its
|
||||
large-result JSONL truncation again prevents automated verification of one ID.
|
||||
|
||||
The initial grader treated grouped path records, extra known coverage counts and
|
||||
an empty per-source contradiction object as mismatches. Task-specific comparison
|
||||
now accepts those lossless report forms. Firmware compares observed offset/length
|
||||
facts, checks any supplied derived end arithmetically, and verifies engine version
|
||||
and executable digest. Run-local UUIDs and differently named display fields are
|
||||
not compared as if they were stable source identity. Original strict grades and
|
||||
pre-equivalence grades remain outside Git, and new grades include a grader digest.
|
||||
These report-format comparisons do not add aliases to any REA tool contract.
|
||||
|
||||
Obsidian bridge discovery takes six searches and two executions. The first returns
|
||||
the entire nested inventory; the second repeats it and computes a focused answer.
|
||||
The live nested Evidence schema describes normalized_result as generic JSON, so
|
||||
derived types alone cannot teach its concrete manifest/occurrences layout. Its
|
||||
recursive record walk happens to match this corpus, but format-specific field
|
||||
selection is the stronger recipe. Both model profiles choose record-and-continue
|
||||
for integrity; deterministic inventory replays used fail. No contradictions were
|
||||
observed, and the exact caller-selected arguments are retained.
|
||||
|
||||
Reference in New Issue
Block a user