diff --git a/experiments/codemode/README.md b/experiments/codemode/README.md index 4baf1067e..eb30d4d12 100644 --- a/experiments/codemode/README.md +++ b/experiments/codemode/README.md @@ -137,6 +137,13 @@ signature with that field omitted fails this stronger assertion. Firmware's run-local region UUIDs remain in answers but are excluded from parity against an independently acquired report. Classification evidence, explicit unknowns and coverage remain part of the requested fact checks. +The question comparison accepts grouped original path records, extra known +coverage counts and explicitly named firmware length/coverage report fields. +Derived extent ends must agree with offset plus reported length. It verifies +engine version and executable digest rather than display capitalization; a manual +review is still needed for limitations and claims beyond these assertions. +Grades record the grader source digest, and earlier strict grades are retained +when the comparison changes. These are report parsers, not REA API aliases. The supplied feedback is a research lead. Its Ghidra timing claim also changed auto-analysis options, so it does not isolate Code Mode's effect. Relevant primary diff --git a/experiments/codemode/grade-trial.py b/experiments/codemode/grade-trial.py index 6439b0637..faa6027bf 100644 --- a/experiments/codemode/grade-trial.py +++ b/experiments/codemode/grade-trial.py @@ -1,6 +1,7 @@ """Check requested JSON facts and real Evidence references against a replay oracle.""" import argparse +import hashlib import json from pathlib import Path @@ -36,27 +37,75 @@ def requested_facts(value, task): 'artifact_sha256', 'entry_address', 'containing_procedure', 'procedure', 'pseudocode_prefix', 'caller_count', 'callee_count']} if task == 'electron-inventory': - return {field: value.get(field) for field in [ + facts = {field: value.get(field) for field in [ 'artifact_sha256', 'manifest', 'entry_kind_counts', 'requested_paths', 'integrity_contradictions', 'coverage', 'unexplored_branches']} + paths = facts['requested_paths'] + if isinstance(paths, dict) and set(paths) == {'package.json', 'main.js', 'preload.js'}: + if all(isinstance(rows, list) and all(isinstance(row, dict) and + row.get('logical_path') == name for row in rows) + for name, rows in paths.items()): + paths = [row for rows in paths.values() for row in rows] + if isinstance(paths, list) and all(isinstance(row, dict) and + isinstance(row.get('logical_path'), str) for row in paths): + paths = sorted(paths, key=lambda row: row['logical_path']) + facts['requested_paths'] = paths + contradictions = facts['integrity_contradictions'] + if isinstance(contradictions, dict) and set(contradictions) == {'inspection', 'inventory'}: + if all(rows == [] for rows in contradictions.values()): + facts['integrity_contradictions'] = [] + branches = facts['unexplored_branches'] + if isinstance(branches, list) and all(isinstance(row, dict) for row in branches): + # Branch identity includes the source Evidence identity. Compare its + # reported meaning here and verify its actual source reference below. + facts['unexplored_branches'] = [{key: row.get(key) for key in [ + 'reason', 'detail', 'next_probe']} for row in branches] + return facts if task == 'firmware': result = value.get('result') if not isinstance(result, dict): result = {} + engine = result.get('engine', result.get('provider')) + if isinstance(engine, dict): + engine = {'version': engine.get('version'), + 'executable_sha256': engine.get('executable_sha256', + engine.get('engine_artifact_sha256'))} regions = result.get('regions') if isinstance(regions, list) and all(isinstance(row, dict) for row in regions): # Binwalk's UUID is run-local; preserve it in the answer but do not # compare it with the independent replay's different occurrence ID. - regions = [{key: row.get(key) for key in [ - 'offset', 'reported_size', 'signature', 'description', - 'reported_confidence', 'size_basis']} for row in regions] + selected_regions = [] + for row in regions: + selected = {key: row.get(key) for key in [ + 'offset', 'signature', 'description', 'reported_confidence']} + if 'reported_size' in row: + size = row['reported_size'] + elif 'reported_extent_bytes' in row: + size = row['reported_extent_bytes'] + else: + extent = row.get('reported_extent') + size = None + if isinstance(extent, dict) and extent.get('offset') == row.get('offset'): + length = extent.get('length') + offset = row.get('offset') + if isinstance(length, int) and isinstance(offset, int) and \ + extent.get('end_exclusive') == offset + length: + size = length + selected['reported_size'] = size + selected_regions.append(selected) + regions = selected_regions return {'artifact_sha256': value.get('artifact_sha256'), - 'engine': result.get('engine'), 'input_size': result.get('input_size'), - 'regions': regions, 'coverage': result.get('coverage')} + 'engine': engine, 'input_size': result.get('input_size', result.get('input_size_bytes')), + 'regions': regions, 'coverage': result.get('coverage', result.get('examined_coverage'))} if task == 'managed': def selected(field, keys): row = value.get(field) return {key: row.get(key) for key in keys} if isinstance(row, dict) else row + coverage = value.get('coverage') + if isinstance(coverage, dict): + coverage = {key: {field: row.get(field) for field in ['state', 'issues']} + if isinstance(row, dict) else row + for key, row in coverage.items() if key in ['managed_artifact', 'native_boundaries']} return { 'artifact_sha256': value.get('artifact_sha256'), 'assembly': selected('assembly', ['name', 'version']), @@ -64,7 +113,7 @@ def requested_facts(value, task): 'classification': value.get('classification'), 'target_frameworks': value.get('target_frameworks'), 'native_boundaries': selected('native_boundaries', ['module_refs', 'pinvoke_imports']), - 'coverage': value.get('coverage'), + 'coverage': coverage, } facts = {field: value.get(field) for field in [ 'artifact_sha256', 'package_name', 'version_name', 'version_code', 'matches', 'onCreate']} @@ -150,11 +199,16 @@ def main(): observed |= ids unreadable += skipped cited = evidence_ids(answer.get('evidence_ids')) + if args.task == 'electron-inventory': + for branch in answer.get('unexplored_branches', []): + if isinstance(branch, dict): + cited |= evidence_ids(branch.get('evidence_id')) if not cited: failures.append({'kind': 'missing_evidence_references'}) if cited - observed: failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)}) result = {'task': args.task, + 'grader_sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), 'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy', 'passed': not failures, 'fact_assertions': len(expected), 'answer_format': answer_format, diff --git a/experiments/codemode/results.md b/experiments/codemode/results.md index b2c414363..b96138f5d 100644 --- a/experiments/codemode/results.md +++ b/experiments/codemode/results.md @@ -252,3 +252,46 @@ sampled peak of 1399996416 bytes and zero observed limit events. Declarations ar cached per exact tool name for this immutable live-catalog connection; repeated discovery does not regenerate them. No provider implementation or published REA API has changed. + +## Typed-discovery serial agent checkpoint + +All rows below use real Codex CLI 0.161.0, Terra high, the same rendered question +within each pair, and the same provider settings. The bridge/runtime/runner source +fingerprints match commit `22326e819`; model-written original arguments can differ. +Comparisons include required mode-specific instructions and are not a controlled +causal decomposition of inference, caching and provider time. + +| Case | Profile | Wall seconds | Input | Cached | Uncached | Output | MCP calls | Fact/reference check | +| ------------------ | ------- | -----------: | -----: | -----: | -------: | -----: | --------: | --------------------------------- | +| android | direct | 67.329 | 107007 | 83712 | 23295 | 1670 | 3 | pass | +| android | bridge | 48.231 | 113256 | 91648 | 21608 | 1456 | 3 | pass | +| managed | bridge | 73.145 | 116974 | 92672 | 24302 | 2065 | 3 | classification field omitted | +| managed | direct | 33.15 | 108096 | 75776 | 32320 | 2315 | 2 | facts match; reference unverified | +| electron-inventory | direct | 89.494 | 266121 | 238848 | 27273 | 3215 | 3 | pass | +| electron-inventory | bridge | 145.577 | 462809 | 407040 | 55769 | 5624 | 8 | pass | +| firmware | bridge | 81.944 | 152886 | 126976 | 25910 | 2192 | 3 | pass | +| firmware | direct | 30.161 | 103224 | 74752 | 28472 | 1761 | 1 | pass | + +The managed bridge omits the producer classification container field (`pe`) while +preserving status/runtime/implementation/architecture and all classification +evidence. Its strict complete-classification assertion fails; this is an omission, +not an incorrect architecture claim. Normal MCP matches requested facts but its +large-result JSONL truncation again prevents automated verification of one ID. + +The initial grader treated grouped path records, extra known coverage counts and +an empty per-source contradiction object as mismatches. Task-specific comparison +now accepts those lossless report forms. Firmware compares observed offset/length +facts, checks any supplied derived end arithmetically, and verifies engine version +and executable digest. Run-local UUIDs and differently named display fields are +not compared as if they were stable source identity. Original strict grades and +pre-equivalence grades remain outside Git, and new grades include a grader digest. +These report-format comparisons do not add aliases to any REA tool contract. + +Obsidian bridge discovery takes six searches and two executions. The first returns +the entire nested inventory; the second repeats it and computes a focused answer. +The live nested Evidence schema describes normalized_result as generic JSON, so +derived types alone cannot teach its concrete manifest/occurrences layout. Its +recursive record walk happens to match this corpus, but format-specific field +selection is the stronger recipe. Both model profiles choose record-and-continue +for integrity; deterministic inventory replays used fail. No contradictions were +observed, and the exact caller-selected arguments are retained.