Files
rea/experiments/codemode/grade-trial.py
T

145 lines
5.9 KiB
Python

"""Check requested JSON facts and real Evidence references against a replay oracle."""
import argparse
import json
from pathlib import Path
def evidence_ids(value):
if isinstance(value, str):
return {value} if value.startswith('ev_') and len(value) == 67 else set()
if isinstance(value, dict):
return set().union(*(evidence_ids(v) for v in value.values()))
if isinstance(value, list):
return set().union(*(evidence_ids(v) for v in value))
return set()
def read_answer(path):
source = path.read_text().strip()
lines = source.splitlines()
# Accept a single complete JSON fence, never repair prose or partial JSON.
if len(lines) >= 3 and lines[0] == '```json' and lines[-1] == '```':
source = '\n'.join(lines[1:-1])
answer_format = 'markdown_json'
else:
answer_format = 'json'
answer = json.loads(source)
if not isinstance(answer, dict):
raise ValueError('Answer is not a JSON object')
return answer, answer_format
def requested_facts(value, task):
if task == 'managed':
def selected(field, keys):
row = value.get(field)
return {key: row.get(key) for key in keys} if isinstance(row, dict) else row
return {
'artifact_sha256': value.get('artifact_sha256'),
'assembly': selected('assembly', ['name', 'version']),
'module': selected('module', ['name', 'mvid']),
'classification': value.get('classification'),
'target_frameworks': value.get('target_frameworks'),
'native_boundaries': selected('native_boundaries', ['module_refs', 'pinvoke_imports']),
'coverage': value.get('coverage'),
}
facts = {field: value.get(field) for field in [
'artifact_sha256', 'package_name', 'version_name', 'version_code', 'matches', 'onCreate']}
methods = facts['onCreate']
if isinstance(methods, list) and all(isinstance(row, dict) for row in methods):
facts['onCreate'] = [{key: row.get(key) for key in [
'reported_signature', 'overload_index', 'dex_descriptor']} for row in methods]
return facts
def producer_ids(reply):
if not isinstance(reply, dict):
return set(), 0
ids = evidence_ids(reply.get('evidence_id'))
for call in reply.get('calls', []):
ids |= evidence_ids(call.get('evidence_id'))
ids |= evidence_ids(call.get('partial_evidence_id'))
unreadable = 0
# SDK envelopes and Codex JSONL use distinct declared field spellings.
for field in ['structuredContent', 'structured_content']:
nested_ids, nested_unreadable = producer_ids(reply.get(field))
ids |= nested_ids
unreadable += nested_unreadable
for content in reply.get('content', []):
if content.get('type') != 'text':
continue
try:
value = json.loads(content['text'])
except ValueError:
unreadable += 1
continue
nested_ids, nested_unreadable = producer_ids(value)
ids |= nested_ids
unreadable += nested_unreadable
return ids, unreadable
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--trial', type=Path, required=True)
parser.add_argument('--oracle', type=Path, required=True)
parser.add_argument('--task', choices=['android', 'managed'], default='android')
args = parser.parse_args()
oracle = json.loads(args.oracle.read_text())
if oracle.get('failure') or not oracle.get('projection'):
raise SystemExit('A failed replay is not a factual oracle')
projection = oracle['projection']
if args.task == 'managed':
projection = {**projection, 'coverage': {
'managed_artifact': projection['coverage'],
'native_boundaries': projection['native_boundaries']['coverage'],
}}
expected = requested_facts(projection, args.task)
# A task explicitly chooses these assertions; do not grade unrequested fields.
if any(value is None for value in expected.values()):
raise SystemExit('Replay does not contain the selected task facts')
failures = []
answer_format = None
answer_path = args.trial / 'answer.txt'
try:
answer, answer_format = read_answer(answer_path)
except (FileNotFoundError, ValueError) as error:
answer = {}
failures.append({'kind': 'answer_parse_failure', 'reason': str(error)})
actual_facts = requested_facts(answer, args.task)
for field, wanted in expected.items():
actual = actual_facts[field]
if actual != wanted:
failures.append({'kind': 'fact_mismatch', 'field': field,
'expected': wanted, 'actual': actual})
observed = set()
unreadable = 0
for line in (args.trial / 'events.jsonl').read_text().splitlines():
event = json.loads(line)
item = event.get('item', {})
if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
continue
# Only producer envelopes and bridge audit IDs establish retention.
ids, skipped = producer_ids(item.get('result') or {})
observed |= ids
unreadable += skipped
cited = evidence_ids(answer.get('evidence_ids'))
if not cited:
failures.append({'kind': 'missing_evidence_references'})
if cited - observed:
failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)})
result = {'task': args.task,
'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy',
'passed': not failures, 'fact_assertions': len(expected),
'answer_format': answer_format,
'unreadable_result_texts': unreadable,
'cited_evidence_ids': sorted(cited), 'failures': failures}
(args.trial / 'grade.json').write_text(json.dumps(result, indent=2) + '\n')
print(json.dumps(result))
raise SystemExit(0 if result['passed'] else 1)
if __name__ == '__main__':
main()