145 lines
5.9 KiB
Python
145 lines
5.9 KiB
Python
"""Check requested JSON facts and real Evidence references against a replay oracle."""
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def evidence_ids(value):
|
|
if isinstance(value, str):
|
|
return {value} if value.startswith('ev_') and len(value) == 67 else set()
|
|
if isinstance(value, dict):
|
|
return set().union(*(evidence_ids(v) for v in value.values()))
|
|
if isinstance(value, list):
|
|
return set().union(*(evidence_ids(v) for v in value))
|
|
return set()
|
|
|
|
|
|
def read_answer(path):
|
|
source = path.read_text().strip()
|
|
lines = source.splitlines()
|
|
# Accept a single complete JSON fence, never repair prose or partial JSON.
|
|
if len(lines) >= 3 and lines[0] == '```json' and lines[-1] == '```':
|
|
source = '\n'.join(lines[1:-1])
|
|
answer_format = 'markdown_json'
|
|
else:
|
|
answer_format = 'json'
|
|
answer = json.loads(source)
|
|
if not isinstance(answer, dict):
|
|
raise ValueError('Answer is not a JSON object')
|
|
return answer, answer_format
|
|
|
|
|
|
def requested_facts(value, task):
|
|
if task == 'managed':
|
|
def selected(field, keys):
|
|
row = value.get(field)
|
|
return {key: row.get(key) for key in keys} if isinstance(row, dict) else row
|
|
return {
|
|
'artifact_sha256': value.get('artifact_sha256'),
|
|
'assembly': selected('assembly', ['name', 'version']),
|
|
'module': selected('module', ['name', 'mvid']),
|
|
'classification': value.get('classification'),
|
|
'target_frameworks': value.get('target_frameworks'),
|
|
'native_boundaries': selected('native_boundaries', ['module_refs', 'pinvoke_imports']),
|
|
'coverage': value.get('coverage'),
|
|
}
|
|
facts = {field: value.get(field) for field in [
|
|
'artifact_sha256', 'package_name', 'version_name', 'version_code', 'matches', 'onCreate']}
|
|
methods = facts['onCreate']
|
|
if isinstance(methods, list) and all(isinstance(row, dict) for row in methods):
|
|
facts['onCreate'] = [{key: row.get(key) for key in [
|
|
'reported_signature', 'overload_index', 'dex_descriptor']} for row in methods]
|
|
return facts
|
|
|
|
|
|
def producer_ids(reply):
|
|
if not isinstance(reply, dict):
|
|
return set(), 0
|
|
ids = evidence_ids(reply.get('evidence_id'))
|
|
for call in reply.get('calls', []):
|
|
ids |= evidence_ids(call.get('evidence_id'))
|
|
ids |= evidence_ids(call.get('partial_evidence_id'))
|
|
unreadable = 0
|
|
# SDK envelopes and Codex JSONL use distinct declared field spellings.
|
|
for field in ['structuredContent', 'structured_content']:
|
|
nested_ids, nested_unreadable = producer_ids(reply.get(field))
|
|
ids |= nested_ids
|
|
unreadable += nested_unreadable
|
|
for content in reply.get('content', []):
|
|
if content.get('type') != 'text':
|
|
continue
|
|
try:
|
|
value = json.loads(content['text'])
|
|
except ValueError:
|
|
unreadable += 1
|
|
continue
|
|
nested_ids, nested_unreadable = producer_ids(value)
|
|
ids |= nested_ids
|
|
unreadable += nested_unreadable
|
|
return ids, unreadable
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--trial', type=Path, required=True)
|
|
parser.add_argument('--oracle', type=Path, required=True)
|
|
parser.add_argument('--task', choices=['android', 'managed'], default='android')
|
|
args = parser.parse_args()
|
|
oracle = json.loads(args.oracle.read_text())
|
|
if oracle.get('failure') or not oracle.get('projection'):
|
|
raise SystemExit('A failed replay is not a factual oracle')
|
|
projection = oracle['projection']
|
|
if args.task == 'managed':
|
|
projection = {**projection, 'coverage': {
|
|
'managed_artifact': projection['coverage'],
|
|
'native_boundaries': projection['native_boundaries']['coverage'],
|
|
}}
|
|
expected = requested_facts(projection, args.task)
|
|
# A task explicitly chooses these assertions; do not grade unrequested fields.
|
|
if any(value is None for value in expected.values()):
|
|
raise SystemExit('Replay does not contain the selected task facts')
|
|
failures = []
|
|
answer_format = None
|
|
answer_path = args.trial / 'answer.txt'
|
|
try:
|
|
answer, answer_format = read_answer(answer_path)
|
|
except (FileNotFoundError, ValueError) as error:
|
|
answer = {}
|
|
failures.append({'kind': 'answer_parse_failure', 'reason': str(error)})
|
|
actual_facts = requested_facts(answer, args.task)
|
|
for field, wanted in expected.items():
|
|
actual = actual_facts[field]
|
|
if actual != wanted:
|
|
failures.append({'kind': 'fact_mismatch', 'field': field,
|
|
'expected': wanted, 'actual': actual})
|
|
observed = set()
|
|
unreadable = 0
|
|
for line in (args.trial / 'events.jsonl').read_text().splitlines():
|
|
event = json.loads(line)
|
|
item = event.get('item', {})
|
|
if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
|
|
continue
|
|
# Only producer envelopes and bridge audit IDs establish retention.
|
|
ids, skipped = producer_ids(item.get('result') or {})
|
|
observed |= ids
|
|
unreadable += skipped
|
|
cited = evidence_ids(answer.get('evidence_ids'))
|
|
if not cited:
|
|
failures.append({'kind': 'missing_evidence_references'})
|
|
if cited - observed:
|
|
failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)})
|
|
result = {'task': args.task,
|
|
'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy',
|
|
'passed': not failures, 'fact_assertions': len(expected),
|
|
'answer_format': answer_format,
|
|
'unreadable_result_texts': unreadable,
|
|
'cited_evidence_ids': sorted(cited), 'failures': failures}
|
|
(args.trial / 'grade.json').write_text(json.dumps(result, indent=2) + '\n')
|
|
print(json.dumps(result))
|
|
raise SystemExit(0 if result['passed'] else 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|