86 lines
3.8 KiB
Python
86 lines
3.8 KiB
Python
"""Check requested JSON facts and real Evidence references against a replay oracle."""
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
def evidence_ids(value):
|
|
if isinstance(value, str):
|
|
return {value} if value.startswith('ev_') and len(value) == 67 else set()
|
|
if isinstance(value, dict):
|
|
return set().union(*(evidence_ids(v) for v in value.values()))
|
|
if isinstance(value, list):
|
|
return set().union(*(evidence_ids(v) for v in value))
|
|
return set()
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--trial', type=Path, required=True)
|
|
parser.add_argument('--oracle', type=Path, required=True)
|
|
args = parser.parse_args()
|
|
oracle = json.loads(args.oracle.read_text())
|
|
if oracle.get('failure') or not oracle.get('projection'):
|
|
raise SystemExit('A failed replay is not a factual oracle')
|
|
expected = oracle['projection']
|
|
# A task explicitly chooses these assertions; do not grade unrequested fields.
|
|
fields = ['artifact_sha256', 'package_name', 'version_name', 'version_code',
|
|
'matches', 'onCreate']
|
|
if not all(field in expected for field in fields):
|
|
raise SystemExit('This grader currently supports the Android question only')
|
|
failures = []
|
|
answer_path = args.trial / 'answer.txt'
|
|
try:
|
|
answer = json.loads(answer_path.read_text())
|
|
if not isinstance(answer, dict):
|
|
raise ValueError('Answer is not a JSON object')
|
|
except (FileNotFoundError, ValueError) as error:
|
|
answer = {}
|
|
failures.append({'kind': 'answer_parse_failure', 'reason': str(error)})
|
|
for field in fields:
|
|
wanted = expected[field]
|
|
actual = answer.get(field)
|
|
if field == 'onCreate':
|
|
selected = ['reported_signature', 'overload_index', 'dex_descriptor']
|
|
wanted = [{k: row[k] for k in selected} for row in wanted]
|
|
if isinstance(actual, list) and all(isinstance(row, dict) for row in actual):
|
|
actual = [{k: row.get(k) for k in selected} for row in actual]
|
|
if actual != wanted:
|
|
failures.append({'kind': 'fact_mismatch', 'field': field,
|
|
'expected': wanted, 'actual': actual})
|
|
observed = set()
|
|
for line in (args.trial / 'events.jsonl').read_text().splitlines():
|
|
event = json.loads(line)
|
|
item = event.get('item', {})
|
|
if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call':
|
|
continue
|
|
result = item.get('result') or {}
|
|
for content in result.get('content', []):
|
|
if content.get('type') == 'text':
|
|
try:
|
|
reply = json.loads(content['text'])
|
|
except ValueError:
|
|
continue
|
|
# Only producer IDs and bridge audit IDs establish retention.
|
|
if isinstance(reply, dict):
|
|
observed |= evidence_ids(reply.get('evidence_id'))
|
|
for call in reply.get('calls', []):
|
|
observed |= evidence_ids(call.get('evidence_id'))
|
|
cited = evidence_ids(answer.get('evidence_ids'))
|
|
if not cited:
|
|
failures.append({'kind': 'missing_evidence_references'})
|
|
if cited - observed:
|
|
failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)})
|
|
result = {'task': 'newpipe_package_and_MainActivity',
|
|
'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy',
|
|
'passed': not failures, 'fact_assertions': len(fields),
|
|
'cited_evidence_ids': sorted(cited), 'failures': failures}
|
|
(args.trial / 'grade.json').write_text(json.dumps(result, indent=2) + '\n')
|
|
print(json.dumps(result))
|
|
raise SystemExit(0 if result['passed'] else 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|