From 3f6dcbc096730c6c6aa18d4c523d8e792d0205b3 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Sun, 11 Oct 2026 18:02:47 +0000 Subject: [PATCH] perf(experiments): stage discovery and verify managed agent facts --- experiments/codemode/README.md | 24 ++++-- experiments/codemode/agent-trial.py | 10 ++- experiments/codemode/bridge.mjs | 16 ++-- experiments/codemode/bridge.test.mjs | 12 +++ experiments/codemode/grade-trial.py | 119 ++++++++++++++++++++------- experiments/codemode/results.md | 37 +++++++++ 6 files changed, 168 insertions(+), 50 deletions(-) diff --git a/experiments/codemode/README.md b/experiments/codemode/README.md index f631311f7..c0041a18b 100644 --- a/experiments/codemode/README.md +++ b/experiments/codemode/README.md @@ -27,7 +27,10 @@ node bridge.mjs /absolute/path/to/node /absolute/path/to/rea/scripts/rea.mjs mcp Register that command as a separate, temporary MCP server for a trial. Existing REA registrations keep their original tools. `rea_search` accepts a query or exact names; use `schemas: false` for the complete lightweight name/description list. -Input schemas are returned by default. Request `output_schemas: true` when a +Query discovery returns names/descriptions by default; exact-name discovery +returns original input schemas. `schemas` explicitly overrides that choice. +Start with one domain word and request all selected exact names in one call. +Request `output_schemas: true` when a result contract is needed; the original output schema is then returned unchanged. `catalog_sha256` identifies the SDK catalog; it must not be substituted for REA's `server_identity.catalog.digests` fields. @@ -87,8 +90,9 @@ from single text content and count code explicitly. Measured findings and failur are recorded in [results.md](results.md). `agent-trial.py` runs a real local Codex CLI with a disposable account/config and -one MCP registration. It copies authentication only, removes the profile on every -exit, and retains JSONL events, the answer and measured API usage outside the repo. +one MCP registration. It copies authentication only, removes the profile on normal +completion, timeout and handled termination, and retains JSONL events, the answer +and measured API usage outside the repo. It defaults to `gpt-5.6-terra` with high reasoning, as requested. Run one task under the resource wrapper: @@ -104,12 +108,14 @@ provider settings. No ambient environment snapshot is retained. A configured fla or a model's `tool_mode` does not prove the model actually composed tools or filtered an intermediate result. Inspect the transcript, and mark accuracy as unassessed until the requested facts and source support have been reviewed. -`grade-trial.py` currently checks the Android task against a successful real-provider -replay and verifies that cited IDs appeared as producer or bridge audit IDs. This -checks producer parity and reference existence; it does not prove Android runtime -behavior or independent correctness of the decompiler. Its strict projection also -checks the provider's overload index, so an otherwise supported method signature -with that field omitted fails this stronger assertion. +`grade-trial.py` checks Android (default) or `--task managed` against a successful +real-provider replay and verifies cited producer or bridge audit IDs. It accepts +JSON or one complete JSON fence, without repairing prose or partial values. This +checks requested producer facts and reference existence, not runtime behavior or +independent decompiler correctness. Unparseable result text is reported explicitly; +IDs in a truncated JSON fragment cannot satisfy the automated reference check. +The Android projection includes the provider's overload index, so a supported +signature with that field omitted fails this stronger assertion. The supplied feedback is a research lead. Its Ghidra timing claim also changed auto-analysis options, so it does not isolate Code Mode's effect. Relevant primary diff --git a/experiments/codemode/agent-trial.py b/experiments/codemode/agent-trial.py index 7e81898bb..54972da72 100644 --- a/experiments/codemode/agent-trial.py +++ b/experiments/codemode/agent-trial.py @@ -6,6 +6,7 @@ import json import os from pathlib import Path import shutil +import signal import subprocess import tempfile import time @@ -22,6 +23,11 @@ def main(): parser.add_argument('--codex', default='/usr/bin/codex') parser.add_argument('--providers', type=Path) args = parser.parse_args() + def terminate(signum, _frame): + # The outer resource scope terminates descendants together. Ensure the + # runner's copied credentials are also removed on a resource abort. + raise SystemExit(128 + signum) + signal.signal(signal.SIGTERM, terminate) repository = Path(__file__).resolve().parents[2] experiment = Path(__file__).resolve().parent node = shutil.which('node') @@ -108,7 +114,9 @@ def main(): 'inspect_android_package', 'search_android_classes', 'inspect_android_class', 'inspect_android_method', 'inspect_firmware_regions', 'inspect_managed_artifact', - 'inspect_managed_member', 'analyze_function', 'close_binary']: + 'inspect_managed_members', 'inspect_managed_native_boundaries', + 'inspect_artifact', 'inspect_wasm_artifact', + 'resolve_containing_procedure', 'analyze_function', 'close_binary']: config += '[mcp_servers.rea.tools.' + tool + ']\napproval_mode = "approve"\n' try: if source_auth.exists(): diff --git a/experiments/codemode/bridge.mjs b/experiments/codemode/bridge.mjs index 5905bed2b..8ebb371fb 100644 --- a/experiments/codemode/bridge.mjs +++ b/experiments/codemode/bridge.mjs @@ -63,12 +63,8 @@ export class CodeModeBridge { } /** Search exposes original schemas and generated input types on demand. */ - async search({ - query = "", - names = [], - schemas = true, - output_schemas = false, - }) { + async search({ query = "", names = [], schemas, output_schemas = false }) { + const includeSchemas = schemas ?? (names.length > 0 || output_schemas); const terms = query.toLowerCase().split(/\s+/).filter(Boolean); const selected = this.upstream.catalog.filter((tool) => names.length @@ -98,7 +94,7 @@ export class CodeModeBridge { .map(({ tool }) => ({ name: tool.name, description: tool.description })); const tools = []; for (const tool of selected) { - const inputType = schemas + const inputType = includeSchemas ? await compile(tool.inputSchema, `${tool.name}_Input`, { bannerComment: "", format: false, @@ -106,7 +102,7 @@ export class CodeModeBridge { }) : undefined; tools.push( - schemas + includeSchemas ? { ...tool, ...(!output_schemas ? { outputSchema: undefined } : {}), @@ -322,11 +318,11 @@ export function createBridgeServer(bridge) { "rea_search", { description: - "Discover original REA tools by all query words or exact names. Empty query lists the complete catalog. Input schemas are exact upstream MCP schemas with generated TypeScript. Request output_schemas=true only when needed. catalog_sha256 hashes the upstream SDK tool catalog, not REA server_identity.catalog.digests. Code Mode calls tools.(original_arguments); results are original MCP envelopes, parsed by json(reply).", + "Discover original REA tools. Start with one domain word, e.g. query=android; query words must all occur. Query/empty discovery defaults to names and descriptions. Then supply all selected exact names together to obtain original input schemas and TypeScript. schemas=true/false overrides this choice. Request output_schemas=true only when needed. catalog_sha256 hashes the SDK catalog, not REA server_identity.catalog.digests. Code calls tools.(original_arguments); json(reply) parses the original MCP value.", inputSchema: z.strictObject({ query: z.string().default(""), names: z.array(z.string()).default([]), - schemas: z.boolean().default(true), + schemas: z.boolean().optional(), output_schemas: z.boolean().default(false), }), annotations: { diff --git a/experiments/codemode/bridge.test.mjs b/experiments/codemode/bridge.test.mjs index 34ab26db1..35130fb81 100644 --- a/experiments/codemode/bridge.test.mjs +++ b/experiments/codemode/bridge.test.mjs @@ -50,6 +50,18 @@ test("discovery preserves exact current REA schemas and names", async () => { all.tools.map((tool) => tool.name), upstream.catalog.map((tool) => tool.name), ); + const query = await bridge.search({ query: "android" }); + assert.ok(query.tools.length > 0); + assert.ok(query.tools.every((tool) => tool.inputSchema === undefined)); + const querySchemas = await bridge.search({ query: "android", schemas: true }); + assert.ok(querySchemas.tools.every((tool) => tool.inputSchema !== undefined)); + const outputRequested = await bridge.search({ + query: "android", + output_schemas: true, + }); + assert.ok( + outputRequested.tools.every((tool) => tool.outputSchema !== undefined), + ); const missed = await bridge.search({ query: "android impossible_query_word", }); diff --git a/experiments/codemode/grade-trial.py b/experiments/codemode/grade-trial.py index fc02d4c29..28b70ccfb 100644 --- a/experiments/codemode/grade-trial.py +++ b/experiments/codemode/grade-trial.py @@ -15,66 +15,125 @@ def evidence_ids(value): return set() +def read_answer(path): + source = path.read_text().strip() + lines = source.splitlines() + # Accept a single complete JSON fence, never repair prose or partial JSON. + if len(lines) >= 3 and lines[0] == '```json' and lines[-1] == '```': + source = '\n'.join(lines[1:-1]) + answer_format = 'markdown_json' + else: + answer_format = 'json' + answer = json.loads(source) + if not isinstance(answer, dict): + raise ValueError('Answer is not a JSON object') + return answer, answer_format + + +def requested_facts(value, task): + if task == 'managed': + def selected(field, keys): + row = value.get(field) + return {key: row.get(key) for key in keys} if isinstance(row, dict) else row + return { + 'artifact_sha256': value.get('artifact_sha256'), + 'assembly': selected('assembly', ['name', 'version']), + 'module': selected('module', ['name', 'mvid']), + 'classification': value.get('classification'), + 'target_frameworks': value.get('target_frameworks'), + 'native_boundaries': selected('native_boundaries', ['module_refs', 'pinvoke_imports']), + 'coverage': value.get('coverage'), + } + facts = {field: value.get(field) for field in [ + 'artifact_sha256', 'package_name', 'version_name', 'version_code', 'matches', 'onCreate']} + methods = facts['onCreate'] + if isinstance(methods, list) and all(isinstance(row, dict) for row in methods): + facts['onCreate'] = [{key: row.get(key) for key in [ + 'reported_signature', 'overload_index', 'dex_descriptor']} for row in methods] + return facts + + +def producer_ids(reply): + if not isinstance(reply, dict): + return set(), 0 + ids = evidence_ids(reply.get('evidence_id')) + for call in reply.get('calls', []): + ids |= evidence_ids(call.get('evidence_id')) + ids |= evidence_ids(call.get('partial_evidence_id')) + unreadable = 0 + # SDK envelopes and Codex JSONL use distinct declared field spellings. + for field in ['structuredContent', 'structured_content']: + nested_ids, nested_unreadable = producer_ids(reply.get(field)) + ids |= nested_ids + unreadable += nested_unreadable + for content in reply.get('content', []): + if content.get('type') != 'text': + continue + try: + value = json.loads(content['text']) + except ValueError: + unreadable += 1 + continue + nested_ids, nested_unreadable = producer_ids(value) + ids |= nested_ids + unreadable += nested_unreadable + return ids, unreadable + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--trial', type=Path, required=True) parser.add_argument('--oracle', type=Path, required=True) + parser.add_argument('--task', choices=['android', 'managed'], default='android') args = parser.parse_args() oracle = json.loads(args.oracle.read_text()) if oracle.get('failure') or not oracle.get('projection'): raise SystemExit('A failed replay is not a factual oracle') - expected = oracle['projection'] + projection = oracle['projection'] + if args.task == 'managed': + projection = {**projection, 'coverage': { + 'managed_artifact': projection['coverage'], + 'native_boundaries': projection['native_boundaries']['coverage'], + }} + expected = requested_facts(projection, args.task) # A task explicitly chooses these assertions; do not grade unrequested fields. - fields = ['artifact_sha256', 'package_name', 'version_name', 'version_code', - 'matches', 'onCreate'] - if not all(field in expected for field in fields): - raise SystemExit('This grader currently supports the Android question only') + if any(value is None for value in expected.values()): + raise SystemExit('Replay does not contain the selected task facts') failures = [] + answer_format = None answer_path = args.trial / 'answer.txt' try: - answer = json.loads(answer_path.read_text()) - if not isinstance(answer, dict): - raise ValueError('Answer is not a JSON object') + answer, answer_format = read_answer(answer_path) except (FileNotFoundError, ValueError) as error: answer = {} failures.append({'kind': 'answer_parse_failure', 'reason': str(error)}) - for field in fields: - wanted = expected[field] - actual = answer.get(field) - if field == 'onCreate': - selected = ['reported_signature', 'overload_index', 'dex_descriptor'] - wanted = [{k: row[k] for k in selected} for row in wanted] - if isinstance(actual, list) and all(isinstance(row, dict) for row in actual): - actual = [{k: row.get(k) for k in selected} for row in actual] + actual_facts = requested_facts(answer, args.task) + for field, wanted in expected.items(): + actual = actual_facts[field] if actual != wanted: failures.append({'kind': 'fact_mismatch', 'field': field, 'expected': wanted, 'actual': actual}) observed = set() + unreadable = 0 for line in (args.trial / 'events.jsonl').read_text().splitlines(): event = json.loads(line) item = event.get('item', {}) if event.get('type') != 'item.completed' or item.get('type') != 'mcp_tool_call': continue - result = item.get('result') or {} - for content in result.get('content', []): - if content.get('type') == 'text': - try: - reply = json.loads(content['text']) - except ValueError: - continue - # Only producer IDs and bridge audit IDs establish retention. - if isinstance(reply, dict): - observed |= evidence_ids(reply.get('evidence_id')) - for call in reply.get('calls', []): - observed |= evidence_ids(call.get('evidence_id')) + # Only producer envelopes and bridge audit IDs establish retention. + ids, skipped = producer_ids(item.get('result') or {}) + observed |= ids + unreadable += skipped cited = evidence_ids(answer.get('evidence_ids')) if not cited: failures.append({'kind': 'missing_evidence_references'}) if cited - observed: failures.append({'kind': 'unobserved_evidence_ids', 'ids': sorted(cited - observed)}) - result = {'task': 'newpipe_package_and_MainActivity', + result = {'task': args.task, 'scope': 'producer_fact_parity_and_reference_existence_not_runtime_accuracy', - 'passed': not failures, 'fact_assertions': len(fields), + 'passed': not failures, 'fact_assertions': len(expected), + 'answer_format': answer_format, + 'unreadable_result_texts': unreadable, 'cited_evidence_ids': sorted(cited), 'failures': failures} (args.trial / 'grade.json').write_text(json.dumps(result, indent=2) + '\n') print(json.dumps(result)) diff --git a/experiments/codemode/results.md b/experiments/codemode/results.md index 45c06260d..60da7278a 100644 --- a/experiments/codemode/results.md +++ b/experiments/codemode/results.md @@ -172,3 +172,40 @@ its queued successor release deferred guest handles before runtime disposal. This test does not simulate or establish new provider behavior. The bridge remains an experiment and has not changed any published REA contract. + +## Existing-client and managed follow-up + +The same Android v3 question with normal MCP, without the experimental Code Mode +feature flags, passes all six fact assertions and reference checks: 75.117 seconds, +103471 API input tokens (73728 cached), 1538 output tokens and three original MCP +calls. This is the strongest measured Android baseline so far. Its wall time also +includes a short boundary-test overlap, so it is not a controlled latency ratio. +The installed model catalog requests code-mode-only tool delivery; flag settings +and nested MCP events alone cannot establish the internal executor's behavior. + +The first real ShareX managed-agent comparison uses the same question with Terra +high and identical provider settings. Normal MCP takes 81.578 seconds, 217573 input +tokens (186368 cached), 2647 output tokens and two original calls. The bridge takes +96.930 seconds, 218089 input tokens (155904 cached), 3247 output tokens, four +searches and two executions. Its first script uses an expression instead of the +required async-body `return`, yielding no projection; it repeats both operations. +Query discovery now defaults to lightweight names/descriptions, while exact-name +discovery supplies input schemas. This run still requested full output schemas +individually and therefore does not establish an efficiency gain from that change. + +Both answers match all seven requested managed fact groups, including exact target +SHA-256, assembly/module identities, classification evidence, framework, declared +ModuleRef/PInvoke arrays and coverage. Those empty arrays concern ShareX.dll only. +The bridge passes cited-reference verification. Codex truncated the original +managed-artifact response in its normal-MCP JSONL at roughly one MiB, leaving an +invalid JSON fragment; the automated grader cannot validate that one reference +from a complete producer envelope. Its reference-failure result is retained, +despite the matching ID being visible in the transcript prefix and replay. This +is a verification limitation, not a demonstrated fabricated ID. + +The runner now handles SIGTERM so copied trial credentials are removed during a +resource abort. A controlled non-model launcher received owned-scope SIGTERM, +exited 143 and left no trial profile while preserving the dummy source auth file. +This verifies credential cleanup only. The eleven actual stdio tests pass after +the adaptive discovery change, with a sampled peak of 1365106688 bytes, 42 tasks +and zero observed memory/task limit events.