From d0b60a97ab2c88044a540b3827f5809bc1462e66 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <1705004398@qq.com> Date: Sun, 11 Oct 2026 16:40:08 +0000 Subject: [PATCH] perf(experiments): trim discovery and record real Codex health trials --- experiments/codemode/README.md | 4 + experiments/codemode/benchmark.mjs | 248 +++++++++++++++++++++++++++ experiments/codemode/bridge.mjs | 16 +- experiments/codemode/bridge.test.mjs | 8 +- experiments/codemode/resource-run.py | 2 +- experiments/codemode/results.md | 47 +++++ 6 files changed, 320 insertions(+), 5 deletions(-) create mode 100644 experiments/codemode/benchmark.mjs create mode 100644 experiments/codemode/results.md diff --git a/experiments/codemode/README.md b/experiments/codemode/README.md index 64deb2e39..d329af87e 100644 --- a/experiments/codemode/README.md +++ b/experiments/codemode/README.md @@ -27,6 +27,10 @@ node bridge.mjs /absolute/path/to/node /absolute/path/to/rea/scripts/rea.mjs mcp Register that command as a separate, temporary MCP server for a trial. Existing REA registrations keep their original tools. `rea_search` accepts a query or exact names; use `schemas: false` for the complete lightweight name/description list. +Input schemas are returned by default. Request `output_schemas: true` when a +result contract is needed; the original output schema is then returned unchanged. +`catalog_sha256` identifies the SDK catalog; it must not be substituted for REA's +`server_identity.catalog.digests` fields. `rea_execute` accepts a JavaScript async function body. For example: ```js diff --git a/experiments/codemode/benchmark.mjs b/experiments/codemode/benchmark.mjs new file mode 100644 index 000000000..e250c76f6 --- /dev/null +++ b/experiments/codemode/benchmark.mjs @@ -0,0 +1,248 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; +import { getEncoding } from "js-tiktoken"; +import { CodeModeBridge, connectUpstream, json } from "./bridge.mjs"; + +// Replay isolates transport/discovery costs. It does not simulate model reasoning. +const workflows = { + "obsidian.asar": { + tools: ["analyze_javascript_application", "inspect_analysis_view"], + question: + "Report application identity, analysis coverage/counts and the first eight module identities.", + code: ` +const summary = await call("analyze_javascript_application", { + input_path: input.path, detail: "summary", max_heap_mb: 1024, + analysis_timeout_ms: 300000, integrity_policy: "fail" +}); +const parent = summary.normalized_result.parent_evidence_id; +const page = await call("inspect_analysis_view", { + source: {kind: "retained-evidence", evidence_id: parent}, + view: {kind: "page", collection: "modules", offset: 0, limit: 8} +}); +return {artifact_sha256: summary.subject?.digest.sha256, + summary: summary.normalized_result, modules: page.normalized_result, + evidence_ids: [summary.evidence_id, parent, page.evidence_id], + limitations: [...summary.limitations, ...page.limitations]};`, + }, + "newpipe.apk": { + tools: [ + "inspect_android_package", + "search_android_classes", + "inspect_android_class", + ], + question: + "Report package/version, exact MainActivity matches and its onCreate method descriptors.", + code: ` +const pkg = await call("inspect_android_package", {path: input.path}); +const found = await call("search_android_classes", {path: input.path, query: "MainActivity"}); +if (!found.normalized_result.matches.includes("org.schabi.newpipe.MainActivity")) + throw new Error("Requested shipped MainActivity was not found"); +const cls = await call("inspect_android_class", { + path: input.path, class_name: "org.schabi.newpipe.MainActivity" +}); +return {artifact_sha256: pkg.subject?.digest.sha256, + package_name: pkg.normalized_result.package_name, + version_name: pkg.normalized_result.version_name, + version_code: pkg.normalized_result.version_code, + matches: found.normalized_result.matches, coverage: found.normalized_result.coverage, + onCreate: cls.normalized_result.methods.filter(x => x.name === "onCreate"), + evidence_ids: [pkg.evidence_id, found.evidence_id, cls.evidence_id], + limitations: [...pkg.limitations, ...found.limitations, ...cls.limitations]};`, + }, + "openwrt.bin": { + tools: ["inspect_firmware_regions"], + question: + "Report firmware identity, every reported region offset/signature and examined coverage.", + code: ` +const inspected = await call("inspect_firmware_regions", {path: input.path}); +return {artifact_sha256: inspected.subject?.digest.sha256, + result: inspected.normalized_result, evidence_ids: [inspected.evidence_id], + limitations: inspected.limitations};`, + }, + jq: { + tools: [ + "open_binary", + "search_procedures", + "analyze_function", + "close_binary", + ], + question: + "Find jq_compile, analyze it and report its first 512 pseudocode units and caller/callee counts.", + code: ` +await call("open_binary", {path: input.path, provider_id: "ghidra"}); +try { + const found = await call("search_procedures", {pattern: "jq_compile", mode: "literal", case_sensitive: true}); + const fn = await call("analyze_function", {procedure: "jq_compile"}); + const n = fn.normalized_result; + return {artifact_sha256: fn.subject?.digest.sha256, + matches: found.normalized_result, procedure: n.procedure, + pseudocode_prefix: n.pseudocode.slice(0, 512), + caller_count: n.callers.length, callee_count: n.callees.length, + evidence_ids: [found.evidence_id, fn.evidence_id], + limitations: [...found.limitations, ...fn.limitations, ...n.limitations]}; +} finally { await call("close_binary", {}); }`, + }, +}; + +const { values } = parseArgs({ + options: { + artifact: { type: "string" }, + mode: { type: "string", default: "direct" }, + "artifact-root": { type: "string" }, + providers: { type: "string" }, + output: { type: "string" }, + }, +}); +if ( + !workflows[values.artifact] || + !["direct", "discovery", "bridge"].includes(values.mode) || + !values["artifact-root"] || + !values.output +) + throw new Error( + "Use --artifact ID --mode direct|discovery|bridge --artifact-root ABS --output NEW_DIR [--providers JSON]", + ); +const experiment = dirname(fileURLToPath(import.meta.url)); +const repository = resolve(experiment, "../.."); +const corpus = JSON.parse( + await readFile(resolve(experiment, "corpus.json"), "utf8"), +); +const artifact = corpus.artifacts.find((x) => x.id === values.artifact); +const artifactPath = resolve(values["artifact-root"], artifact.file); +const bytes = await readFile(artifactPath); +const sha256 = createHash("sha256").update(bytes).digest("hex"); +if (sha256 !== artifact.sha256 || bytes.length !== artifact.bytes) + throw new Error("Artifact identity differs from frozen corpus"); +await mkdir(values.output, { recursive: false, mode: 0o700 }); +const encoding = getEncoding("cl100k_base"); +const measure = (value) => { + const text = typeof value === "string" ? value : JSON.stringify(value); + return { + bytes: Buffer.byteLength(text), + cl100k_estimated_tokens: encoding.encode(text).length, + }; +}; +const environment = values.providers + ? JSON.parse(await readFile(values.providers, "utf8")) + : {}; +const started = performance.now(); +const upstream = await connectUpstream({ + command: process.execPath, + args: [resolve(repository, "scripts/rea.mjs"), "mcp"], + env: environment, +}); +const connected = performance.now(); +const records = []; +const realCall = upstream.call; +upstream.call = async (name, args, options) => { + const began = performance.now(); + try { + const reply = await realCall(name, args, { timeout: 600_000, ...options }); + const parsed = json(reply); + records.push({ + name, + arguments: args, + elapsed_ms: performance.now() - began, + is_error: reply.isError === true, + input: measure(args), + output: measure(reply), + evidence_id: parsed.evidence_id, + reply, + }); + return reply; + } catch (error) { + records.push({ + name, + arguments: args, + elapsed_ms: performance.now() - began, + failure: String(error), + input: measure(args), + }); + throw error; + } +}; +const bridge = new CodeModeBridge(upstream); +const workflow = workflows[values.artifact]; +const input = { path: artifactPath }; +const code = `const input = ${JSON.stringify(input)}; +async function call(name, args) { + const reply = await tools[name](args); + const parsed = json(reply); + if (reply.isError) throw new Error(JSON.stringify(parsed)); + return parsed; +} +${workflow.code}`; +let discovery; +let value; +let failure; +let bridgeReply; +const operationStarted = performance.now(); +try { + if (values.mode !== "direct") + discovery = await bridge.search({ names: workflow.tools }); + if (values.mode === "bridge") { + bridgeReply = await bridge.execute({ code, timeout_ms: 900_000 }); + value = bridgeReply.value; + failure = bridgeReply.bridge_error; + } else { + const tools = Object.fromEntries( + upstream.catalog.map((tool) => [ + tool.name, + (args) => upstream.call(tool.name, args), + ]), + ); + const AsyncFunction = Object.getPrototypeOf( + async function () {}, + ).constructor; + value = await new AsyncFunction("tools", "json", code)(tools, json); + } +} catch (error) { + failure = String(error); +} finally { + await upstream.close(); +} +const catalogExposure = values.mode === "direct" ? upstream.catalog : discovery; +const summary = { + scope: "deterministic_real_provider_replay_not_model_accuracy", + baseline_sha: "fb94d9dde0ccda83ae520ac6fb22b1d283562cf3", + artifact: artifact.id, + artifact_sha256: sha256, + mode: values.mode, + question: workflow.question, + startup_ms: connected - started, + operation_ms: performance.now() - operationStarted, + original_calls: records.map(({ reply: _reply, ...metadata }) => metadata), + exposed_catalog: catalogExposure ? measure(catalogExposure) : null, + execution_code: values.mode === "bridge" ? measure(code) : null, + exposed_tool_results: + values.mode === "bridge" + ? [measure(bridgeReply)] + : records.filter((x) => x.output).map((x) => x.output), + projection: value, + failure: failure ?? null, + correctness: failure + ? "failed_workflow" + : "requires_cross_profile_fact_review", + estimator: + "cl100k_base_compact_MCP_JSON_including_envelope_not_actual_model_tokens", +}; +await writeFile( + resolve(values.output, "summary.json"), + JSON.stringify(summary, null, 2), +); +await writeFile(resolve(values.output, "calls.json"), JSON.stringify(records)); +await writeFile(resolve(values.output, "code.js"), code); +console.log( + JSON.stringify({ + artifact: artifact.id, + mode: values.mode, + calls: records.length, + startup_ms: summary.startup_ms, + operation_ms: summary.operation_ms, + failure: summary.failure, + }), +); +if (failure) process.exitCode = 1; diff --git a/experiments/codemode/bridge.mjs b/experiments/codemode/bridge.mjs index 4b933fd89..bc5e5921d 100644 --- a/experiments/codemode/bridge.mjs +++ b/experiments/codemode/bridge.mjs @@ -63,7 +63,12 @@ export class CodeModeBridge { } /** Search exposes original schemas and generated input types on demand. */ - async search({ query = "", names = [], schemas = true }) { + async search({ + query = "", + names = [], + schemas = true, + output_schemas = false, + }) { const terms = query.toLowerCase().split(/\s+/).filter(Boolean); const selected = this.upstream.catalog.filter((tool) => names.length @@ -85,7 +90,11 @@ export class CodeModeBridge { : undefined; tools.push( schemas - ? { ...tool, typescript: inputType } + ? { + ...tool, + ...(!output_schemas ? { outputSchema: undefined } : {}), + typescript: inputType, + } : { name: tool.name, description: tool.description, @@ -292,11 +301,12 @@ export function createBridgeServer(bridge) { "rea_search", { description: - "Discover original REA tools by all query words or exact names. Empty query lists the complete catalog. Schemas are exact upstream MCP schemas; generated TypeScript inputs are optional. Code Mode calls tools.(original_arguments); results are original MCP envelopes, parsed by json(reply).", + "Discover original REA tools by all query words or exact names. Empty query lists the complete catalog. Input schemas are exact upstream MCP schemas with generated TypeScript. Request output_schemas=true only when needed. catalog_sha256 hashes the upstream SDK tool catalog, not REA server_identity.catalog.digests. Code Mode calls tools.(original_arguments); results are original MCP envelopes, parsed by json(reply).", inputSchema: z.strictObject({ query: z.string().default(""), names: z.array(z.string()).default([]), schemas: z.boolean().default(true), + output_schemas: z.boolean().default(false), }), annotations: { readOnlyHint: true, diff --git a/experiments/codemode/bridge.test.mjs b/experiments/codemode/bridge.test.mjs index 1468087cd..92f10e1f8 100644 --- a/experiments/codemode/bridge.test.mjs +++ b/experiments/codemode/bridge.test.mjs @@ -31,7 +31,10 @@ after(async () => { }); test("discovery preserves exact current REA schemas and names", async () => { - const result = await bridge.search({ names: ["binary_session"] }); + const result = await bridge.search({ + names: ["binary_session"], + output_schemas: true, + }); assert.equal(result.total_tools, upstream.catalog.length); const original = upstream.catalog.find( (tool) => tool.name === "binary_session", @@ -39,6 +42,9 @@ test("discovery preserves exact current REA schemas and names", async () => { const { typescript, ...discovered } = result.tools[0]; assert.deepEqual(discovered, original); assert.match(typescript, /export interface BinarySession_Input/); + const inputOnly = await bridge.search({ names: ["binary_session"] }); + assert.deepEqual(inputOnly.tools[0].inputSchema, original.inputSchema); + assert.equal(inputOnly.tools[0].outputSchema, undefined); const all = await bridge.search({ schemas: false }); assert.deepEqual( all.tools.map((tool) => tool.name), diff --git a/experiments/codemode/resource-run.py b/experiments/codemode/resource-run.py index 14383cfbf..f4855a2fa 100644 --- a/experiments/codemode/resource-run.py +++ b/experiments/codemode/resource-run.py @@ -29,7 +29,7 @@ def main(): cpu = max(os.sched_getaffinity(0)) command = [ 'systemd-run', '--user', '--scope', '--unit=' + unit, - '-p', 'MemoryMax=4G', '-p', 'MemoryHigh=2G', '-p', 'MemorySwapMax=0', + '-p', 'MemoryMax=4G', '-p', 'MemoryHigh=3G', '-p', 'MemorySwapMax=0', '-p', 'CPUQuota=100%', '-p', 'TasksMax=96', 'taskset', '-c', str(cpu), *args, ] diff --git a/experiments/codemode/results.md b/experiments/codemode/results.md new file mode 100644 index 000000000..a0072aeda --- /dev/null +++ b/experiments/codemode/results.md @@ -0,0 +1,47 @@ +# Preliminary measurements + +These are real Codex CLI 0.161.0 trials on the frozen checkout, using +`gpt-5.6-terra`, high reasoning, one CPU and a guarded memory scope. The task +called `binary_session` once and requested open state and server identity. This +is a health probe, not an investigation benchmark. API input tokens are cumulative +across model steps; cached tokens are a subset, not an additional cost. + +| Profile | Seconds | Input tokens | Cached input | Output tokens | Outcome | +| ---------------------------------- | ------: | -----------: | -----------: | ------------: | --------------------------------------------------------------- | +| Direct MCP | 47.635 | 103154 | 70656 | 940 | Correct closed state; identity omitted after preview truncation | +| Client Code Mode flag | 67.345 | 197315 | 157696 | 1458 | Correct closed state; extra failed resource lookup | +| Client Code Mode only flag | 54.276 | 126609 | 84992 | 939 | Correct closed state; identity omitted | +| Bridge, original prompt | 300.173 | unavailable | unavailable | unavailable | Timeout before any completed tool call | +| Bridge, explicit projection prompt | 84.790 | 209248 | 175616 | 2125 | Correct closed state/version; conflated catalog identities | + +These prompts and flags do not establish a controlled comparison. The projection +trial used a more explicit prompt. The client trials expose nested original MCP +calls in their transcripts; the flags alone do not establish code composition or +intermediate-result filtering. The timeout is retained as a failed trial without +an attributed root cause. + +The direct tool delivered the original session result, including package version +6.4.0 and REA's catalog digests. In the bridge projection trial the script selected +the nonexistent `server_identity.catalog.digest`, which JavaScript omitted. The +answer instead used the bridge's SDK catalog hash. That is not the requested REA +identity and must fail that factual check. No unsupported digest is used as an +oracle for subsequent trials. + +Discovery was unnecessarily returning entire output schemas. For example, the +live `binary_session` output schema occupies 18039 compact JSON bytes, compared +with 859 for its input schema. `analyze_javascript_application` uses 41662 and +2863 bytes respectively. The complete SDK catalog contains 172 tools and 1986857 +compact JSON bytes. The bridge now defaults to input schemas and makes unchanged +output schemas available explicitly. Actual model savings from this change remain +unmeasured. + +The upstream/downstream stdio boundary suite passed all nine tests. The repository +typecheck and lint passed under a one-CPU resource scope with a sampled peak of +2934038528 bytes. A 2 GiB soft limit caused heavy reclaim during native TypeScript +checking; the wrapper's soft limit is now aligned with its 3 GiB early-stop +threshold. Its 4 GiB hard ceiling and host-available-memory checks remain in force. +An earlier cached-build attempt was killed inside its 2 GiB scope; that is a +failed build attempt, not evidence of a host OOM or a benchmark result. + +Real shipped-artifact investigations and provider parity are still unverified. +The bridge is an experiment and has not changed any published REA contract.