diff --git a/experiments/codemode/README.md b/experiments/codemode/README.md index 9e5c09180..f631311f7 100644 --- a/experiments/codemode/README.md +++ b/experiments/codemode/README.md @@ -46,9 +46,10 @@ Calls and executions sharing one active target are serialized; `Promise.all` currently provides composition rather than parallel provider execution. `npm test` runs actual upstream and downstream stdio MCP against this checkout's -compiled runtime. The standalone experiment's nine tests cover discovery parity, +compiled runtime. The standalone experiment's eleven tests cover discovery parity, original results/errors, bulk promise settlement, host capability absence, CPU -deadline recovery, retained Evidence after an error, unresolved promises and the +deadline recovery, retained Evidence after an error or provider timeout, +queued-call cancellation, unresolved promises and the two-tool downstream boundary. These tests do not establish real native/Android/ firmware provider behavior. A lifecycle regression revealed that the QuickJS job pump can allocate an additional context: the execution scope now owns every diff --git a/experiments/codemode/benchmark.mjs b/experiments/codemode/benchmark.mjs index 70aaf82f9..c24843c38 100644 --- a/experiments/codemode/benchmark.mjs +++ b/experiments/codemode/benchmark.mjs @@ -217,6 +217,7 @@ let discovery; let value; let failure; let bridgeReply; +const recovery = []; const operationStarted = performance.now(); try { if (values.mode !== "direct") @@ -240,6 +241,39 @@ try { } catch (error) { failure = String(error); } finally { + const partials = new Set( + records.flatMap((record) => { + if (!record.reply?.isError) return []; + try { + const partial = json(record.reply).error?.details?.partial_observation; + return partial?.kind === "retained-evidence" + ? [partial.evidence_id] + : []; + } catch { + return []; // Preserve non-JSON failures in calls.json, without inventing IDs. + } + }), + ); + for (const evidence_id of partials) { + // Capture already retained partial facts before disposing the replay session. + // Recovery is not a successful completion of the failed analyst workflow. + try { + const reply = await realCall( + "inspect_analysis_view", + { + source: { kind: "retained-evidence", evidence_id }, + view: { kind: "summary" }, + }, + { timeout: 60_000 }, + ); + recovery.push({ parent_evidence_id: evidence_id, reply }); + } catch (error) { + recovery.push({ + parent_evidence_id: evidence_id, + failure: String(error), + }); + } + } await upstream.close(); } const catalogExposure = values.mode === "direct" ? upstream.catalog : discovery; @@ -272,6 +306,7 @@ const summary = { : records.filter((x) => x.text_output).map((x) => x.text_output), projection: value, failure: failure ?? null, + partial_recovery: recovery, correctness: failure ? "failed_workflow" : "requires_cross_profile_fact_review", diff --git a/experiments/codemode/bridge.mjs b/experiments/codemode/bridge.mjs index cba2d3e44..5905bed2b 100644 --- a/experiments/codemode/bridge.mjs +++ b/experiments/codemode/bridge.mjs @@ -196,9 +196,9 @@ export class CodeModeBridge { const audit = { name, arguments_sha256: digest(args) }; calls.push(audit); const work = toolQueue.then(async () => { - signal.throwIfAborted(); const started = performance.now(); try { + signal.throwIfAborted(); const reply = await this.upstream.call(name, args, { signal, timeout: Math.max(1, deadline - Date.now()), @@ -210,6 +210,9 @@ export class CodeModeBridge { try { const record = json(reply); if (record.evidence_id) audit.evidence_id = record.evidence_id; + const partial = record.error?.details?.partial_observation; + if (partial?.kind === "retained-evidence") + audit.partial_evidence_id = partial.evidence_id; } catch { /* Non-JSON MCP content is still preserved in reply. */ } diff --git a/experiments/codemode/bridge.test.mjs b/experiments/codemode/bridge.test.mjs index 756ed637c..34ab26db1 100644 --- a/experiments/codemode/bridge.test.mjs +++ b/experiments/codemode/bridge.test.mjs @@ -132,6 +132,58 @@ test("error after successful call preserves original Evidence reference", async ); }); +test("provider timeout retains its actual partial Evidence reference", async () => { + const result = await bridge.execute({ + code: `const reply = await tools.analyze_javascript_application(${JSON.stringify({ ...analysisArgs, analysis_timeout_ms: 1 })}); return json(reply);`, + }); + const partial = result.value.error.details.partial_observation; + assert.equal(partial.kind, "retained-evidence"); + assert.equal(result.calls[0].is_error, true); + assert.equal(result.calls[0].partial_evidence_id, partial.evidence_id); + const view = json( + await upstream.call("inspect_analysis_view", { + source: partial, + view: { kind: "summary" }, + }), + ); + assert.equal(view.normalized_result.parent_evidence_id, partial.evidence_id); +}); + +test("cancelling queued calls releases deferred handles and permits reuse", async () => { + let callStarted; + const started = new Promise((resolve) => { + callStarted = resolve; + }); + const cancelled = new AbortController(); + const controlled = new CodeModeBridge({ + catalog: upstream.catalog, + async call(name, args, options) { + const reply = await upstream.call(name, args, options); + callStarted(); + // A controlled delivery delay after a real result makes the queued + // cancellation deterministic, without simulating provider behavior. + await new Promise((resolve) => setTimeout(resolve, 50)); + return reply; + }, + }); + const execution = controlled.execute( + { + code: "return await Promise.all([tools.binary_session({}), tools.binary_session({})]);", + }, + { signal: cancelled.signal }, + ); + await started; + cancelled.abort(new Error("controlled queued cancellation")); + const result = await execution; + assert.ok(result.bridge_error); + assert.equal(result.calls.length, 2); + assert.match( + result.calls[1].transport_error, + /controlled queued cancellation/, + ); + assert.equal((await controlled.execute({ code: "return 42;" })).value, 42); +}); + test("a guest promise without any producer fails without hanging", async () => { const result = await bridge.execute({ code: "return await new Promise(() => {});", diff --git a/experiments/codemode/results.md b/experiments/codemode/results.md index fbdf47b67..45c06260d 100644 --- a/experiments/codemode/results.md +++ b/experiments/codemode/results.md @@ -78,7 +78,9 @@ jq's original trial failed because the stripped shipped executable has no `jq_compile` symbol. The corrected direct trial uses the observed ELF entry `0x403a50`, resolves it to Ghidra's `entry` function and analyzes that function. It succeeds with retained Evidence and a closed ephemeral provider session. -Bridge parity for the corrected workflow is pending. This small entry function +The corrected bridge returns identical selected facts and limitations. Its final +result estimate is 1544 tokens versus 10189 for direct single-text results, plus +1610 for discovery and 267 for code. This small entry function does not establish large-function or complete native-analysis coverage. OpenWrt's original direct trial returned `unreadable_output` without the expected @@ -86,15 +88,22 @@ report. Twelve independent bounded Binwalk invocations produced reports. An observer launcher and a subsequent unchanged real Binwalk bridge trial succeeded, reporting uImage at offset 0 and SquashFS at 2741985 with `signature_scan` coverage. The observer is a diagnostic with a different launcher identity, not a benchmark -baseline. The original failure remains unexplained; direct parity is pending. +baseline. The original failure remains unexplained. Subsequent direct and bridge +workflows agree on all region facts and coverage; random upstream region IDs vary +and are preserved. Single-text estimates are 1387 tokens direct and 827 projected, +so this one-operation task provides much less composition benefit than Android. SQLite's actual WASM fails the frozen REA WABT output parser with `Unrecognized WABT section row`. WABT 1.0.42 prints the nine-character DataCount label without leading padding; the owning parser currently requires leading whitespace. No downstream shim is used, and no successful WASM provider claim is made. Obsidian's initial real-model analysis ended without Evidence after Codex's host -closed stdout; a standalone SDK replay is pending. Neither failure is a Code Mode -speedup. +closed stdout. Its standalone SDK replay returns a real provider timeout with a +retained partial Evidence ID. This was observed before automatic partial-summary +recovery was added, so the transcript retains that reference but does not export +its payload. A source-fixture production-boundary test now verifies that provider +timeout IDs survive auditing and resolve through the retained-evidence view. +Neither failure is a Code Mode speedup. ## Real Android agents @@ -141,4 +150,25 @@ work and differing prompts/instructions. They cannot support a controlled speed ratio. A reproducible efficiency conclusion requires successful equal-task trials and serial repetitions with the strongest existing-client baseline. +The stronger client trial uses an explicit instruction to compose and project the +existing operations, and passes all six checks: 72.341 seconds, 161370 input +tokens (120576 cached), 1664 output tokens and three completed MCP calls. An +equivalent question with the bridge passes in 115.962 seconds, 321295 input tokens +(283648 cached), 2274 output tokens, five discovery calls and one execution. The +native transcript does not expose executor source, so exact internal composition +is unverified. Nevertheless, this measured client profile is a materially stronger +baseline than the original repeated-call trial. Total input, uncached input, +output and wall time must be considered separately; cached-token differences make +total input an incomplete cost metric. + +This reverses the preliminary temptation to equate replay compression with the +best Codex experience. Existing client support with explicit Evidence-aware +recipes may be enough. An optional bridge needs to justify discovery overhead and +remains useful to test for clients without equivalent execution support. + +Eleven actual SDK/stdio tests pass, including a controlled delivery delay after a +real upstream result to exercise queued cancellation. Both the delayed call and +its queued successor release deferred guest handles before runtime disposal. +This test does not simulate or establish new provider behavior. + The bridge remains an experiment and has not changed any published REA contract.