fix(experiments): preserve partial evidence and queued cancellation ownership

This commit is contained in:
N0zoM1z0
2026-10-11 17:26:35 +00:00
parent 15cc7ce6a6
commit 89a7ca8486
5 changed files with 128 additions and 7 deletions
+3 -2
View File
@@ -46,9 +46,10 @@ Calls and executions sharing one active target are serialized; `Promise.all`
currently provides composition rather than parallel provider execution.
`npm test` runs actual upstream and downstream stdio MCP against this checkout's
compiled runtime. The standalone experiment's nine tests cover discovery parity,
compiled runtime. The standalone experiment's eleven tests cover discovery parity,
original results/errors, bulk promise settlement, host capability absence, CPU
deadline recovery, retained Evidence after an error, unresolved promises and the
deadline recovery, retained Evidence after an error or provider timeout,
queued-call cancellation, unresolved promises and the
two-tool downstream boundary. These tests do not establish real native/Android/
firmware provider behavior. A lifecycle regression revealed that the QuickJS job
pump can allocate an additional context: the execution scope now owns every
+35
View File
@@ -217,6 +217,7 @@ let discovery;
let value;
let failure;
let bridgeReply;
const recovery = [];
const operationStarted = performance.now();
try {
if (values.mode !== "direct")
@@ -240,6 +241,39 @@ try {
} catch (error) {
failure = String(error);
} finally {
const partials = new Set(
records.flatMap((record) => {
if (!record.reply?.isError) return [];
try {
const partial = json(record.reply).error?.details?.partial_observation;
return partial?.kind === "retained-evidence"
? [partial.evidence_id]
: [];
} catch {
return []; // Preserve non-JSON failures in calls.json, without inventing IDs.
}
}),
);
for (const evidence_id of partials) {
// Capture already retained partial facts before disposing the replay session.
// Recovery is not a successful completion of the failed analyst workflow.
try {
const reply = await realCall(
"inspect_analysis_view",
{
source: { kind: "retained-evidence", evidence_id },
view: { kind: "summary" },
},
{ timeout: 60_000 },
);
recovery.push({ parent_evidence_id: evidence_id, reply });
} catch (error) {
recovery.push({
parent_evidence_id: evidence_id,
failure: String(error),
});
}
}
await upstream.close();
}
const catalogExposure = values.mode === "direct" ? upstream.catalog : discovery;
@@ -272,6 +306,7 @@ const summary = {
: records.filter((x) => x.text_output).map((x) => x.text_output),
projection: value,
failure: failure ?? null,
partial_recovery: recovery,
correctness: failure
? "failed_workflow"
: "requires_cross_profile_fact_review",
+4 -1
View File
@@ -196,9 +196,9 @@ export class CodeModeBridge {
const audit = { name, arguments_sha256: digest(args) };
calls.push(audit);
const work = toolQueue.then(async () => {
signal.throwIfAborted();
const started = performance.now();
try {
signal.throwIfAborted();
const reply = await this.upstream.call(name, args, {
signal,
timeout: Math.max(1, deadline - Date.now()),
@@ -210,6 +210,9 @@ export class CodeModeBridge {
try {
const record = json(reply);
if (record.evidence_id) audit.evidence_id = record.evidence_id;
const partial = record.error?.details?.partial_observation;
if (partial?.kind === "retained-evidence")
audit.partial_evidence_id = partial.evidence_id;
} catch {
/* Non-JSON MCP content is still preserved in reply. */
}
+52
View File
@@ -132,6 +132,58 @@ test("error after successful call preserves original Evidence reference", async
);
});
test("provider timeout retains its actual partial Evidence reference", async () => {
const result = await bridge.execute({
code: `const reply = await tools.analyze_javascript_application(${JSON.stringify({ ...analysisArgs, analysis_timeout_ms: 1 })}); return json(reply);`,
});
const partial = result.value.error.details.partial_observation;
assert.equal(partial.kind, "retained-evidence");
assert.equal(result.calls[0].is_error, true);
assert.equal(result.calls[0].partial_evidence_id, partial.evidence_id);
const view = json(
await upstream.call("inspect_analysis_view", {
source: partial,
view: { kind: "summary" },
}),
);
assert.equal(view.normalized_result.parent_evidence_id, partial.evidence_id);
});
test("cancelling queued calls releases deferred handles and permits reuse", async () => {
let callStarted;
const started = new Promise((resolve) => {
callStarted = resolve;
});
const cancelled = new AbortController();
const controlled = new CodeModeBridge({
catalog: upstream.catalog,
async call(name, args, options) {
const reply = await upstream.call(name, args, options);
callStarted();
// A controlled delivery delay after a real result makes the queued
// cancellation deterministic, without simulating provider behavior.
await new Promise((resolve) => setTimeout(resolve, 50));
return reply;
},
});
const execution = controlled.execute(
{
code: "return await Promise.all([tools.binary_session({}), tools.binary_session({})]);",
},
{ signal: cancelled.signal },
);
await started;
cancelled.abort(new Error("controlled queued cancellation"));
const result = await execution;
assert.ok(result.bridge_error);
assert.equal(result.calls.length, 2);
assert.match(
result.calls[1].transport_error,
/controlled queued cancellation/,
);
assert.equal((await controlled.execute({ code: "return 42;" })).value, 42);
});
test("a guest promise without any producer fails without hanging", async () => {
const result = await bridge.execute({
code: "return await new Promise(() => {});",
+34 -4
View File
@@ -78,7 +78,9 @@ jq's original trial failed because the stripped shipped executable has no
`jq_compile` symbol. The corrected direct trial uses the observed ELF entry
`0x403a50`, resolves it to Ghidra's `entry` function and analyzes that function.
It succeeds with retained Evidence and a closed ephemeral provider session.
Bridge parity for the corrected workflow is pending. This small entry function
The corrected bridge returns identical selected facts and limitations. Its final
result estimate is 1544 tokens versus 10189 for direct single-text results, plus
1610 for discovery and 267 for code. This small entry function
does not establish large-function or complete native-analysis coverage.
OpenWrt's original direct trial returned `unreadable_output` without the expected
@@ -86,15 +88,22 @@ report. Twelve independent bounded Binwalk invocations produced reports. An
observer launcher and a subsequent unchanged real Binwalk bridge trial succeeded,
reporting uImage at offset 0 and SquashFS at 2741985 with `signature_scan` coverage.
The observer is a diagnostic with a different launcher identity, not a benchmark
baseline. The original failure remains unexplained; direct parity is pending.
baseline. The original failure remains unexplained. Subsequent direct and bridge
workflows agree on all region facts and coverage; random upstream region IDs vary
and are preserved. Single-text estimates are 1387 tokens direct and 827 projected,
so this one-operation task provides much less composition benefit than Android.
SQLite's actual WASM fails the frozen REA WABT output parser with `Unrecognized
WABT section row`. WABT 1.0.42 prints the nine-character DataCount label without
leading padding; the owning parser currently requires leading whitespace. No
downstream shim is used, and no successful WASM provider claim is made.
Obsidian's initial real-model analysis ended without Evidence after Codex's host
closed stdout; a standalone SDK replay is pending. Neither failure is a Code Mode
speedup.
closed stdout. Its standalone SDK replay returns a real provider timeout with a
retained partial Evidence ID. This was observed before automatic partial-summary
recovery was added, so the transcript retains that reference but does not export
its payload. A source-fixture production-boundary test now verifies that provider
timeout IDs survive auditing and resolve through the retained-evidence view.
Neither failure is a Code Mode speedup.
## Real Android agents
@@ -141,4 +150,25 @@ work and differing prompts/instructions. They cannot support a controlled speed
ratio. A reproducible efficiency conclusion requires successful equal-task trials
and serial repetitions with the strongest existing-client baseline.
The stronger client trial uses an explicit instruction to compose and project the
existing operations, and passes all six checks: 72.341 seconds, 161370 input
tokens (120576 cached), 1664 output tokens and three completed MCP calls. An
equivalent question with the bridge passes in 115.962 seconds, 321295 input tokens
(283648 cached), 2274 output tokens, five discovery calls and one execution. The
native transcript does not expose executor source, so exact internal composition
is unverified. Nevertheless, this measured client profile is a materially stronger
baseline than the original repeated-call trial. Total input, uncached input,
output and wall time must be considered separately; cached-token differences make
total input an incomplete cost metric.
This reverses the preliminary temptation to equate replay compression with the
best Codex experience. Existing client support with explicit Evidence-aware
recipes may be enough. An optional bridge needs to justify discovery overhead and
remains useful to test for clients without equivalent execution support.
Eleven actual SDK/stdio tests pass, including a controlled delivery delay after a
real upstream result to exercise queued cancellation. Both the delayed call and
its queued successor release deferred guest handles before runtime disposal.
This test does not simulate or establish new provider behavior.
The bridge remains an experiment and has not changed any published REA contract.