fix(experiments): preserve partial evidence and queued cancellation ownership
This commit is contained in:
@@ -46,9 +46,10 @@ Calls and executions sharing one active target are serialized; `Promise.all`
|
||||
currently provides composition rather than parallel provider execution.
|
||||
|
||||
`npm test` runs actual upstream and downstream stdio MCP against this checkout's
|
||||
compiled runtime. The standalone experiment's nine tests cover discovery parity,
|
||||
compiled runtime. The standalone experiment's eleven tests cover discovery parity,
|
||||
original results/errors, bulk promise settlement, host capability absence, CPU
|
||||
deadline recovery, retained Evidence after an error, unresolved promises and the
|
||||
deadline recovery, retained Evidence after an error or provider timeout,
|
||||
queued-call cancellation, unresolved promises and the
|
||||
two-tool downstream boundary. These tests do not establish real native/Android/
|
||||
firmware provider behavior. A lifecycle regression revealed that the QuickJS job
|
||||
pump can allocate an additional context: the execution scope now owns every
|
||||
|
||||
@@ -217,6 +217,7 @@ let discovery;
|
||||
let value;
|
||||
let failure;
|
||||
let bridgeReply;
|
||||
const recovery = [];
|
||||
const operationStarted = performance.now();
|
||||
try {
|
||||
if (values.mode !== "direct")
|
||||
@@ -240,6 +241,39 @@ try {
|
||||
} catch (error) {
|
||||
failure = String(error);
|
||||
} finally {
|
||||
const partials = new Set(
|
||||
records.flatMap((record) => {
|
||||
if (!record.reply?.isError) return [];
|
||||
try {
|
||||
const partial = json(record.reply).error?.details?.partial_observation;
|
||||
return partial?.kind === "retained-evidence"
|
||||
? [partial.evidence_id]
|
||||
: [];
|
||||
} catch {
|
||||
return []; // Preserve non-JSON failures in calls.json, without inventing IDs.
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const evidence_id of partials) {
|
||||
// Capture already retained partial facts before disposing the replay session.
|
||||
// Recovery is not a successful completion of the failed analyst workflow.
|
||||
try {
|
||||
const reply = await realCall(
|
||||
"inspect_analysis_view",
|
||||
{
|
||||
source: { kind: "retained-evidence", evidence_id },
|
||||
view: { kind: "summary" },
|
||||
},
|
||||
{ timeout: 60_000 },
|
||||
);
|
||||
recovery.push({ parent_evidence_id: evidence_id, reply });
|
||||
} catch (error) {
|
||||
recovery.push({
|
||||
parent_evidence_id: evidence_id,
|
||||
failure: String(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
await upstream.close();
|
||||
}
|
||||
const catalogExposure = values.mode === "direct" ? upstream.catalog : discovery;
|
||||
@@ -272,6 +306,7 @@ const summary = {
|
||||
: records.filter((x) => x.text_output).map((x) => x.text_output),
|
||||
projection: value,
|
||||
failure: failure ?? null,
|
||||
partial_recovery: recovery,
|
||||
correctness: failure
|
||||
? "failed_workflow"
|
||||
: "requires_cross_profile_fact_review",
|
||||
|
||||
@@ -196,9 +196,9 @@ export class CodeModeBridge {
|
||||
const audit = { name, arguments_sha256: digest(args) };
|
||||
calls.push(audit);
|
||||
const work = toolQueue.then(async () => {
|
||||
signal.throwIfAborted();
|
||||
const started = performance.now();
|
||||
try {
|
||||
signal.throwIfAborted();
|
||||
const reply = await this.upstream.call(name, args, {
|
||||
signal,
|
||||
timeout: Math.max(1, deadline - Date.now()),
|
||||
@@ -210,6 +210,9 @@ export class CodeModeBridge {
|
||||
try {
|
||||
const record = json(reply);
|
||||
if (record.evidence_id) audit.evidence_id = record.evidence_id;
|
||||
const partial = record.error?.details?.partial_observation;
|
||||
if (partial?.kind === "retained-evidence")
|
||||
audit.partial_evidence_id = partial.evidence_id;
|
||||
} catch {
|
||||
/* Non-JSON MCP content is still preserved in reply. */
|
||||
}
|
||||
|
||||
@@ -132,6 +132,58 @@ test("error after successful call preserves original Evidence reference", async
|
||||
);
|
||||
});
|
||||
|
||||
test("provider timeout retains its actual partial Evidence reference", async () => {
|
||||
const result = await bridge.execute({
|
||||
code: `const reply = await tools.analyze_javascript_application(${JSON.stringify({ ...analysisArgs, analysis_timeout_ms: 1 })}); return json(reply);`,
|
||||
});
|
||||
const partial = result.value.error.details.partial_observation;
|
||||
assert.equal(partial.kind, "retained-evidence");
|
||||
assert.equal(result.calls[0].is_error, true);
|
||||
assert.equal(result.calls[0].partial_evidence_id, partial.evidence_id);
|
||||
const view = json(
|
||||
await upstream.call("inspect_analysis_view", {
|
||||
source: partial,
|
||||
view: { kind: "summary" },
|
||||
}),
|
||||
);
|
||||
assert.equal(view.normalized_result.parent_evidence_id, partial.evidence_id);
|
||||
});
|
||||
|
||||
test("cancelling queued calls releases deferred handles and permits reuse", async () => {
|
||||
let callStarted;
|
||||
const started = new Promise((resolve) => {
|
||||
callStarted = resolve;
|
||||
});
|
||||
const cancelled = new AbortController();
|
||||
const controlled = new CodeModeBridge({
|
||||
catalog: upstream.catalog,
|
||||
async call(name, args, options) {
|
||||
const reply = await upstream.call(name, args, options);
|
||||
callStarted();
|
||||
// A controlled delivery delay after a real result makes the queued
|
||||
// cancellation deterministic, without simulating provider behavior.
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
return reply;
|
||||
},
|
||||
});
|
||||
const execution = controlled.execute(
|
||||
{
|
||||
code: "return await Promise.all([tools.binary_session({}), tools.binary_session({})]);",
|
||||
},
|
||||
{ signal: cancelled.signal },
|
||||
);
|
||||
await started;
|
||||
cancelled.abort(new Error("controlled queued cancellation"));
|
||||
const result = await execution;
|
||||
assert.ok(result.bridge_error);
|
||||
assert.equal(result.calls.length, 2);
|
||||
assert.match(
|
||||
result.calls[1].transport_error,
|
||||
/controlled queued cancellation/,
|
||||
);
|
||||
assert.equal((await controlled.execute({ code: "return 42;" })).value, 42);
|
||||
});
|
||||
|
||||
test("a guest promise without any producer fails without hanging", async () => {
|
||||
const result = await bridge.execute({
|
||||
code: "return await new Promise(() => {});",
|
||||
|
||||
@@ -78,7 +78,9 @@ jq's original trial failed because the stripped shipped executable has no
|
||||
`jq_compile` symbol. The corrected direct trial uses the observed ELF entry
|
||||
`0x403a50`, resolves it to Ghidra's `entry` function and analyzes that function.
|
||||
It succeeds with retained Evidence and a closed ephemeral provider session.
|
||||
Bridge parity for the corrected workflow is pending. This small entry function
|
||||
The corrected bridge returns identical selected facts and limitations. Its final
|
||||
result estimate is 1544 tokens versus 10189 for direct single-text results, plus
|
||||
1610 for discovery and 267 for code. This small entry function
|
||||
does not establish large-function or complete native-analysis coverage.
|
||||
|
||||
OpenWrt's original direct trial returned `unreadable_output` without the expected
|
||||
@@ -86,15 +88,22 @@ report. Twelve independent bounded Binwalk invocations produced reports. An
|
||||
observer launcher and a subsequent unchanged real Binwalk bridge trial succeeded,
|
||||
reporting uImage at offset 0 and SquashFS at 2741985 with `signature_scan` coverage.
|
||||
The observer is a diagnostic with a different launcher identity, not a benchmark
|
||||
baseline. The original failure remains unexplained; direct parity is pending.
|
||||
baseline. The original failure remains unexplained. Subsequent direct and bridge
|
||||
workflows agree on all region facts and coverage; random upstream region IDs vary
|
||||
and are preserved. Single-text estimates are 1387 tokens direct and 827 projected,
|
||||
so this one-operation task provides much less composition benefit than Android.
|
||||
|
||||
SQLite's actual WASM fails the frozen REA WABT output parser with `Unrecognized
|
||||
WABT section row`. WABT 1.0.42 prints the nine-character DataCount label without
|
||||
leading padding; the owning parser currently requires leading whitespace. No
|
||||
downstream shim is used, and no successful WASM provider claim is made.
|
||||
Obsidian's initial real-model analysis ended without Evidence after Codex's host
|
||||
closed stdout; a standalone SDK replay is pending. Neither failure is a Code Mode
|
||||
speedup.
|
||||
closed stdout. Its standalone SDK replay returns a real provider timeout with a
|
||||
retained partial Evidence ID. This was observed before automatic partial-summary
|
||||
recovery was added, so the transcript retains that reference but does not export
|
||||
its payload. A source-fixture production-boundary test now verifies that provider
|
||||
timeout IDs survive auditing and resolve through the retained-evidence view.
|
||||
Neither failure is a Code Mode speedup.
|
||||
|
||||
## Real Android agents
|
||||
|
||||
@@ -141,4 +150,25 @@ work and differing prompts/instructions. They cannot support a controlled speed
|
||||
ratio. A reproducible efficiency conclusion requires successful equal-task trials
|
||||
and serial repetitions with the strongest existing-client baseline.
|
||||
|
||||
The stronger client trial uses an explicit instruction to compose and project the
|
||||
existing operations, and passes all six checks: 72.341 seconds, 161370 input
|
||||
tokens (120576 cached), 1664 output tokens and three completed MCP calls. An
|
||||
equivalent question with the bridge passes in 115.962 seconds, 321295 input tokens
|
||||
(283648 cached), 2274 output tokens, five discovery calls and one execution. The
|
||||
native transcript does not expose executor source, so exact internal composition
|
||||
is unverified. Nevertheless, this measured client profile is a materially stronger
|
||||
baseline than the original repeated-call trial. Total input, uncached input,
|
||||
output and wall time must be considered separately; cached-token differences make
|
||||
total input an incomplete cost metric.
|
||||
|
||||
This reverses the preliminary temptation to equate replay compression with the
|
||||
best Codex experience. Existing client support with explicit Evidence-aware
|
||||
recipes may be enough. An optional bridge needs to justify discovery overhead and
|
||||
remains useful to test for clients without equivalent execution support.
|
||||
|
||||
Eleven actual SDK/stdio tests pass, including a controlled delivery delay after a
|
||||
real upstream result to exercise queued cancellation. Both the delayed call and
|
||||
its queued successor release deferred guest handles before runtime disposal.
|
||||
This test does not simulate or establish new provider behavior.
|
||||
|
||||
The bridge remains an experiment and has not changed any published REA contract.
|
||||
|
||||
Reference in New Issue
Block a user