diff --git a/src/application/DirectAnalysisCleanup.ts b/src/application/DirectAnalysisCleanup.ts index 573dea7c0..3f3c54425 100644 --- a/src/application/DirectAnalysisCleanup.ts +++ b/src/application/DirectAnalysisCleanup.ts @@ -6,6 +6,7 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js"; import { err, type Result } from "../domain/result.js"; import type { AnalysisExecution } from "./AnalysisProvider.js"; import type { BinarySession } from "./binary/BinarySession.js"; +import { analysisErrorWithCleanupFailure } from "./binary/AnalysisClientCleanup.js"; /** Await one-shot cleanup and retain either the outcome or both failures. */ export const withSessionCleanup = async ( @@ -77,19 +78,19 @@ export const managedAnalysisCleanupFailure = ( result: Result, error: AnalysisError, ): Result => { + if (!result.ok) + return err(analysisErrorWithCleanupFailure(result.error, error)); const cleanup = projectAnalysisError(error); return err( new ProviderAdapterError( error instanceof ProviderAdapterError ? error.providerId : "rea", "close_binary", { - cause: result.ok ? error : result.error, + cause: error, cleanup: { reason: cleanup.message, resources: error.cleanupResources }, diagnostics: { cleanup_error: cleanup, - ...(result.ok - ? { partial_observation: jsonValueSchema.parse(result.value) } - : { primary_error: projectAnalysisError(result.error) }), + partial_observation: jsonValueSchema.parse(result.value), }, }, ), diff --git a/src/application/binary/AnalysisClientCleanup.ts b/src/application/binary/AnalysisClientCleanup.ts index a21a33b4b..ab7dd3534 100644 --- a/src/application/binary/AnalysisClientCleanup.ts +++ b/src/application/binary/AnalysisClientCleanup.ts @@ -1,4 +1,6 @@ import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { projectAnalysisError } from "../../domain/analysisErrorProjection.js"; import { err, ok, type Result } from "../../domain/result.js"; import { AnalysisError } from "../../domain/analysisErrorBase.js"; import type { AnalysisClient, ExecutionOptions } from "../AnalysisProvider.js"; @@ -30,3 +32,27 @@ export const closeAnalysisClient = async ( ); } }; + +/** Preserve the original lifecycle failure alongside incomplete cleanup. */ +export const analysisErrorWithCleanupFailure = ( + primary: AnalysisError, + cleanup: AnalysisError, + operation = "close_binary", +): AnalysisError => { + const projected = projectAnalysisError(cleanup); + return new ProviderAdapterError( + cleanup instanceof ProviderAdapterError ? cleanup.providerId : "rea", + operation, + { + cause: primary, + cleanup: { + reason: projected.message, + resources: cleanup.cleanupResources, + }, + diagnostics: { + primary_error: projectAnalysisError(primary), + cleanup_error: projected, + }, + }, + ); +}; diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index 184fea127..ddbc24554 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -44,7 +44,10 @@ import { commitExecutionProfile, prepareSessionExecution, } from "./BinarySessionExecution.js"; -import { closeAnalysisClient } from "./AnalysisClientCleanup.js"; +import { + analysisErrorWithCleanupFailure, + closeAnalysisClient, +} from "./AnalysisClientCleanup.js"; export type { BinarySessionPort } from "./BinarySessionPort.js"; const OFFICIAL_OPERATIONS: ReadonlySet = new Set( OFFICIAL_TOOL_CONTRACTS.map(({ name }) => name), @@ -53,6 +56,15 @@ const ENHANCED_OPERATIONS: ReadonlySet = new Set( ENHANCED_TOOL_CONTRACTS.map(({ name }) => name), ); +interface SessionBinding { + readonly target: BinaryTarget; + readonly client: AnalysisClient; + readonly profile: AnalysisProfileCommitment | null; + readonly compatibility: Readonly>; + readonly route: SessionProviderRoute; + readonly runId: string; +} + /** * Owns the single active target shared by CLI and MCP adapters. * @@ -64,14 +76,12 @@ export class BinarySession extends BinarySessionRecords implements BinarySessionPort { - #active: + #active: SessionBinding | undefined; + #pendingCleanup: | { - readonly target: BinaryTarget; readonly client: AnalysisClient; - readonly profile: AnalysisProfileCommitment | null; - readonly compatibility: Readonly>; - readonly route: SessionProviderRoute; - readonly runId: string; + readonly providerId: string; + readonly retainDocument: boolean; } | undefined; #transition: Promise = Promise.resolve(); @@ -240,6 +250,10 @@ export class BinarySession ? profile === null : profile !== null && analysisProfilesEqual(activeProfile, profile); await this.#drainCalls(admittedThrough); + if (isAborted(options.signal)) + return err(new AnalysisCancelledError("open_binary")); + const pendingClosed = await this.#retryPendingCleanup(); + if (!pendingClosed.ok) return pendingClosed; if (isAborted(options.signal)) return err(new AnalysisCancelledError("open_binary")); if (sameTarget && sameProfile) { @@ -252,7 +266,7 @@ export class BinarySession const previous = this.#active; this.#active = undefined; if (previous !== undefined) { - const closed = await closeAnalysisClient( + const closed = await this.#retireClient( previous.client, previous.route.identity.id, ); @@ -262,18 +276,35 @@ export class BinarySession return closed; } } + if (isAborted(options.signal)) + return err( + await this.#restoreAfterFailedOpen( + previous, + new AnalysisCancelledError("open_binary"), + ), + ); const runId = randomUUID(); const client = route.createClient(target, { runId }); const started = await client.execute("health", {}, options); if (!started.ok) { - await client.close(); - await this.#restore(previous); - return started; + return err( + await this.#failedOpen( + client, + route.identity.id, + previous, + started.error, + ), + ); } if (isAborted(options.signal)) { - await client.close(); - await this.#restore(previous); - return err(new AnalysisCancelledError("open_binary")); + return err( + await this.#failedOpen( + client, + route.identity.id, + previous, + new AnalysisCancelledError("open_binary"), + ), + ); } this.#active = { target, @@ -293,9 +324,14 @@ export class BinarySession const imported = this.importAnalysisSnapshot(options.snapshot); if (!imported.ok) { this.#active = undefined; - await client.close(); - await this.#restore(previous); - return imported; + return err( + await this.#failedOpen( + client, + route.identity.id, + previous, + imported.error, + ), + ); } } return ok(target); @@ -346,8 +382,11 @@ export class BinarySession this.#active = undefined; const closed = previous === undefined - ? ok(null) - : await closeAnalysisClient( + ? await this.#retryPendingCleanup( + options.progress, + options.retainProviderDocuments, + ) + : await this.#retireClient( previous.client, previous.route.identity.id, { @@ -364,6 +403,69 @@ export class BinarySession return closed; } + async #retireClient( + client: AnalysisClient, + providerId: string, + options: Pick & { + readonly retainDocument?: boolean; + } = {}, + ): Promise> { + this.#pendingCleanup = { + client, + providerId, + retainDocument: options.retainDocument === true, + }; + return this.#retryPendingCleanup(options.progress); + } + + async #retryPendingCleanup( + progress?: ExecutionOptions["progress"], + retainDocument?: boolean, + ): Promise> { + let pending = this.#pendingCleanup; + if (pending === undefined) return ok(null); + if (retainDocument !== undefined) { + pending = { ...pending, retainDocument }; + this.#pendingCleanup = pending; + } + const closed = await closeAnalysisClient( + pending.client, + pending.providerId, + { + ...(progress === undefined ? {} : { progress }), + ...(pending.retainDocument ? { retainDocument: true } : {}), + }, + ); + if (closed.ok) this.#pendingCleanup = undefined; + return closed; + } + + async #failedOpen( + client: AnalysisClient, + providerId: string, + previous: SessionBinding | undefined, + primary: AnalysisError, + ): Promise { + const closed = await this.#retireClient(client, providerId); + if (!closed.ok) + return analysisErrorWithCleanupFailure( + primary, + closed.error, + "open_binary", + ); + return this.#restoreAfterFailedOpen(previous, primary); + } + + async #restoreAfterFailedOpen( + previous: SessionBinding | undefined, + primary: AnalysisError, + ): Promise { + const restored = await this.#restore(previous); + return restored.ok + ? primary + : analysisErrorWithCleanupFailure(primary, restored.error, "open_binary"); + } + /** Describe the active binary session. */ status(): JsonValue { return binarySessionStatus({ @@ -580,22 +682,13 @@ export class BinarySession } async #restore( - previous: - | { - readonly target: BinaryTarget; - readonly client: AnalysisClient; - readonly profile: AnalysisProfileCommitment | null; - readonly compatibility: Readonly>; - readonly route: SessionProviderRoute; - readonly runId: string; - } - | undefined, - ): Promise { - if (previous === undefined) return; + previous: SessionBinding | undefined, + ): Promise> { + if (previous === undefined) return ok(null); const runId = randomUUID(); const client = previous.route.createClient(previous.target, { runId }); const started = await client.execute("health", {}); - if (started.ok) + if (started.ok) { this.#active = { target: previous.target, client, @@ -604,7 +697,18 @@ export class BinarySession route: previous.route, runId, }; - else await client.close(); + return ok(null); + } + const closed = await this.#retireClient(client, previous.route.identity.id); + return closed.ok + ? ok(null) + : err( + analysisErrorWithCleanupFailure( + started.error, + closed.error, + "open_binary", + ), + ); } async #waitForTransition( diff --git a/tests/composition/analysis-sessions/binarySessionCleanupRetry.test.ts b/tests/composition/analysis-sessions/binarySessionCleanupRetry.test.ts new file mode 100644 index 000000000..b1244ce84 --- /dev/null +++ b/tests/composition/analysis-sessions/binarySessionCleanupRetry.test.ts @@ -0,0 +1,273 @@ +import { access, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect, it, onTestFinished } from "vitest"; + +import type { AnalysisClient } from "../../../src/application/AnalysisProvider.js"; +import { HopperStartError } from "../../../src/domain/hopperErrors.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { ProviderCleanupError } from "../../../src/domain/providerCleanupError.js"; +import { err, ok as resultOk } from "../../../src/domain/result.js"; +import { observed as ok } from "../../fixtures/analysisExecution.js"; +import { + createBinarySessionTargets, + createDeferred, + createTestBinarySession, +} from "../../fixtures/binarySession.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const retryFixture = async ( + execute: AnalysisClient["execute"] = () => Promise.resolve(ok(null)), +) => { + const parent = await createTestTempDirectory("rea-session-cleanup-retry-"); + const runtime = join(parent, "owned-runtime"); + const document = join(parent, "retained-document"); + await mkdir(runtime); + await writeFile(join(runtime, "observation"), "owned observation"); + await writeFile(document, "caller-retained document"); + const targets = await createBinarySessionTargets(); + let allowed = false; + let created = 0; + const cleanupError = new ProviderCleanupError("fixture", [runtime], { + reason: "owned runtime removal denied", + leftover_paths: [runtime], + }); + const client: AnalysisClient = { + execute, + close: async () => { + if (!allowed) throw cleanupError; + await rm(runtime, { recursive: true, force: true }); + }, + closeWithOutcome: async (options) => { + if (!allowed) return err(cleanupError); + await client.close(); + if (options?.retainDocument !== true) await rm(document, { force: true }); + return resultOk(null); + }, + }; + const session = createTestBinarySession(() => { + created += 1; + return created === 1 + ? client + : { + execute: () => Promise.resolve(ok(null)), + close: () => Promise.resolve(), + }; + }); + onTestFinished(async () => { + allowed = true; + await session.close(); + }); + return { + session, + targets, + runtime, + document, + cleanupError, + allowCleanup: () => { + allowed = true; + }, + created: () => created, + }; +}; + +it.each(["close", "switch", "open after close"] as const)( + "retries owned cleanup before completing %s", + async (transition) => { + const fixture = await retryFixture(); + const { + session, + targets: [first, second], + runtime, + cleanupError, + } = fixture; + expect((await session.open(first)).ok).toBe(true); + const attempt = () => + transition === "close" ? session.close() : session.open(second); + const initial = + transition === "switch" ? await attempt() : await session.close(); + expect(initial).toEqual(err(cleanupError)); + expect(session.activeTarget()).toBeUndefined(); + expect(await session.execute("address_name", {})).toMatchObject({ + ok: false, + }); + await access(runtime); + + expect(await attempt()).toEqual(err(cleanupError)); + expect(fixture.created()).toBe(1); + await access(runtime); + + fixture.allowCleanup(); + expect((await attempt()).ok).toBe(true); + await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); + expect(fixture.created()).toBe(transition === "close" ? 1 : 2); + }, +); + +it("keeps a failed candidate for cleanup and preserves its startup failure", async () => { + const primary = new HopperStartError(); + const fixture = await retryFixture(() => Promise.resolve(err(primary))); + const result = await fixture.session.open(fixture.targets[0]); + expect(result).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + cleanupResources: [fixture.runtime], + diagnostics: { + primary_error: projectAnalysisError(primary), + cleanup_error: projectAnalysisError(fixture.cleanupError), + }, + }, + }); + expect(fixture.session.activeTarget()).toBeUndefined(); + fixture.allowCleanup(); + expect((await fixture.session.close()).ok).toBe(true); + await expect(access(fixture.runtime)).rejects.toMatchObject({ + code: "ENOENT", + }); +}); + +it("retains cancelled startup cleanup and the cancellation reason", async () => { + const controller = new AbortController(); + const fixture = await retryFixture(() => { + controller.abort(); + return Promise.resolve(ok(null)); + }); + expect( + await fixture.session.open(fixture.targets[0], { + signal: controller.signal, + }), + ).toMatchObject({ + ok: false, + error: { + cleanupIncomplete: true, + cleanupResources: [fixture.runtime], + diagnostics: { primary_error: { code: "cancelled" } }, + }, + }); + fixture.allowCleanup(); + expect((await fixture.session.close()).ok).toBe(true); + await expect(access(fixture.runtime)).rejects.toMatchObject({ + code: "ENOENT", + }); +}); + +it("preserves caller-selected document retention when retrying cleanup", async () => { + const fixture = await retryFixture(); + expect((await fixture.session.open(fixture.targets[0])).ok).toBe(true); + expect( + await fixture.session.close({ retainProviderDocuments: true }), + ).toEqual(err(fixture.cleanupError)); + fixture.allowCleanup(); + expect((await fixture.session.close()).ok).toBe(true); + await expect(access(fixture.runtime)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await readFile(fixture.document, "utf8")).toBe( + "caller-retained document", + ); +}); + +it("honors an explicit retention choice on a cleanup retry", async () => { + const fixture = await retryFixture(); + expect((await fixture.session.open(fixture.targets[0])).ok).toBe(true); + expect(await fixture.session.close()).toEqual(err(fixture.cleanupError)); + fixture.allowCleanup(); + expect( + (await fixture.session.close({ retainProviderDocuments: true })).ok, + ).toBe(true); + await expect(access(fixture.runtime)).rejects.toMatchObject({ + code: "ENOENT", + }); + expect(await readFile(fixture.document, "utf8")).toBe( + "caller-retained document", + ); +}); + +it("keeps failed restoration resources available for a close retry", async () => { + const parent = await createTestTempDirectory("rea-restore-cleanup-retry-"); + const runtime = join(parent, "restore-runtime"); + await mkdir(runtime); + await writeFile(join(runtime, "observation"), "restore observation"); + const [first, second] = await createBinarySessionTargets(); + let created = 0; + let allowed = false; + const session = createTestBinarySession(() => { + const index = ++created; + return { + execute: () => + Promise.resolve(index === 1 ? ok(null) : err(new HopperStartError())), + close: async () => { + if (index !== 3) return; + if (!allowed) + throw new ProviderCleanupError("fixture", [runtime], { + reason: "restoration cleanup denied", + }); + await rm(runtime, { recursive: true, force: true }); + }, + }; + }); + onTestFinished(async () => { + allowed = true; + await session.close(); + }); + expect((await session.open(first)).ok).toBe(true); + expect(await session.open(second)).toMatchObject({ + ok: false, + error: { cleanupIncomplete: true, cleanupResources: [runtime] }, + }); + expect(session.activeTarget()).toBeUndefined(); + allowed = true; + expect((await session.close()).ok).toBe(true); + await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); +}); + +it("does not start a replacement cancelled while pending cleanup completes", async () => { + const parent = await createTestTempDirectory("rea-cancel-cleanup-retry-"); + const runtime = join(parent, "owned-runtime"); + await mkdir(runtime); + await writeFile(join(runtime, "observation"), "owned observation"); + const [first, second] = await createBinarySessionTargets(); + const entered = createDeferred(); + const release = createDeferred(); + let allowed = false; + let created = 0; + const session = createTestBinarySession(() => { + const index = ++created; + return { + execute: () => Promise.resolve(ok(null)), + closeWithOutcome: async () => { + if (index !== 1) return resultOk(null); + if (!allowed) + return err( + new ProviderCleanupError("fixture", [runtime], { + reason: "cleanup denied", + }), + ); + entered.resolve(); + await release.promise; + await rm(runtime, { recursive: true, force: true }); + return resultOk(null); + }, + close: () => Promise.resolve(), + }; + }); + onTestFinished(async () => { + allowed = true; + release.resolve(); + await session.close(); + }); + expect((await session.open(first)).ok).toBe(true); + expect((await session.close()).ok).toBe(false); + allowed = true; + const controller = new AbortController(); + const opening = session.open(second, { signal: controller.signal }); + await entered.promise; + controller.abort(); + release.resolve(); + expect(await opening).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError" }, + }); + expect(created).toBe(1); + await expect(access(runtime)).rejects.toMatchObject({ code: "ENOENT" }); +});