fix(providers): retain worker cleanup owners through shutdown

Validation: integrated changes passed formatting, typecheck, lint, resolved module-boundary verification and git diff --check. Tests and real-provider workflows were not run.
This commit is contained in:
morluto
2026-10-11 00:03:21 +08:00
parent a01148553e
commit d1235fadc9
24 changed files with 688 additions and 120 deletions
+20 -11
View File
@@ -1,6 +1,6 @@
import { createHash } from "node:crypto";
import { constants } from "node:fs";
import { lstat, open, readdir, writeFile } from "node:fs/promises";
import { lstat, open, opendir, writeFile } from "node:fs/promises";
import { isAbsolute, join } from "node:path";
import { TextDecoder } from "node:util";
import {
@@ -138,18 +138,27 @@ export const inventoryRecoveryFiles = async (
OPERATION,
`Unsafe staging directory: ${directory}`,
);
for (const entry of await readdir(directory, { withFileTypes: true })) {
const handle = await opendir(directory);
const children: string[] = [];
try {
for await (const entry of handle) {
checkRecoveryCancellation(signal);
entries++;
if (entries > RECOVERY_LIMITS.outputEntries)
throw new AnalysisOutputError(
OPERATION,
"Recovery staging exceeds the 10000-entry resource limit",
);
children.push(entry.name);
}
} finally {
await handle.close().catch(() => undefined);
}
for (const name of children) {
checkRecoveryCancellation(signal);
entries++;
const relativePath =
prefix === "" ? entry.name : `${prefix}/${entry.name}`;
const path = join(directory, entry.name);
const relativePath = prefix === "" ? name : `${prefix}/${name}`;
const path = join(directory, name);
const info = await lstat(path);
if (entries > RECOVERY_LIMITS.outputEntries)
throw new AnalysisOutputError(
OPERATION,
"Recovery staging exceeds the 10000-entry resource limit",
);
if (info.isSymbolicLink() || (!info.isDirectory() && !info.isFile()))
throw new AnalysisOutputError(
OPERATION,
+33 -7
View File
@@ -33,6 +33,31 @@ export type WakaruLauncher = (
options: OwnedProviderProcessSpawnOptions,
) => Promise<SpawnedOwnedProviderProcess>;
/** Cleanup failure carrying the live supervisor so the provider can retry it. */
export class WakaruCleanupFailure extends ProviderCleanupError {
constructor(
readonly cleanupOwner: ProviderProcessSupervisor,
readonly runId: string,
readonly cwd: string,
reason: string,
previousError: unknown,
snapshot: ReturnType<ProviderProcessSupervisor["snapshot"]>,
) {
super(
"wakaru",
[runId, cwd],
{
reason,
previous_error:
previousError instanceof Error ? previousError.message : null,
exit_code: snapshot.exitCode ?? null,
signal: snapshot.signal ?? null,
},
{ operation: OPERATION, cause: previousError },
);
}
}
/** Resolve and fingerprint explicitly configured tools; acquire nothing at startup. */
export const resolveWakaruCommand = async (
environment: Readonly<Record<string, string | undefined>>,
@@ -158,14 +183,15 @@ export const runWakaruCommand = async (context: WakaruCommandContext) => {
}
const stopped = await supervisor.stop();
const snapshot = supervisor.snapshot();
supervisor.dispose();
if (stopped.status === "incomplete")
throw new ProviderCleanupError("wakaru", [runId, context.cwd], {
reason: stopped.reason,
previous_error: failure instanceof Error ? failure.message : null,
exit_code: snapshot.exitCode ?? null,
signal: snapshot.signal ?? null,
});
throw new WakaruCleanupFailure(
supervisor,
runId,
context.cwd,
stopped.reason,
failure,
snapshot,
);
if (failure !== undefined) throw failure;
if (context.signal?.aborted === true)
throw new AnalysisCancelledError(OPERATION);
+148 -11
View File
@@ -20,14 +20,30 @@ import {
recoveryInputFailure,
snapshotRecoveryInput,
} from "./RecoveryFiles.js";
import { resolveWakaruCommand, type WakaruLauncher } from "./WakaruCommand.js";
import {
resolveWakaruCommand,
type WakaruLauncher,
WakaruCleanupFailure,
} from "./WakaruCommand.js";
import type { ProviderProcessSupervisor } from "../../process/ProviderProcess.js";
import { prepareWakaruExecution } from "./WakaruExecution.js";
import { RECOVERY_LIMITS } from "./WakaruRelease.js";
const OPERATION = "recover_javascript_sources";
/** Stateless adapter; each recovery owns its snapshot, process and publication. */
/** Serialized recovery adapter retaining failed process, workspace and publication cleanup. */
export class WakaruProvider implements JavaScriptRecoveryPort {
#tail: Promise<void> = Promise.resolve();
#closing = false;
#closePromise: Promise<void> | undefined;
#pendingCleanup:
| {
root: string | undefined;
process: ProviderProcessSupervisor | undefined;
tree: SafeOutputTree | undefined;
}
| undefined;
constructor(
readonly environment: Readonly<
Record<string, string | undefined>
@@ -36,10 +52,46 @@ export class WakaruProvider implements JavaScriptRecoveryPort {
) {}
/** Recover one script using validated upstream reports and verified output bytes. */
async recover(
recover(
input: JavaScriptRecoveryInput,
options?: ExecutionOptions,
): Promise<Result<AnalysisExecution, AnalysisError>> {
if (this.#closing)
return Promise.resolve(
err(
new ProviderCleanupError(
"wakaru",
[],
{ reason: "Provider is closing" },
{ operation: OPERATION },
),
),
);
const operation = this.#tail.then(() =>
this.#closing
? err(
new ProviderCleanupError(
"wakaru",
[],
{ reason: "Provider is closing" },
{ operation: OPERATION },
),
)
: this.#recover(input, options),
);
this.#tail = operation.then(
() => undefined,
() => undefined,
);
return operation;
}
async #recover(
input: JavaScriptRecoveryInput,
options?: ExecutionOptions,
): Promise<Result<AnalysisExecution, AnalysisError>> {
const pending = await this.#retire();
if (pending !== undefined) return err(pending);
let root: string | undefined;
let tree: SafeOutputTree | undefined;
let failure: unknown;
@@ -88,20 +140,42 @@ export class WakaruProvider implements JavaScriptRecoveryPort {
}
if (failure !== undefined) {
const residuals: string[] = [];
const cleanupOwner =
failure instanceof WakaruCleanupFailure
? failure.cleanupOwner
: undefined;
let failedTree: SafeOutputTree | undefined;
try {
await tree?.rollback();
} catch {
if (tree !== undefined) residuals.push(tree.outputRoot);
if (tree !== undefined) {
failedTree = tree;
residuals.push(tree.outputRoot);
}
}
if (cleanupOwner !== undefined)
residuals.push(cleanupOwner.launch.ownership?.runId ?? "wakaru-worker");
if (root !== undefined && cleanupOwner === undefined) {
const failedRoot = root;
try {
await rm(failedRoot, { recursive: true, force: true });
root = undefined;
} catch {
residuals.push(failedRoot);
}
}
if (
root !== undefined &&
!(failure instanceof AnalysisError && failure.cleanupIncomplete)
cleanupOwner !== undefined ||
failedTree !== undefined ||
root !== undefined
) {
try {
await rm(root, { recursive: true, force: true });
} catch {
residuals.push(root);
}
this.#pendingCleanup = {
root,
process: cleanupOwner,
tree: failedTree,
};
if (root !== undefined) residuals.push(root);
root = undefined;
}
if (residuals.length > 0)
return err(
@@ -124,4 +198,67 @@ export class WakaruProvider implements JavaScriptRecoveryPort {
)
: ok(execution);
}
async close(): Promise<void> {
this.#closing = true;
this.#closePromise ??= this.#tail
.then(async () => {
const failure = await this.#retire();
if (failure !== undefined) throw failure;
})
.catch((cause: unknown) => {
this.#closePromise = undefined;
throw cause;
});
return this.#closePromise;
}
async #retire(): Promise<ProviderCleanupError | undefined> {
const pending = this.#pendingCleanup;
if (pending === undefined) return undefined;
if (pending.process !== undefined) {
const stopped = await pending.process.stop();
if (stopped.status === "incomplete")
return new ProviderCleanupError(
"wakaru",
[
pending.process.launch.ownership?.runId ?? "wakaru-worker",
...(pending.root === undefined ? [] : [pending.root]),
],
{ reason: stopped.reason },
{ operation: OPERATION },
);
pending.process = undefined;
}
if (pending.tree !== undefined) {
const tree = pending.tree;
try {
await tree.rollback();
pending.tree = undefined;
} catch (cause: unknown) {
return new ProviderCleanupError(
"wakaru",
[tree.outputRoot],
{ reason: recoveryFailureMessage(cause) },
{ operation: OPERATION, cause },
);
}
}
if (pending.root !== undefined) {
const root = pending.root;
try {
await rm(root, { recursive: true, force: true });
pending.root = undefined;
} catch (cause: unknown) {
return new ProviderCleanupError(
"wakaru",
[root],
{ reason: recoveryFailureMessage(cause) },
{ operation: OPERATION, cause },
);
}
}
this.#pendingCleanup = undefined;
return undefined;
}
}
+27 -5
View File
@@ -12,6 +12,7 @@ import {
} from "../../domain/analysisErrorCore.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js";
import {
@@ -48,6 +49,22 @@ export interface SourceMapDecoderDependencies {
readonly prepareOwnership?: (signal?: AbortSignal) => Promise<void>;
}
export interface SourceMapCleanupOwner {
close(
previousError: string | null,
): Promise<ProviderCleanupError | undefined>;
}
export class SourceMapCleanupFailure extends ProviderCleanupError {
constructor(
readonly cleanupOwner: SourceMapCleanupOwner,
resources: readonly string[],
diagnostics: Readonly<Record<string, JsonValue>>,
) {
super(PROVIDER, resources, diagnostics, { operation: OPERATION });
}
}
/** Run a trusted codec command and release its process/root before returning output. */
export const runSourceMapCommand = async (
request: string,
@@ -94,7 +111,7 @@ export const runSourceMapCommand = async (
: result;
};
class OwnedCodec {
class OwnedCodec implements SourceMapCleanupOwner {
readonly #runId = `rea-source-map-${randomUUID()}`;
#runtime: SourceMapCodecRuntime | undefined;
#supervisor: ProviderProcessSupervisor | undefined;
@@ -231,21 +248,27 @@ class OwnedCodec {
previousError: string | null,
): Promise<ProviderCleanupError | undefined> {
const failures: string[] = [];
let processReleased = this.#supervisor === undefined;
if (this.#supervisor !== undefined) {
const stopped = await this.#supervisor.stop();
if (stopped.status === "incomplete") failures.push(stopped.reason);
else {
processReleased = true;
this.#supervisor = undefined;
}
}
if (this.#runtime !== undefined) {
if (processReleased && this.#runtime !== undefined) {
try {
await this.#runtime.close();
this.#runtime = undefined;
} catch (cause: unknown) {
failures.push(message(cause));
}
}
return failures.length === 0
? undefined
: new ProviderCleanupError(
PROVIDER,
: new SourceMapCleanupFailure(
this,
[
this.#runId,
...(this.#runtime === undefined ? [] : [this.#runtime.path]),
@@ -256,7 +279,6 @@ class OwnedCodec {
previous_error: previousError,
codec_pid: this.#supervisor?.launch.process.pid ?? null,
},
{ operation: OPERATION },
);
}
}
+94 -14
View File
@@ -18,19 +18,64 @@ import {
sourceMapCodecReplySchema,
} from "./SourceMapCodecProtocol.js";
import {
SourceMapCleanupFailure,
runSourceMapCommand,
type SourceMapCleanupOwner,
type SourceMapDecoderDependencies,
} from "./SourceMapCommand.js";
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import { jsonParts } from "../../domain/jsonSerialization.js";
const OPERATION = "trace_web_source_location";
/** Integrate the pinned upstream codec through an independently bounded owned process. */
export class SourceMapDecoder implements WebSourceMapPort {
#tail: Promise<void> = Promise.resolve();
#closing = false;
#closePromise: Promise<void> | undefined;
#pendingCleanup: SourceMapCleanupOwner | undefined;
constructor(readonly dependencies: SourceMapDecoderDependencies = {}) {}
/** Return complete point evidence after process and private-root cleanup. */
async trace(
trace(
input: Parameters<WebSourceMapPort["trace"]>[0],
options?: ExecutionOptions,
): Promise<Result<WebSourceMapReport, AnalysisError>> {
if (this.#closing)
return Promise.resolve(
err(
new ProviderCleanupError(
"source-map-decoder",
[],
{ reason: "Provider is closing" },
{ operation: OPERATION },
),
),
);
const operation = this.#tail.then(() =>
this.#closing
? err(
new ProviderCleanupError(
"source-map-decoder",
[],
{ reason: "Provider is closing" },
{ operation: OPERATION },
),
)
: this.#trace(input, options),
);
this.#tail = operation.then(
() => undefined,
() => undefined,
);
return operation;
}
async #trace(
input: Parameters<WebSourceMapPort["trace"]>[0],
options?: ExecutionOptions,
): Promise<Result<WebSourceMapReport, AnalysisError>> {
const cleanupFailure = await this.#retryCleanup();
if (cleanupFailure !== undefined) return err(cleanupFailure);
if (options?.signal?.aborted)
return err(new AnalysisCancelledError(OPERATION));
const request = sourceMapCodecInputSchema.safeParse({
@@ -40,26 +85,38 @@ export class SourceMapDecoder implements WebSourceMapPort {
});
if (!request.success)
return err(new AnalysisInputError(OPERATION, { cause: request.error }));
const serialized = JSON.stringify(request.data);
if (
Buffer.byteLength(input.text) > WEB_SOURCE_MAP_LIMITS.mapBytes ||
Buffer.byteLength(serialized) > WEB_SOURCE_MAP_LIMITS.outputBytes
)
return err(
new ArtifactOperationError(
OPERATION,
"limit",
undefined,
`${input.path}: Source-map input exceeds the 4 MiB map or 32 MiB encoded request budget.`,
),
const capacityFailure = () =>
new ArtifactOperationError(
OPERATION,
"limit",
undefined,
`${input.path}: Source-map input exceeds the 4 MiB map or 32 MiB encoded request budget.`,
);
if (Buffer.byteLength(input.text) > WEB_SOURCE_MAP_LIMITS.mapBytes)
return err(capacityFailure());
const encoded: string[] = [];
let encodedBytes = 0;
for (const part of jsonParts(request.data)) {
const bytes = Buffer.byteLength(part);
if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes - encodedBytes)
return err(capacityFailure());
encoded.push(part);
encodedBytes += bytes;
}
if (options?.signal?.aborted)
return err(new AnalysisCancelledError(OPERATION));
const serialized = encoded.join("");
const response = await runSourceMapCommand(
serialized,
input.path,
options,
this.dependencies,
);
if (!response.ok) return response;
if (!response.ok) {
if (response.error instanceof SourceMapCleanupFailure)
this.#pendingCleanup = response.error.cleanupOwner;
return response;
}
let value: unknown;
try {
value = JSON.parse(response.value);
@@ -92,6 +149,29 @@ export class SourceMapDecoder implements WebSourceMapPort {
);
return ok(reply.data.report);
}
async close(): Promise<void> {
this.#closing = true;
this.#closePromise ??= this.#tail
.then(async () => {
const failure = await this.#retryCleanup();
if (failure !== undefined) throw failure;
})
.catch((cause: unknown) => {
this.#closePromise = undefined;
throw cause;
});
return this.#closePromise;
}
async #retryCleanup(): Promise<AnalysisError | undefined> {
const pending = this.#pendingCleanup;
if (pending === undefined) return undefined;
const failure = await pending.close(null);
if (failure !== undefined) return failure;
this.#pendingCleanup = undefined;
return undefined;
}
}
const codecFailure = (
path: string,