fix(providers): retain worker cleanup owners through shutdown
Validation: integrated changes passed formatting, typecheck, lint, resolved module-boundary verification and git diff --check. Tests and real-provider workflows were not run.
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { constants } from "node:fs";
|
||||
import { lstat, open, readdir, writeFile } from "node:fs/promises";
|
||||
import { lstat, open, opendir, writeFile } from "node:fs/promises";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import { TextDecoder } from "node:util";
|
||||
import {
|
||||
@@ -138,18 +138,27 @@ export const inventoryRecoveryFiles = async (
|
||||
OPERATION,
|
||||
`Unsafe staging directory: ${directory}`,
|
||||
);
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const handle = await opendir(directory);
|
||||
const children: string[] = [];
|
||||
try {
|
||||
for await (const entry of handle) {
|
||||
checkRecoveryCancellation(signal);
|
||||
entries++;
|
||||
if (entries > RECOVERY_LIMITS.outputEntries)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"Recovery staging exceeds the 10000-entry resource limit",
|
||||
);
|
||||
children.push(entry.name);
|
||||
}
|
||||
} finally {
|
||||
await handle.close().catch(() => undefined);
|
||||
}
|
||||
for (const name of children) {
|
||||
checkRecoveryCancellation(signal);
|
||||
entries++;
|
||||
const relativePath =
|
||||
prefix === "" ? entry.name : `${prefix}/${entry.name}`;
|
||||
const path = join(directory, entry.name);
|
||||
const relativePath = prefix === "" ? name : `${prefix}/${name}`;
|
||||
const path = join(directory, name);
|
||||
const info = await lstat(path);
|
||||
if (entries > RECOVERY_LIMITS.outputEntries)
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
"Recovery staging exceeds the 10000-entry resource limit",
|
||||
);
|
||||
if (info.isSymbolicLink() || (!info.isDirectory() && !info.isFile()))
|
||||
throw new AnalysisOutputError(
|
||||
OPERATION,
|
||||
|
||||
@@ -33,6 +33,31 @@ export type WakaruLauncher = (
|
||||
options: OwnedProviderProcessSpawnOptions,
|
||||
) => Promise<SpawnedOwnedProviderProcess>;
|
||||
|
||||
/** Cleanup failure carrying the live supervisor so the provider can retry it. */
|
||||
export class WakaruCleanupFailure extends ProviderCleanupError {
|
||||
constructor(
|
||||
readonly cleanupOwner: ProviderProcessSupervisor,
|
||||
readonly runId: string,
|
||||
readonly cwd: string,
|
||||
reason: string,
|
||||
previousError: unknown,
|
||||
snapshot: ReturnType<ProviderProcessSupervisor["snapshot"]>,
|
||||
) {
|
||||
super(
|
||||
"wakaru",
|
||||
[runId, cwd],
|
||||
{
|
||||
reason,
|
||||
previous_error:
|
||||
previousError instanceof Error ? previousError.message : null,
|
||||
exit_code: snapshot.exitCode ?? null,
|
||||
signal: snapshot.signal ?? null,
|
||||
},
|
||||
{ operation: OPERATION, cause: previousError },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Resolve and fingerprint explicitly configured tools; acquire nothing at startup. */
|
||||
export const resolveWakaruCommand = async (
|
||||
environment: Readonly<Record<string, string | undefined>>,
|
||||
@@ -158,14 +183,15 @@ export const runWakaruCommand = async (context: WakaruCommandContext) => {
|
||||
}
|
||||
const stopped = await supervisor.stop();
|
||||
const snapshot = supervisor.snapshot();
|
||||
supervisor.dispose();
|
||||
if (stopped.status === "incomplete")
|
||||
throw new ProviderCleanupError("wakaru", [runId, context.cwd], {
|
||||
reason: stopped.reason,
|
||||
previous_error: failure instanceof Error ? failure.message : null,
|
||||
exit_code: snapshot.exitCode ?? null,
|
||||
signal: snapshot.signal ?? null,
|
||||
});
|
||||
throw new WakaruCleanupFailure(
|
||||
supervisor,
|
||||
runId,
|
||||
context.cwd,
|
||||
stopped.reason,
|
||||
failure,
|
||||
snapshot,
|
||||
);
|
||||
if (failure !== undefined) throw failure;
|
||||
if (context.signal?.aborted === true)
|
||||
throw new AnalysisCancelledError(OPERATION);
|
||||
|
||||
@@ -20,14 +20,30 @@ import {
|
||||
recoveryInputFailure,
|
||||
snapshotRecoveryInput,
|
||||
} from "./RecoveryFiles.js";
|
||||
import { resolveWakaruCommand, type WakaruLauncher } from "./WakaruCommand.js";
|
||||
import {
|
||||
resolveWakaruCommand,
|
||||
type WakaruLauncher,
|
||||
WakaruCleanupFailure,
|
||||
} from "./WakaruCommand.js";
|
||||
import type { ProviderProcessSupervisor } from "../../process/ProviderProcess.js";
|
||||
import { prepareWakaruExecution } from "./WakaruExecution.js";
|
||||
import { RECOVERY_LIMITS } from "./WakaruRelease.js";
|
||||
|
||||
const OPERATION = "recover_javascript_sources";
|
||||
|
||||
/** Stateless adapter; each recovery owns its snapshot, process and publication. */
|
||||
/** Serialized recovery adapter retaining failed process, workspace and publication cleanup. */
|
||||
export class WakaruProvider implements JavaScriptRecoveryPort {
|
||||
#tail: Promise<void> = Promise.resolve();
|
||||
#closing = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
#pendingCleanup:
|
||||
| {
|
||||
root: string | undefined;
|
||||
process: ProviderProcessSupervisor | undefined;
|
||||
tree: SafeOutputTree | undefined;
|
||||
}
|
||||
| undefined;
|
||||
|
||||
constructor(
|
||||
readonly environment: Readonly<
|
||||
Record<string, string | undefined>
|
||||
@@ -36,10 +52,46 @@ export class WakaruProvider implements JavaScriptRecoveryPort {
|
||||
) {}
|
||||
|
||||
/** Recover one script using validated upstream reports and verified output bytes. */
|
||||
async recover(
|
||||
recover(
|
||||
input: JavaScriptRecoveryInput,
|
||||
options?: ExecutionOptions,
|
||||
): Promise<Result<AnalysisExecution, AnalysisError>> {
|
||||
if (this.#closing)
|
||||
return Promise.resolve(
|
||||
err(
|
||||
new ProviderCleanupError(
|
||||
"wakaru",
|
||||
[],
|
||||
{ reason: "Provider is closing" },
|
||||
{ operation: OPERATION },
|
||||
),
|
||||
),
|
||||
);
|
||||
const operation = this.#tail.then(() =>
|
||||
this.#closing
|
||||
? err(
|
||||
new ProviderCleanupError(
|
||||
"wakaru",
|
||||
[],
|
||||
{ reason: "Provider is closing" },
|
||||
{ operation: OPERATION },
|
||||
),
|
||||
)
|
||||
: this.#recover(input, options),
|
||||
);
|
||||
this.#tail = operation.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
return operation;
|
||||
}
|
||||
|
||||
async #recover(
|
||||
input: JavaScriptRecoveryInput,
|
||||
options?: ExecutionOptions,
|
||||
): Promise<Result<AnalysisExecution, AnalysisError>> {
|
||||
const pending = await this.#retire();
|
||||
if (pending !== undefined) return err(pending);
|
||||
let root: string | undefined;
|
||||
let tree: SafeOutputTree | undefined;
|
||||
let failure: unknown;
|
||||
@@ -88,20 +140,42 @@ export class WakaruProvider implements JavaScriptRecoveryPort {
|
||||
}
|
||||
if (failure !== undefined) {
|
||||
const residuals: string[] = [];
|
||||
const cleanupOwner =
|
||||
failure instanceof WakaruCleanupFailure
|
||||
? failure.cleanupOwner
|
||||
: undefined;
|
||||
let failedTree: SafeOutputTree | undefined;
|
||||
try {
|
||||
await tree?.rollback();
|
||||
} catch {
|
||||
if (tree !== undefined) residuals.push(tree.outputRoot);
|
||||
if (tree !== undefined) {
|
||||
failedTree = tree;
|
||||
residuals.push(tree.outputRoot);
|
||||
}
|
||||
}
|
||||
if (cleanupOwner !== undefined)
|
||||
residuals.push(cleanupOwner.launch.ownership?.runId ?? "wakaru-worker");
|
||||
if (root !== undefined && cleanupOwner === undefined) {
|
||||
const failedRoot = root;
|
||||
try {
|
||||
await rm(failedRoot, { recursive: true, force: true });
|
||||
root = undefined;
|
||||
} catch {
|
||||
residuals.push(failedRoot);
|
||||
}
|
||||
}
|
||||
if (
|
||||
root !== undefined &&
|
||||
!(failure instanceof AnalysisError && failure.cleanupIncomplete)
|
||||
cleanupOwner !== undefined ||
|
||||
failedTree !== undefined ||
|
||||
root !== undefined
|
||||
) {
|
||||
try {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
} catch {
|
||||
residuals.push(root);
|
||||
}
|
||||
this.#pendingCleanup = {
|
||||
root,
|
||||
process: cleanupOwner,
|
||||
tree: failedTree,
|
||||
};
|
||||
if (root !== undefined) residuals.push(root);
|
||||
root = undefined;
|
||||
}
|
||||
if (residuals.length > 0)
|
||||
return err(
|
||||
@@ -124,4 +198,67 @@ export class WakaruProvider implements JavaScriptRecoveryPort {
|
||||
)
|
||||
: ok(execution);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closing = true;
|
||||
this.#closePromise ??= this.#tail
|
||||
.then(async () => {
|
||||
const failure = await this.#retire();
|
||||
if (failure !== undefined) throw failure;
|
||||
})
|
||||
.catch((cause: unknown) => {
|
||||
this.#closePromise = undefined;
|
||||
throw cause;
|
||||
});
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #retire(): Promise<ProviderCleanupError | undefined> {
|
||||
const pending = this.#pendingCleanup;
|
||||
if (pending === undefined) return undefined;
|
||||
if (pending.process !== undefined) {
|
||||
const stopped = await pending.process.stop();
|
||||
if (stopped.status === "incomplete")
|
||||
return new ProviderCleanupError(
|
||||
"wakaru",
|
||||
[
|
||||
pending.process.launch.ownership?.runId ?? "wakaru-worker",
|
||||
...(pending.root === undefined ? [] : [pending.root]),
|
||||
],
|
||||
{ reason: stopped.reason },
|
||||
{ operation: OPERATION },
|
||||
);
|
||||
pending.process = undefined;
|
||||
}
|
||||
if (pending.tree !== undefined) {
|
||||
const tree = pending.tree;
|
||||
try {
|
||||
await tree.rollback();
|
||||
pending.tree = undefined;
|
||||
} catch (cause: unknown) {
|
||||
return new ProviderCleanupError(
|
||||
"wakaru",
|
||||
[tree.outputRoot],
|
||||
{ reason: recoveryFailureMessage(cause) },
|
||||
{ operation: OPERATION, cause },
|
||||
);
|
||||
}
|
||||
}
|
||||
if (pending.root !== undefined) {
|
||||
const root = pending.root;
|
||||
try {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
pending.root = undefined;
|
||||
} catch (cause: unknown) {
|
||||
return new ProviderCleanupError(
|
||||
"wakaru",
|
||||
[root],
|
||||
{ reason: recoveryFailureMessage(cause) },
|
||||
{ operation: OPERATION, cause },
|
||||
);
|
||||
}
|
||||
}
|
||||
this.#pendingCleanup = undefined;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
} from "../../domain/analysisErrorCore.js";
|
||||
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
|
||||
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
|
||||
import type { JsonValue } from "../../domain/jsonValue.js";
|
||||
import { err, ok, type Result } from "../../domain/result.js";
|
||||
import { WEB_SOURCE_MAP_LIMITS } from "../../domain/webSourceLocation.js";
|
||||
import {
|
||||
@@ -48,6 +49,22 @@ export interface SourceMapDecoderDependencies {
|
||||
readonly prepareOwnership?: (signal?: AbortSignal) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface SourceMapCleanupOwner {
|
||||
close(
|
||||
previousError: string | null,
|
||||
): Promise<ProviderCleanupError | undefined>;
|
||||
}
|
||||
|
||||
export class SourceMapCleanupFailure extends ProviderCleanupError {
|
||||
constructor(
|
||||
readonly cleanupOwner: SourceMapCleanupOwner,
|
||||
resources: readonly string[],
|
||||
diagnostics: Readonly<Record<string, JsonValue>>,
|
||||
) {
|
||||
super(PROVIDER, resources, diagnostics, { operation: OPERATION });
|
||||
}
|
||||
}
|
||||
|
||||
/** Run a trusted codec command and release its process/root before returning output. */
|
||||
export const runSourceMapCommand = async (
|
||||
request: string,
|
||||
@@ -94,7 +111,7 @@ export const runSourceMapCommand = async (
|
||||
: result;
|
||||
};
|
||||
|
||||
class OwnedCodec {
|
||||
class OwnedCodec implements SourceMapCleanupOwner {
|
||||
readonly #runId = `rea-source-map-${randomUUID()}`;
|
||||
#runtime: SourceMapCodecRuntime | undefined;
|
||||
#supervisor: ProviderProcessSupervisor | undefined;
|
||||
@@ -231,21 +248,27 @@ class OwnedCodec {
|
||||
previousError: string | null,
|
||||
): Promise<ProviderCleanupError | undefined> {
|
||||
const failures: string[] = [];
|
||||
let processReleased = this.#supervisor === undefined;
|
||||
if (this.#supervisor !== undefined) {
|
||||
const stopped = await this.#supervisor.stop();
|
||||
if (stopped.status === "incomplete") failures.push(stopped.reason);
|
||||
else {
|
||||
processReleased = true;
|
||||
this.#supervisor = undefined;
|
||||
}
|
||||
}
|
||||
if (this.#runtime !== undefined) {
|
||||
if (processReleased && this.#runtime !== undefined) {
|
||||
try {
|
||||
await this.#runtime.close();
|
||||
this.#runtime = undefined;
|
||||
} catch (cause: unknown) {
|
||||
failures.push(message(cause));
|
||||
}
|
||||
}
|
||||
return failures.length === 0
|
||||
? undefined
|
||||
: new ProviderCleanupError(
|
||||
PROVIDER,
|
||||
: new SourceMapCleanupFailure(
|
||||
this,
|
||||
[
|
||||
this.#runId,
|
||||
...(this.#runtime === undefined ? [] : [this.#runtime.path]),
|
||||
@@ -256,7 +279,6 @@ class OwnedCodec {
|
||||
previous_error: previousError,
|
||||
codec_pid: this.#supervisor?.launch.process.pid ?? null,
|
||||
},
|
||||
{ operation: OPERATION },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,19 +18,64 @@ import {
|
||||
sourceMapCodecReplySchema,
|
||||
} from "./SourceMapCodecProtocol.js";
|
||||
import {
|
||||
SourceMapCleanupFailure,
|
||||
runSourceMapCommand,
|
||||
type SourceMapCleanupOwner,
|
||||
type SourceMapDecoderDependencies,
|
||||
} from "./SourceMapCommand.js";
|
||||
import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
|
||||
import { jsonParts } from "../../domain/jsonSerialization.js";
|
||||
const OPERATION = "trace_web_source_location";
|
||||
|
||||
/** Integrate the pinned upstream codec through an independently bounded owned process. */
|
||||
export class SourceMapDecoder implements WebSourceMapPort {
|
||||
#tail: Promise<void> = Promise.resolve();
|
||||
#closing = false;
|
||||
#closePromise: Promise<void> | undefined;
|
||||
#pendingCleanup: SourceMapCleanupOwner | undefined;
|
||||
|
||||
constructor(readonly dependencies: SourceMapDecoderDependencies = {}) {}
|
||||
/** Return complete point evidence after process and private-root cleanup. */
|
||||
async trace(
|
||||
trace(
|
||||
input: Parameters<WebSourceMapPort["trace"]>[0],
|
||||
options?: ExecutionOptions,
|
||||
): Promise<Result<WebSourceMapReport, AnalysisError>> {
|
||||
if (this.#closing)
|
||||
return Promise.resolve(
|
||||
err(
|
||||
new ProviderCleanupError(
|
||||
"source-map-decoder",
|
||||
[],
|
||||
{ reason: "Provider is closing" },
|
||||
{ operation: OPERATION },
|
||||
),
|
||||
),
|
||||
);
|
||||
const operation = this.#tail.then(() =>
|
||||
this.#closing
|
||||
? err(
|
||||
new ProviderCleanupError(
|
||||
"source-map-decoder",
|
||||
[],
|
||||
{ reason: "Provider is closing" },
|
||||
{ operation: OPERATION },
|
||||
),
|
||||
)
|
||||
: this.#trace(input, options),
|
||||
);
|
||||
this.#tail = operation.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
return operation;
|
||||
}
|
||||
|
||||
async #trace(
|
||||
input: Parameters<WebSourceMapPort["trace"]>[0],
|
||||
options?: ExecutionOptions,
|
||||
): Promise<Result<WebSourceMapReport, AnalysisError>> {
|
||||
const cleanupFailure = await this.#retryCleanup();
|
||||
if (cleanupFailure !== undefined) return err(cleanupFailure);
|
||||
if (options?.signal?.aborted)
|
||||
return err(new AnalysisCancelledError(OPERATION));
|
||||
const request = sourceMapCodecInputSchema.safeParse({
|
||||
@@ -40,26 +85,38 @@ export class SourceMapDecoder implements WebSourceMapPort {
|
||||
});
|
||||
if (!request.success)
|
||||
return err(new AnalysisInputError(OPERATION, { cause: request.error }));
|
||||
const serialized = JSON.stringify(request.data);
|
||||
if (
|
||||
Buffer.byteLength(input.text) > WEB_SOURCE_MAP_LIMITS.mapBytes ||
|
||||
Buffer.byteLength(serialized) > WEB_SOURCE_MAP_LIMITS.outputBytes
|
||||
)
|
||||
return err(
|
||||
new ArtifactOperationError(
|
||||
OPERATION,
|
||||
"limit",
|
||||
undefined,
|
||||
`${input.path}: Source-map input exceeds the 4 MiB map or 32 MiB encoded request budget.`,
|
||||
),
|
||||
const capacityFailure = () =>
|
||||
new ArtifactOperationError(
|
||||
OPERATION,
|
||||
"limit",
|
||||
undefined,
|
||||
`${input.path}: Source-map input exceeds the 4 MiB map or 32 MiB encoded request budget.`,
|
||||
);
|
||||
if (Buffer.byteLength(input.text) > WEB_SOURCE_MAP_LIMITS.mapBytes)
|
||||
return err(capacityFailure());
|
||||
const encoded: string[] = [];
|
||||
let encodedBytes = 0;
|
||||
for (const part of jsonParts(request.data)) {
|
||||
const bytes = Buffer.byteLength(part);
|
||||
if (bytes > WEB_SOURCE_MAP_LIMITS.outputBytes - encodedBytes)
|
||||
return err(capacityFailure());
|
||||
encoded.push(part);
|
||||
encodedBytes += bytes;
|
||||
}
|
||||
if (options?.signal?.aborted)
|
||||
return err(new AnalysisCancelledError(OPERATION));
|
||||
const serialized = encoded.join("");
|
||||
const response = await runSourceMapCommand(
|
||||
serialized,
|
||||
input.path,
|
||||
options,
|
||||
this.dependencies,
|
||||
);
|
||||
if (!response.ok) return response;
|
||||
if (!response.ok) {
|
||||
if (response.error instanceof SourceMapCleanupFailure)
|
||||
this.#pendingCleanup = response.error.cleanupOwner;
|
||||
return response;
|
||||
}
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(response.value);
|
||||
@@ -92,6 +149,29 @@ export class SourceMapDecoder implements WebSourceMapPort {
|
||||
);
|
||||
return ok(reply.data.report);
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.#closing = true;
|
||||
this.#closePromise ??= this.#tail
|
||||
.then(async () => {
|
||||
const failure = await this.#retryCleanup();
|
||||
if (failure !== undefined) throw failure;
|
||||
})
|
||||
.catch((cause: unknown) => {
|
||||
this.#closePromise = undefined;
|
||||
throw cause;
|
||||
});
|
||||
return this.#closePromise;
|
||||
}
|
||||
|
||||
async #retryCleanup(): Promise<AnalysisError | undefined> {
|
||||
const pending = this.#pendingCleanup;
|
||||
if (pending === undefined) return undefined;
|
||||
const failure = await pending.close(null);
|
||||
if (failure !== undefined) return failure;
|
||||
this.#pendingCleanup = undefined;
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
const codecFailure = (
|
||||
path: string,
|
||||
|
||||
Reference in New Issue
Block a user