Files
rea/src/process/ArtifactAnalysisResources.ts
T
9958754f6c fix(process): preserve discovery ownership across cleanup retries (#2643)
Bind retained descriptors and runtime resources to their admitting operation until cleanup succeeds, then rebind newly admitted work. Verify failed retries, successful recovery, shutdown and CLI projection.

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
2026-10-13 13:43:06 +00:00

300 lines
9.0 KiB
TypeScript

import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js";
import { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js";
import { readStableArtifact } from "../artifacts/readStableArtifact.js";
import {
AnalysisError,
type AnalysisCapturedOutput,
type AnalysisPartialObservation,
} from "../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisResourceConstraintError,
AnalysisOutputError,
AnalysisTimeoutError,
} from "../domain/analysisErrorCore.js";
import { analysisErrorWithCleanupFailure } from "../domain/analysisErrorCleanup.js";
import { projectAnalysisError } from "../domain/analysisErrorProjection.js";
import { ProviderAdapterError } from "../domain/providerAdapterError.js";
import { ProviderCleanupError } from "../domain/providerCleanupError.js";
import type { JsonValue } from "../domain/jsonValue.js";
import { err, type Result } from "../domain/result.js";
import {
readLinuxResourceLimits,
type LinuxResourceLimits,
} from "./LinuxResourceLimits.js";
import type { PrivateRuntimeRoot } from "./PrivateRuntimeRoot.js";
import type { ProviderProcessSupervisor } from "./ProviderProcess.js";
import { OwnedCommandFailure } from "./OwnedCommand.js";
export type ArtifactAnalysisRuntime = Pick<
PrivateRuntimeRoot,
"path" | "close"
>;
/** Own admission, artifact descriptors, producer cleanup and private workspaces together. */
export class ArtifactAnalysisResources {
readonly #artifacts: {
readonly scope: ArtifactResourceScope;
operation: string;
};
readonly #pending = new Map<
ArtifactAnalysisRuntime,
{
readonly owner: ProviderProcessSupervisor | undefined;
readonly operation: string;
}
>();
#tail: Promise<void> = Promise.resolve();
#busy = false;
#closed = false;
constructor(
readonly providerId: string,
readonly operation: string,
readonly readArtifact: typeof readStableArtifact,
readonly readKernelLimits: typeof readLinuxResourceLimits,
readonly capacity: {
readonly message: string;
readonly details: Readonly<Record<string, JsonValue>>;
},
) {
this.#artifacts = { scope: new ArtifactResourceScope(), operation };
}
run<T>(
work: () => Promise<Result<T, AnalysisError>>,
signal?: AbortSignal,
operation = this.operation,
): Promise<Result<T, AnalysisError>> {
if (this.#closed || signal?.aborted)
return Promise.resolve(err(new AnalysisCancelledError(operation)));
if (this.#busy)
return Promise.resolve(
err(
new AnalysisResourceConstraintError(
operation,
"capacity",
this.capacity.message,
this.capacity.details,
),
),
);
this.#busy = true;
const running = this.#tail
.then(() =>
this.#artifacts.scope.run(() => {
// Failed descriptor cleanup still belongs to the preceding request until
// this scope has drained it and admits acquisition for the new request.
this.#artifacts.operation = operation;
return work();
}),
)
.catch((cause: unknown) => err(this.#artifactFailure(cause)))
.finally(() => {
this.#busy = false;
});
this.#tail = running.then(
() => undefined,
() => undefined,
);
return running;
}
read(path: string, maximumBytes: number, signal?: AbortSignal) {
return this.readArtifact(
path,
maximumBytes,
signal,
undefined,
this.#artifacts.scope,
);
}
readLimits(limits: LinuxResourceLimits): Promise<LinuxResourceLimits> {
return this.readKernelLimits(limits, this.#artifacts.scope);
}
retain(
root: ArtifactAnalysisRuntime,
owner?: ProviderProcessSupervisor,
): void {
this.#pending.set(root, {
owner,
operation:
this.#pending.get(root)?.operation ?? this.#artifacts.operation,
});
}
get pendingPaths(): readonly string[] {
return [...this.#pending.keys()].map((root) => root.path);
}
async release(): Promise<void> {
for (const [root, retained] of this.#pending) {
try {
const { owner } = retained;
if (owner !== undefined) {
const stopped = await owner.stop();
if (stopped.status === "incomplete") throw new Error(stopped.reason);
this.#pending.set(root, { ...retained, owner: undefined });
}
await root.close();
this.#pending.delete(root);
} catch (cause: unknown) {
throw new ProviderCleanupError(
this.providerId,
[root.path],
{ reason: String(cause) },
{ operation: retained.operation, cause },
);
}
}
}
cleanupFailure(
observation: NonNullable<ArtifactReaderFailure["cleanup"]>,
cause: unknown,
capturedOutput?: AnalysisCapturedOutput,
operation = this.operation,
): ProviderCleanupError {
return new ProviderCleanupError(
this.providerId,
observation.resources,
{
reason: observation.reason,
...(capturedOutput === undefined
? {}
: { captured_output: { ...capturedOutput } }),
},
{ operation, cause },
);
}
commandFailure(
cause: OwnedCommandFailure,
limits: {
readonly command_timeout_ms: number;
readonly output_bytes: number;
},
effectiveLimits: LinuxResourceLimits | undefined,
operation = this.operation,
): AnalysisError {
const capturedOutput =
cause.snapshot === null
? undefined
: {
stdout: cause.snapshot.stdout.text,
stderr: cause.snapshot.stderr.text,
truncated: cause.snapshot.diagnosticTruncated ?? false,
exit_code: cause.snapshot.exitCode ?? null,
signal: cause.snapshot.signal ?? null,
};
const detail = {
cause,
...(capturedOutput === undefined ? {} : { capturedOutput }),
};
if (cause.reason === "cancelled")
return new AnalysisCancelledError(operation, detail);
if (cause.reason === "timeout")
return new AnalysisTimeoutError(
operation,
limits.command_timeout_ms,
detail,
);
if (cause.reason === "output-limit")
return new AnalysisResourceConstraintError(
operation,
"memory",
cause.message,
{ output_bytes: limits.output_bytes },
detail,
);
if (
cause.reason === "process" &&
(cause.snapshot?.signal === "SIGXFSZ" ||
cause.snapshot?.signal === "SIGXCPU")
)
return new AnalysisResourceConstraintError(
operation,
cause.snapshot.signal === "SIGXFSZ" ? "file-size" : "cpu",
`Producer terminated with ${cause.snapshot.signal}; the signal's exact cause is not independently established.`,
effectiveLimits === undefined ? null : { ...effectiveLimits },
detail,
);
return new AnalysisOutputError(operation, cause.message, detail);
}
async finish<T>(
result: Result<T, AnalysisError>,
observation: (value: T) => AnalysisPartialObservation,
operation = this.operation,
): Promise<Result<T, AnalysisError>> {
try {
await this.release();
return result;
} catch (cause: unknown) {
const partialObservation = result.ok
? observation(result.value)
: result.error.partialObservation;
return err(
new ProviderCleanupError(
this.providerId,
this.pendingPaths,
{
reason: String(cause),
previous_error: result.ok
? null
: projectAnalysisError(result.error),
},
{
operation,
cause,
...(partialObservation === undefined ? {} : { partialObservation }),
},
),
);
}
}
close(): Promise<void> {
this.#closed = true;
const work = this.#tail.then(() => this.#closeResources());
this.#tail = work.catch(() => undefined);
return work;
}
async #closeResources(): Promise<void> {
let descriptorFailure: AnalysisError | undefined;
try {
await this.#artifacts.scope.close();
} catch (cause: unknown) {
descriptorFailure = this.#artifactFailure(cause);
}
try {
await this.release();
} catch (cause: unknown) {
if (descriptorFailure !== undefined && cause instanceof AnalysisError)
throw analysisErrorWithCleanupFailure(
descriptorFailure,
cause,
this.#artifacts.operation,
);
throw cause;
}
if (descriptorFailure !== undefined) throw descriptorFailure;
}
#artifactFailure(
cause: unknown,
operation = this.#artifacts.operation,
): AnalysisError {
if (cause instanceof AnalysisError) return cause;
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
return this.cleanupFailure(cause.cleanup, cause, undefined, operation);
return new ProviderAdapterError(this.providerId, operation, {
cause,
diagnostics: { stage: "artifact-resources", reason: String(cause) },
});
}
}