fix(process): preserve discovery ownership across cleanup retries (#2643)
Bind retained descriptors and runtime resources to their admitting operation until cleanup succeeds, then rebind newly admitted work. Verify failed retries, successful recovery, shutdown and CLI projection. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
@@ -175,7 +175,7 @@ export class TsharkCaptureProvider implements PacketCapturePort {
|
||||
];
|
||||
phase = "workspace";
|
||||
root = await this.createRuntime();
|
||||
this.#resources.retain(root, undefined, operation);
|
||||
this.#resources.retain(root, undefined);
|
||||
const snapshotPath = join(root.path, "capture");
|
||||
const configurationPath = join(root.path, "configuration");
|
||||
await mkdir(configurationPath, { mode: 0o700 });
|
||||
@@ -406,7 +406,7 @@ export class TsharkCaptureProvider implements PacketCapturePort {
|
||||
cause instanceof OwnedCommandFailure &&
|
||||
cause.cleanupOwner !== undefined
|
||||
)
|
||||
this.#resources.retain(root, cause.cleanupOwner, operation);
|
||||
this.#resources.retain(root, cause.cleanupOwner);
|
||||
result = err(
|
||||
this.#failure(
|
||||
cause,
|
||||
|
||||
@@ -33,7 +33,10 @@ export type ArtifactAnalysisRuntime = Pick<
|
||||
|
||||
/** Own admission, artifact descriptors, producer cleanup and private workspaces together. */
|
||||
export class ArtifactAnalysisResources {
|
||||
readonly #artifacts = new ArtifactResourceScope();
|
||||
readonly #artifacts: {
|
||||
readonly scope: ArtifactResourceScope;
|
||||
operation: string;
|
||||
};
|
||||
readonly #pending = new Map<
|
||||
ArtifactAnalysisRuntime,
|
||||
{
|
||||
@@ -54,7 +57,9 @@ export class ArtifactAnalysisResources {
|
||||
readonly message: string;
|
||||
readonly details: Readonly<Record<string, JsonValue>>;
|
||||
},
|
||||
) {}
|
||||
) {
|
||||
this.#artifacts = { scope: new ArtifactResourceScope(), operation };
|
||||
}
|
||||
|
||||
run<T>(
|
||||
work: () => Promise<Result<T, AnalysisError>>,
|
||||
@@ -76,8 +81,15 @@ export class ArtifactAnalysisResources {
|
||||
);
|
||||
this.#busy = true;
|
||||
const running = this.#tail
|
||||
.then(() => this.#artifacts.run(work))
|
||||
.catch((cause: unknown) => err(this.#artifactFailure(cause, operation)))
|
||||
.then(() =>
|
||||
this.#artifacts.scope.run(() => {
|
||||
// Failed descriptor cleanup still belongs to the preceding request until
|
||||
// this scope has drained it and admits acquisition for the new request.
|
||||
this.#artifacts.operation = operation;
|
||||
return work();
|
||||
}),
|
||||
)
|
||||
.catch((cause: unknown) => err(this.#artifactFailure(cause)))
|
||||
.finally(() => {
|
||||
this.#busy = false;
|
||||
});
|
||||
@@ -94,20 +106,23 @@ export class ArtifactAnalysisResources {
|
||||
maximumBytes,
|
||||
signal,
|
||||
undefined,
|
||||
this.#artifacts,
|
||||
this.#artifacts.scope,
|
||||
);
|
||||
}
|
||||
|
||||
readLimits(limits: LinuxResourceLimits): Promise<LinuxResourceLimits> {
|
||||
return this.readKernelLimits(limits, this.#artifacts);
|
||||
return this.readKernelLimits(limits, this.#artifacts.scope);
|
||||
}
|
||||
|
||||
retain(
|
||||
root: ArtifactAnalysisRuntime,
|
||||
owner?: ProviderProcessSupervisor,
|
||||
operation = this.operation,
|
||||
): void {
|
||||
this.#pending.set(root, { owner, operation });
|
||||
this.#pending.set(root, {
|
||||
owner,
|
||||
operation:
|
||||
this.#pending.get(root)?.operation ?? this.#artifacts.operation,
|
||||
});
|
||||
}
|
||||
|
||||
get pendingPaths(): readonly string[] {
|
||||
@@ -251,7 +266,7 @@ export class ArtifactAnalysisResources {
|
||||
async #closeResources(): Promise<void> {
|
||||
let descriptorFailure: AnalysisError | undefined;
|
||||
try {
|
||||
await this.#artifacts.close();
|
||||
await this.#artifacts.scope.close();
|
||||
} catch (cause: unknown) {
|
||||
descriptorFailure = this.#artifactFailure(cause);
|
||||
}
|
||||
@@ -262,14 +277,17 @@ export class ArtifactAnalysisResources {
|
||||
throw analysisErrorWithCleanupFailure(
|
||||
descriptorFailure,
|
||||
cause,
|
||||
this.operation,
|
||||
this.#artifacts.operation,
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
if (descriptorFailure !== undefined) throw descriptorFailure;
|
||||
}
|
||||
|
||||
#artifactFailure(cause: unknown, operation = this.operation): AnalysisError {
|
||||
#artifactFailure(
|
||||
cause: unknown,
|
||||
operation = this.#artifacts.operation,
|
||||
): AnalysisError {
|
||||
if (cause instanceof AnalysisError) return cause;
|
||||
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
|
||||
return this.cleanupFailure(cause.cleanup, cause, undefined, operation);
|
||||
|
||||
@@ -173,7 +173,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort {
|
||||
];
|
||||
phase = "workspace";
|
||||
root = await this.createRuntime();
|
||||
this.#resources.retain(root, undefined, operation);
|
||||
this.#resources.retain(root, undefined);
|
||||
const snapshotPath = join(root.path, "recording");
|
||||
const execute = async (args: readonly string[]) => {
|
||||
const output = await runOwnedCommand(
|
||||
@@ -371,7 +371,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort {
|
||||
cause instanceof OwnedCommandFailure &&
|
||||
cause.cleanupOwner !== undefined
|
||||
)
|
||||
this.#resources.retain(root, cause.cleanupOwner, operation);
|
||||
this.#resources.retain(root, cause.cleanupOwner);
|
||||
result = err(
|
||||
this.#failure(
|
||||
cause,
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
descriptorFailurePath,
|
||||
expectRetainedDescriptor,
|
||||
} from "../../fixtures/descriptorCloseFailure.js";
|
||||
import { expect, it, onTestFinished, vi } from "vitest";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
@@ -15,27 +19,22 @@ const cleanup = vi.hoisted(() => ({
|
||||
}));
|
||||
vi.mock("node:fs/promises", async (importOriginal) => {
|
||||
const filesystem = await importOriginal<typeof import("node:fs/promises")>();
|
||||
return {
|
||||
...filesystem,
|
||||
open: async (...args: Parameters<typeof filesystem.open>) => {
|
||||
const handle = await filesystem.open(...args);
|
||||
if (args[0] === cleanup.path) {
|
||||
cleanup.handles.push(handle);
|
||||
const close = handle.close.bind(handle);
|
||||
handle.close = async () => {
|
||||
if (cleanup.denied && handle.fd >= 0)
|
||||
throw new Error("Descriptor close deliberately denied");
|
||||
await close();
|
||||
};
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
};
|
||||
const { withDescriptorCloseFailure } =
|
||||
await import("../../fixtures/descriptorCloseFailure.js");
|
||||
return withDescriptorCloseFailure(filesystem, cleanup);
|
||||
});
|
||||
|
||||
it
|
||||
.runIf(process.platform === "linux")
|
||||
.each(["capture", "tool", "limiter", "kernel"])(
|
||||
.each([
|
||||
"capture",
|
||||
"tool",
|
||||
"limiter",
|
||||
"kernel",
|
||||
"field-tool",
|
||||
"field-limiter",
|
||||
"field-kernel",
|
||||
])(
|
||||
"retains failed %s descriptors and retries them before further acquisition and shutdown",
|
||||
async (member) => {
|
||||
const directory = await createTestTempDirectory("rea-capture-descriptor-");
|
||||
@@ -67,14 +66,16 @@ it
|
||||
},
|
||||
() => PrivateRuntimeRoot.create({ parent: directory }),
|
||||
);
|
||||
cleanup.path =
|
||||
member === "capture"
|
||||
? path
|
||||
: member === "tool"
|
||||
? command
|
||||
: member === "limiter"
|
||||
? limiter
|
||||
: "/proc/self/limits";
|
||||
const discovery = member.startsWith("field-");
|
||||
const resource = discovery ? member.slice(6) : member;
|
||||
const operation = discovery
|
||||
? "inspect_packet_fields"
|
||||
: "inspect_packet_capture";
|
||||
cleanup.path = descriptorFailurePath(resource, {
|
||||
capture: path,
|
||||
tool: command,
|
||||
limiter,
|
||||
});
|
||||
cleanup.denied = true;
|
||||
cleanup.handles = [];
|
||||
onTestFinished(async () => {
|
||||
@@ -84,24 +85,45 @@ it
|
||||
cleanup.path = "";
|
||||
});
|
||||
const service = new PacketCaptureService(provider);
|
||||
const first = await service.inspect({ path, fields: ["frame.number"] });
|
||||
expect(first).toMatchObject({
|
||||
ok: false,
|
||||
error: { cleanupIncomplete: true, cleanupResources: [cleanup.path] },
|
||||
});
|
||||
expect(cleanup.handles).toHaveLength(1);
|
||||
expect(cleanup.handles[0]?.fd).toBeGreaterThanOrEqual(0);
|
||||
const first = discovery
|
||||
? await service.inspectFields({ query: "usb" })
|
||||
: await service.inspect({ path, fields: ["frame.number"] });
|
||||
expectRetainedDescriptor(first, cleanup, operation);
|
||||
const launchCount = launches;
|
||||
expect(
|
||||
await service.inspect({ path, fields: ["frame.number"] }),
|
||||
).toMatchObject({ ok: false, error: { cleanupIncomplete: true } });
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { cleanupIncomplete: true, operation },
|
||||
});
|
||||
expect(cleanup.handles).toHaveLength(1);
|
||||
expect(launches).toBe(launchCount);
|
||||
let closingOperation = operation;
|
||||
if (discovery) {
|
||||
cleanup.denied = false;
|
||||
const recovered = await service.inspect({
|
||||
path,
|
||||
fields: ["frame.number"],
|
||||
});
|
||||
expect(recovered.ok).toBe(true);
|
||||
expect(cleanup.handles.every((handle) => handle.fd === -1)).toBe(true);
|
||||
expect(launches).toBeGreaterThan(launchCount);
|
||||
cleanup.denied = true;
|
||||
expect(
|
||||
await service.inspect({ path, fields: ["frame.number"] }),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { cleanupIncomplete: true, operation: "inspect_packet_capture" },
|
||||
});
|
||||
expect(cleanup.handles.at(-1)?.fd).toBeGreaterThanOrEqual(0);
|
||||
closingOperation = "inspect_packet_capture";
|
||||
}
|
||||
await expect(provider.close()).rejects.toMatchObject({
|
||||
cleanupIncomplete: true,
|
||||
operation: closingOperation,
|
||||
});
|
||||
cleanup.denied = false;
|
||||
await provider.close();
|
||||
expect(cleanup.handles[0]?.fd).toBe(-1);
|
||||
expect(cleanup.handles.every((handle) => handle.fd === -1)).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -11,6 +11,9 @@ import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js";
|
||||
import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { createDeferred } from "../../fixtures/binarySession.js";
|
||||
import { Cli } from "incur";
|
||||
import { registerCaptureCommands } from "../../../src/cli/captureCommands.js";
|
||||
import { silentLogger } from "../../../src/logger.js";
|
||||
|
||||
// A command-boundary simulator: real process ownership/filesystem, not real TShark coverage.
|
||||
const fixture = async (mode = "ok", failCleanup = false) => {
|
||||
@@ -625,3 +628,34 @@ it("retains completed discovery on late cancellation and cleanup failure", async
|
||||
await f.service.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves discovery resource ownership through CLI shutdown retries", async () => {
|
||||
const f = await fixture("cleanup", true);
|
||||
const cli = Cli.create("discovery-cleanup", { sync: false });
|
||||
registerCaptureCommands(cli, silentLogger, {}, f.service);
|
||||
let stdout = "";
|
||||
try {
|
||||
await cli.serve(
|
||||
["inspect-packet-fields", "--query", "usb", "--format", "json"],
|
||||
{
|
||||
stdout: (text) => {
|
||||
stdout += text;
|
||||
},
|
||||
exit: () => {},
|
||||
},
|
||||
);
|
||||
expect(JSON.parse(stdout)).toMatchObject({
|
||||
code: "cleanup_incomplete",
|
||||
details: {
|
||||
diagnostics: {
|
||||
cleanup_error: {
|
||||
details: { operation: "inspect_packet_fields" },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
f.allowCleanup();
|
||||
await f.service.close();
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
import {
|
||||
descriptorFailurePath,
|
||||
expectRetainedDescriptor,
|
||||
} from "../../fixtures/descriptorCloseFailure.js";
|
||||
import { expect, it, onTestFinished, vi } from "vitest";
|
||||
import { writeFile } from "node:fs/promises";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
@@ -15,27 +19,22 @@ const cleanup = vi.hoisted(() => ({
|
||||
}));
|
||||
vi.mock("node:fs/promises", async (importOriginal) => {
|
||||
const filesystem = await importOriginal<typeof import("node:fs/promises")>();
|
||||
return {
|
||||
...filesystem,
|
||||
open: async (...args: Parameters<typeof filesystem.open>) => {
|
||||
const handle = await filesystem.open(...args);
|
||||
if (args[0] === cleanup.path) {
|
||||
cleanup.handles.push(handle);
|
||||
const close = handle.close.bind(handle);
|
||||
handle.close = async () => {
|
||||
if (cleanup.denied && handle.fd >= 0)
|
||||
throw new Error("Descriptor close deliberately denied");
|
||||
await close();
|
||||
};
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
};
|
||||
const { withDescriptorCloseFailure } =
|
||||
await import("../../fixtures/descriptorCloseFailure.js");
|
||||
return withDescriptorCloseFailure(filesystem, cleanup);
|
||||
});
|
||||
|
||||
it
|
||||
.runIf(process.platform === "linux")
|
||||
.each(["capture", "tool", "limiter", "kernel"])(
|
||||
.each([
|
||||
"capture",
|
||||
"tool",
|
||||
"limiter",
|
||||
"kernel",
|
||||
"field-tool",
|
||||
"field-limiter",
|
||||
"field-kernel",
|
||||
])(
|
||||
"retains failed %s descriptors and retries them before further acquisition and shutdown",
|
||||
async (member) => {
|
||||
const directory = await createTestTempDirectory("rea-signal-descriptor-");
|
||||
@@ -67,14 +66,16 @@ it
|
||||
},
|
||||
() => PrivateRuntimeRoot.create({ parent: directory }),
|
||||
);
|
||||
cleanup.path =
|
||||
member === "capture"
|
||||
? path
|
||||
: member === "tool"
|
||||
? command
|
||||
: member === "limiter"
|
||||
? limiter
|
||||
: "/proc/self/limits";
|
||||
const discovery = member.startsWith("field-");
|
||||
const resource = discovery ? member.slice(6) : member;
|
||||
const operation = discovery
|
||||
? "inspect_signal_decoders"
|
||||
: "decode_signal_recording";
|
||||
cleanup.path = descriptorFailurePath(resource, {
|
||||
capture: path,
|
||||
tool: command,
|
||||
limiter,
|
||||
});
|
||||
cleanup.denied = true;
|
||||
cleanup.handles = [];
|
||||
onTestFinished(async () => {
|
||||
@@ -84,22 +85,20 @@ it
|
||||
cleanup.path = "";
|
||||
});
|
||||
const service = new SignalRecordingService(provider);
|
||||
const first = await service.decode({ path, decoders: "uart" });
|
||||
expect(first).toMatchObject({
|
||||
ok: false,
|
||||
error: { cleanupIncomplete: true, cleanupResources: [cleanup.path] },
|
||||
});
|
||||
expect(cleanup.handles).toHaveLength(1);
|
||||
expect(cleanup.handles[0]?.fd).toBeGreaterThanOrEqual(0);
|
||||
const first = discovery
|
||||
? await service.inspectDecoders({ query: "uart" })
|
||||
: await service.decode({ path, decoders: "uart" });
|
||||
expectRetainedDescriptor(first, cleanup, operation);
|
||||
const launchCount = launches;
|
||||
expect(await service.decode({ path, decoders: "uart" })).toMatchObject({
|
||||
ok: false,
|
||||
error: { cleanupIncomplete: true },
|
||||
error: { cleanupIncomplete: true, operation },
|
||||
});
|
||||
expect(cleanup.handles).toHaveLength(1);
|
||||
expect(launches).toBe(launchCount);
|
||||
await expect(provider.close()).rejects.toMatchObject({
|
||||
cleanupIncomplete: true,
|
||||
operation,
|
||||
});
|
||||
cleanup.denied = false;
|
||||
await provider.close();
|
||||
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import { expect } from "vitest";
|
||||
import type { Result } from "../../src/domain/result.js";
|
||||
import type { AnalysisError } from "../../src/domain/analysisErrorBase.js";
|
||||
|
||||
export interface DescriptorCloseFailure {
|
||||
path: string;
|
||||
denied: boolean;
|
||||
handles: FileHandle[];
|
||||
}
|
||||
|
||||
/** Public failure attribution and a still-live OS handle establish retained ownership. */
|
||||
export const expectRetainedDescriptor = (
|
||||
result: Result<unknown, AnalysisError>,
|
||||
cleanup: DescriptorCloseFailure,
|
||||
operation: string,
|
||||
): void => {
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
cleanupIncomplete: true,
|
||||
cleanupResources: [cleanup.path],
|
||||
operation,
|
||||
},
|
||||
});
|
||||
expect(cleanup.handles).toHaveLength(1);
|
||||
expect(cleanup.handles[0]?.fd).toBeGreaterThanOrEqual(0);
|
||||
};
|
||||
|
||||
export const descriptorFailurePath = (
|
||||
resource: string,
|
||||
paths: Readonly<Record<"capture" | "tool" | "limiter", string>>,
|
||||
): string => {
|
||||
switch (resource) {
|
||||
case "capture":
|
||||
case "tool":
|
||||
case "limiter":
|
||||
return paths[resource];
|
||||
case "kernel":
|
||||
return "/proc/self/limits";
|
||||
default:
|
||||
throw new Error(`Unknown descriptor failure resource: ${resource}`);
|
||||
}
|
||||
};
|
||||
|
||||
/** Inject a retryable close failure while retaining the actual OS descriptor. */
|
||||
export const withDescriptorCloseFailure = (
|
||||
filesystem: typeof import("node:fs/promises"),
|
||||
cleanup: DescriptorCloseFailure,
|
||||
) => ({
|
||||
...filesystem,
|
||||
open: async (...args: Parameters<typeof filesystem.open>) => {
|
||||
const handle = await filesystem.open(...args);
|
||||
if (args[0] === cleanup.path) {
|
||||
cleanup.handles.push(handle);
|
||||
const close = handle.close.bind(handle);
|
||||
handle.close = async () => {
|
||||
if (cleanup.denied && handle.fd >= 0)
|
||||
throw new Error("Descriptor close deliberately denied");
|
||||
await close();
|
||||
};
|
||||
}
|
||||
return handle;
|
||||
},
|
||||
});
|
||||
Reference in New Issue
Block a user