fix(process): preserve discovery ownership across cleanup retries (#2643)

Bind retained descriptors and runtime resources to their admitting operation until cleanup succeeds, then rebind newly admitted work. Verify failed retries, successful recovery, shutdown and CLI projection.

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
Ysopwnable
2026-10-13 13:43:06 +00:00
committed by GitHub
co-authored by Kaoru0822
parent a4b8e9ccee
commit 9958754f6c
7 changed files with 220 additions and 82 deletions
+2 -2
View File
@@ -175,7 +175,7 @@ export class TsharkCaptureProvider implements PacketCapturePort {
];
phase = "workspace";
root = await this.createRuntime();
this.#resources.retain(root, undefined, operation);
this.#resources.retain(root, undefined);
const snapshotPath = join(root.path, "capture");
const configurationPath = join(root.path, "configuration");
await mkdir(configurationPath, { mode: 0o700 });
@@ -406,7 +406,7 @@ export class TsharkCaptureProvider implements PacketCapturePort {
cause instanceof OwnedCommandFailure &&
cause.cleanupOwner !== undefined
)
this.#resources.retain(root, cause.cleanupOwner, operation);
this.#resources.retain(root, cause.cleanupOwner);
result = err(
this.#failure(
cause,
+29 -11
View File
@@ -33,7 +33,10 @@ export type ArtifactAnalysisRuntime = Pick<
/** Own admission, artifact descriptors, producer cleanup and private workspaces together. */
export class ArtifactAnalysisResources {
readonly #artifacts = new ArtifactResourceScope();
readonly #artifacts: {
readonly scope: ArtifactResourceScope;
operation: string;
};
readonly #pending = new Map<
ArtifactAnalysisRuntime,
{
@@ -54,7 +57,9 @@ export class ArtifactAnalysisResources {
readonly message: string;
readonly details: Readonly<Record<string, JsonValue>>;
},
) {}
) {
this.#artifacts = { scope: new ArtifactResourceScope(), operation };
}
run<T>(
work: () => Promise<Result<T, AnalysisError>>,
@@ -76,8 +81,15 @@ export class ArtifactAnalysisResources {
);
this.#busy = true;
const running = this.#tail
.then(() => this.#artifacts.run(work))
.catch((cause: unknown) => err(this.#artifactFailure(cause, operation)))
.then(() =>
this.#artifacts.scope.run(() => {
// Failed descriptor cleanup still belongs to the preceding request until
// this scope has drained it and admits acquisition for the new request.
this.#artifacts.operation = operation;
return work();
}),
)
.catch((cause: unknown) => err(this.#artifactFailure(cause)))
.finally(() => {
this.#busy = false;
});
@@ -94,20 +106,23 @@ export class ArtifactAnalysisResources {
maximumBytes,
signal,
undefined,
this.#artifacts,
this.#artifacts.scope,
);
}
readLimits(limits: LinuxResourceLimits): Promise<LinuxResourceLimits> {
return this.readKernelLimits(limits, this.#artifacts);
return this.readKernelLimits(limits, this.#artifacts.scope);
}
retain(
root: ArtifactAnalysisRuntime,
owner?: ProviderProcessSupervisor,
operation = this.operation,
): void {
this.#pending.set(root, { owner, operation });
this.#pending.set(root, {
owner,
operation:
this.#pending.get(root)?.operation ?? this.#artifacts.operation,
});
}
get pendingPaths(): readonly string[] {
@@ -251,7 +266,7 @@ export class ArtifactAnalysisResources {
async #closeResources(): Promise<void> {
let descriptorFailure: AnalysisError | undefined;
try {
await this.#artifacts.close();
await this.#artifacts.scope.close();
} catch (cause: unknown) {
descriptorFailure = this.#artifactFailure(cause);
}
@@ -262,14 +277,17 @@ export class ArtifactAnalysisResources {
throw analysisErrorWithCleanupFailure(
descriptorFailure,
cause,
this.operation,
this.#artifacts.operation,
);
throw cause;
}
if (descriptorFailure !== undefined) throw descriptorFailure;
}
#artifactFailure(cause: unknown, operation = this.operation): AnalysisError {
#artifactFailure(
cause: unknown,
operation = this.#artifacts.operation,
): AnalysisError {
if (cause instanceof AnalysisError) return cause;
if (cause instanceof ArtifactReaderFailure && cause.cleanup !== undefined)
return this.cleanupFailure(cause.cleanup, cause, undefined, operation);
+2 -2
View File
@@ -173,7 +173,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort {
];
phase = "workspace";
root = await this.createRuntime();
this.#resources.retain(root, undefined, operation);
this.#resources.retain(root, undefined);
const snapshotPath = join(root.path, "recording");
const execute = async (args: readonly string[]) => {
const output = await runOwnedCommand(
@@ -371,7 +371,7 @@ export class SigrokRecordingProvider implements SignalRecordingPort {
cause instanceof OwnedCommandFailure &&
cause.cleanupOwner !== undefined
)
this.#resources.retain(root, cause.cleanupOwner, operation);
this.#resources.retain(root, cause.cleanupOwner);
result = err(
this.#failure(
cause,
@@ -1,3 +1,7 @@
import {
descriptorFailurePath,
expectRetainedDescriptor,
} from "../../fixtures/descriptorCloseFailure.js";
import { expect, it, onTestFinished, vi } from "vitest";
import { writeFile } from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
@@ -15,27 +19,22 @@ const cleanup = vi.hoisted(() => ({
}));
vi.mock("node:fs/promises", async (importOriginal) => {
const filesystem = await importOriginal<typeof import("node:fs/promises")>();
return {
...filesystem,
open: async (...args: Parameters<typeof filesystem.open>) => {
const handle = await filesystem.open(...args);
if (args[0] === cleanup.path) {
cleanup.handles.push(handle);
const close = handle.close.bind(handle);
handle.close = async () => {
if (cleanup.denied && handle.fd >= 0)
throw new Error("Descriptor close deliberately denied");
await close();
};
}
return handle;
},
};
const { withDescriptorCloseFailure } =
await import("../../fixtures/descriptorCloseFailure.js");
return withDescriptorCloseFailure(filesystem, cleanup);
});
it
.runIf(process.platform === "linux")
.each(["capture", "tool", "limiter", "kernel"])(
.each([
"capture",
"tool",
"limiter",
"kernel",
"field-tool",
"field-limiter",
"field-kernel",
])(
"retains failed %s descriptors and retries them before further acquisition and shutdown",
async (member) => {
const directory = await createTestTempDirectory("rea-capture-descriptor-");
@@ -67,14 +66,16 @@ it
},
() => PrivateRuntimeRoot.create({ parent: directory }),
);
cleanup.path =
member === "capture"
? path
: member === "tool"
? command
: member === "limiter"
? limiter
: "/proc/self/limits";
const discovery = member.startsWith("field-");
const resource = discovery ? member.slice(6) : member;
const operation = discovery
? "inspect_packet_fields"
: "inspect_packet_capture";
cleanup.path = descriptorFailurePath(resource, {
capture: path,
tool: command,
limiter,
});
cleanup.denied = true;
cleanup.handles = [];
onTestFinished(async () => {
@@ -84,24 +85,45 @@ it
cleanup.path = "";
});
const service = new PacketCaptureService(provider);
const first = await service.inspect({ path, fields: ["frame.number"] });
expect(first).toMatchObject({
ok: false,
error: { cleanupIncomplete: true, cleanupResources: [cleanup.path] },
});
expect(cleanup.handles).toHaveLength(1);
expect(cleanup.handles[0]?.fd).toBeGreaterThanOrEqual(0);
const first = discovery
? await service.inspectFields({ query: "usb" })
: await service.inspect({ path, fields: ["frame.number"] });
expectRetainedDescriptor(first, cleanup, operation);
const launchCount = launches;
expect(
await service.inspect({ path, fields: ["frame.number"] }),
).toMatchObject({ ok: false, error: { cleanupIncomplete: true } });
).toMatchObject({
ok: false,
error: { cleanupIncomplete: true, operation },
});
expect(cleanup.handles).toHaveLength(1);
expect(launches).toBe(launchCount);
let closingOperation = operation;
if (discovery) {
cleanup.denied = false;
const recovered = await service.inspect({
path,
fields: ["frame.number"],
});
expect(recovered.ok).toBe(true);
expect(cleanup.handles.every((handle) => handle.fd === -1)).toBe(true);
expect(launches).toBeGreaterThan(launchCount);
cleanup.denied = true;
expect(
await service.inspect({ path, fields: ["frame.number"] }),
).toMatchObject({
ok: false,
error: { cleanupIncomplete: true, operation: "inspect_packet_capture" },
});
expect(cleanup.handles.at(-1)?.fd).toBeGreaterThanOrEqual(0);
closingOperation = "inspect_packet_capture";
}
await expect(provider.close()).rejects.toMatchObject({
cleanupIncomplete: true,
operation: closingOperation,
});
cleanup.denied = false;
await provider.close();
expect(cleanup.handles[0]?.fd).toBe(-1);
expect(cleanup.handles.every((handle) => handle.fd === -1)).toBe(true);
},
);
@@ -11,6 +11,9 @@ import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js";
import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
import { createDeferred } from "../../fixtures/binarySession.js";
import { Cli } from "incur";
import { registerCaptureCommands } from "../../../src/cli/captureCommands.js";
import { silentLogger } from "../../../src/logger.js";
// A command-boundary simulator: real process ownership/filesystem, not real TShark coverage.
const fixture = async (mode = "ok", failCleanup = false) => {
@@ -625,3 +628,34 @@ it("retains completed discovery on late cancellation and cleanup failure", async
await f.service.close();
}
});
it("preserves discovery resource ownership through CLI shutdown retries", async () => {
const f = await fixture("cleanup", true);
const cli = Cli.create("discovery-cleanup", { sync: false });
registerCaptureCommands(cli, silentLogger, {}, f.service);
let stdout = "";
try {
await cli.serve(
["inspect-packet-fields", "--query", "usb", "--format", "json"],
{
stdout: (text) => {
stdout += text;
},
exit: () => {},
},
);
expect(JSON.parse(stdout)).toMatchObject({
code: "cleanup_incomplete",
details: {
diagnostics: {
cleanup_error: {
details: { operation: "inspect_packet_fields" },
},
},
},
});
} finally {
f.allowCleanup();
await f.service.close();
}
});
@@ -1,3 +1,7 @@
import {
descriptorFailurePath,
expectRetainedDescriptor,
} from "../../fixtures/descriptorCloseFailure.js";
import { expect, it, onTestFinished, vi } from "vitest";
import { writeFile } from "node:fs/promises";
import type { FileHandle } from "node:fs/promises";
@@ -15,27 +19,22 @@ const cleanup = vi.hoisted(() => ({
}));
vi.mock("node:fs/promises", async (importOriginal) => {
const filesystem = await importOriginal<typeof import("node:fs/promises")>();
return {
...filesystem,
open: async (...args: Parameters<typeof filesystem.open>) => {
const handle = await filesystem.open(...args);
if (args[0] === cleanup.path) {
cleanup.handles.push(handle);
const close = handle.close.bind(handle);
handle.close = async () => {
if (cleanup.denied && handle.fd >= 0)
throw new Error("Descriptor close deliberately denied");
await close();
};
}
return handle;
},
};
const { withDescriptorCloseFailure } =
await import("../../fixtures/descriptorCloseFailure.js");
return withDescriptorCloseFailure(filesystem, cleanup);
});
it
.runIf(process.platform === "linux")
.each(["capture", "tool", "limiter", "kernel"])(
.each([
"capture",
"tool",
"limiter",
"kernel",
"field-tool",
"field-limiter",
"field-kernel",
])(
"retains failed %s descriptors and retries them before further acquisition and shutdown",
async (member) => {
const directory = await createTestTempDirectory("rea-signal-descriptor-");
@@ -67,14 +66,16 @@ it
},
() => PrivateRuntimeRoot.create({ parent: directory }),
);
cleanup.path =
member === "capture"
? path
: member === "tool"
? command
: member === "limiter"
? limiter
: "/proc/self/limits";
const discovery = member.startsWith("field-");
const resource = discovery ? member.slice(6) : member;
const operation = discovery
? "inspect_signal_decoders"
: "decode_signal_recording";
cleanup.path = descriptorFailurePath(resource, {
capture: path,
tool: command,
limiter,
});
cleanup.denied = true;
cleanup.handles = [];
onTestFinished(async () => {
@@ -84,22 +85,20 @@ it
cleanup.path = "";
});
const service = new SignalRecordingService(provider);
const first = await service.decode({ path, decoders: "uart" });
expect(first).toMatchObject({
ok: false,
error: { cleanupIncomplete: true, cleanupResources: [cleanup.path] },
});
expect(cleanup.handles).toHaveLength(1);
expect(cleanup.handles[0]?.fd).toBeGreaterThanOrEqual(0);
const first = discovery
? await service.inspectDecoders({ query: "uart" })
: await service.decode({ path, decoders: "uart" });
expectRetainedDescriptor(first, cleanup, operation);
const launchCount = launches;
expect(await service.decode({ path, decoders: "uart" })).toMatchObject({
ok: false,
error: { cleanupIncomplete: true },
error: { cleanupIncomplete: true, operation },
});
expect(cleanup.handles).toHaveLength(1);
expect(launches).toBe(launchCount);
await expect(provider.close()).rejects.toMatchObject({
cleanupIncomplete: true,
operation,
});
cleanup.denied = false;
await provider.close();
+65
View File
@@ -0,0 +1,65 @@
import type { FileHandle } from "node:fs/promises";
import { expect } from "vitest";
import type { Result } from "../../src/domain/result.js";
import type { AnalysisError } from "../../src/domain/analysisErrorBase.js";
export interface DescriptorCloseFailure {
path: string;
denied: boolean;
handles: FileHandle[];
}
/** Public failure attribution and a still-live OS handle establish retained ownership. */
export const expectRetainedDescriptor = (
result: Result<unknown, AnalysisError>,
cleanup: DescriptorCloseFailure,
operation: string,
): void => {
expect(result).toMatchObject({
ok: false,
error: {
cleanupIncomplete: true,
cleanupResources: [cleanup.path],
operation,
},
});
expect(cleanup.handles).toHaveLength(1);
expect(cleanup.handles[0]?.fd).toBeGreaterThanOrEqual(0);
};
export const descriptorFailurePath = (
resource: string,
paths: Readonly<Record<"capture" | "tool" | "limiter", string>>,
): string => {
switch (resource) {
case "capture":
case "tool":
case "limiter":
return paths[resource];
case "kernel":
return "/proc/self/limits";
default:
throw new Error(`Unknown descriptor failure resource: ${resource}`);
}
};
/** Inject a retryable close failure while retaining the actual OS descriptor. */
export const withDescriptorCloseFailure = (
filesystem: typeof import("node:fs/promises"),
cleanup: DescriptorCloseFailure,
) => ({
...filesystem,
open: async (...args: Parameters<typeof filesystem.open>) => {
const handle = await filesystem.open(...args);
if (args[0] === cleanup.path) {
cleanup.handles.push(handle);
const close = handle.close.bind(handle);
handle.close = async () => {
if (cleanup.denied && handle.fd >= 0)
throw new Error("Descriptor close deliberately denied");
await close();
};
}
return handle;
},
});