fix(process): retain owned command cleanup diagnostics (#1907)
Preserve per-member verification failures and sweep uncertainty across supervisor stop and owned-command failures. Report the observed token mismatch with its PID coordinates instead of implying PID reuse. Keep unresolved ownership unsignaled and retain the supervisor for a later verified stop. Exercise cancellation, timeout and cleanup revalidation with source-owned process fixtures and cooperative settlement. Closes #1856 Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
Kaoru0822
parent
23d50ea3be
commit
38539a2f4a
@@ -7,7 +7,10 @@ import {
|
||||
ProviderProcessSupervisor,
|
||||
spawnOwnedProviderProcess,
|
||||
} from "./ProviderProcess.js";
|
||||
import { cleanupOwnedProcessGroup } from "./ProcessOwnership.js";
|
||||
import {
|
||||
cleanupOwnedProcessGroup,
|
||||
type ProcessCleanupFailure,
|
||||
} from "./ProcessOwnership.js";
|
||||
import {
|
||||
ProviderStartupDeadline,
|
||||
waitForAbortableDelay,
|
||||
@@ -15,16 +18,19 @@ import {
|
||||
|
||||
/** A short-lived command failed its lifecycle or complete-output contract. */
|
||||
export class OwnedCommandFailure extends Error {
|
||||
readonly cleanupResult: ProcessCleanupFailure | undefined;
|
||||
|
||||
constructor(
|
||||
readonly reason: "cancelled" | "timeout" | "output-limit" | "process",
|
||||
message: string,
|
||||
readonly snapshot: ProviderProcessSnapshot | null = null,
|
||||
readonly cleanupFailure: string | null = null,
|
||||
options?: ErrorOptions,
|
||||
options?: ErrorOptions & { readonly cleanupResult?: ProcessCleanupFailure },
|
||||
readonly resources: readonly string[] = [],
|
||||
readonly cleanupOwner?: ProviderProcessSupervisor,
|
||||
) {
|
||||
super(message, options);
|
||||
this.cleanupResult = options?.cleanupResult;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,7 +196,12 @@ const collectOwnedCommand = async (
|
||||
failure?.message ?? "Command cleanup failed.",
|
||||
snapshot,
|
||||
stopped.reason,
|
||||
failure === undefined ? undefined : { cause: failure },
|
||||
{
|
||||
...(failure === undefined ? {} : { cause: failure }),
|
||||
...(stopped.cleanupResult === undefined
|
||||
? {}
|
||||
: { cleanupResult: stopped.cleanupResult }),
|
||||
},
|
||||
[
|
||||
spawn.runId,
|
||||
...(spawn.cwd === undefined ? [] : [spawn.cwd]),
|
||||
@@ -205,7 +216,12 @@ const collectOwnedCommand = async (
|
||||
failure.message,
|
||||
snapshot,
|
||||
failure.cleanupFailure,
|
||||
{ cause: failure },
|
||||
{
|
||||
cause: failure,
|
||||
...(failure.cleanupResult === undefined
|
||||
? {}
|
||||
: { cleanupResult: failure.cleanupResult }),
|
||||
},
|
||||
failure.resources,
|
||||
);
|
||||
if (callerSignal?.aborted)
|
||||
|
||||
@@ -111,7 +111,7 @@ describe("owned process-group cleanup discovery", () => {
|
||||
await expect(cleanupOwnedProcessGroup(ownership, adapter)).resolves.toEqual(
|
||||
{
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 104",
|
||||
failures: [{ pid: 104, reason: "run-token-mismatch" }],
|
||||
},
|
||||
);
|
||||
|
||||
@@ -331,7 +331,7 @@ describe("sampled detached process groups during cleanup", () => {
|
||||
|
||||
expect(result).toMatchObject({
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 200",
|
||||
failures: [{ pid: 200, reason: "run-token-mismatch" }],
|
||||
});
|
||||
expect(signalGroup).toHaveBeenCalledTimes(1);
|
||||
@@ -369,7 +369,7 @@ describe("sampled detached process groups during cleanup", () => {
|
||||
|
||||
expect(result).toMatchObject({
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 200",
|
||||
failures: [{ pid: 200, reason: "run-token-mismatch" }],
|
||||
});
|
||||
expect(signalGroup).not.toHaveBeenCalled();
|
||||
|
||||
@@ -157,19 +157,15 @@ export type ProcessCleanupResult =
|
||||
readonly signaled: boolean;
|
||||
readonly unverified?: readonly OwnershipSweepUnverifiedProcess[];
|
||||
}
|
||||
| {
|
||||
readonly cleaned: false;
|
||||
readonly reason: string;
|
||||
readonly failures?: readonly {
|
||||
readonly pid: number;
|
||||
readonly reason:
|
||||
| "environment-unreadable"
|
||||
| "run-token-mismatch"
|
||||
| "process-identity-unavailable";
|
||||
readonly diagnostic?: string;
|
||||
}[];
|
||||
readonly unverified?: readonly OwnershipSweepUnverifiedProcess[];
|
||||
};
|
||||
| ProcessCleanupFailure;
|
||||
|
||||
/** Retained verification failures for one cleanup attempt, without additional signaling authority. */
|
||||
export interface ProcessCleanupFailure {
|
||||
readonly cleaned: false;
|
||||
readonly reason: string;
|
||||
readonly failures?: readonly ProcessOwnershipValidationFailure[];
|
||||
readonly unverified?: readonly OwnershipSweepUnverifiedProcess[];
|
||||
}
|
||||
|
||||
/** Token-verified liveness result used by post-root-exit settlement. */
|
||||
export type ProcessGroupObservation =
|
||||
@@ -591,7 +587,15 @@ const cleanupValidationFailure = (
|
||||
if (failures.some(({ reason }) => reason === "run-token-mismatch"))
|
||||
return {
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: `process ownership token mismatch for PID(s): ${[
|
||||
...new Set(
|
||||
failures
|
||||
.filter(({ reason }) => reason === "run-token-mismatch")
|
||||
.map(({ pid }) => pid),
|
||||
),
|
||||
]
|
||||
.sort((left, right) => left - right)
|
||||
.join(", ")}`,
|
||||
failures,
|
||||
...unverifiedEntry,
|
||||
};
|
||||
|
||||
@@ -74,7 +74,7 @@ describe("owned process-group cleanup validation: ownership and lineage", () =>
|
||||
});
|
||||
expect(await cleanupOwnedProcessGroup(ownership, adapter)).toEqual({
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 101",
|
||||
failures: [{ pid: 101, reason: "run-token-mismatch" }],
|
||||
});
|
||||
expect(signalGroup).not.toHaveBeenCalled();
|
||||
@@ -106,7 +106,7 @@ describe("owned process-group cleanup validation: ownership and lineage", () =>
|
||||
};
|
||||
expect(await cleanupOwnedProcessGroup(ownership, adapter)).toEqual({
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 101",
|
||||
failures: [
|
||||
expect.objectContaining({
|
||||
pid: 100,
|
||||
@@ -189,7 +189,7 @@ describe("owned process-group cleanup liveness rechecks", () => {
|
||||
cleanupOwnedProcessGroup(ownership, adapter),
|
||||
).resolves.toEqual({
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 101",
|
||||
failures: recheckFails
|
||||
? [
|
||||
{
|
||||
@@ -345,7 +345,7 @@ describe("owned process-group cleanup validation: launcher exit races", () => {
|
||||
if (remaining === "foreign")
|
||||
expect(result).toMatchObject({
|
||||
cleaned: false,
|
||||
reason: "process tree contains an unowned or PID-reused process",
|
||||
reason: "process ownership token mismatch for PID(s): 101",
|
||||
});
|
||||
else
|
||||
expect(result).toEqual({
|
||||
|
||||
@@ -6,6 +6,7 @@ import { WindowsOwnedProcess } from "../windows/WindowsOwnedProcess.js";
|
||||
|
||||
import type {
|
||||
OwnedProcessGroup,
|
||||
ProcessCleanupFailure,
|
||||
ProcessCleanupResult,
|
||||
} from "./ProcessOwnership.js";
|
||||
import { prepareProcessOwnershipInspection } from "./ProcessOwnershipObservation.js";
|
||||
@@ -143,7 +144,11 @@ export type ProviderProcessStopResult =
|
||||
| "terminated"
|
||||
| "killed";
|
||||
}
|
||||
| { readonly status: "incomplete"; readonly reason: string };
|
||||
| {
|
||||
readonly status: "incomplete";
|
||||
readonly reason: string;
|
||||
readonly cleanupResult?: ProcessCleanupFailure;
|
||||
};
|
||||
|
||||
/**
|
||||
* Spawn a provider in an owned Windows job or dedicated POSIX process group.
|
||||
@@ -373,13 +378,21 @@ export class ProviderProcessSupervisor {
|
||||
if (this.launch.cleanup !== undefined) {
|
||||
const cleanup = await this.#ownedCleanup(options.cleanupResult);
|
||||
if (!cleanup.cleaned)
|
||||
return { status: "incomplete", reason: cleanup.reason };
|
||||
return {
|
||||
status: "incomplete",
|
||||
reason: cleanup.reason,
|
||||
cleanupResult: cleanup,
|
||||
};
|
||||
const graceMs = options.killGraceMs ?? DEFAULT_KILL_GRACE_MS;
|
||||
if (await this.waitForExit(graceMs))
|
||||
return { status: "verified-cleanup" };
|
||||
const retried = await this.launch.cleanup();
|
||||
if (!retried.cleaned)
|
||||
return { status: "incomplete", reason: retried.reason };
|
||||
return {
|
||||
status: "incomplete",
|
||||
reason: retried.reason,
|
||||
cleanupResult: retried,
|
||||
};
|
||||
if (await this.waitForExit(graceMs))
|
||||
return { status: "verified-cleanup" };
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user