feat(process): bound a finalization interval before forced capture timeout
Add optional finalization_ms to keep observing a captured root after SIGTERM, record the actual finalizer exit and each signal attempt, and escalate once when needed. Bind the initial root identity to captured ownership and preserve observations through cleanup uncertainty. Fixes #1253. Co-authored-by: Mykola Dementiev <16836724+nikolay-dementiev@users.noreply.github.com> Co-authored-by: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com>
This commit is contained in:
co-authored by
Mykola Dementiev
N0zoM1z0
parent
1e51c732bb
commit
c1efa6467d
@@ -431,10 +431,15 @@ export const createSystemProcessOwnershipHost = (
|
||||
|
||||
export const systemProcessOwnershipHost = createSystemProcessOwnershipHost();
|
||||
|
||||
/** Observe the current OS start identity for one live PID, when supported. */
|
||||
/**
|
||||
* Observe a live PID's start identity, optionally binding it to an owned run.
|
||||
* Token validation is bracketed by identity reads so PID reuse cannot supply
|
||||
* the first identity of a different process for an owned launch.
|
||||
*/
|
||||
export const observeProcessStartIdentity = async (
|
||||
pid: number,
|
||||
host: ProcessOwnershipHost = systemProcessOwnershipHost,
|
||||
expectedRunId?: string,
|
||||
): Promise<ProcessIdentityObservation | undefined> => {
|
||||
const process = (await host.listProcesses()).find(
|
||||
(entry) => entry.pid === pid,
|
||||
@@ -443,12 +448,42 @@ export const observeProcessStartIdentity = async (
|
||||
return undefined;
|
||||
if (host.processIdentities === undefined)
|
||||
return { state: "unavailable", reason: "process identity is unsupported" };
|
||||
return (
|
||||
(await host.processIdentities([process])).get(pid) ?? {
|
||||
const identity: ProcessIdentityObservation = (
|
||||
await host.processIdentities([process])
|
||||
).get(pid) ?? {
|
||||
state: "unavailable",
|
||||
reason: "process identity was not returned",
|
||||
};
|
||||
if (expectedRunId === undefined || identity.state !== "readable")
|
||||
return identity;
|
||||
let owned = false;
|
||||
for await (const { observation } of readProcessRunTokens(host, [process])) {
|
||||
if (observation.state === "unavailable")
|
||||
return {
|
||||
state: "unavailable",
|
||||
reason: `process ownership token could not be read for PID ${String(pid)}: ${observation.reason}`,
|
||||
};
|
||||
if (observation.runId !== expectedRunId)
|
||||
return {
|
||||
state: "unavailable",
|
||||
reason: `process ownership token did not match the captured run for PID ${String(pid)}`,
|
||||
};
|
||||
owned = true;
|
||||
}
|
||||
if (!owned)
|
||||
return {
|
||||
state: "unavailable",
|
||||
reason: "process identity was not returned",
|
||||
}
|
||||
);
|
||||
reason: `process ownership token was not returned for PID ${String(pid)}`,
|
||||
};
|
||||
const confirmed = await observeProcessStartIdentity(pid, host);
|
||||
if (confirmed === undefined || confirmed.state !== "readable")
|
||||
return confirmed;
|
||||
return confirmed.identity === identity.identity
|
||||
? identity
|
||||
: {
|
||||
state: "unavailable",
|
||||
reason: `process identity changed while validating the captured run token for PID ${String(pid)}`,
|
||||
};
|
||||
};
|
||||
|
||||
/** Revalidate a launch-time start identity immediately before signaling a PID. */
|
||||
|
||||
@@ -37,6 +37,54 @@ describe("process start identity lease", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("does not adopt a replacement PID's first identity for an owned launch", async () => {
|
||||
const replacement: ProcessOwnershipHost = {
|
||||
...host("replacement-start"),
|
||||
environment: async () => ({ REA_PROCESS_RUN_ID: "another-run" }),
|
||||
};
|
||||
await expect(
|
||||
observeProcessStartIdentity(42, replacement, "captured-run"),
|
||||
).resolves.toMatchObject({
|
||||
state: "unavailable",
|
||||
reason: expect.stringContaining("ownership"),
|
||||
});
|
||||
});
|
||||
|
||||
it("does not establish an owned launch when its token is unreadable", async () => {
|
||||
const unreadable: ProcessOwnershipHost = {
|
||||
...host("start-1"),
|
||||
runTokens: async () =>
|
||||
new Map([
|
||||
[42, { state: "unavailable", reason: "environment_errno_EACCES" }],
|
||||
]),
|
||||
};
|
||||
await expect(
|
||||
observeProcessStartIdentity(42, unreadable, "captured-run"),
|
||||
).resolves.toMatchObject({
|
||||
state: "unavailable",
|
||||
reason: expect.stringContaining("environment_errno_EACCES"),
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects an identity that changes while the captured run token is read", async () => {
|
||||
let identity = "start-1";
|
||||
const changing: ProcessOwnershipHost = {
|
||||
...host("start-1"),
|
||||
processIdentities: async () =>
|
||||
new Map([[42, { state: "readable", identity }]]),
|
||||
environment: async () => {
|
||||
identity = "replacement-start";
|
||||
return { REA_PROCESS_RUN_ID: "captured-run" };
|
||||
},
|
||||
};
|
||||
await expect(
|
||||
observeProcessStartIdentity(42, changing, "captured-run"),
|
||||
).resolves.toMatchObject({
|
||||
state: "unavailable",
|
||||
reason: expect.stringContaining("identity changed"),
|
||||
});
|
||||
});
|
||||
|
||||
it("signals the PID when its launch identity still matches", async () => {
|
||||
const sendSignal = vi.fn();
|
||||
await expect(
|
||||
|
||||
@@ -0,0 +1,548 @@
|
||||
import { expect, it } from "vitest";
|
||||
import type { IPty } from "@lydell/node-pty";
|
||||
|
||||
import { parseProcessScenario } from "../../domain/process/processScenario.js";
|
||||
import type { ProcessCaptureFinalization } from "../../domain/process/processCapture.js";
|
||||
import { awaitTerminalExit } from "./ProcessCaptureLifecycle.js";
|
||||
import type { ProcessTimer } from "./ProcessTimer.js";
|
||||
|
||||
type Delivery = ProcessCaptureFinalization["signals"][number]["delivery"];
|
||||
|
||||
interface FakeExit {
|
||||
readonly exitCode: number;
|
||||
readonly signal?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Terminal whose exit notification the test delivers when it chooses, plus
|
||||
* the identity-checked signaller that finalization uses instead of a bare
|
||||
* terminal kill. Both record into `signals`.
|
||||
*/
|
||||
const fakeTerminal = (
|
||||
deliveries: Partial<Record<"SIGTERM" | "SIGKILL", Delivery>> = {},
|
||||
hold?: Promise<void>,
|
||||
signalFailure?: Error,
|
||||
) => {
|
||||
let listener: ((exit: FakeExit) => void) | undefined;
|
||||
const signals: string[] = [];
|
||||
const terminal: Pick<IPty, "onExit" | "kill"> = {
|
||||
onExit: (next) => {
|
||||
listener = next;
|
||||
return { dispose: () => undefined };
|
||||
},
|
||||
kill: (signal) => {
|
||||
signals.push(`terminal:${signal ?? "SIGHUP"}`);
|
||||
},
|
||||
};
|
||||
const signalTarget = async (
|
||||
signal: "SIGTERM" | "SIGKILL",
|
||||
): Promise<{ readonly delivery: Delivery }> => {
|
||||
signals.push(signal);
|
||||
await hold;
|
||||
if (signalFailure !== undefined) throw signalFailure;
|
||||
return { delivery: deliveries[signal] ?? "signaled" };
|
||||
};
|
||||
return {
|
||||
signals,
|
||||
terminal,
|
||||
signalTarget,
|
||||
deliverExit: (exit: FakeExit) => listener?.(exit),
|
||||
};
|
||||
};
|
||||
|
||||
const waitFor = async (condition: () => boolean): Promise<void> => {
|
||||
const deadline = Date.now() + 3_000;
|
||||
while (!condition()) {
|
||||
if (Date.now() > deadline) throw new Error("condition not reached");
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
};
|
||||
|
||||
const snapshots: ProcessCaptureFinalization[] = [];
|
||||
|
||||
const observed = <Result extends { readonly unobserved?: boolean }>(
|
||||
result: Result,
|
||||
): Exclude<Result, { readonly unobserved: true }> => {
|
||||
if (result.unobserved === true) throw new Error("exit was not observed");
|
||||
return result as Exclude<Result, { readonly unobserved: true }>;
|
||||
};
|
||||
|
||||
const start = (
|
||||
fake: ReturnType<typeof fakeTerminal>,
|
||||
scenario: Readonly<Record<string, unknown>>,
|
||||
signal?: AbortSignal,
|
||||
) =>
|
||||
awaitTerminalExit({
|
||||
terminal: fake.terminal,
|
||||
scenario: parseProcessScenario({
|
||||
executable: "unused",
|
||||
idle_timeout_ms: 10_000,
|
||||
...scenario,
|
||||
}),
|
||||
started: Date.now(),
|
||||
lastOutput: () => Date.now(),
|
||||
signal,
|
||||
signalTarget: fake.signalTarget,
|
||||
recordFinalization: (snapshot) => {
|
||||
snapshots.push(snapshot);
|
||||
},
|
||||
timers: new Set<ProcessTimer>(),
|
||||
interactions: [],
|
||||
dispatchedEventIndexes: new Set<number>(),
|
||||
recordEvent: () => undefined,
|
||||
});
|
||||
|
||||
it("records attempts when a self-exit notification arrives after escalation", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
await new Promise((resolve) => setTimeout(resolve, 150));
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals.map(({ signal }) => signal),
|
||||
"both attempts are observed in order",
|
||||
).toEqual(["SIGTERM", "SIGKILL"]);
|
||||
expect(exit.signal, "the observed exit remains independent of attempts").toBe(
|
||||
0,
|
||||
);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal === "SIGKILL"),
|
||||
"the escalation is sent once, not on every poll",
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal === "SIGTERM"),
|
||||
"the finalization signal is sent once, not on every poll",
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("kills at once when cancelled during the finalization interval", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const controller = new AbortController();
|
||||
const pending = start(
|
||||
fake,
|
||||
{ timeout_ms: 100, finalization_ms: 60_000 },
|
||||
controller.signal,
|
||||
);
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
controller.abort();
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(exit.reason, "cancellation replaces the initiating deadline").toBe(
|
||||
"cancelled",
|
||||
);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal === "SIGKILL"),
|
||||
"the abort branch itself sent one SIGKILL, long before the interval ended",
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
exit.finalization?.signals.map(({ signal }) => signal),
|
||||
"cancellation records both attempts",
|
||||
).toEqual(["SIGTERM", "SIGKILL"]);
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal.startsWith("terminal:")),
|
||||
"cancellation during finalization never uses a bare terminal kill",
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("records escalation attempts alongside an observed SIGKILL exit", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals,
|
||||
"both finalization attempts are recorded",
|
||||
).toMatchObject([
|
||||
{ signal: "SIGTERM", delivery: "signaled" },
|
||||
{ signal: "SIGKILL", delivery: "signaled" },
|
||||
]);
|
||||
expect(exit.signal, "the observed exit signal is retained separately").toBe(
|
||||
9,
|
||||
);
|
||||
expect(
|
||||
exit.finalization?.elapsed_ms,
|
||||
"the escalation waited for the whole interval",
|
||||
).toBeGreaterThanOrEqual(100);
|
||||
});
|
||||
|
||||
it("records a scripted SIGKILL exit without inferring its cause", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
fake.signals,
|
||||
"the harness itself never escalated inside the interval",
|
||||
).not.toContain("SIGKILL");
|
||||
expect(
|
||||
exit.finalization?.signals,
|
||||
"only the SIGTERM attempt was made",
|
||||
).toMatchObject([{ signal: "SIGTERM", delivery: "signaled" }]);
|
||||
expect(exit.signal, "the observed SIGKILL remains an exit fact").toBe(9);
|
||||
});
|
||||
|
||||
it.each([
|
||||
[3, 3],
|
||||
[0, 0],
|
||||
[7, 7],
|
||||
] as const)(
|
||||
"records exit code %i of a finalizer that exits on its own as %i",
|
||||
async (exitCode, expected) => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.exit_code,
|
||||
"the PTY-reported code of an exit not caused by a signal is recorded",
|
||||
).toBe(expected);
|
||||
expect(exit.reason, "the initiating deadline stays the exit reason").toBe(
|
||||
"timeout",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("records no exit code when the finalizing target was ended by a signal", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.exit_code,
|
||||
"a code reported next to a terminating signal is never recorded",
|
||||
).toBeNull();
|
||||
expect(exit.signal, "the signal remains the observed exit fact").toBe(9);
|
||||
});
|
||||
|
||||
it("records no exit code when the PTY reports a negative code without a signal", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: -1, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.exit_code,
|
||||
"a negative PTY code is not a process exit code and is never recorded",
|
||||
).toBeNull();
|
||||
expect(exit.reason, "the initiating deadline stays the exit reason").toBe(
|
||||
"timeout",
|
||||
);
|
||||
});
|
||||
|
||||
it("records no exit code when no exit was observed", async () => {
|
||||
const fake = fakeTerminal({}, new Promise<void>(() => undefined));
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(result.unobserved, "no exit is invented").toBe(true);
|
||||
expect(
|
||||
result.finalization?.exit_code,
|
||||
"an unobserved exit has no code to record",
|
||||
).toBeNull();
|
||||
}, 10_000);
|
||||
|
||||
it("records every delivery result and publishes attempts incrementally", async () => {
|
||||
snapshots.length = 0;
|
||||
const fake = fakeTerminal({ SIGTERM: "gone", SIGKILL: "gone" });
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals.map(({ delivery }) => delivery),
|
||||
"both delivery results are kept as reported",
|
||||
).toEqual(["gone", "gone"]);
|
||||
expect(
|
||||
snapshots.map(({ signals }) => signals.length),
|
||||
"the first attempt was published before the escalation existed",
|
||||
).toContain(1);
|
||||
expect(
|
||||
snapshots.at(-1)?.signals.length,
|
||||
"the last published record holds both attempts",
|
||||
).toBe(2);
|
||||
});
|
||||
|
||||
it.each(["identity-changed", "unverified"] as const)(
|
||||
"stops waiting for an exit when the escalation reports %s",
|
||||
async (delivery) => {
|
||||
snapshots.length = 0;
|
||||
const fake = fakeTerminal({ SIGTERM: "signaled", SIGKILL: delivery });
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(result.unobserved, "no exit is invented").toBe(true);
|
||||
expect(result.reason, "the initiating deadline is retained").toBe(
|
||||
"timeout",
|
||||
);
|
||||
expect(
|
||||
result.finalization,
|
||||
"attempts and the undelivered result are kept",
|
||||
).toMatchObject({
|
||||
elapsed_ms: null,
|
||||
signals: [
|
||||
{ signal: "SIGTERM", delivery: "signaled" },
|
||||
{ signal: "SIGKILL", delivery },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
fake.signals.filter((signal) => signal.startsWith("terminal:")),
|
||||
"a failed identity check never falls back to a bare terminal kill",
|
||||
).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
||||
it("returns at once when a cancellation kill cannot be delivered", async () => {
|
||||
const fake = fakeTerminal({ SIGKILL: "unverified" });
|
||||
const controller = new AbortController();
|
||||
const pending = start(
|
||||
fake,
|
||||
{ timeout_ms: 100, finalization_ms: 60_000 },
|
||||
controller.signal,
|
||||
);
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
controller.abort();
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(result.reason, "cancellation replaces the deadline").toBe("cancelled");
|
||||
expect(result.unobserved, "the capture does not wait for an exit").toBe(true);
|
||||
});
|
||||
|
||||
it("waits for a pending delivery before reporting an observed exit", async () => {
|
||||
let release: () => void = () => undefined;
|
||||
const hold = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const fake = fakeTerminal({ SIGTERM: "signaled" }, hold);
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
let settled = false;
|
||||
void pending.then(() => {
|
||||
settled = true;
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
expect(settled, "the result waits for the outstanding delivery").toBe(false);
|
||||
|
||||
release();
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals,
|
||||
"the delivery result is complete in the published record",
|
||||
).toMatchObject([{ signal: "SIGTERM", delivery: "signaled" }]);
|
||||
});
|
||||
|
||||
it("keeps the zero-interval deadline as a bare SIGKILL", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const pending = start(fake, { timeout_ms: 60, finalization_ms: 0 });
|
||||
await waitFor(() => fake.signals.includes("terminal:SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
fake.signals.filter((signal) => !signal.startsWith("terminal:")),
|
||||
"the identity-checked signaller is never used without an interval",
|
||||
).toEqual([]);
|
||||
expect(exit.finalization, "no finalization record exists").toBeUndefined();
|
||||
});
|
||||
|
||||
it("stops waiting for a hung delivery once the exit is observed", async () => {
|
||||
const fake = fakeTerminal({}, new Promise<void>(() => undefined));
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals,
|
||||
"an undelivered attempt stays recorded as unverified",
|
||||
).toMatchObject([
|
||||
{
|
||||
signal: "SIGTERM",
|
||||
delivery: "unverified",
|
||||
reason: "delivery did not settle within 1000 ms",
|
||||
},
|
||||
]);
|
||||
expect(exit.signal, "the observed exit is kept").toBe(0);
|
||||
}, 10_000);
|
||||
|
||||
it("retains a thrown signal error as an unverified delivery reason", async () => {
|
||||
const fake = fakeTerminal(
|
||||
{},
|
||||
undefined,
|
||||
new Error("signal transport failed"),
|
||||
);
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals,
|
||||
"a rejected signal retains its error message",
|
||||
).toMatchObject([
|
||||
{
|
||||
signal: "SIGTERM",
|
||||
delivery: "unverified",
|
||||
reason: "signal transport failed",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps an observed exit when a late kill delivery reports unverified", async () => {
|
||||
let release: () => void = () => undefined;
|
||||
const hold = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
const fake = fakeTerminal({ SIGKILL: "unverified" }, hold);
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
await waitFor(() => fake.signals.includes("SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
release();
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(
|
||||
result.unobserved,
|
||||
"an exit that was already observed is never turned into an unobserved one",
|
||||
).not.toBe(true);
|
||||
expect(observed(result).signal, "the observed exit signal is retained").toBe(
|
||||
9,
|
||||
);
|
||||
});
|
||||
|
||||
it("stops waiting when the escalation delivery never settles and no exit arrives", async () => {
|
||||
const fake = fakeTerminal({}, new Promise<void>(() => undefined));
|
||||
const waitStarted = Date.now();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 100 });
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(
|
||||
Date.now() - waitStarted,
|
||||
"the wait is bounded by the one-second cleanup grace after the interval",
|
||||
).toBeLessThan(2_500);
|
||||
expect(result.unobserved, "no exit is invented").toBe(true);
|
||||
expect(
|
||||
result.finalization?.signals.at(-1),
|
||||
"the undelivered escalation stays recorded as unverified",
|
||||
).toMatchObject({
|
||||
signal: "SIGKILL",
|
||||
delivery: "unverified",
|
||||
reason: "delivery did not settle within 1000 ms",
|
||||
});
|
||||
expect(result.finalization?.elapsed_ms, "no exit was observed").toBeNull();
|
||||
}, 10_000);
|
||||
|
||||
const activeTimers = (): number =>
|
||||
process.getActiveResourcesInfo().filter((name) => name === "Timeout").length;
|
||||
|
||||
it("leaves no referenced timer behind after a zero-interval exit", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const before = activeTimers();
|
||||
const pending = start(fake, { timeout_ms: 60, finalization_ms: 0 });
|
||||
await waitFor(() => fake.signals.includes("terminal:SIGKILL"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 9 });
|
||||
|
||||
await pending;
|
||||
|
||||
expect(
|
||||
activeTimers(),
|
||||
"the zero path resolves without a bounded-wait timer",
|
||||
).toBeLessThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("cancels the bounded-wait timer once deliveries settle after an exit", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const before = activeTimers();
|
||||
const pending = start(fake, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => fake.signals.includes("SIGTERM"));
|
||||
fake.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
await pending;
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
|
||||
expect(
|
||||
activeTimers(),
|
||||
"no one-second grace timer outlives the finalized exit",
|
||||
).toBeLessThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("keeps a non-empty reason when a signal delivery throws without a message", async () => {
|
||||
const fake = fakeTerminal();
|
||||
const throwing = {
|
||||
...fake,
|
||||
signalTarget: async (): Promise<never> => {
|
||||
throw new Error();
|
||||
},
|
||||
};
|
||||
const pending = start(throwing, { timeout_ms: 100, finalization_ms: 5_000 });
|
||||
await waitFor(() => throwing.signals.length >= 0);
|
||||
await new Promise((resolve) => setTimeout(resolve, 150));
|
||||
throwing.deliverExit({ exitCode: 0, signal: 0 });
|
||||
|
||||
const exit = observed(await pending);
|
||||
|
||||
expect(
|
||||
exit.finalization?.signals[0]?.reason?.length,
|
||||
"a thrown error without a message still leaves a named reason",
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("never publishes an unverified attempt without a reason, also while it is in flight", async () => {
|
||||
const fake = fakeTerminal();
|
||||
let calls = 0;
|
||||
const custom = {
|
||||
...fake,
|
||||
signalTarget: (signal: "SIGTERM" | "SIGKILL") => {
|
||||
calls += 1;
|
||||
fake.signals.push(signal);
|
||||
return calls === 1
|
||||
? new Promise<never>(() => undefined)
|
||||
: Promise.resolve({ delivery: "identity-changed" as const });
|
||||
},
|
||||
};
|
||||
const pending = start(custom, { timeout_ms: 50, finalization_ms: 20 });
|
||||
|
||||
const result = await pending;
|
||||
|
||||
expect(result.unobserved, "the undeliverable escalation ends the wait").toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
result.finalization?.signals[0],
|
||||
"the still-undelivered SIGTERM carries a reason, as the capture schema requires",
|
||||
).toMatchObject({ signal: "SIGTERM", delivery: "unverified" });
|
||||
expect(
|
||||
result.finalization?.signals[0]?.reason?.length,
|
||||
"the in-flight attempt is never published without a reason",
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
@@ -52,6 +52,7 @@ import { scheduleProcessInterval, type ProcessTimer } from "./ProcessTimer.js";
|
||||
import type {
|
||||
FilesystemCheckpoint,
|
||||
InteractionEvent,
|
||||
ProcessCaptureFinalization,
|
||||
UnverifiedProcessCapture,
|
||||
ProcessSample,
|
||||
ProcessSettlement,
|
||||
@@ -66,11 +67,20 @@ import type {
|
||||
} from "../../domain/process/processCapture.js";
|
||||
import { partialProcessCaptureObservationSchema } from "../../domain/process/processCapture.js";
|
||||
interface TerminalExitOptions {
|
||||
readonly terminal: IPty;
|
||||
readonly terminal: Pick<IPty, "onExit" | "kill">;
|
||||
readonly scenario: ProcessScenario;
|
||||
readonly started: number;
|
||||
readonly lastOutput: () => number;
|
||||
readonly signal: AbortSignal | undefined;
|
||||
/** Identity-checked signal to the captured root, used only during finalization. */
|
||||
readonly signalTarget: (signal: FinalizationSignal) => Promise<{
|
||||
readonly delivery: FinalizationDelivery;
|
||||
readonly reason?: string | undefined;
|
||||
}>;
|
||||
/** Receives the attempts so far, so a later failure keeps them. */
|
||||
readonly recordFinalization: (
|
||||
finalization: ProcessCaptureFinalization,
|
||||
) => void;
|
||||
readonly timers: Set<ProcessTimer>;
|
||||
readonly interactions: InteractionEvent[];
|
||||
readonly dispatchedEventIndexes: ReadonlySet<number>;
|
||||
@@ -78,12 +88,33 @@ interface TerminalExitOptions {
|
||||
readonly onLiveProgress?: () => void;
|
||||
}
|
||||
|
||||
type FinalizationSignal =
|
||||
ProcessCaptureFinalization["signals"][number]["signal"];
|
||||
type FinalizationDelivery =
|
||||
ProcessCaptureFinalization["signals"][number]["delivery"];
|
||||
|
||||
/** The capture's exit, or the facts known when no exit could be observed. */
|
||||
export type TerminalExitResult =
|
||||
| {
|
||||
readonly unobserved?: false;
|
||||
readonly exitCode: number;
|
||||
readonly signal?: number;
|
||||
readonly reason: "exited" | "timeout" | "idle_timeout" | "cancelled";
|
||||
readonly finalization?: ProcessCaptureFinalization;
|
||||
}
|
||||
| {
|
||||
readonly unobserved: true;
|
||||
readonly reason: "timeout" | "idle_timeout" | "cancelled";
|
||||
readonly finalization: ProcessCaptureFinalization;
|
||||
};
|
||||
|
||||
interface CaptureResultOptions {
|
||||
readonly frames: readonly TerminalFrame[];
|
||||
readonly exit: {
|
||||
readonly exitCode: number;
|
||||
readonly signal?: number;
|
||||
readonly reason: "exited" | "timeout" | "idle_timeout";
|
||||
readonly finalization?: ProcessCaptureFinalization;
|
||||
};
|
||||
readonly samples: readonly ProcessSample[];
|
||||
readonly before: ProcessFilesystemSnapshot;
|
||||
@@ -130,6 +161,7 @@ export interface ProcessCaptureObservationBuffer {
|
||||
UnverifiedProcessCapture["interaction_events"]
|
||||
>;
|
||||
exit: IncompleteProcessCaptureObservations["exit"];
|
||||
finalization?: IncompleteProcessCaptureObservations["finalization"];
|
||||
settlement: IncompleteProcessCaptureObservations["settlement"];
|
||||
process_samples: PartialProcessObservationField<readonly ProcessSample[]>;
|
||||
filesystem_snapshots: {
|
||||
@@ -149,6 +181,8 @@ export const createProcessCaptureObservationBuffer = (options: {
|
||||
readonly samples: readonly ProcessSample[];
|
||||
readonly eventJournal: readonly ProcessCaptureEventJournalEntry[];
|
||||
readonly before: ProcessFilesystemSnapshot;
|
||||
/** Seed a finalization observation only for a scenario with an interval. */
|
||||
readonly finalizationEnabled?: boolean;
|
||||
}): ProcessCaptureObservationBuffer => ({
|
||||
target_pid: {
|
||||
state: "unavailable",
|
||||
@@ -165,6 +199,15 @@ export const createProcessCaptureObservationBuffer = (options: {
|
||||
state: "unavailable",
|
||||
reason: "Terminal exit was not observed before the run failed.",
|
||||
},
|
||||
...(options.finalizationEnabled === true
|
||||
? {
|
||||
finalization: {
|
||||
state: "unavailable" as const,
|
||||
reason:
|
||||
"Finalization was not observed before capture completion finished.",
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
settlement: {
|
||||
state: "unavailable",
|
||||
reason:
|
||||
@@ -271,6 +314,9 @@ export const buildCaptureResult = (
|
||||
: null,
|
||||
signal: options.exit.signal ?? null,
|
||||
reason: options.exit.reason,
|
||||
...(options.exit.finalization === undefined
|
||||
? {}
|
||||
: { finalization: options.exit.finalization }),
|
||||
},
|
||||
settlement: options.settlement,
|
||||
process_samples: normalizeProcessSamples(
|
||||
@@ -644,29 +690,172 @@ export const observeSettlement = async (
|
||||
return { state: "alive_at_deadline", elapsed_ms: Date.now() - started };
|
||||
};
|
||||
|
||||
/** Name why an attempt is unverified; a blank message never reaches the capture schema. */
|
||||
export const describeUnverifiedCause = (
|
||||
cause: unknown,
|
||||
fallback: string,
|
||||
): string => {
|
||||
const message = cause instanceof Error ? cause.message : String(cause);
|
||||
return message.trim() === "" ? fallback : message;
|
||||
};
|
||||
|
||||
/**
|
||||
* Wait for the captured terminal to exit and report how it ended.
|
||||
*
|
||||
* A timeout or idle deadline kills the root at once when `finalization_ms` is
|
||||
* 0. With a positive interval it sends SIGTERM through `signalTarget`, keeps
|
||||
* observing, and sends SIGKILL once if the interval ends first; cancellation
|
||||
* during the interval sends that SIGKILL at once. Every attempt is recorded
|
||||
* with its delivery result before the delivery is awaited, and each wait is
|
||||
* bounded by the cleanup verification grace. If the SIGKILL cannot be
|
||||
* delivered the wait ends with `unobserved: true` and no invented exit, so the
|
||||
* caller can keep its observations and release the run through cleanup.
|
||||
*/
|
||||
export const awaitTerminalExit = async ({
|
||||
terminal,
|
||||
scenario,
|
||||
started,
|
||||
lastOutput,
|
||||
signal,
|
||||
signalTarget,
|
||||
recordFinalization,
|
||||
timers,
|
||||
interactions,
|
||||
dispatchedEventIndexes,
|
||||
recordEvent,
|
||||
onLiveProgress,
|
||||
}: TerminalExitOptions): Promise<{
|
||||
exitCode: number;
|
||||
signal?: number;
|
||||
reason: "exited" | "timeout" | "idle_timeout" | "cancelled";
|
||||
}> =>
|
||||
}: TerminalExitOptions): Promise<TerminalExitResult> =>
|
||||
new Promise((resolveExit) => {
|
||||
// The kill caused by a deadline is observed later as an ordinary PTY exit.
|
||||
// Keep the initiating lifecycle reason so comparisons distinguish a target
|
||||
// exit from harness-owned timeout, idle-timeout, and cancellation cleanup.
|
||||
let reason: "exited" | "timeout" | "idle_timeout" | "cancelled" = "exited";
|
||||
onLiveProgress?.();
|
||||
// A deadline with a finalization interval sends SIGTERM first and keeps
|
||||
// observing; SIGKILL follows only when the interval elapses or the request
|
||||
// is cancelled. Both go through the identity-checked signaller.
|
||||
// The interval is measured on the monotonic clock so a wall-clock change
|
||||
// cannot shorten it or produce a negative elapsed time.
|
||||
let finalizationStartedAt: number | undefined;
|
||||
let escalationSent = false;
|
||||
let settled = false;
|
||||
let exitObserved = false;
|
||||
const finalizationSignals: {
|
||||
signal: FinalizationSignal;
|
||||
sent_at_ms: number;
|
||||
delivery: FinalizationDelivery;
|
||||
reason?: string | undefined;
|
||||
}[] = [];
|
||||
const pendingDeliveries: Promise<void>[] = [];
|
||||
const snapshot = (
|
||||
elapsed: number | null,
|
||||
exitCode: number | null = null,
|
||||
): ProcessCaptureFinalization => ({
|
||||
requested_ms: scenario.finalization_ms,
|
||||
elapsed_ms: elapsed,
|
||||
exit_code: exitCode,
|
||||
signals: finalizationSignals.map((attempt) => ({ ...attempt })),
|
||||
});
|
||||
const unobservedExit = (): void => {
|
||||
if (settled || exitObserved || finalizationStartedAt === undefined)
|
||||
return;
|
||||
settled = true;
|
||||
for (const timer of timers) timer.cancel();
|
||||
timers.clear();
|
||||
resolveExit({
|
||||
unobserved: true,
|
||||
reason: reason === "exited" ? "timeout" : reason,
|
||||
finalization: snapshot(null),
|
||||
});
|
||||
};
|
||||
// An attempt is recorded before its delivery is awaited, so a hung or
|
||||
// failed delivery still leaves the attempt in the observations.
|
||||
const attempt = (finalizationSignal: FinalizationSignal): void => {
|
||||
if (finalizationStartedAt === undefined) return;
|
||||
const record: (typeof finalizationSignals)[number] = {
|
||||
signal: finalizationSignal,
|
||||
sent_at_ms: Math.round(performance.now() - finalizationStartedAt),
|
||||
delivery: "unverified" as FinalizationDelivery,
|
||||
reason: "delivery had not settled when the record was published",
|
||||
};
|
||||
finalizationSignals.push(record);
|
||||
recordFinalization(snapshot(null));
|
||||
const deliveryBound = new AbortController();
|
||||
// The wait is bounded by the cleanup verification grace: a delivery that
|
||||
// never settles is recorded as unverified instead of holding the capture.
|
||||
pendingDeliveries.push(
|
||||
Promise.race([
|
||||
signalTarget(finalizationSignal),
|
||||
delay(
|
||||
PROCESS_CLEANUP_VERIFICATION_GRACE_MS,
|
||||
{
|
||||
delivery: "unverified" as const,
|
||||
reason: `delivery did not settle within ${String(PROCESS_CLEANUP_VERIFICATION_GRACE_MS)} ms`,
|
||||
},
|
||||
{
|
||||
ref: false,
|
||||
signal: deliveryBound.signal,
|
||||
},
|
||||
).catch(() => ({
|
||||
delivery: "unverified" as const,
|
||||
reason: "delivery wait was interrupted",
|
||||
})),
|
||||
])
|
||||
.finally(() => {
|
||||
deliveryBound.abort();
|
||||
})
|
||||
.then(
|
||||
({ delivery, reason: deliveryReason }) => {
|
||||
record.delivery = delivery;
|
||||
if (deliveryReason !== undefined) record.reason = deliveryReason;
|
||||
else delete record.reason;
|
||||
},
|
||||
(cause: unknown) => {
|
||||
record.delivery = "unverified";
|
||||
record.reason = describeUnverifiedCause(
|
||||
cause,
|
||||
"signal delivery failed without a message",
|
||||
);
|
||||
},
|
||||
)
|
||||
.then(() => {
|
||||
recordFinalization(snapshot(null));
|
||||
// A kill that cannot reach the root leaves no exit to wait for.
|
||||
if (
|
||||
finalizationSignal === "SIGKILL" &&
|
||||
record.delivery !== "signaled" &&
|
||||
record.delivery !== "gone"
|
||||
)
|
||||
unobservedExit();
|
||||
}),
|
||||
);
|
||||
};
|
||||
const forceKill = (): void => {
|
||||
if (finalizationStartedAt !== undefined) {
|
||||
if (escalationSent) return;
|
||||
escalationSent = true;
|
||||
attempt("SIGKILL");
|
||||
return;
|
||||
}
|
||||
terminal.kill("SIGKILL");
|
||||
};
|
||||
const terminateAtDeadline = (): void => {
|
||||
if (scenario.finalization_ms === 0) {
|
||||
terminal.kill("SIGKILL");
|
||||
return;
|
||||
}
|
||||
finalizationStartedAt = performance.now();
|
||||
attempt("SIGTERM");
|
||||
};
|
||||
terminal.onExit((exit) => {
|
||||
if (settled) return;
|
||||
exitObserved = true;
|
||||
// Measures when the exit was observed, not when the operating system
|
||||
// ended the process; the PTY layer can deliver the notification late.
|
||||
const finalizationElapsedMs =
|
||||
finalizationStartedAt === undefined
|
||||
? undefined
|
||||
: Math.round(performance.now() - finalizationStartedAt);
|
||||
recordEvent("lifecycle", 0);
|
||||
for (const [eventIndex, event] of scenario.events.entries()) {
|
||||
if (dispatchedEventIndexes.has(eventIndex)) continue;
|
||||
@@ -690,19 +879,60 @@ export const awaitTerminalExit = async ({
|
||||
timer.cancel();
|
||||
}
|
||||
timers.clear();
|
||||
resolveExit({ ...exit, reason });
|
||||
const finish = (): void => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
resolveExit({
|
||||
...exit,
|
||||
reason,
|
||||
...(finalizationElapsedMs === undefined
|
||||
? {}
|
||||
: {
|
||||
finalization: snapshot(
|
||||
finalizationElapsedMs,
|
||||
exit.exitCode >= 0 && (exit.signal ?? 0) === 0
|
||||
? exit.exitCode
|
||||
: null,
|
||||
),
|
||||
}),
|
||||
});
|
||||
};
|
||||
if (pendingDeliveries.length === 0) {
|
||||
finish();
|
||||
return;
|
||||
}
|
||||
// A delivery that never settles must not hold back an observed exit;
|
||||
// its attempt stays recorded as unverified.
|
||||
const grace = new AbortController();
|
||||
void Promise.race([
|
||||
Promise.allSettled(pendingDeliveries),
|
||||
delay(PROCESS_CLEANUP_VERIFICATION_GRACE_MS, undefined, {
|
||||
ref: false,
|
||||
signal: grace.signal,
|
||||
}).catch(() => undefined),
|
||||
])
|
||||
.finally(() => {
|
||||
grace.abort();
|
||||
})
|
||||
.then(finish);
|
||||
});
|
||||
const timeout = scheduleProcessInterval(() => {
|
||||
onLiveProgress?.();
|
||||
if (signal?.aborted === true) {
|
||||
reason = "cancelled";
|
||||
terminal.kill("SIGKILL");
|
||||
forceKill();
|
||||
} else if (finalizationStartedAt !== undefined) {
|
||||
if (
|
||||
performance.now() - finalizationStartedAt >=
|
||||
scenario.finalization_ms
|
||||
)
|
||||
forceKill();
|
||||
} else if (Date.now() - started >= scenario.timeout_ms) {
|
||||
reason = "timeout";
|
||||
terminal.kill("SIGKILL");
|
||||
terminateAtDeadline();
|
||||
} else if (Date.now() - lastOutput() >= scenario.idle_timeout_ms) {
|
||||
reason = "idle_timeout";
|
||||
terminal.kill("SIGKILL");
|
||||
terminateAtDeadline();
|
||||
}
|
||||
}, 20);
|
||||
timers.add(timeout);
|
||||
@@ -1268,6 +1498,7 @@ export const prepareProcessCapture = async (
|
||||
samples: [],
|
||||
eventJournal: [],
|
||||
before,
|
||||
finalizationEnabled: scenario.finalization_ms > 0,
|
||||
}),
|
||||
{ scenario },
|
||||
);
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { expect, it, vi } from "vitest";
|
||||
import { processScenarioSchema } from "../../domain/process/processScenario.js";
|
||||
import { AnalysisCapabilityUnavailableError } from "../../domain/analysisErrorCore.js";
|
||||
import { captureProcessScenario } from "./ProcessHarness.js";
|
||||
import {
|
||||
captureProcessScenario,
|
||||
retainRootSignaller,
|
||||
} from "./ProcessHarness.js";
|
||||
import { settleProcessCaptureJournal } from "./ProcessCaptureLifecycle.js";
|
||||
import { parseProcessCapture } from "../../domain/process/processCaptureParsing.js";
|
||||
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../../domain/process/processCaptureExample.js";
|
||||
@@ -116,6 +119,13 @@ it("keeps prelaunch snapshot evidence and the ownership failure when root cleanu
|
||||
} catch (cause: unknown) {
|
||||
if (!(cause instanceof ProcessCaptureError)) throw cause;
|
||||
expect(cause.cause).toBe(ownershipFailure);
|
||||
const incomplete = cause.partialObservation;
|
||||
if (incomplete === undefined || !("observations" in incomplete))
|
||||
throw new Error("expected incomplete preparation observations");
|
||||
expect(
|
||||
Object.keys(incomplete.observations),
|
||||
"a default scenario keeps the base observation keys through the preparation path",
|
||||
).not.toContain("finalization");
|
||||
expect(
|
||||
analysisErrorProjectionSchema.parse(projectAnalysisError(cause)),
|
||||
).toMatchObject({
|
||||
@@ -795,3 +805,173 @@ it("fails closed on Windows before resolving or launching scenario paths", async
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("retains the root start identity once at launch and signals only with it", async () => {
|
||||
const observe = vi.fn(async () => ({
|
||||
state: "readable" as const,
|
||||
identity: "id-launch",
|
||||
}));
|
||||
const signal = vi.fn(async () => "signaled" as const);
|
||||
const signalRoot = retainRootSignaller(4242, 500, "captured-run", {
|
||||
observe,
|
||||
signal,
|
||||
});
|
||||
|
||||
expect(
|
||||
observe,
|
||||
"the identity is read at launch, before any signal",
|
||||
).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
await signalRoot("SIGTERM"),
|
||||
"a matching identity is signalled",
|
||||
).toEqual({
|
||||
delivery: "signaled",
|
||||
});
|
||||
await signalRoot("SIGKILL");
|
||||
|
||||
expect(
|
||||
observe,
|
||||
"later signals never re-read the identity",
|
||||
).toHaveBeenCalledTimes(1);
|
||||
expect(
|
||||
signal.mock.calls,
|
||||
"both signals carry the pid and the identity retained at launch",
|
||||
).toEqual([
|
||||
[4242, "id-launch", "SIGTERM"],
|
||||
[4242, "id-launch", "SIGKILL"],
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a vanished root", async () => undefined, { delivery: "gone" }],
|
||||
[
|
||||
"an unreadable identity",
|
||||
async () => ({ state: "unavailable" as const, reason: "unsupported" }),
|
||||
{ delivery: "unverified", reason: "unsupported" },
|
||||
],
|
||||
[
|
||||
"a failed identity read",
|
||||
async () => {
|
||||
throw new Error("ps failed");
|
||||
},
|
||||
{ delivery: "unverified", reason: "ps failed" },
|
||||
],
|
||||
])(
|
||||
"never signals without a retained identity: %s",
|
||||
async (_label, read, delivery) => {
|
||||
const signal = vi.fn(async () => "signaled" as const);
|
||||
const signalRoot = retainRootSignaller(4242, 500, "captured-run", {
|
||||
observe: read,
|
||||
signal,
|
||||
});
|
||||
|
||||
expect(
|
||||
await signalRoot("SIGTERM"),
|
||||
"the delivery result is recorded",
|
||||
).toEqual(delivery);
|
||||
expect(signal, "no identity means no signal at all").not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not read an identity when there is no finalization interval", async () => {
|
||||
const observe = vi.fn(async () => undefined);
|
||||
const signalRoot = retainRootSignaller(4242, 0, "captured-run", {
|
||||
observe,
|
||||
signal: vi.fn(async () => "signaled" as const),
|
||||
});
|
||||
|
||||
expect(
|
||||
observe,
|
||||
"the zero-interval path never inspects the root",
|
||||
).not.toHaveBeenCalled();
|
||||
expect(
|
||||
await signalRoot("SIGKILL"),
|
||||
"an unretained identity stays unverified",
|
||||
).toEqual({
|
||||
delivery: "unverified",
|
||||
reason: "root identity was not retained",
|
||||
});
|
||||
});
|
||||
|
||||
it.each(["signaled", "gone", "identity-changed", "unverified"] as const)(
|
||||
"reports the identity-checked delivery result %s unchanged",
|
||||
async (delivery) => {
|
||||
const signalRoot = retainRootSignaller(4242, 500, "captured-run", {
|
||||
observe: async () => ({ state: "readable", identity: "id-launch" }),
|
||||
signal: async () => delivery,
|
||||
});
|
||||
|
||||
expect(
|
||||
await signalRoot("SIGKILL"),
|
||||
"the signaller result is not coerced",
|
||||
).toEqual(
|
||||
delivery === "unverified"
|
||||
? {
|
||||
delivery,
|
||||
reason:
|
||||
"start identity could not be verified or the signal call failed",
|
||||
}
|
||||
: { delivery },
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("seeds a finalization observation only for a scenario with an interval", () => {
|
||||
const seed = {
|
||||
frames: [],
|
||||
interactions: [],
|
||||
samples: [],
|
||||
eventJournal: [],
|
||||
before: {
|
||||
files: [],
|
||||
truncated: false,
|
||||
completeRoots: [],
|
||||
coverage: emptyFilesystemCoverage,
|
||||
},
|
||||
};
|
||||
|
||||
expect(
|
||||
Object.keys(createProcessCaptureObservationBuffer(seed)),
|
||||
"a default scenario keeps the base observation keys",
|
||||
).not.toContain("finalization");
|
||||
expect(
|
||||
Object.keys(
|
||||
createProcessCaptureObservationBuffer({
|
||||
...seed,
|
||||
finalizationEnabled: true,
|
||||
}),
|
||||
),
|
||||
"a scenario with an interval carries the finalization observation",
|
||||
).toContain("finalization");
|
||||
});
|
||||
|
||||
it("keeps a non-empty reason when the identity inspection fails without a message", async () => {
|
||||
const signalRoot = retainRootSignaller(4242, 500, "captured-run", {
|
||||
observe: async () => {
|
||||
throw new Error();
|
||||
},
|
||||
signal: vi.fn(async () => "signaled" as const),
|
||||
});
|
||||
|
||||
const result = await signalRoot("SIGTERM");
|
||||
|
||||
expect(result.delivery, "the attempt is unverified").toBe("unverified");
|
||||
expect(
|
||||
result.reason?.length,
|
||||
"the reason is never empty, which the capture schema would reject",
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("keeps a non-empty reason when the identity diagnostic is empty", async () => {
|
||||
const signalRoot = retainRootSignaller(4242, 500, "captured-run", {
|
||||
observe: async () => ({ state: "unavailable" as const, reason: "" }),
|
||||
signal: vi.fn(async () => "signaled" as const),
|
||||
});
|
||||
|
||||
const result = await signalRoot("SIGTERM");
|
||||
|
||||
expect(
|
||||
result.reason?.length,
|
||||
"an empty diagnostic is replaced by a named fallback",
|
||||
).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
@@ -43,6 +43,7 @@ import {
|
||||
type ProcessCaptureCleanupHost,
|
||||
type ProcessCaptureObservationBuffer,
|
||||
type PendingProcessCapture,
|
||||
describeUnverifiedCause,
|
||||
type ProcessCaptureResourceCleanupOptions,
|
||||
ProcessCaptureResourceScope,
|
||||
} from "./ProcessCaptureLifecycle.js";
|
||||
@@ -65,6 +66,10 @@ import {
|
||||
type ProcessCaptureProgressTracker,
|
||||
} from "./ProcessCaptureProgress.js";
|
||||
import type { ProcessOwnershipBaseline } from "../ProcessOwnership.js";
|
||||
import {
|
||||
observeProcessStartIdentity,
|
||||
signalProcessWithStartIdentity,
|
||||
} from "../ProcessOwnershipObservation.js";
|
||||
|
||||
interface StartedCaptureRuntime {
|
||||
readonly renderer: TerminalRenderer;
|
||||
@@ -75,8 +80,79 @@ interface StartedCaptureRuntime {
|
||||
readonly lastOutput: () => number;
|
||||
readonly rawTerminalRetention: () => TerminalRetention;
|
||||
readonly stopSampler: ReturnType<typeof startProcessSampler>;
|
||||
/** Signals the captured root only while its launch-time start identity still matches. */
|
||||
readonly signalRoot: (signal: "SIGTERM" | "SIGKILL") => Promise<{
|
||||
readonly delivery: "signaled" | "gone" | "identity-changed" | "unverified";
|
||||
readonly reason?: string | undefined;
|
||||
}>;
|
||||
}
|
||||
|
||||
/** Identity observation and signalling used for finalization signals; injectable for tests. */
|
||||
export interface FinalizationHost {
|
||||
readonly observe: typeof observeProcessStartIdentity;
|
||||
readonly signal: typeof signalProcessWithStartIdentity;
|
||||
}
|
||||
|
||||
/** Bind the captured root's initial identity to its run token before retaining it. */
|
||||
export const retainRootSignaller = (
|
||||
pid: number,
|
||||
finalizationMs: number,
|
||||
runId: string,
|
||||
host: FinalizationHost = {
|
||||
observe: (rootPid) =>
|
||||
observeProcessStartIdentity(rootPid, undefined, runId),
|
||||
signal: signalProcessWithStartIdentity,
|
||||
},
|
||||
): StartedCaptureRuntime["signalRoot"] => {
|
||||
const retained =
|
||||
finalizationMs > 0
|
||||
? host.observe(pid).then(
|
||||
(observation) =>
|
||||
observation === undefined
|
||||
? ({ state: "gone" } as const)
|
||||
: observation.state === "readable"
|
||||
? ({
|
||||
state: "identity",
|
||||
identity: observation.identity,
|
||||
} as const)
|
||||
: ({
|
||||
state: "unverified",
|
||||
reason:
|
||||
observation.reason.trim() === ""
|
||||
? "start identity is unavailable without a diagnostic"
|
||||
: observation.reason,
|
||||
} as const),
|
||||
(cause: unknown) =>
|
||||
({
|
||||
state: "unverified",
|
||||
reason: describeUnverifiedCause(
|
||||
cause,
|
||||
"start identity inspection failed without a message",
|
||||
),
|
||||
}) as const,
|
||||
)
|
||||
: undefined;
|
||||
return async (signal) => {
|
||||
const identity = await retained;
|
||||
if (identity === undefined)
|
||||
return {
|
||||
delivery: "unverified",
|
||||
reason: "root identity was not retained",
|
||||
};
|
||||
if (identity.state === "unverified")
|
||||
return { delivery: "unverified", reason: identity.reason };
|
||||
if (identity.state === "gone") return { delivery: "gone" };
|
||||
const delivery = await host.signal(pid, identity.identity, signal);
|
||||
return delivery === "unverified"
|
||||
? {
|
||||
delivery,
|
||||
reason:
|
||||
"start identity could not be verified or the signal call failed",
|
||||
}
|
||||
: { delivery };
|
||||
};
|
||||
};
|
||||
|
||||
const cleanupFailedStartup = async (options: {
|
||||
readonly cause: unknown;
|
||||
readonly timers: Set<ProcessTimer>;
|
||||
@@ -176,6 +252,7 @@ interface StartCaptureRuntimeOptions {
|
||||
readonly dispatchedEventIndexes: Set<number>;
|
||||
readonly recordEvent: RecordProcessCaptureEvent;
|
||||
readonly signal?: AbortSignal;
|
||||
readonly finalizationHost?: FinalizationHost;
|
||||
}
|
||||
|
||||
const startCaptureRuntime = async (
|
||||
@@ -245,6 +322,12 @@ const startCaptureRuntime = async (
|
||||
lastOutput: () => lastOutput,
|
||||
rawTerminalRetention,
|
||||
stopSampler,
|
||||
signalRoot: retainRootSignaller(
|
||||
terminal.pid,
|
||||
scenario.finalization_ms,
|
||||
options.runId,
|
||||
options.finalizationHost,
|
||||
),
|
||||
};
|
||||
} catch (cause: unknown) {
|
||||
return cleanupFailedStartup({
|
||||
@@ -371,7 +454,10 @@ const completeCapture = async (options: {
|
||||
readonly frames: readonly TerminalFrame[];
|
||||
readonly samples: readonly ProcessSample[];
|
||||
readonly interactions: readonly InteractionEvent[];
|
||||
readonly exit: Awaited<ReturnType<typeof awaitTerminalExit>>;
|
||||
readonly exit: Extract<
|
||||
Awaited<ReturnType<typeof awaitTerminalExit>>,
|
||||
{ readonly unobserved?: false }
|
||||
>;
|
||||
readonly signal?: AbortSignal;
|
||||
readonly captureSnapshot: typeof snapshotRoots;
|
||||
readonly eventJournal: readonly ProcessCaptureEventJournalEntry[];
|
||||
@@ -532,6 +618,7 @@ const runProcessScenario = async (
|
||||
cleanupHost?: ProcessCaptureCleanupHost,
|
||||
progress?: ProcessCaptureProgress,
|
||||
resourceScope?: ProcessCaptureResourceScope,
|
||||
finalizationHost?: FinalizationHost,
|
||||
): Promise<ProcessCapture> => {
|
||||
const progressTracker = createProcessCaptureProgressTracker(progress);
|
||||
const frames: TerminalFrame[] = [];
|
||||
@@ -584,6 +671,7 @@ const runProcessScenario = async (
|
||||
samples,
|
||||
eventJournal,
|
||||
before,
|
||||
finalizationEnabled: scenario.finalization_ms > 0,
|
||||
});
|
||||
let runtime: StartedCaptureRuntime | undefined;
|
||||
let actualRootPid: number | undefined;
|
||||
@@ -613,6 +701,7 @@ const runProcessScenario = async (
|
||||
dispatchedEventIndexes,
|
||||
recordEvent,
|
||||
...(signal === undefined ? {} : { signal }),
|
||||
...(finalizationHost === undefined ? {} : { finalizationHost }),
|
||||
});
|
||||
stopSampler = runtime.stopSampler;
|
||||
const exit = await awaitTerminalExit({
|
||||
@@ -621,12 +710,35 @@ const runProcessScenario = async (
|
||||
started: runtime.started,
|
||||
lastOutput: runtime.lastOutput,
|
||||
signal,
|
||||
signalTarget: runtime.signalRoot,
|
||||
recordFinalization: (finalization) => {
|
||||
observations.finalization = { state: "available", value: finalization };
|
||||
},
|
||||
timers,
|
||||
interactions,
|
||||
dispatchedEventIndexes,
|
||||
recordEvent,
|
||||
...(progress === undefined ? {} : { onLiveProgress: reportRunning }),
|
||||
});
|
||||
if (exit.unobserved === true) {
|
||||
observations.finalization = {
|
||||
state: "available",
|
||||
value: exit.finalization,
|
||||
};
|
||||
observations.exit = {
|
||||
state: "unavailable",
|
||||
reason: `The captured process exit was not observed because the finalization SIGKILL could not be delivered (initiating reason: ${exit.reason}).`,
|
||||
};
|
||||
if (exit.reason === "cancelled") throw processCaptureCancelled();
|
||||
throw new Error(
|
||||
"The captured process exit was not observed because the finalization SIGKILL could not be delivered.",
|
||||
);
|
||||
}
|
||||
if (exit.finalization !== undefined)
|
||||
observations.finalization = {
|
||||
state: "available",
|
||||
value: exit.finalization,
|
||||
};
|
||||
observations.exit = {
|
||||
state: "available",
|
||||
value: {
|
||||
@@ -634,6 +746,9 @@ const runProcessScenario = async (
|
||||
exit.reason === "exited" && exit.exitCode >= 0 ? exit.exitCode : null,
|
||||
signal: exit.signal ?? null,
|
||||
reason: exit.reason,
|
||||
...(exit.finalization === undefined
|
||||
? {}
|
||||
: { finalization: exit.finalization }),
|
||||
},
|
||||
};
|
||||
capture = await completeCapture({
|
||||
@@ -710,6 +825,7 @@ export const captureProcessScenario = async (
|
||||
cleanupHost?: ProcessCaptureCleanupHost,
|
||||
progress?: ProcessCaptureProgress,
|
||||
resourceScope?: ProcessCaptureResourceScope,
|
||||
finalizationHost?: FinalizationHost,
|
||||
): Promise<Result<ProcessCapture, ProcessCaptureError | AnalysisError>> => {
|
||||
const ownershipReason = processCaptureOwnershipUnavailableReason(platform);
|
||||
if (ownershipReason !== undefined)
|
||||
@@ -740,6 +856,7 @@ export const captureProcessScenario = async (
|
||||
cleanupHost,
|
||||
progress,
|
||||
runResourceScope,
|
||||
finalizationHost,
|
||||
);
|
||||
return ok(capture);
|
||||
} catch (cause: unknown) {
|
||||
|
||||
Reference in New Issue
Block a user