feat(process): record provider and verifier run lineage
This commit is contained in:
+329
-79
@@ -15,8 +15,8 @@ export interface OwnedProcessGroup {
|
||||
readonly expectedParentPid?: number;
|
||||
}
|
||||
|
||||
/** One operating-system observation of a process-group member. */
|
||||
interface ProcessGroupMember {
|
||||
/** One entry from an operating-system process-table snapshot. */
|
||||
export interface ProcessTableEntry {
|
||||
readonly pid: number;
|
||||
readonly parentPid: number;
|
||||
readonly processGroupId: number;
|
||||
@@ -24,17 +24,39 @@ interface ProcessGroupMember {
|
||||
readonly command: string;
|
||||
}
|
||||
|
||||
/** Narrow operating-system seam used to inspect and signal process groups. */
|
||||
export interface ProcessOwnershipHost {
|
||||
listMembers(processGroupId: number): Promise<
|
||||
readonly {
|
||||
readonly pid: number;
|
||||
readonly parentPid: number;
|
||||
/** Token-verified process lineage retained for one owned provider run. */
|
||||
export interface OwnedProcessLineage {
|
||||
readonly schemaVersion: 1;
|
||||
readonly runId: string;
|
||||
readonly launcherPid: number;
|
||||
readonly launcherParentPid: number;
|
||||
readonly processGroupId: number;
|
||||
readonly descendants: readonly {
|
||||
readonly pid: number;
|
||||
readonly parentPid: number;
|
||||
readonly processGroupId: number;
|
||||
}[];
|
||||
}
|
||||
|
||||
/** Result of observing owned lineage without signaling any process. */
|
||||
export type ProcessLineageObservation =
|
||||
| {
|
||||
readonly status: "verified";
|
||||
readonly observedAt: string;
|
||||
readonly lineage: OwnedProcessLineage;
|
||||
}
|
||||
| {
|
||||
readonly status: "unavailable";
|
||||
readonly observedAt: string;
|
||||
readonly runId: string;
|
||||
readonly launcherPid: number;
|
||||
readonly processGroupId: number;
|
||||
readonly state: string;
|
||||
readonly command: string;
|
||||
}[]
|
||||
>;
|
||||
readonly reason: string;
|
||||
};
|
||||
|
||||
/** Narrow operating-system seam used to inspect processes and signal groups. */
|
||||
export interface ProcessOwnershipHost {
|
||||
listProcesses(): Promise<readonly ProcessTableEntry[]>;
|
||||
environment(pid: number): Promise<Readonly<Record<string, string>>>;
|
||||
signalGroup(processGroupId: number, signal: NodeJS.Signals): void;
|
||||
}
|
||||
@@ -126,7 +148,7 @@ export const parseProcessEnvironment = (
|
||||
);
|
||||
|
||||
const systemHost: ProcessOwnershipHost = {
|
||||
async listMembers(processGroupId) {
|
||||
async listProcesses() {
|
||||
if (process.platform === "win32") return [];
|
||||
const { stdout } = await execFileAsync("ps", [
|
||||
"-axo",
|
||||
@@ -142,8 +164,7 @@ const systemHost: ProcessOwnershipHost = {
|
||||
processGroupId: Number(match[3]),
|
||||
state: match[4] ?? "",
|
||||
command: match[5] ?? "",
|
||||
}))
|
||||
.filter((member) => member.processGroupId === processGroupId);
|
||||
}));
|
||||
},
|
||||
async environment(pid) {
|
||||
if (process.platform === "linux") {
|
||||
@@ -192,73 +213,186 @@ export const readProcessRunId = async (
|
||||
): Promise<string | undefined> =>
|
||||
(await host.environment(pid)).REA_PROCESS_RUN_ID;
|
||||
|
||||
/**
|
||||
* Verify run-token ownership before signaling a POSIX process group.
|
||||
*
|
||||
* Process IDs and group IDs can be reused. REA therefore re-reads every live
|
||||
* member's environment immediately before signaling and fails closed if any
|
||||
* member cannot be inspected or lacks the per-capture run token.
|
||||
*/
|
||||
/** Token-validate every rooted POSIX process group before signaling any. */
|
||||
export const cleanupOwnedProcessGroup = async (
|
||||
ownership: OwnedProcessGroup,
|
||||
host: ProcessOwnershipHost = systemHost,
|
||||
): Promise<ProcessCleanupResult> => {
|
||||
let members: readonly ProcessGroupMember[];
|
||||
let processes: readonly ProcessTableEntry[];
|
||||
try {
|
||||
members = await host.listMembers(ownership.processGroupId);
|
||||
processes = liveProcesses(await host.listProcesses());
|
||||
} catch {
|
||||
return { cleaned: false, reason: "process group could not be inspected" };
|
||||
return { cleaned: false, reason: "process table could not be inspected" };
|
||||
}
|
||||
const liveMembers = liveProcessGroupMembers(members);
|
||||
if (liveMembers.length === 0) return { cleaned: true, signaled: false };
|
||||
const leader = liveMembers.find(
|
||||
(member) => member.pid === ownership.leaderPid,
|
||||
);
|
||||
if (leader !== undefined) {
|
||||
if (
|
||||
ownership.expectedParentPid !== undefined &&
|
||||
leader.parentPid !== ownership.expectedParentPid
|
||||
)
|
||||
return {
|
||||
cleaned: false,
|
||||
reason: "owned launcher parent identity did not match",
|
||||
};
|
||||
if (
|
||||
ownership.expectedCommand !== undefined &&
|
||||
!commandMatches(leader.command, ownership.expectedCommand)
|
||||
)
|
||||
return {
|
||||
cleaned: false,
|
||||
reason: "owned launcher command identity did not match",
|
||||
};
|
||||
}
|
||||
const failures: ProcessOwnershipValidationFailure[] = [];
|
||||
for (const member of liveMembers) {
|
||||
const plan = await createOwnedCleanupPlan(ownership, processes, host);
|
||||
if ("cleaned" in plan) return plan;
|
||||
let signaled = false;
|
||||
for (const processGroupId of plan.signalOrder) {
|
||||
const revalidation = await revalidateOwnedProcessGroup(
|
||||
processGroupId,
|
||||
ownership,
|
||||
host,
|
||||
);
|
||||
if ("cleaned" in revalidation) return revalidation;
|
||||
if (revalidation.empty) continue;
|
||||
try {
|
||||
const environment = await host.environment(member.pid);
|
||||
if (environment.REA_PROCESS_RUN_ID !== ownership.runId)
|
||||
host.signalGroup(processGroupId, "SIGKILL");
|
||||
signaled = true;
|
||||
} catch (cause: unknown) {
|
||||
const code =
|
||||
cause instanceof Error && "code" in cause ? cause.code : undefined;
|
||||
if (code !== "ESRCH")
|
||||
return { cleaned: false, reason: "owned process group signal failed" };
|
||||
}
|
||||
}
|
||||
return { cleaned: true, signaled };
|
||||
};
|
||||
|
||||
interface OwnedCleanupPlan {
|
||||
readonly signalOrder: readonly number[];
|
||||
}
|
||||
|
||||
const createOwnedCleanupPlan = async (
|
||||
ownership: OwnedProcessGroup,
|
||||
processes: readonly ProcessTableEntry[],
|
||||
host: ProcessOwnershipHost,
|
||||
): Promise<OwnedCleanupPlan | ProcessCleanupResult> => {
|
||||
const launcher = processes.find(({ pid }) => pid === ownership.leaderPid);
|
||||
const rootMembers = processes.filter(
|
||||
({ processGroupId }) => processGroupId === ownership.processGroupId,
|
||||
);
|
||||
if (launcher === undefined && rootMembers.length === 0)
|
||||
return { cleaned: true, signaled: false };
|
||||
let descendants: readonly ProcessTableEntry[] = [];
|
||||
if (launcher !== undefined) {
|
||||
const identityFailure = launcherIdentityFailure(launcher, ownership);
|
||||
if (identityFailure !== null)
|
||||
return { cleaned: false, reason: identityFailure };
|
||||
descendants = descendantsOf(launcher.pid, processes);
|
||||
}
|
||||
const descendantPids = new Set(descendants.map(({ pid }) => pid));
|
||||
const processGroupIds = new Set<number>([ownership.processGroupId]);
|
||||
for (const descendant of descendants)
|
||||
processGroupIds.add(descendant.processGroupId);
|
||||
for (const processGroupId of processGroupIds) {
|
||||
if (processGroupId === ownership.processGroupId) continue;
|
||||
const groupLeader = processes.find(
|
||||
({ pid, processGroupId: observedGroupId }) =>
|
||||
pid === processGroupId && observedGroupId === processGroupId,
|
||||
);
|
||||
if (groupLeader === undefined || !descendantPids.has(groupLeader.pid))
|
||||
return {
|
||||
cleaned: false,
|
||||
reason:
|
||||
"descendant process-group leader identity could not be verified",
|
||||
};
|
||||
}
|
||||
const liveMembers = processes.filter(({ processGroupId }) =>
|
||||
processGroupIds.has(processGroupId),
|
||||
);
|
||||
const failures = await processOwnershipFailures(
|
||||
liveMembers,
|
||||
ownership.runId,
|
||||
host,
|
||||
);
|
||||
if (failures.length > 0) return cleanupValidationFailure(failures);
|
||||
return {
|
||||
signalOrder: [ownership.processGroupId].concat(
|
||||
[...processGroupIds]
|
||||
.filter((processGroupId) => processGroupId !== ownership.processGroupId)
|
||||
.sort((left, right) => left - right),
|
||||
),
|
||||
};
|
||||
};
|
||||
|
||||
const launcherIdentityFailure = (
|
||||
launcher: ProcessTableEntry,
|
||||
ownership: OwnedProcessGroup,
|
||||
): string | null => {
|
||||
if (launcher.processGroupId !== ownership.processGroupId)
|
||||
return "owned launcher process-group identity did not match";
|
||||
if (
|
||||
ownership.expectedParentPid !== undefined &&
|
||||
launcher.parentPid !== ownership.expectedParentPid
|
||||
)
|
||||
return "owned launcher parent identity did not match";
|
||||
if (
|
||||
ownership.expectedCommand !== undefined &&
|
||||
!commandMatches(launcher.command, ownership.expectedCommand)
|
||||
)
|
||||
return "owned launcher command identity did not match";
|
||||
return null;
|
||||
};
|
||||
|
||||
const revalidateOwnedProcessGroup = async (
|
||||
processGroupId: number,
|
||||
ownership: OwnedProcessGroup,
|
||||
host: ProcessOwnershipHost,
|
||||
): Promise<{ readonly empty: boolean } | ProcessCleanupResult> => {
|
||||
let members: readonly ProcessTableEntry[];
|
||||
try {
|
||||
members = liveProcesses(await host.listProcesses()).filter(
|
||||
({ processGroupId: observedGroupId }) =>
|
||||
observedGroupId === processGroupId,
|
||||
);
|
||||
} catch {
|
||||
return {
|
||||
cleaned: false,
|
||||
reason: "process ownership could not be revalidated",
|
||||
};
|
||||
}
|
||||
const failures = await processOwnershipFailures(
|
||||
members,
|
||||
ownership.runId,
|
||||
host,
|
||||
);
|
||||
if (failures.length > 0) return cleanupValidationFailure(failures);
|
||||
if (processGroupId === ownership.processGroupId) {
|
||||
const launcher = members.find(({ pid }) => pid === ownership.leaderPid);
|
||||
if (launcher !== undefined) {
|
||||
const identityFailure = launcherIdentityFailure(launcher, ownership);
|
||||
if (identityFailure !== null)
|
||||
return { cleaned: false, reason: identityFailure };
|
||||
}
|
||||
}
|
||||
if (
|
||||
processGroupId !== ownership.processGroupId &&
|
||||
members.length > 0 &&
|
||||
!members.some(({ pid }) => pid === processGroupId)
|
||||
)
|
||||
return {
|
||||
cleaned: false,
|
||||
reason:
|
||||
"descendant process-group leader identity could not be revalidated",
|
||||
};
|
||||
return { empty: members.length === 0 };
|
||||
};
|
||||
|
||||
const cleanupValidationFailure = (
|
||||
failures: readonly ProcessOwnershipValidationFailure[],
|
||||
): ProcessCleanupResult => ({
|
||||
cleaned: false,
|
||||
reason: failures.some(({ reason }) => reason === "run-token-mismatch")
|
||||
? "process tree contains an unowned or PID-reused process"
|
||||
: "process ownership could not be revalidated",
|
||||
failures,
|
||||
});
|
||||
|
||||
const processOwnershipFailures = async (
|
||||
members: readonly ProcessTableEntry[],
|
||||
runId: string,
|
||||
host: ProcessOwnershipHost,
|
||||
): Promise<readonly ProcessOwnershipValidationFailure[]> => {
|
||||
const failures: ProcessOwnershipValidationFailure[] = [];
|
||||
for (const member of members) {
|
||||
try {
|
||||
if ((await host.environment(member.pid)).REA_PROCESS_RUN_ID !== runId)
|
||||
failures.push({ pid: member.pid, reason: "run-token-mismatch" });
|
||||
} catch {
|
||||
failures.push({ pid: member.pid, reason: "environment-unreadable" });
|
||||
}
|
||||
}
|
||||
if (failures.length > 0)
|
||||
return {
|
||||
cleaned: false,
|
||||
reason: failures.some(({ reason }) => reason === "run-token-mismatch")
|
||||
? "process group contains an unowned or PID-reused process"
|
||||
: "process ownership could not be revalidated",
|
||||
failures,
|
||||
};
|
||||
try {
|
||||
host.signalGroup(ownership.processGroupId, "SIGKILL");
|
||||
} catch (cause: unknown) {
|
||||
const code =
|
||||
cause instanceof Error && "code" in cause ? cause.code : undefined;
|
||||
if (code !== "ESRCH")
|
||||
return { cleaned: false, reason: "owned process group signal failed" };
|
||||
}
|
||||
return { cleaned: true, signaled: true };
|
||||
return failures;
|
||||
};
|
||||
|
||||
/** Observe one group without signaling it, failing closed on identity doubt. */
|
||||
@@ -266,16 +400,18 @@ export const observeOwnedProcessGroup = async (
|
||||
ownership: OwnedProcessGroup,
|
||||
host: ProcessOwnershipHost = systemHost,
|
||||
): Promise<ProcessGroupObservation> => {
|
||||
let members: readonly ProcessGroupMember[];
|
||||
let members: readonly ProcessTableEntry[];
|
||||
try {
|
||||
members = await host.listMembers(ownership.processGroupId);
|
||||
members = (await host.listProcesses()).filter(
|
||||
({ processGroupId }) => processGroupId === ownership.processGroupId,
|
||||
);
|
||||
} catch {
|
||||
return {
|
||||
state: "unverifiable",
|
||||
reason: "process group could not be inspected",
|
||||
};
|
||||
}
|
||||
const liveMembers = liveProcessGroupMembers(members);
|
||||
const liveMembers = liveProcesses(members);
|
||||
if (liveMembers.length === 0) return { state: "empty" };
|
||||
for (const member of liveMembers) {
|
||||
try {
|
||||
@@ -297,16 +433,130 @@ export const observeOwnedProcessGroup = async (
|
||||
return { state: "alive" };
|
||||
};
|
||||
|
||||
const liveProcessGroupMembers = (
|
||||
members: readonly ProcessGroupMember[],
|
||||
): readonly ProcessGroupMember[] =>
|
||||
/**
|
||||
* Record the live launcher and descendant lineage after run-token validation.
|
||||
*
|
||||
* The observation is intentionally point-in-time. A verified empty descendant
|
||||
* list means no descendants were live during this observation, not that the
|
||||
* run never created a short-lived child.
|
||||
*/
|
||||
export const observeOwnedProcessLineage = async (
|
||||
ownership: OwnedProcessGroup,
|
||||
host: ProcessOwnershipHost = systemHost,
|
||||
): Promise<ProcessLineageObservation> => {
|
||||
let processes: readonly ProcessTableEntry[];
|
||||
try {
|
||||
processes = liveProcesses(await host.listProcesses());
|
||||
} catch {
|
||||
return unavailableLineage(
|
||||
ownership,
|
||||
"process table could not be inspected",
|
||||
);
|
||||
}
|
||||
const launcher = processes.find(({ pid }) => pid === ownership.leaderPid);
|
||||
if (launcher === undefined)
|
||||
return unavailableLineage(ownership, "owned launcher is not live");
|
||||
if (launcher.processGroupId !== ownership.processGroupId)
|
||||
return unavailableLineage(
|
||||
ownership,
|
||||
"owned launcher process-group identity did not match",
|
||||
);
|
||||
if (
|
||||
ownership.expectedParentPid !== undefined &&
|
||||
launcher.parentPid !== ownership.expectedParentPid
|
||||
)
|
||||
return unavailableLineage(
|
||||
ownership,
|
||||
"owned launcher parent identity did not match",
|
||||
);
|
||||
if (
|
||||
ownership.expectedCommand !== undefined &&
|
||||
!commandMatches(launcher.command, ownership.expectedCommand)
|
||||
)
|
||||
return unavailableLineage(
|
||||
ownership,
|
||||
"owned launcher command identity did not match",
|
||||
);
|
||||
const descendants = descendantsOf(launcher.pid, processes);
|
||||
for (const member of [launcher, ...descendants]) {
|
||||
try {
|
||||
if (
|
||||
(await host.environment(member.pid)).REA_PROCESS_RUN_ID !==
|
||||
ownership.runId
|
||||
)
|
||||
return unavailableLineage(
|
||||
ownership,
|
||||
"process lineage contains an unowned or PID-reused process",
|
||||
);
|
||||
} catch {
|
||||
return unavailableLineage(
|
||||
ownership,
|
||||
"process ownership could not be revalidated",
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
status: "verified",
|
||||
observedAt: new Date().toISOString(),
|
||||
lineage: {
|
||||
schemaVersion: 1,
|
||||
runId: ownership.runId,
|
||||
launcherPid: launcher.pid,
|
||||
launcherParentPid: launcher.parentPid,
|
||||
processGroupId: launcher.processGroupId,
|
||||
descendants: descendants
|
||||
.sort((left, right) => left.pid - right.pid)
|
||||
.map(({ pid, parentPid, processGroupId }) => ({
|
||||
pid,
|
||||
parentPid,
|
||||
processGroupId,
|
||||
})),
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
const descendantsOf = (
|
||||
launcherPid: number,
|
||||
processes: readonly ProcessTableEntry[],
|
||||
): ProcessTableEntry[] => {
|
||||
const childrenByParent = new Map<number, ProcessTableEntry[]>();
|
||||
for (const process of processes) {
|
||||
const siblings = childrenByParent.get(process.parentPid) ?? [];
|
||||
siblings.push(process);
|
||||
childrenByParent.set(process.parentPid, siblings);
|
||||
}
|
||||
const descendants: ProcessTableEntry[] = [];
|
||||
const pending = [...(childrenByParent.get(launcherPid) ?? [])];
|
||||
const visited = new Set<number>([launcherPid]);
|
||||
for (const process of pending) {
|
||||
if (visited.has(process.pid)) continue;
|
||||
visited.add(process.pid);
|
||||
descendants.push(process);
|
||||
pending.push(...(childrenByParent.get(process.pid) ?? []));
|
||||
}
|
||||
return descendants;
|
||||
};
|
||||
|
||||
const unavailableLineage = (
|
||||
ownership: OwnedProcessGroup,
|
||||
reason: string,
|
||||
): Extract<ProcessLineageObservation, { readonly status: "unavailable" }> => ({
|
||||
status: "unavailable",
|
||||
observedAt: new Date().toISOString(),
|
||||
runId: ownership.runId,
|
||||
launcherPid: ownership.leaderPid,
|
||||
processGroupId: ownership.processGroupId,
|
||||
reason,
|
||||
});
|
||||
|
||||
const liveProcesses = (
|
||||
members: readonly ProcessTableEntry[],
|
||||
): readonly ProcessTableEntry[] =>
|
||||
members.filter(({ state }) => !state.startsWith("Z"));
|
||||
|
||||
const commandMatches = (actual: string, expected: string): boolean => {
|
||||
const normalizedActual = actual.trim();
|
||||
const normalizedExpected = expected.trim();
|
||||
return (
|
||||
normalizedActual === normalizedExpected ||
|
||||
normalizedActual.startsWith(`${normalizedExpected} `)
|
||||
actual.trim() === expected.trim() ||
|
||||
actual.trim().startsWith(`${expected.trim()} `)
|
||||
);
|
||||
};
|
||||
|
||||
@@ -33,6 +33,7 @@ export interface SpawnedOwnedProviderProcess {
|
||||
export interface ProviderProcessLaunch {
|
||||
readonly process: ChildProcess;
|
||||
readonly ownsProcessLifetime: boolean;
|
||||
readonly ownership?: OwnedProcessGroup;
|
||||
readonly cleanup?: () => Promise<ProcessCleanupResult>;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { ProcessLineageObservation } from "./ProcessOwnership.js";
|
||||
import { observeOwnedProcessLineage } from "./ProcessOwnership.js";
|
||||
import type { ProviderProcessLaunch } from "./ProviderProcess.js";
|
||||
|
||||
/** Retain the latest truthful process-lineage observation for one client. */
|
||||
export class ProviderRunLineage {
|
||||
#observation: ProcessLineageObservation | null = null;
|
||||
|
||||
/** Clear stale lineage before a new launch attempt. */
|
||||
reset(): void {
|
||||
this.#observation = null;
|
||||
}
|
||||
|
||||
/** Observe a token-owned launcher without signaling it. */
|
||||
async observe(launch: ProviderProcessLaunch | undefined): Promise<void> {
|
||||
if (launch?.ownership === undefined) return;
|
||||
this.#observation = await observeOwnedProcessLineage(launch.ownership);
|
||||
}
|
||||
|
||||
/** Return a detached copy suitable for caller-visible status. */
|
||||
snapshot(): ProcessLineageObservation | null {
|
||||
return this.#observation === null
|
||||
? null
|
||||
: structuredClone(this.#observation);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user