feat(process): record provider and verifier run lineage

This commit is contained in:
morluto
2026-07-22 14:01:47 +00:00
parent f5b3e22e72
commit 513fa294a4
200 changed files with 2808 additions and 546 deletions
+329 -79
View File
@@ -15,8 +15,8 @@ export interface OwnedProcessGroup {
readonly expectedParentPid?: number;
}
/** One operating-system observation of a process-group member. */
interface ProcessGroupMember {
/** One entry from an operating-system process-table snapshot. */
export interface ProcessTableEntry {
readonly pid: number;
readonly parentPid: number;
readonly processGroupId: number;
@@ -24,17 +24,39 @@ interface ProcessGroupMember {
readonly command: string;
}
/** Narrow operating-system seam used to inspect and signal process groups. */
export interface ProcessOwnershipHost {
listMembers(processGroupId: number): Promise<
readonly {
readonly pid: number;
readonly parentPid: number;
/** Token-verified process lineage retained for one owned provider run. */
export interface OwnedProcessLineage {
readonly schemaVersion: 1;
readonly runId: string;
readonly launcherPid: number;
readonly launcherParentPid: number;
readonly processGroupId: number;
readonly descendants: readonly {
readonly pid: number;
readonly parentPid: number;
readonly processGroupId: number;
}[];
}
/** Result of observing owned lineage without signaling any process. */
export type ProcessLineageObservation =
| {
readonly status: "verified";
readonly observedAt: string;
readonly lineage: OwnedProcessLineage;
}
| {
readonly status: "unavailable";
readonly observedAt: string;
readonly runId: string;
readonly launcherPid: number;
readonly processGroupId: number;
readonly state: string;
readonly command: string;
}[]
>;
readonly reason: string;
};
/** Narrow operating-system seam used to inspect processes and signal groups. */
export interface ProcessOwnershipHost {
listProcesses(): Promise<readonly ProcessTableEntry[]>;
environment(pid: number): Promise<Readonly<Record<string, string>>>;
signalGroup(processGroupId: number, signal: NodeJS.Signals): void;
}
@@ -126,7 +148,7 @@ export const parseProcessEnvironment = (
);
const systemHost: ProcessOwnershipHost = {
async listMembers(processGroupId) {
async listProcesses() {
if (process.platform === "win32") return [];
const { stdout } = await execFileAsync("ps", [
"-axo",
@@ -142,8 +164,7 @@ const systemHost: ProcessOwnershipHost = {
processGroupId: Number(match[3]),
state: match[4] ?? "",
command: match[5] ?? "",
}))
.filter((member) => member.processGroupId === processGroupId);
}));
},
async environment(pid) {
if (process.platform === "linux") {
@@ -192,73 +213,186 @@ export const readProcessRunId = async (
): Promise<string | undefined> =>
(await host.environment(pid)).REA_PROCESS_RUN_ID;
/**
* Verify run-token ownership before signaling a POSIX process group.
*
* Process IDs and group IDs can be reused. REA therefore re-reads every live
* member's environment immediately before signaling and fails closed if any
* member cannot be inspected or lacks the per-capture run token.
*/
/** Token-validate every rooted POSIX process group before signaling any. */
export const cleanupOwnedProcessGroup = async (
ownership: OwnedProcessGroup,
host: ProcessOwnershipHost = systemHost,
): Promise<ProcessCleanupResult> => {
let members: readonly ProcessGroupMember[];
let processes: readonly ProcessTableEntry[];
try {
members = await host.listMembers(ownership.processGroupId);
processes = liveProcesses(await host.listProcesses());
} catch {
return { cleaned: false, reason: "process group could not be inspected" };
return { cleaned: false, reason: "process table could not be inspected" };
}
const liveMembers = liveProcessGroupMembers(members);
if (liveMembers.length === 0) return { cleaned: true, signaled: false };
const leader = liveMembers.find(
(member) => member.pid === ownership.leaderPid,
);
if (leader !== undefined) {
if (
ownership.expectedParentPid !== undefined &&
leader.parentPid !== ownership.expectedParentPid
)
return {
cleaned: false,
reason: "owned launcher parent identity did not match",
};
if (
ownership.expectedCommand !== undefined &&
!commandMatches(leader.command, ownership.expectedCommand)
)
return {
cleaned: false,
reason: "owned launcher command identity did not match",
};
}
const failures: ProcessOwnershipValidationFailure[] = [];
for (const member of liveMembers) {
const plan = await createOwnedCleanupPlan(ownership, processes, host);
if ("cleaned" in plan) return plan;
let signaled = false;
for (const processGroupId of plan.signalOrder) {
const revalidation = await revalidateOwnedProcessGroup(
processGroupId,
ownership,
host,
);
if ("cleaned" in revalidation) return revalidation;
if (revalidation.empty) continue;
try {
const environment = await host.environment(member.pid);
if (environment.REA_PROCESS_RUN_ID !== ownership.runId)
host.signalGroup(processGroupId, "SIGKILL");
signaled = true;
} catch (cause: unknown) {
const code =
cause instanceof Error && "code" in cause ? cause.code : undefined;
if (code !== "ESRCH")
return { cleaned: false, reason: "owned process group signal failed" };
}
}
return { cleaned: true, signaled };
};
interface OwnedCleanupPlan {
readonly signalOrder: readonly number[];
}
const createOwnedCleanupPlan = async (
ownership: OwnedProcessGroup,
processes: readonly ProcessTableEntry[],
host: ProcessOwnershipHost,
): Promise<OwnedCleanupPlan | ProcessCleanupResult> => {
const launcher = processes.find(({ pid }) => pid === ownership.leaderPid);
const rootMembers = processes.filter(
({ processGroupId }) => processGroupId === ownership.processGroupId,
);
if (launcher === undefined && rootMembers.length === 0)
return { cleaned: true, signaled: false };
let descendants: readonly ProcessTableEntry[] = [];
if (launcher !== undefined) {
const identityFailure = launcherIdentityFailure(launcher, ownership);
if (identityFailure !== null)
return { cleaned: false, reason: identityFailure };
descendants = descendantsOf(launcher.pid, processes);
}
const descendantPids = new Set(descendants.map(({ pid }) => pid));
const processGroupIds = new Set<number>([ownership.processGroupId]);
for (const descendant of descendants)
processGroupIds.add(descendant.processGroupId);
for (const processGroupId of processGroupIds) {
if (processGroupId === ownership.processGroupId) continue;
const groupLeader = processes.find(
({ pid, processGroupId: observedGroupId }) =>
pid === processGroupId && observedGroupId === processGroupId,
);
if (groupLeader === undefined || !descendantPids.has(groupLeader.pid))
return {
cleaned: false,
reason:
"descendant process-group leader identity could not be verified",
};
}
const liveMembers = processes.filter(({ processGroupId }) =>
processGroupIds.has(processGroupId),
);
const failures = await processOwnershipFailures(
liveMembers,
ownership.runId,
host,
);
if (failures.length > 0) return cleanupValidationFailure(failures);
return {
signalOrder: [ownership.processGroupId].concat(
[...processGroupIds]
.filter((processGroupId) => processGroupId !== ownership.processGroupId)
.sort((left, right) => left - right),
),
};
};
const launcherIdentityFailure = (
launcher: ProcessTableEntry,
ownership: OwnedProcessGroup,
): string | null => {
if (launcher.processGroupId !== ownership.processGroupId)
return "owned launcher process-group identity did not match";
if (
ownership.expectedParentPid !== undefined &&
launcher.parentPid !== ownership.expectedParentPid
)
return "owned launcher parent identity did not match";
if (
ownership.expectedCommand !== undefined &&
!commandMatches(launcher.command, ownership.expectedCommand)
)
return "owned launcher command identity did not match";
return null;
};
const revalidateOwnedProcessGroup = async (
processGroupId: number,
ownership: OwnedProcessGroup,
host: ProcessOwnershipHost,
): Promise<{ readonly empty: boolean } | ProcessCleanupResult> => {
let members: readonly ProcessTableEntry[];
try {
members = liveProcesses(await host.listProcesses()).filter(
({ processGroupId: observedGroupId }) =>
observedGroupId === processGroupId,
);
} catch {
return {
cleaned: false,
reason: "process ownership could not be revalidated",
};
}
const failures = await processOwnershipFailures(
members,
ownership.runId,
host,
);
if (failures.length > 0) return cleanupValidationFailure(failures);
if (processGroupId === ownership.processGroupId) {
const launcher = members.find(({ pid }) => pid === ownership.leaderPid);
if (launcher !== undefined) {
const identityFailure = launcherIdentityFailure(launcher, ownership);
if (identityFailure !== null)
return { cleaned: false, reason: identityFailure };
}
}
if (
processGroupId !== ownership.processGroupId &&
members.length > 0 &&
!members.some(({ pid }) => pid === processGroupId)
)
return {
cleaned: false,
reason:
"descendant process-group leader identity could not be revalidated",
};
return { empty: members.length === 0 };
};
const cleanupValidationFailure = (
failures: readonly ProcessOwnershipValidationFailure[],
): ProcessCleanupResult => ({
cleaned: false,
reason: failures.some(({ reason }) => reason === "run-token-mismatch")
? "process tree contains an unowned or PID-reused process"
: "process ownership could not be revalidated",
failures,
});
const processOwnershipFailures = async (
members: readonly ProcessTableEntry[],
runId: string,
host: ProcessOwnershipHost,
): Promise<readonly ProcessOwnershipValidationFailure[]> => {
const failures: ProcessOwnershipValidationFailure[] = [];
for (const member of members) {
try {
if ((await host.environment(member.pid)).REA_PROCESS_RUN_ID !== runId)
failures.push({ pid: member.pid, reason: "run-token-mismatch" });
} catch {
failures.push({ pid: member.pid, reason: "environment-unreadable" });
}
}
if (failures.length > 0)
return {
cleaned: false,
reason: failures.some(({ reason }) => reason === "run-token-mismatch")
? "process group contains an unowned or PID-reused process"
: "process ownership could not be revalidated",
failures,
};
try {
host.signalGroup(ownership.processGroupId, "SIGKILL");
} catch (cause: unknown) {
const code =
cause instanceof Error && "code" in cause ? cause.code : undefined;
if (code !== "ESRCH")
return { cleaned: false, reason: "owned process group signal failed" };
}
return { cleaned: true, signaled: true };
return failures;
};
/** Observe one group without signaling it, failing closed on identity doubt. */
@@ -266,16 +400,18 @@ export const observeOwnedProcessGroup = async (
ownership: OwnedProcessGroup,
host: ProcessOwnershipHost = systemHost,
): Promise<ProcessGroupObservation> => {
let members: readonly ProcessGroupMember[];
let members: readonly ProcessTableEntry[];
try {
members = await host.listMembers(ownership.processGroupId);
members = (await host.listProcesses()).filter(
({ processGroupId }) => processGroupId === ownership.processGroupId,
);
} catch {
return {
state: "unverifiable",
reason: "process group could not be inspected",
};
}
const liveMembers = liveProcessGroupMembers(members);
const liveMembers = liveProcesses(members);
if (liveMembers.length === 0) return { state: "empty" };
for (const member of liveMembers) {
try {
@@ -297,16 +433,130 @@ export const observeOwnedProcessGroup = async (
return { state: "alive" };
};
const liveProcessGroupMembers = (
members: readonly ProcessGroupMember[],
): readonly ProcessGroupMember[] =>
/**
* Record the live launcher and descendant lineage after run-token validation.
*
* The observation is intentionally point-in-time. A verified empty descendant
* list means no descendants were live during this observation, not that the
* run never created a short-lived child.
*/
export const observeOwnedProcessLineage = async (
ownership: OwnedProcessGroup,
host: ProcessOwnershipHost = systemHost,
): Promise<ProcessLineageObservation> => {
let processes: readonly ProcessTableEntry[];
try {
processes = liveProcesses(await host.listProcesses());
} catch {
return unavailableLineage(
ownership,
"process table could not be inspected",
);
}
const launcher = processes.find(({ pid }) => pid === ownership.leaderPid);
if (launcher === undefined)
return unavailableLineage(ownership, "owned launcher is not live");
if (launcher.processGroupId !== ownership.processGroupId)
return unavailableLineage(
ownership,
"owned launcher process-group identity did not match",
);
if (
ownership.expectedParentPid !== undefined &&
launcher.parentPid !== ownership.expectedParentPid
)
return unavailableLineage(
ownership,
"owned launcher parent identity did not match",
);
if (
ownership.expectedCommand !== undefined &&
!commandMatches(launcher.command, ownership.expectedCommand)
)
return unavailableLineage(
ownership,
"owned launcher command identity did not match",
);
const descendants = descendantsOf(launcher.pid, processes);
for (const member of [launcher, ...descendants]) {
try {
if (
(await host.environment(member.pid)).REA_PROCESS_RUN_ID !==
ownership.runId
)
return unavailableLineage(
ownership,
"process lineage contains an unowned or PID-reused process",
);
} catch {
return unavailableLineage(
ownership,
"process ownership could not be revalidated",
);
}
}
return {
status: "verified",
observedAt: new Date().toISOString(),
lineage: {
schemaVersion: 1,
runId: ownership.runId,
launcherPid: launcher.pid,
launcherParentPid: launcher.parentPid,
processGroupId: launcher.processGroupId,
descendants: descendants
.sort((left, right) => left.pid - right.pid)
.map(({ pid, parentPid, processGroupId }) => ({
pid,
parentPid,
processGroupId,
})),
},
};
};
const descendantsOf = (
launcherPid: number,
processes: readonly ProcessTableEntry[],
): ProcessTableEntry[] => {
const childrenByParent = new Map<number, ProcessTableEntry[]>();
for (const process of processes) {
const siblings = childrenByParent.get(process.parentPid) ?? [];
siblings.push(process);
childrenByParent.set(process.parentPid, siblings);
}
const descendants: ProcessTableEntry[] = [];
const pending = [...(childrenByParent.get(launcherPid) ?? [])];
const visited = new Set<number>([launcherPid]);
for (const process of pending) {
if (visited.has(process.pid)) continue;
visited.add(process.pid);
descendants.push(process);
pending.push(...(childrenByParent.get(process.pid) ?? []));
}
return descendants;
};
const unavailableLineage = (
ownership: OwnedProcessGroup,
reason: string,
): Extract<ProcessLineageObservation, { readonly status: "unavailable" }> => ({
status: "unavailable",
observedAt: new Date().toISOString(),
runId: ownership.runId,
launcherPid: ownership.leaderPid,
processGroupId: ownership.processGroupId,
reason,
});
const liveProcesses = (
members: readonly ProcessTableEntry[],
): readonly ProcessTableEntry[] =>
members.filter(({ state }) => !state.startsWith("Z"));
const commandMatches = (actual: string, expected: string): boolean => {
const normalizedActual = actual.trim();
const normalizedExpected = expected.trim();
return (
normalizedActual === normalizedExpected ||
normalizedActual.startsWith(`${normalizedExpected} `)
actual.trim() === expected.trim() ||
actual.trim().startsWith(`${expected.trim()} `)
);
};
+1
View File
@@ -33,6 +33,7 @@ export interface SpawnedOwnedProviderProcess {
export interface ProviderProcessLaunch {
readonly process: ChildProcess;
readonly ownsProcessLifetime: boolean;
readonly ownership?: OwnedProcessGroup;
readonly cleanup?: () => Promise<ProcessCleanupResult>;
}
+26
View File
@@ -0,0 +1,26 @@
import type { ProcessLineageObservation } from "./ProcessOwnership.js";
import { observeOwnedProcessLineage } from "./ProcessOwnership.js";
import type { ProviderProcessLaunch } from "./ProviderProcess.js";
/** Retain the latest truthful process-lineage observation for one client. */
export class ProviderRunLineage {
#observation: ProcessLineageObservation | null = null;
/** Clear stale lineage before a new launch attempt. */
reset(): void {
this.#observation = null;
}
/** Observe a token-owned launcher without signaling it. */
async observe(launch: ProviderProcessLaunch | undefined): Promise<void> {
if (launch?.ownership === undefined) return;
this.#observation = await observeOwnedProcessLineage(launch.ownership);
}
/** Return a detached copy suitable for caller-visible status. */
snapshot(): ProcessLineageObservation | null {
return this.#observation === null
? null
: structuredClone(this.#observation);
}
}