chore(main): merge pinned CI checkpoint 2895a18

Keep a test-owned Linux subreaper until token-verified process-group cleanup completes, then wait for each adopted child. Observe from a separate verifier so the orphan remains outside its lineage. Assert disappearance from procfs after reaping.

Reproduced two failures and eight retained zombies without an init reaper. The corrected fourteen process cases pass with no residual children; typecheck and lint pass.

Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com>

scripts/lib/clean-linux-hopper-runtime.mjs

@@ -1,8 +1,36 @@
-import { execFile } from "node:child_process";
-import { promisify } from "node:util";
-import { readdir, stat } from "node:fs/promises";
+import { readFile, readdir, stat } from "node:fs/promises";
+import { basename } from "node:path";
 
-const execFileAsync = promisify(execFile);
+// CPython's interface option ends interpreter parsing; following values are
+// script arguments. -W/-X consume either an attached value or the next operand.
+// https://docs.python.org/3.12/using/cmdline.html
+const pythonScriptOperand = (arguments_) => {
+  for (let index = 1; index < arguments_.length; index++) {
+    const option = arguments_[index];
+    if (option === "--") return arguments_[index + 1];
+    if (option === "-" || option === "") return undefined;
+    if (!option.startsWith("-")) return option;
+    if (option === "--check-hash-based-pycs") {
+      index++;
+      continue;
+    }
+    if (/^--(?:help(?:-env|-xoptions|-all)?|version)$/u.test(option))
+      return undefined;
+    if (option.startsWith("--"))
+      throw new Error(`Unknown Python helper invocation option: ${option}`);
+    for (let character = 1; character < option.length; character++) {
+      const flag = option[character];
+      if ("cmh?V".includes(flag)) return undefined;
+      if (flag === "W" || flag === "X") {
+        if (character === option.length - 1) index++;
+        break;
+      }
+      if (!"bBdEiIOPqRsSuvx".includes(flag))
+        throw new Error(`Unknown Python helper invocation option: ${option}`);
+    }
+  }
+  return undefined;
+};
 
 /**
  * Observe leftover demo processes on a dedicated clean Linux runner, including
@@ -18,9 +46,6 @@ export const assertCleanLinuxHopperRuntime = async (
   ],
   procRoot = "/proc",
 ) => {
-  const { stdout } = await execFileAsync("ps", ["-ax", "-o", "command="]);
-  if (/\/opt\/hopper\/bin\/Hopper|\bXvfb\b|hopper-demo-x11\.py/u.test(stdout))
-    throw new Error("Clean Linux runner retained a Hopper demo process");
   const launchers = [];
   for (const path of launcherPaths) {
     try {
@@ -29,20 +54,55 @@ export const assertCleanLinuxHopperRuntime = async (
       if (error?.code !== "ENOENT") throw error;
     }
   }
-  if (launchers.length === 0) return;
   // /proc exposes the executed file even when argv[0] names a removed symlink.
   // Compare file identity, not a display path or a process name.
   let inspectionError;
   for (const entry of await readdir(procRoot, { withFileTypes: true })) {
     if (!entry.isDirectory() || !/^\d+$/u.test(entry.name)) continue;
-    let executable;
+    let retainedRuntime;
     try {
-      executable = await stat(`${procRoot}/${entry.name}/exe`, {
-        bigint: true,
-      });
+      // procfs preserves argument boundaries. A caller merely mentioning a
+      // runtime in its script text or other arguments is not that runtime.
+      const commandLine = await readFile(
+        `${procRoot}/${entry.name}/cmdline`,
+        "utf8",
+      );
+      const arguments_ = commandLine.split("\0");
+      // The kernel command field identifies renamed runtime processes such
+      // as Xvfb. Keep it separate from arguments so ordinary script text does
+      // not count as a retained runtime.
+      const kernelCommand = (
+        await readFile(`${procRoot}/${entry.name}/comm`, "utf8")
+      ).replace(/\n$/u, "");
+      if (
+        commandLine.length === 0 &&
+        /^python3(?:\.\d+)?$/u.test(kernelCommand)
+      )
+        throw new Error(
+          "Cannot identify a command-less Python process on the clean Linux runner",
+        );
+      retainedRuntime =
+        kernelCommand === "Hopper" ||
+        kernelCommand === "Xvfb" ||
+        arguments_[0] === "/opt/hopper/bin/Hopper" ||
+        basename(arguments_[0] ?? "") === "Xvfb" ||
+        (/^python3(?:\.\d+)?$/u.test(basename(arguments_[0] ?? "")) &&
+          arguments_.some(
+            (value) => basename(value) === "hopper-demo-x11.py",
+          ) &&
+          basename(pythonScriptOperand(arguments_) ?? "") ===
+            "hopper-demo-x11.py");
+      if (!retainedRuntime && launchers.length > 0) {
+        const executable = await stat(`${procRoot}/${entry.name}/exe`, {
+          bigint: true,
+        });
+        retainedRuntime = launchers.some(
+          ({ dev, ino }) => dev === executable.dev && ino === executable.ino,
+        );
+      }
     } catch (error) {
       if (error?.code === "ENOENT" || error?.code === "ESRCH") continue;
-      // Other users' executables may be unreadable. The setup lane launches
+      // Other users' process facts may be unreadable. The setup lane launches
       // providers as the caller; unreadable caller-owned processes fail closed.
       if (error?.code === "EACCES") {
         const owner = await stat(`${procRoot}/${entry.name}`).catch((cause) => {
@@ -56,11 +116,7 @@ export const assertCleanLinuxHopperRuntime = async (
       inspectionError ??= error;
       continue;
     }
-    if (
-      launchers.some(
-        ({ dev, ino }) => dev === executable.dev && ino === executable.ino,
-      )
-    )
+    if (retainedRuntime)
       throw new Error("Clean Linux runner retained a Hopper demo process");
   }
   if (inspectionError !== undefined) throw inspectionError;

scripts/lib/real-hopper-search.d.mts

@@ -1,3 +1,7 @@
+import type { z } from "zod";
+
+import type { analysisStringSchema } from "../../src/domain/hopperValues.js";
+
 type VerifierCall = (
   name: string,
   arguments_?: Record<string, unknown>,
@@ -25,6 +29,3 @@ export function verifyHopperStringObjects(
   }[];
   readonly displayEscapesExcludedFromMatching: true;
 }>;
-import type { z } from "zod";
-
-import type { analysisStringSchema } from "../../src/domain/hopperValues.js";

tests/boundary/process/cleanLinuxHopperRuntime.test.ts

@@ -0,0 +1,411 @@
+import { execFile } from "node:child_process";
+import { randomUUID } from "node:crypto";
+import {
+  chmod,
+  copyFile,
+  mkdir,
+  readFile,
+  symlink,
+  unlink,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+import { promisify } from "node:util";
+
+import { expect, it } from "vitest";
+
+import { assertCleanLinuxHopperRuntime } from "../../../scripts/lib/clean-linux-hopper-runtime.mjs";
+import { cleanupOwnedProcessGroup } from "../../../src/process/ProcessOwnership.js";
+import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
+
+const execFileAsync = promisify(execFile);
+
+it.runIf(process.platform === "linux").each([
+  ["Xvfb", "Clean Linux runner retained a Hopper demo process"],
+  ["Hopper", "Clean Linux runner retained a Hopper demo process"],
+  ["python3", "Cannot identify a command-less Python process"],
+  ["unrelated", undefined],
+] as const)(
+  "observes an unreaped %s without signaling or reaping it",
+  async (name, refusal) => {
+    const directory = await createTestTempDirectory("rea-runtime-zombie-");
+    const procRoot = join(directory, "proc");
+    await mkdir(procRoot);
+    const executable = join(directory, name);
+    await copyFile("/usr/bin/true", executable);
+    await chmod(executable, 0o700);
+    // WNOWAIT observes exit without reaping. The parent keeps the zombie until
+    // the test closes its stdin, then waitpid performs the actual cleanup.
+    const parent = execFileAsync(
+      "/usr/bin/python3",
+      [
+        "-c",
+        `
+import os, sys
+pid = os.fork()
+if pid == 0:
+    os.execl(sys.argv[1], sys.argv[1])
+try:
+    os.waitid(os.P_PID, pid, os.WEXITED | os.WNOWAIT)
+    print(pid, flush=True)
+    sys.stdin.buffer.read(1)
+finally:
+    os.waitpid(pid, 0)
+`,
+        executable,
+      ],
+      { timeout: 10_000 },
+    );
+    const ready = new Promise<number>((resolve, reject) => {
+      parent.child.stdout?.once("data", (data: Buffer) =>
+        resolve(Number(data.toString().trim())),
+      );
+      parent.catch(reject);
+    });
+    let zombiePid: number | undefined;
+    try {
+      zombiePid = await ready;
+      expect(Number.isSafeInteger(zombiePid)).toBe(true);
+      const prefix = `/proc/${String(zombiePid)}`;
+      const procEntry = join(procRoot, String(zombiePid));
+      await mkdir(procEntry);
+      for (const name of ["cmdline", "comm"])
+        await symlink(`${prefix}/${name}`, join(procEntry, name));
+      expect(await readFile(`${prefix}/cmdline`, "utf8")).toBe("");
+      expect((await readFile(`${prefix}/comm`, "utf8")).trim()).toBe(name);
+      const observation = assertCleanLinuxHopperRuntime([], procRoot);
+      if (refusal === undefined)
+        await expect(observation).resolves.toBeUndefined();
+      else await expect(observation).rejects.toThrow(refusal);
+      expect(await readFile(`${prefix}/stat`, "utf8")).toContain(
+        `(${name}) Z `,
+      );
+    } finally {
+      parent.child.stdin?.end("reap");
+      await parent;
+      if (zombiePid !== undefined)
+        await expect(
+          readFile(`/proc/${String(zombiePid)}/stat`, "utf8"),
+        ).rejects.toMatchObject({ code: "ENOENT" });
+    }
+  },
+);
+
+it.runIf(process.platform === "linux")(
+  "does not treat runtime names in another process's arguments as a runtime",
+  async () => {
+    const directory = await createTestTempDirectory("rea-runtime-arguments-");
+    const procRoot = join(directory, "proc");
+    await mkdir(procRoot);
+    const observer = new URL(
+      "../../../scripts/lib/clean-linux-hopper-runtime.mjs",
+      import.meta.url,
+    ).href;
+    const code = `
+      import { mkdir, symlink } from "node:fs/promises";
+      import { join } from "node:path";
+      const { assertCleanLinuxHopperRuntime } = await import(${JSON.stringify(observer)});
+      const entry = join(${JSON.stringify(procRoot)}, String(process.pid));
+      await mkdir(entry);
+      await symlink("/proc/" + process.pid + "/cmdline", join(entry, "cmdline"));
+      await symlink("/proc/" + process.pid + "/comm", join(entry, "comm"));
+      await assertCleanLinuxHopperRuntime([], ${JSON.stringify(procRoot)});
+      console.log('clean');
+    `;
+    const { stdout } = await execFileAsync(
+      process.execPath,
+      [
+        "--input-type=module",
+        "-e",
+        code,
+        procRoot,
+        "Xvfb",
+        "/opt/hopper/bin/Hopper",
+        "hopper-demo-x11.py",
+      ],
+      { timeout: 10_000 },
+    );
+    expect(stdout.trim()).toBe("clean");
+  },
+);
+
+it.runIf(process.platform === "linux")(
+  "distinguishes an ordinary Node script with the helper's filename",
+  async () => {
+    const directory = await createTestTempDirectory("rea-runtime-arguments-");
+    const procRoot = join(directory, "proc");
+    await mkdir(procRoot);
+    const script = join(directory, "hopper-demo-x11.py");
+    const observer = new URL(
+      "../../../scripts/lib/clean-linux-hopper-runtime.mjs",
+      import.meta.url,
+    ).href;
+    const code = `
+      import { mkdir, symlink } from "node:fs/promises";
+      import { join } from "node:path";
+      const entry = join(${JSON.stringify(procRoot)}, String(process.pid));
+      await mkdir(entry);
+      await symlink("/proc/" + process.pid + "/cmdline", join(entry, "cmdline"));
+      await symlink("/proc/" + process.pid + "/comm", join(entry, "comm"));
+      const { assertCleanLinuxHopperRuntime } = await import(${JSON.stringify(observer)});
+      await assertCleanLinuxHopperRuntime([], ${JSON.stringify(procRoot)});
+      console.log('clean');
+    `;
+    const { stdout } = await execFileAsync(
+      process.execPath,
+      ["--input-type=module", "--eval", code, script],
+      { timeout: 10_000 },
+    );
+    expect(stdout.trim()).toBe("clean");
+  },
+);
+
+const launchReapedOrphan = (runId: string, parentCode: string) => {
+  // The short-lived Node launcher still exits before observation. A retained
+  // subreaper adopts its child and waits after owned cleanup, even when the
+  // container's PID 1 does not reap. It is a sibling of the verifier below.
+  const reaper = execFileAsync(
+    "/usr/bin/python3",
+    [
+      "-c",
+      `
+import ctypes, json, os, subprocess, sys
+libc = ctypes.CDLL(None, use_errno=True)
+if libc.prctl(36, 1, 0, 0, 0) != 0: # PR_SET_CHILD_SUBREAPER
+    raise OSError(ctypes.get_errno(), "PR_SET_CHILD_SUBREAPER")
+launcher = subprocess.run(sys.argv[1:], capture_output=True, text=True, check=True)
+child = json.loads(launcher.stdout)
+try:
+    print(json.dumps(child), flush=True)
+    sys.stdin.buffer.read(1)
+finally:
+    os.waitpid(child["pid"], 0)
+`,
+      process.execPath,
+      "-e",
+      parentCode,
+    ],
+    {
+      env: { ...process.env, REA_PROCESS_RUN_ID: runId },
+      timeout: 40_000,
+    },
+  );
+  const ready = new Promise<{ pid: number; parentPid: number }>(
+    (resolve, reject) => {
+      reaper.child.stdout?.once("data", (data: Buffer) => {
+        try {
+          resolve(JSON.parse(data.toString()));
+        } catch (error) {
+          reject(error);
+        }
+      });
+      reaper.then(
+        () => reject(new Error("Fixture reaper exited before readiness")),
+        reject,
+      );
+    },
+  );
+  return { reaper, ready };
+};
+
+const observeFromSeparateVerifier = async (
+  runId: string,
+  launcherPaths: string[],
+  procRoot: string,
+) => {
+  const observer = new URL(
+    "../../../scripts/lib/clean-linux-hopper-runtime.mjs",
+    import.meta.url,
+  ).href;
+  const verifierRun = new URL(
+    "../../../scripts/lib/verifier-run.mjs",
+    import.meta.url,
+  ).href;
+  const { stdout: observed } = await execFileAsync(
+    process.execPath,
+    [
+      "--input-type=module",
+      "-e",
+      `
+const { assertCleanLinuxHopperRuntime } = await import(${JSON.stringify(observer)});
+const { completeVerifierRun } = await import(${JSON.stringify(verifierRun)});
+const { process_lineage } = await completeVerifierRun({
+  run_id: ${JSON.stringify(runId)}, verifier_pid: process.pid, parent_pid: process.ppid,
+});
+let refusal;
+try { await assertCleanLinuxHopperRuntime(${JSON.stringify(launcherPaths)}, ${JSON.stringify(procRoot)}); }
+catch (error) { refusal = error.message; }
+console.log(JSON.stringify({ process_lineage, refusal }));
+`,
+    ],
+    { timeout: 10_000 },
+  );
+  return JSON.parse(observed) as {
+    process_lineage: {
+      status: string;
+      descendants: { pid: number }[];
+    };
+    refusal?: string;
+  };
+};
+
+it.runIf(process.platform === "linux").each([
+  ["Xvfb-named Node", null, true],
+  ["Python helper", [], true],
+  ["Python helper with -u", ["-u"], true],
+  [
+    "Python helper with combined/value options",
+    ["-Bu", "-W", "ignore", "-X", "dev", "--check-hash-based-pycs", "always"],
+    true,
+  ],
+  ["Python helper after --", ["--"], true],
+  [
+    "Python -c script argument",
+    ["-c", "import time; print('ready', flush=True); time.sleep(30)"],
+    false,
+  ],
+  ["Python -m script argument", ["-m", "unrelated"], false],
+  ["alternate launcher", null, true],
+] as const)(
+  "classifies a reparented %s without signaling it",
+  async (mode, pythonOptions, retained) => {
+    const runId = randomUUID();
+    const childCode =
+      "process.title = 'Xvfb'; setInterval(() => {}, 1000); process.send('ready');";
+    const directory = await createTestTempDirectory("rea-clean-runtime-test-");
+    const procRoot = join(directory, "proc");
+    await mkdir(procRoot);
+    const launcher = join(directory, "Hopper");
+    const alternate = join(directory, "Hopper-alternate");
+    const helper = join(directory, "hopper-demo-x11.py");
+    const alternateMode = mode === "alternate launcher";
+    const pythonMode = pythonOptions !== null;
+    if (alternateMode) {
+      await copyFile("/usr/bin/sleep", launcher);
+      await chmod(launcher, 0o700);
+      await symlink(launcher, alternate);
+    }
+    if (pythonMode) {
+      await writeFile(
+        helper,
+        "import time\nprint('ready', flush=True)\ntime.sleep(30)\n",
+      );
+      await copyFile(helper, join(directory, "unrelated.py"));
+    }
+    // The fixture must be the cause of a refusal; a retained host runtime
+    // would otherwise mask a broken classification or executable-identity check.
+    await assertCleanLinuxHopperRuntime(
+      alternateMode ? [launcher] : [],
+      procRoot,
+    );
+    const command = alternateMode
+      ? alternate
+      : pythonMode
+        ? "/usr/bin/python3"
+        : process.execPath;
+    const arguments_ = alternateMode
+      ? ["30"]
+      : pythonMode
+        ? [...pythonOptions, helper]
+        : ["-e", childCode];
+    const stdio = alternateMode
+      ? "ignore"
+      : pythonMode
+        ? ["ignore", "pipe", "ignore"]
+        : ["ignore", "ignore", "ignore", "ipc"];
+    const parentCode = `
+      const { spawn } = require('node:child_process');
+      const child = spawn(${JSON.stringify(command)}, ${JSON.stringify(arguments_)}, {
+        detached: true, stdio: ${JSON.stringify(stdio)}, cwd: ${JSON.stringify(directory)}
+      });
+      child.once('error', () => process.exit(1));
+      const ready = () => {
+        process.stdout.write(JSON.stringify({ pid: child.pid, parentPid: process.pid }));
+        if (child.connected) child.disconnect();
+        child.stdout?.destroy();
+        child.unref();
+      };
+      ${pythonMode ? "child.stdout.once('data', ready);" : `child.once(${JSON.stringify(alternateMode ? "spawn" : "message")}, ready);`}
+    `;
+    const { reaper, ready } = launchReapedOrphan(runId, parentCode);
+    let orphanPid: number | undefined;
+    try {
+      const { pid, parentPid } = await ready;
+      orphanPid = pid;
+      const procEntry = join(procRoot, String(pid));
+      await mkdir(procEntry);
+      for (const name of ["cmdline", "comm", "exe"])
+        await symlink(`/proc/${String(pid)}/${name}`, join(procEntry, name));
+      const { stdout: coordinates } = await execFileAsync("ps", [
+        "-p",
+        String(pid),
+        "-o",
+        "ppid=,pgid=",
+      ]);
+      const [currentParent, group] = coordinates
+        .trim()
+        .split(/\s+/u)
+        .map(Number);
+      expect(currentParent).not.toBe(parentPid);
+      expect(currentParent).toBe(reaper.child.pid);
+      expect(group).toBe(pid);
+
+      if (alternateMode) {
+        const commandLine = await readFile(
+          `/proc/${String(pid)}/cmdline`,
+          "utf8",
+        );
+        expect(commandLine.split("\0")[0]).toBe(alternate);
+        await unlink(alternate);
+      }
+      if (pythonMode) {
+        const commandLine = await readFile(
+          `/proc/${String(pid)}/cmdline`,
+          "utf8",
+        );
+        expect(
+          commandLine.split("\0").slice(0, pythonOptions.length + 2),
+        ).toEqual([command, ...pythonOptions, helper]);
+      }
+      const { process_lineage: lineage, refusal } =
+        await observeFromSeparateVerifier(
+          runId,
+          alternateMode ? [launcher] : [],
+          procRoot,
+        );
+      expect(lineage.status).toBe("verified");
+      expect(lineage.descendants.some((p) => p.pid === pid)).toBe(false);
+      if (retained)
+        expect(refusal).toBe(
+          "Clean Linux runner retained a Hopper demo process",
+        );
+      else expect(refusal).toBeUndefined();
+      // Observation must not terminate the fixture, even though it resembles Xvfb.
+      const { stdout: alive } = await execFileAsync("ps", [
+        "-p",
+        String(pid),
+        "-o",
+        "stat=",
+      ]);
+      expect(alive.trim()).not.toMatch(/^Z/u);
+    } finally {
+      try {
+        if (orphanPid !== undefined) {
+          const cleanup = await cleanupOwnedProcessGroup({
+            runId,
+            leaderPid: orphanPid,
+            processGroupId: orphanPid,
+          });
+          expect(cleanup.cleaned, JSON.stringify(cleanup)).toBe(true);
+        }
+      } finally {
+        reaper.child.stdin?.end("reap");
+        await reaper;
+        if (orphanPid !== undefined)
+          await expect(
+            readFile(`/proc/${String(orphanPid)}/stat`, "utf8"),
+          ).rejects.toMatchObject({ code: "ENOENT" });
+      }
+    }
+  },
+);

tests/process-global/cleanLinuxHopperRuntime.test.ts

@@ -74,7 +74,11 @@ it
       if (alternateMode) {
         const processEntry = join(procRoot, String(pid));
         await mkdir(processEntry);
-        await symlink(`/proc/${String(pid)}/exe`, join(processEntry, "exe"));
+        for (const name of ["cmdline", "comm", "exe"])
+          await symlink(
+            `/proc/${String(pid)}/${name}`,
+            join(processEntry, name),
+          );
       }
       const { stdout: coordinates } = await execFileAsync("ps", [
         "-p",