refactor: govern background job execution

backend/jobs/runner.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from typing import Any
+
+from backend.jobs.registry import JobRegistry
+from backend.jobs.repository import SQLiteJobRunRepository
+
+
+JobAction = Callable[[], Any]
+
+
+class InProcessJobRunner:
+    def __init__(self, registry: JobRegistry, repository: SQLiteJobRunRepository) -> None:
+        self.registry = registry
+        self.repository = repository
+        self._locks: dict[str, threading.Lock] = {}
+        self._locks_guard = threading.Lock()
+
+    def submit(
+        self, job_id: str, idempotency_key: str, action: JobAction,
+        metadata: dict[str, Any] | None = None,
+    ) -> bool:
+        definition = self.registry.get(job_id)
+        if self.repository.completed(job_id, idempotency_key):
+            return False
+        lock = self._lock(definition.lock_key)
+        if not lock.acquire(blocking=False):
+            return False
+        thread = threading.Thread(
+            target=self._execute_locked,
+            args=(job_id, idempotency_key, action, metadata, lock),
+            name=f"job-{job_id}-{idempotency_key}"[:80],
+            daemon=True,
+        )
+        thread.start()
+        return True
+
+    def run_inline(
+        self, job_id: str, idempotency_key: str, action: JobAction,
+        metadata: dict[str, Any] | None = None,
+    ) -> bool:
+        definition = self.registry.get(job_id)
+        if self.repository.completed(job_id, idempotency_key):
+            return False
+        lock = self._lock(definition.lock_key)
+        if not lock.acquire(blocking=False):
+            return False
+        self._execute_locked(job_id, idempotency_key, action, metadata, lock)
+        return True
+
+    def start_scheduler(
+        self, callback: Callable[[], None], stop_event: threading.Event,
+        interval_seconds: float, initial_delay_seconds: float = 0,
+    ) -> threading.Thread:
+        def schedule_loop() -> None:
+            if stop_event.wait(initial_delay_seconds):
+                return
+            while not stop_event.is_set():
+                try:
+                    callback()
+                except Exception:
+                    # Submitted jobs persist their own failures; the scheduler must stay alive.
+                    pass
+                stop_event.wait(interval_seconds)
+
+        thread = threading.Thread(
+            target=schedule_loop,
+            name="background-job-scheduler",
+            daemon=True,
+        )
+        thread.start()
+        return thread
+
+    def wait_for_idle(self, timeout_seconds: float = 5) -> bool:
+        deadline = time.monotonic() + max(0, timeout_seconds)
+        while time.monotonic() <= deadline:
+            with self._locks_guard:
+                busy = any(lock.locked() for lock in self._locks.values())
+            if not busy:
+                return True
+            time.sleep(0.01)
+        return False
+
+    def _execute_locked(
+        self, job_id: str, idempotency_key: str, action: JobAction,
+        metadata: dict[str, Any] | None, lock: threading.Lock,
+    ) -> None:
+        definition = self.registry.get(job_id)
+        try:
+            for attempt in range(1, definition.max_attempts + 1):
+                run_id = self.repository.start(
+                    job_id, idempotency_key, definition.output_version, metadata
+                )
+                started = time.perf_counter()
+                try:
+                    result = action()
+                    if isinstance(result, dict) and result.get("status") == "failed":
+                        raise RuntimeError(str(result.get("error") or "Job reported failure"))
+                    elapsed_ms = round((time.perf_counter() - started) * 1000)
+                    self.repository.finish(run_id, "success", elapsed_ms)
+                    return
+                except Exception as exc:
+                    elapsed_ms = round((time.perf_counter() - started) * 1000)
+                    self.repository.finish(
+                        run_id, "failed", elapsed_ms,
+                        type(exc).__name__, str(exc),
+                    )
+                    if attempt >= definition.max_attempts:
+                        return
+        finally:
+            lock.release()
+
+    def _lock(self, lock_key: str) -> threading.Lock:
+        with self._locks_guard:
+            return self._locks.setdefault(lock_key, threading.Lock())