refactor: govern background job execution

backend/jobs/__init__.py

@@ -0,0 +1,5 @@
+from .registry import JobDefinition, JobRegistry
+from .repository import SQLiteJobRunRepository
+from .runner import InProcessJobRunner
+
+__all__ = ["InProcessJobRunner", "JobDefinition", "JobRegistry", "SQLiteJobRunRepository"]

backend/jobs/registry.py

@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from app_config import APP_DIR
+
+
+@dataclass(frozen=True)
+class JobDefinition:
+    job_id: str
+    schedule: str
+    input_date_policy: str
+    dependencies: tuple[str, ...]
+    lock_key: str
+    timeout_seconds: int
+    max_attempts: int
+    output_version: str
+
+
+class JobRegistry:
+    def __init__(self, definitions: tuple[JobDefinition, ...]) -> None:
+        self.definitions = definitions
+        self._by_id = {item.job_id: item for item in definitions}
+        if len(self._by_id) != len(definitions):
+            raise ValueError("Background job IDs must be unique")
+
+    @classmethod
+    def load(cls, path: Path | None = None) -> "JobRegistry":
+        config_path = path or APP_DIR / "config" / "jobs.config.json"
+        payload = json.loads(config_path.read_text(encoding="utf-8"))
+        definitions = tuple(
+            JobDefinition(
+                job_id=str(item["id"]),
+                schedule=str(item["schedule"]),
+                input_date_policy=str(item["input_date_policy"]),
+                dependencies=tuple(str(value) for value in item.get("dependencies") or []),
+                lock_key=str(item["lock_key"]),
+                timeout_seconds=max(1, int(item["timeout_seconds"])),
+                max_attempts=max(1, int(item["max_attempts"])),
+                output_version=str(item["output_version"]),
+            )
+            for item in payload.get("jobs") or []
+        )
+        if not definitions:
+            raise ValueError("Background job registry is empty")
+        return cls(definitions)
+
+    def get(self, job_id: str) -> JobDefinition:
+        try:
+            return self._by_id[job_id]
+        except KeyError as exc:
+            raise ValueError(f"Background job is not registered: {job_id}") from exc

backend/jobs/repository.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from database import ReviewDatabase
+
+
+@dataclass(frozen=True)
+class SQLiteJobRunRepository:
+    database: ReviewDatabase
+
+    def completed(self, job_id: str, idempotency_key: str) -> bool:
+        with self.database.connect() as connection:
+            row = connection.execute(
+                """
+                SELECT 1 FROM job_runs
+                WHERE job_id = ? AND idempotency_key = ? AND status = 'success'
+                LIMIT 1
+                """,
+                (job_id, idempotency_key),
+            ).fetchone()
+        return row is not None
+
+    def start(
+        self, job_id: str, idempotency_key: str, output_version: str,
+        metadata: dict[str, Any] | None = None,
+    ) -> int:
+        now = datetime.now().astimezone().isoformat(timespec="seconds")
+        with self.database.connect() as connection:
+            row = connection.execute(
+                """
+                SELECT COALESCE(MAX(attempt), 0) + 1 AS attempt FROM job_runs
+                WHERE job_id = ? AND idempotency_key = ?
+                """,
+                (job_id, idempotency_key),
+            ).fetchone()
+            cursor = connection.execute(
+                """
+                INSERT INTO job_runs
+                    (job_id, idempotency_key, status, attempt, started_at,
+                     output_version, metadata)
+                VALUES (?, ?, 'running', ?, ?, ?, ?)
+                """,
+                (
+                    job_id, idempotency_key, int(row["attempt"]), now,
+                    output_version,
+                    json.dumps(metadata or {}, ensure_ascii=False, separators=(",", ":")),
+                ),
+            )
+            connection.execute(
+                """
+                DELETE FROM job_runs
+                WHERE id < (SELECT COALESCE(MAX(id), 0) - 20000 FROM job_runs)
+                  AND status != 'running'
+                """
+            )
+            return int(cursor.lastrowid)
+
+    def finish(
+        self, run_id: int, status: str, elapsed_ms: int,
+        error_code: str = "", message: str = "",
+    ) -> None:
+        now = datetime.now().astimezone().isoformat(timespec="seconds")
+        with self.database.connect() as connection:
+            connection.execute(
+                """
+                UPDATE job_runs
+                SET status = ?, finished_at = ?, elapsed_ms = ?,
+                    error_code = ?, message = ?
+                WHERE id = ?
+                """,
+                (status, now, elapsed_ms, error_code, message[:1000], int(run_id)),
+            )
+
+    def recent(self, limit: int = 20) -> list[dict[str, Any]]:
+        with self.database.connect() as connection:
+            rows = connection.execute(
+                """
+                SELECT id, job_id, idempotency_key, status, attempt, started_at,
+                       finished_at, elapsed_ms, error_code, message, output_version
+                FROM job_runs ORDER BY id DESC LIMIT ?
+                """,
+                (max(1, min(100, int(limit))),),
+            ).fetchall()
+        return [dict(row) for row in rows]

backend/jobs/runner.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from typing import Any
+
+from backend.jobs.registry import JobRegistry
+from backend.jobs.repository import SQLiteJobRunRepository
+
+
+JobAction = Callable[[], Any]
+
+
+class InProcessJobRunner:
+    def __init__(self, registry: JobRegistry, repository: SQLiteJobRunRepository) -> None:
+        self.registry = registry
+        self.repository = repository
+        self._locks: dict[str, threading.Lock] = {}
+        self._locks_guard = threading.Lock()
+
+    def submit(
+        self, job_id: str, idempotency_key: str, action: JobAction,
+        metadata: dict[str, Any] | None = None,
+    ) -> bool:
+        definition = self.registry.get(job_id)
+        if self.repository.completed(job_id, idempotency_key):
+            return False
+        lock = self._lock(definition.lock_key)
+        if not lock.acquire(blocking=False):
+            return False
+        thread = threading.Thread(
+            target=self._execute_locked,
+            args=(job_id, idempotency_key, action, metadata, lock),
+            name=f"job-{job_id}-{idempotency_key}"[:80],
+            daemon=True,
+        )
+        thread.start()
+        return True
+
+    def run_inline(
+        self, job_id: str, idempotency_key: str, action: JobAction,
+        metadata: dict[str, Any] | None = None,
+    ) -> bool:
+        definition = self.registry.get(job_id)
+        if self.repository.completed(job_id, idempotency_key):
+            return False
+        lock = self._lock(definition.lock_key)
+        if not lock.acquire(blocking=False):
+            return False
+        self._execute_locked(job_id, idempotency_key, action, metadata, lock)
+        return True
+
+    def start_scheduler(
+        self, callback: Callable[[], None], stop_event: threading.Event,
+        interval_seconds: float, initial_delay_seconds: float = 0,
+    ) -> threading.Thread:
+        def schedule_loop() -> None:
+            if stop_event.wait(initial_delay_seconds):
+                return
+            while not stop_event.is_set():
+                try:
+                    callback()
+                except Exception:
+                    # Submitted jobs persist their own failures; the scheduler must stay alive.
+                    pass
+                stop_event.wait(interval_seconds)
+
+        thread = threading.Thread(
+            target=schedule_loop,
+            name="background-job-scheduler",
+            daemon=True,
+        )
+        thread.start()
+        return thread
+
+    def wait_for_idle(self, timeout_seconds: float = 5) -> bool:
+        deadline = time.monotonic() + max(0, timeout_seconds)
+        while time.monotonic() <= deadline:
+            with self._locks_guard:
+                busy = any(lock.locked() for lock in self._locks.values())
+            if not busy:
+                return True
+            time.sleep(0.01)
+        return False
+
+    def _execute_locked(
+        self, job_id: str, idempotency_key: str, action: JobAction,
+        metadata: dict[str, Any] | None, lock: threading.Lock,
+    ) -> None:
+        definition = self.registry.get(job_id)
+        try:
+            for attempt in range(1, definition.max_attempts + 1):
+                run_id = self.repository.start(
+                    job_id, idempotency_key, definition.output_version, metadata
+                )
+                started = time.perf_counter()
+                try:
+                    result = action()
+                    if isinstance(result, dict) and result.get("status") == "failed":
+                        raise RuntimeError(str(result.get("error") or "Job reported failure"))
+                    elapsed_ms = round((time.perf_counter() - started) * 1000)
+                    self.repository.finish(run_id, "success", elapsed_ms)
+                    return
+                except Exception as exc:
+                    elapsed_ms = round((time.perf_counter() - started) * 1000)
+                    self.repository.finish(
+                        run_id, "failed", elapsed_ms,
+                        type(exc).__name__, str(exc),
+                    )
+                    if attempt >= definition.max_attempts:
+                        return
+        finally:
+            lock.release()
+
+    def _lock(self, lock_key: str) -> threading.Lock:
+        with self._locks_guard:
+            return self._locks.setdefault(lock_key, threading.Lock())