BAI-3: harden live readiness diagnostics

Co-authored-by: multica-agent <github@multica.ai>

app/backend/jobs/repository.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Any
 
 from database import ReviewDatabase
@@ -27,9 +27,37 @@ class SQLiteJobRunRepository:
     def start(
         self, job_id: str, idempotency_key: str, output_version: str,
         metadata: dict[str, Any] | None = None,
-    ) -> int:
-        now = datetime.now().astimezone().isoformat(timespec="seconds")
+        stale_after_seconds: int = 0,
+    ) -> int | None:
+        current = datetime.now().astimezone()
+        now = current.isoformat(timespec="seconds")
+        stale_before = (
+            current - timedelta(seconds=max(1, int(stale_after_seconds or 1)))
+        ).isoformat(timespec="seconds")
         with self.database.connect() as connection:
+            # The process-local runner lock cannot protect two server processes.
+            connection.execute("BEGIN IMMEDIATE")
+            connection.execute(
+                """
+                UPDATE job_runs
+                SET status = 'failed', finished_at = ?, error_code = 'StaleRun',
+                    message = 'Previous run exceeded its execution window.'
+                WHERE job_id = ? AND idempotency_key = ? AND status = 'running'
+                  AND started_at < ?
+                """,
+                (now, job_id, idempotency_key, stale_before),
+            )
+            claimed = connection.execute(
+                """
+                SELECT 1 FROM job_runs
+                WHERE job_id = ? AND idempotency_key = ?
+                  AND status IN ('running', 'success')
+                LIMIT 1
+                """,
+                (job_id, idempotency_key),
+            ).fetchone()
+            if claimed:
+                return None
             row = connection.execute(
                 """
                 SELECT COALESCE(MAX(attempt), 0) + 1 AS attempt FROM job_runs

app/backend/jobs/runner.py

@@ -51,8 +51,7 @@ class InProcessJobRunner:
         lock = self._lock(definition.lock_key)
         if not lock.acquire(blocking=False):
             return False
-        self._execute_locked(job_id, idempotency_key, action, metadata, lock)
-        return True
+        return self._execute_locked(job_id, idempotency_key, action, metadata, lock)
 
     def start_scheduler(
         self, callback: Callable[[], None], interval_seconds: float,
@@ -105,13 +104,19 @@ class InProcessJobRunner:
     def _execute_locked(
         self, job_id: str, idempotency_key: str, action: JobAction,
         metadata: dict[str, Any] | None, lock: threading.Lock,
-    ) -> None:
+    ) -> bool:
         definition = self.registry.get(job_id)
         try:
             for attempt in range(1, definition.max_attempts + 1):
                 run_id = self.repository.start(
-                    job_id, idempotency_key, definition.output_version, metadata
+                    job_id,
+                    idempotency_key,
+                    definition.output_version,
+                    metadata,
+                    definition.timeout_seconds,
                 )
+                if run_id is None:
+                    return False
                 started = time.perf_counter()
                 try:
                     result = action()
@@ -119,7 +124,7 @@ class InProcessJobRunner:
                         raise RuntimeError(str(result.get("error") or "Job reported failure"))
                     elapsed_ms = round((time.perf_counter() - started) * 1000)
                     self.repository.finish(run_id, "success", elapsed_ms)
-                    return
+                    return True
                 except Exception as exc:
                     elapsed_ms = round((time.perf_counter() - started) * 1000)
                     self.repository.finish(
@@ -127,7 +132,8 @@ class InProcessJobRunner:
                         type(exc).__name__, str(exc),
                     )
                     if attempt >= definition.max_attempts:
-                        return
+                        return True
+            return True
         finally:
             lock.release()