BAI-3: harden live readiness diagnostics

Co-authored-by: multica-agent <github@multica.ai>

app/backend/application.py

@@ -43,6 +43,7 @@ from backend.features.screener.service import (
 from backend.features.sentiment import SentimentServiceMixin
 from backend.features.sentiment.routes import SentimentRoutesMixin
 from backend.features.system import SystemHttpMixin
+from backend.features.system.health import HealthServiceMixin
 from backend.features.system.routes import SystemRoutesMixin
 from backend.features.system.service import SystemServiceMixin
 from backend.features.themes import ThemeServiceMixin
@@ -77,6 +78,7 @@ LEGACY_SECRET_KEYS = {
 
 
 class DashboardService(
+    HealthServiceMixin,
     SystemServiceMixin,
     AccountApplicationMixin,
     JobServiceMixin,

app/backend/features/system/health.py

@@ -0,0 +1,94 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+
+class HealthServiceMixin:
+    def health_status(self) -> dict[str, Any]:
+        """Return a cheap public readiness summary without provider secrets."""
+        database_ready = False
+        database_status: dict[str, Any] = {}
+        try:
+            with self.database.connect() as connection:
+                database_ready = connection.execute("SELECT 1").fetchone() is not None
+            database_status = self.database.status()
+        except Exception:
+            database_ready = False
+
+        try:
+            recent_jobs = self.jobs.repository.recent(12)
+        except Exception:
+            recent_jobs = []
+
+        last_sync = database_status.get("last_sync") or {}
+        snapshot_dates = int(database_status.get("snapshot_dates") or 0)
+        data_configured = bool(self.configured)
+        last_sync_success = last_sync.get("status") == "success"
+        if data_configured and snapshot_dates and last_sync_success:
+            data_status = "ready"
+        elif snapshot_dates:
+            data_status = "degraded"
+        else:
+            data_status = "unavailable"
+
+        latest_job = recent_jobs[0] if recent_jobs else {}
+        failed_jobs = sum(1 for job in recent_jobs if job.get("status") == "failed")
+        if not database_ready:
+            jobs_status = "unavailable"
+        elif latest_job.get("status") == "running":
+            jobs_status = "running"
+        elif latest_job.get("status") == "failed":
+            jobs_status = "degraded"
+        else:
+            jobs_status = "ready"
+
+        platform = self._platform_llm_profile()
+        primary_ready = self._profile_configured(platform["primary"])
+        fallback_ready = self._profile_configured(platform["fallback"])
+        models_status = "ready" if primary_ready else "unavailable"
+
+        component_states = (
+            "ready",
+            "ready" if database_ready else "unavailable",
+            data_status,
+            jobs_status,
+            models_status,
+        )
+        return {
+            "ok": database_ready,
+            "status": (
+                "ready"
+                if all(state in {"ready", "running"} for state in component_states)
+                else "degraded"
+            ),
+            "account_required": True,
+            "time": datetime.now().astimezone().isoformat(timespec="seconds"),
+            "components": {
+                "process": {"status": "ready"},
+                "database": {
+                    "status": "ready" if database_ready else "unavailable",
+                    "snapshot_dates": snapshot_dates,
+                    "last_snapshot_at": database_status.get("updated_at") or "",
+                },
+                "data_sources": {
+                    "status": data_status,
+                    "configured": data_configured,
+                    "snapshot_available": snapshot_dates > 0,
+                    "latest_trade_date": str(last_sync.get("trade_date") or ""),
+                    "last_success_at": (
+                        str(last_sync.get("finished_at") or "") if last_sync_success else ""
+                    ),
+                },
+                "jobs": {
+                    "status": jobs_status,
+                    "recent_failures": failed_jobs,
+                    "latest_finished_at": str(latest_job.get("finished_at") or ""),
+                },
+                "models": {
+                    "status": models_status,
+                    "primary": "ready" if primary_ready else "unavailable",
+                    "fallback": "ready" if fallback_ready else "disabled",
+                },
+            },
+        }

app/backend/features/system/routes.py

@@ -1,20 +1,12 @@
 from __future__ import annotations
 
-from datetime import datetime
 from http import HTTPStatus
 
 
 class SystemRoutesMixin:
     def _handle_system_public_get(self, parsed) -> bool:
         if parsed.path == "/api/health":
-            self.send_json(
-                {
-                    "ok": True,
-                    "storage": "sqlite",
-                    "account_required": True,
-                    "time": datetime.now().astimezone().isoformat(timespec="seconds"),
-                }
-            )
+            self.send_json(self.application_service.health_status())
             return True
         return False
 

app/backend/jobs/repository.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timedelta
 from typing import Any
 
 from database import ReviewDatabase
@@ -27,9 +27,37 @@ class SQLiteJobRunRepository:
     def start(
         self, job_id: str, idempotency_key: str, output_version: str,
         metadata: dict[str, Any] | None = None,
-    ) -> int:
-        now = datetime.now().astimezone().isoformat(timespec="seconds")
+        stale_after_seconds: int = 0,
+    ) -> int | None:
+        current = datetime.now().astimezone()
+        now = current.isoformat(timespec="seconds")
+        stale_before = (
+            current - timedelta(seconds=max(1, int(stale_after_seconds or 1)))
+        ).isoformat(timespec="seconds")
         with self.database.connect() as connection:
+            # The process-local runner lock cannot protect two server processes.
+            connection.execute("BEGIN IMMEDIATE")
+            connection.execute(
+                """
+                UPDATE job_runs
+                SET status = 'failed', finished_at = ?, error_code = 'StaleRun',
+                    message = 'Previous run exceeded its execution window.'
+                WHERE job_id = ? AND idempotency_key = ? AND status = 'running'
+                  AND started_at < ?
+                """,
+                (now, job_id, idempotency_key, stale_before),
+            )
+            claimed = connection.execute(
+                """
+                SELECT 1 FROM job_runs
+                WHERE job_id = ? AND idempotency_key = ?
+                  AND status IN ('running', 'success')
+                LIMIT 1
+                """,
+                (job_id, idempotency_key),
+            ).fetchone()
+            if claimed:
+                return None
             row = connection.execute(
                 """
                 SELECT COALESCE(MAX(attempt), 0) + 1 AS attempt FROM job_runs

app/backend/jobs/runner.py

@@ -51,8 +51,7 @@ class InProcessJobRunner:
         lock = self._lock(definition.lock_key)
         if not lock.acquire(blocking=False):
             return False
-        self._execute_locked(job_id, idempotency_key, action, metadata, lock)
-        return True
+        return self._execute_locked(job_id, idempotency_key, action, metadata, lock)
 
     def start_scheduler(
         self, callback: Callable[[], None], interval_seconds: float,
@@ -105,13 +104,19 @@ class InProcessJobRunner:
     def _execute_locked(
         self, job_id: str, idempotency_key: str, action: JobAction,
         metadata: dict[str, Any] | None, lock: threading.Lock,
-    ) -> None:
+    ) -> bool:
         definition = self.registry.get(job_id)
         try:
             for attempt in range(1, definition.max_attempts + 1):
                 run_id = self.repository.start(
-                    job_id, idempotency_key, definition.output_version, metadata
+                    job_id,
+                    idempotency_key,
+                    definition.output_version,
+                    metadata,
+                    definition.timeout_seconds,
                 )
+                if run_id is None:
+                    return False
                 started = time.perf_counter()
                 try:
                     result = action()
@@ -119,7 +124,7 @@ class InProcessJobRunner:
                         raise RuntimeError(str(result.get("error") or "Job reported failure"))
                     elapsed_ms = round((time.perf_counter() - started) * 1000)
                     self.repository.finish(run_id, "success", elapsed_ms)
-                    return
+                    return True
                 except Exception as exc:
                     elapsed_ms = round((time.perf_counter() - started) * 1000)
                     self.repository.finish(
@@ -127,7 +132,8 @@ class InProcessJobRunner:
                         type(exc).__name__, str(exc),
                     )
                     if attempt >= definition.max_attempts:
-                        return
+                        return True
+            return True
         finally:
             lock.release()