HEL-543: add data hub observability side-channel (provider status, source catalog, lineage)

- New provider_call_log/provider_health tables (additive-only schema),
  wired via a fail-open observability.observe()/record_call() helper.
- Tushare pipeline keeps its existing src_calls record unchanged and now
  also feeds the unified provider_health/provider_call_log side channel.
- Eastmoney/Tencent realtime_serve.py call sites and the iFinD steward
  call site are wrapped with observability.observe() at the call site
  only; no adapter internals, routing, fallback order, or return values
  are touched.
- New read-only admin API endpoints: /admin/api/providers/status,
  /admin/api/source-catalog, /admin/api/lineage,
  /admin/api/lineage/affected.
- New static, read-only source_catalog.py and lineage.py registries
  documenting existing providers/interfaces/datasets and known
  main-site consumers (cited against backend/features/screener and
  backend/features/heaven call sites).
- provider_call_log is purged by the existing pipeline.cleanup() job
  alongside src_calls/job_runs.
- 47 new unit/integration tests covering classification, fail-open
  behavior under DB/log failures, unchanged payloads/exceptions on
  success and failure paths, and the new HTTP endpoints. Full suite:
  173 tests, all green.

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>

xiaobai-datahub/datahub/admin_api.py

@@ -6,9 +6,11 @@ from typing import Any
 from datahub.adapters import RESERVED
 from datahub.auth import AuthService
 from datahub.db import HubDB
+from datahub import lineage as lineage_module
 from datahub.pipeline import OFFICIAL_DATASETS, STOCKS_DATASET, Pipeline
 from datahub.scheduler import Scheduler
 from datahub.serving import ApiError
+from datahub import source_catalog
 from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd
 
 
@@ -105,6 +107,40 @@ class AdminAPI:
             raise ApiError("INVALID_ARGUMENT", f"unknown provider: {provider}")
         return adapter.probe()
 
+    # ------------------------------------------------------------------
+    # HEL-543: read-only side-channel status/catalog/lineage. These never
+    # change routing, credentials, or adapters; they only read the
+    # provider_call_log/provider_health tables (observability.py) plus the
+    # static registries in source_catalog.py / lineage.py.
+    # ------------------------------------------------------------------
+    def providers_status(self, provider: str = "", limit: int = 50) -> dict[str, Any]:
+        limit = max(1, min(int(limit or 50), 200))
+        health_sql = "SELECT * FROM provider_health"
+        params: tuple[Any, ...] = ()
+        if provider:
+            health_sql += " WHERE provider = ?"
+            params = (provider,)
+        health_sql += " ORDER BY provider, interface"
+        health = self.db.fetchall(health_sql, params)
+        calls_sql = "SELECT * FROM provider_call_log"
+        if provider:
+            calls_sql += " WHERE provider = ?"
+        calls_sql += " ORDER BY id DESC LIMIT ?"
+        recent = self.db.fetchall(calls_sql, (*params, limit))
+        return {"health": health, "recent_calls": recent}
+
+    def source_catalog(self) -> dict[str, Any]:
+        return {"items": source_catalog.snapshot(self.db, self.auth)}
+
+    def lineage(self, trade_date: str = "") -> dict[str, Any]:
+        day = yyyymmdd(trade_date) if trade_date else yyyymmdd(now_shanghai())
+        return {"trade_date": day, "items": lineage_module.snapshot(self.db, day)}
+
+    def lineage_affected(self, provider: str = "", interface: str = "") -> dict[str, Any]:
+        if not provider:
+            raise ApiError("INVALID_ARGUMENT", "provider is required")
+        return {"provider": provider, "interface": interface, "items": lineage_module.affected(self.db, provider, interface)}
+
     def jobs(self) -> dict[str, Any]:
         runs = self.db.fetchall("SELECT * FROM job_runs ORDER BY id DESC LIMIT 100")
         stocks_times = "/".join(self.pipeline.settings.stocks_refresh_times) or "20:00"