HEL-543: add data hub observability side-channel (provider status, source catalog, lineage)
- New provider_call_log/provider_health tables (additive-only schema), wired via a fail-open observability.observe()/record_call() helper. - Tushare pipeline keeps its existing src_calls record unchanged and now also feeds the unified provider_health/provider_call_log side channel. - Eastmoney/Tencent realtime_serve.py call sites and the iFinD steward call site are wrapped with observability.observe() at the call site only; no adapter internals, routing, fallback order, or return values are touched. - New read-only admin API endpoints: /admin/api/providers/status, /admin/api/source-catalog, /admin/api/lineage, /admin/api/lineage/affected. - New static, read-only source_catalog.py and lineage.py registries documenting existing providers/interfaces/datasets and known main-site consumers (cited against backend/features/screener and backend/features/heaven call sites). - provider_call_log is purged by the existing pipeline.cleanup() job alongside src_calls/job_runs. - 47 new unit/integration tests covering classification, fail-open behavior under DB/log failures, unchanged payloads/exceptions on success and failure paths, and the new HTTP endpoints. Full suite: 173 tests, all green. Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
co-authored by
Cursor
multica-agent
parent
4a90c32fcc
commit
f014eb11bd
@@ -122,6 +122,26 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
if path == "/admin/api/sources" and method == "GET":
|
||||
self._json(self.hub.admin.sources(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/providers/status" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
provider = (query.get("provider") or [""])[0]
|
||||
limit = (query.get("limit") or ["50"])[0]
|
||||
self._json(self.hub.admin.providers_status(provider, int(limit or 50)), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/source-catalog" and method == "GET":
|
||||
self._json(self.hub.admin.source_catalog(), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/lineage" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
date = (query.get("date") or [""])[0]
|
||||
self._json(self.hub.admin.lineage(date), HTTPStatus.OK)
|
||||
return
|
||||
if path == "/admin/api/lineage/affected" and method == "GET":
|
||||
query = parse_query(urlparse(self.path).query)
|
||||
provider = (query.get("provider") or [""])[0]
|
||||
interface = (query.get("interface") or [""])[0]
|
||||
self._json(self.hub.admin.lineage_affected(provider, interface), HTTPStatus.OK)
|
||||
return
|
||||
if path.startswith("/admin/api/sources/") and path.endswith("/probe") and method == "POST":
|
||||
provider = path.split("/")[4]
|
||||
self._json(self.hub.admin.probe(provider), HTTPStatus.OK)
|
||||
|
||||
Reference in New Issue
Block a user