leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

HEL-543: add data hub observability side-channel (provider status, source catalog, lineage)

Browse Source 
- New provider_call_log/provider_health tables (additive-only schema),
  wired via a fail-open observability.observe()/record_call() helper.
- Tushare pipeline keeps its existing src_calls record unchanged and now
  also feeds the unified provider_health/provider_call_log side channel.
- Eastmoney/Tencent realtime_serve.py call sites and the iFinD steward
  call site are wrapped with observability.observe() at the call site
  only; no adapter internals, routing, fallback order, or return values
  are touched.
- New read-only admin API endpoints: /admin/api/providers/status,
  /admin/api/source-catalog, /admin/api/lineage,
  /admin/api/lineage/affected.
- New static, read-only source_catalog.py and lineage.py registries
  documenting existing providers/interfaces/datasets and known
  main-site consumers (cited against backend/features/screener and
  backend/features/heaven call sites).
- provider_call_log is purged by the existing pipeline.cleanup() job
  alongside src_calls/job_runs.
- 47 new unit/integration tests covering classification, fail-open
  behavior under DB/log failures, unchanged payloads/exceptions on
  success and failure paths, and the new HTTP endpoints. Full suite:
  173 tests, all green.

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
2026-09-14 00:59:03 +08:00
co-authored by Cursor multica-agent
parent 4a90c32fcc
commit f014eb11bd
14 changed files with 1568 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/datahub/realtime_serve.py
+58 -11
View File
@@ -18,6 +18,7 @@ from datahub.adapters.tencent import TencentAdapter
from datahub.codes import resolve_code
from datahub.db import HubDB
from datahub.governance.lkg import LastKnownGood
from datahub import observability
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
QUOTE_TTL = 60
@@ -25,6 +26,23 @@ INDEX_TTL = 60
INTRADAY_TTL = 20
QUOTE_BATCH = 60
# HEL-543 side-channel classifiers. These only *read* an already-successful
# payload to decide what to log; they never change the payload itself and a
# classifier exception always degrades to "ok" (see observability.classify_rows).
def _classify_quote_rows(rows: Any) -> tuple[str, str, int | None]:
return observability.classify_rows(rows, freshness_field="quote_time_epoch")
def _classify_rows_no_freshness(rows: Any) -> tuple[str, str, int | None]:
return observability.classify_rows(rows, freshness_field=None)
def _classify_intraday_payload(data: Any) -> tuple[str, str, int | None]:
points = data.get("points") if isinstance(data, dict) else None
return observability.classify_rows(points or [], freshness_field=None)
class RealtimeApiError(RuntimeError):
def __init__(self, code: str, message: str) -> None:
@@ -44,12 +62,17 @@ def fetch_index_quotes(db: HubDB) -> dict[str, Any]:
cached = _read_cache(db, cache_key)
if cached is not None:
return cached
eastmoney = EastmoneyAdapter()
try:
rows = eastmoney.fetch_indices()
rows = observability.observe(
db, "eastmoney", "indices", lambda: EastmoneyAdapter().fetch_indices(),
classify=_classify_quote_rows,
)
source = "eastmoney:ulist"
except Exception:
rows = TencentAdapter().fetch_indices()
rows = observability.observe(
db, "tencent", "indices", lambda: TencentAdapter().fetch_indices(),
classify=_classify_quote_rows,
)
source = "tencent:qt"
if len(rows) < 3:
raise RealtimeApiError("SOURCE_UNAVAILABLE", "index quotes incomplete")
@@ -77,7 +100,10 @@ def fetch_market_quotes(db: HubDB) -> dict[str, Any]:
rows: list[dict[str, Any]] = []
source = ""
try:
rows = EastmoneyAdapter().fetch_market_quotes()
rows = observability.observe(
db, "eastmoney", "market_quotes", lambda: EastmoneyAdapter().fetch_market_quotes(),
classify=_classify_quote_rows,
)
source = "eastmoney:clist"
except Exception as exc:
errors.append(f"eastmoney:{exc}")
@@ -85,7 +111,10 @@ def fetch_market_quotes(db: HubDB) -> dict[str, Any]:
listed = _listed_ts_codes(db)
if not listed:
raise AdapterError("no local stock master for tencent market snapshot")
rows = _tencent_named_quotes(listed)
rows = observability.observe(
db, "tencent", "market_quotes_fallback", lambda: _tencent_named_quotes(listed),
classify=_classify_quote_rows,
)
if len(rows) < 200:
raise AdapterError(f"Tencent market snapshot too small: {len(rows)}")
source = "tencent:qt"
@@ -130,7 +159,10 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
missing = [code for code in resolved if code not in by_code]
try:
rows = _eastmoney_named_quotes(missing)
rows = observability.observe(
db, "eastmoney", "named_quotes", lambda: _eastmoney_named_quotes(missing),
classify=_classify_quote_rows,
)
by_code.update(_quote_map(rows, missing))
if rows:
sources.append("eastmoney:ulist")
@@ -140,7 +172,10 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
missing = [code for code in resolved if code not in by_code]
if missing:
try:
rows = _tencent_named_quotes(missing)
rows = observability.observe(
db, "tencent", "named_quotes", lambda: _tencent_named_quotes(missing),
classify=_classify_quote_rows,
)
by_code.update(_quote_map(rows, missing))
if rows:
sources.append("tencent:qt")
@@ -203,7 +238,10 @@ def fetch_sector_quote(db: HubDB, code: str, expected_date: str = "") -> dict[st
return cached
errors: list[str] = []
try:
row = EastmoneyAdapter().fetch_shenwan_quote(ts_code)
row = observability.observe(
db, "eastmoney", "sector_quote", lambda: EastmoneyAdapter().fetch_shenwan_quote(ts_code),
classify=_classify_quote_rows,
)
if not _sector_row_matches(row, canonical_name):
raise AdapterError(
f"industry name mismatch: expected {canonical_name}, got {row.get('name') or '--'}"
@@ -270,7 +308,10 @@ def fetch_limit_pool(db: HubDB, trade_date: str = "") -> dict[str, Any]:
if cached is not None:
return cached
try:
rows = EastmoneyAdapter().fetch_limit_pool(day)
rows = observability.observe(
db, "eastmoney", "limit_pool", lambda: EastmoneyAdapter().fetch_limit_pool(day),
classify=_classify_rows_no_freshness,
)
source = "eastmoney:zt_pool"
except Exception as exc:
recovered = _load_quotes_lkg(db, cache_key)
@@ -530,7 +571,10 @@ def warm_realtime(db: HubDB) -> dict[str, Any]:
if master_code and master_name:
canonical_names.setdefault(master_code, master_name)
codes = list(canonical_names)
fetched_sector_rows = _eastmoney_sector_quotes(codes)
fetched_sector_rows = observability.observe(
db, "eastmoney", "sector_quotes_batch", lambda: _eastmoney_sector_quotes(codes),
classify=_classify_quote_rows,
)
for row in fetched_sector_rows:
if _row_quote_date(row, today) != today:
continue
@@ -588,7 +632,10 @@ def fetch_intraday(db: HubDB, code: str, date: str = "") -> dict[str, Any]:
return cached
adapter = EastmoneyAdapter()
try:
payload_data = adapter.fetch_intraday(ts_code, date)
payload_data = observability.observe(
db, "eastmoney", "intraday", lambda: adapter.fetch_intraday(ts_code, date),
classify=_classify_intraday_payload,
)
source = "eastmoney:trends2"
except Exception as exc:
recovered = _load_intraday_lkg(db, ts_code, date)