from __future__ import annotations import logging import sys from threading import Lock from typing import Any, Callable, ClassVar from backend.data.datahub.client import DatahubClient, DatahubResponse from backend.data.datahub.compare import compare_rows from backend.data.datahub.errors import DatahubError from backend.data.datahub.native import ( API_TO_DATASET, filter_calendar_rows, filter_stock_rows, project_fields, to_native_rows, yyyymmdd, ) from backend.data.datahub.redact import redact_text, redact_value from backend.data.datahub.route_state import LEDGER from backend.data.datahub.settings import DatahubSettings from backend.data.providers.tushare_daily import DailyMarketMixin from backend.data.providers.tushare_dashboard import DashboardMixin from backend.data.providers.tushare_dragon_tiger import DragonTigerMixin from backend.data.providers.tushare_indices import IndexMixin from backend.data.providers.tushare_industries import ShenwanIndustryMixin from backend.data.providers.tushare_sectors import SectorMixin from backend.data.providers.tushare_stocks import StockMixin from backend.data.providers.tushare_transport import TushareError LOGGER = logging.getLogger("xiaobai.datahub") ShadowSink = Callable[[dict[str, Any]], None] def _usable_intraday_points(rows: list[Any]) -> list[dict[str, Any]]: points: list[dict[str, Any]] = [] for row in rows: if not isinstance(row, dict): continue try: close = float(row.get("close") or 0) except (TypeError, ValueError): close = 0.0 if close <= 0: continue point = dict(row) if "average" not in point and point.get("avg_price") is not None: point["average"] = point.get("avg_price") points.append(point) return points EMPTY_FAIL_DATASETS = { "stocks", "daily", "index_daily", "valuation", "moneyflow", "auction", "limit_events", "sector_daily", } def looks_like_heaven(module_name: str, filename: str = "") -> bool: """问天调用栈识别（诊断用）。问天按数据集依赖接入，不再整栈强制旧链路。""" path = filename.replace("\\", "/") return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path def caller_is_heaven(depth: int = 24) -> bool: frame = sys._getframe(1) for _ in range(depth): frame = frame.f_back if frame is not None else None if frame is None: return False name = str(frame.f_globals.get("__name__") or "") filename = str(frame.f_code.co_filename or "") if looks_like_heaven(name, filename): return True return False class DatahubBridge: def __init__( self, settings: DatahubSettings, client: DatahubClient, shadow_sink: ShadowSink | None = None, heaven_guard: Callable[[], bool] | None = None, ) -> None: self.settings = settings self.client = client self.shadow_sink = shadow_sink self.heaven_guard = heaven_guard or caller_is_heaven def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None: flags = self.settings.flags("status") if not flags.read and not flags.shadow: return None try: response = self._require_fresh(self.client.dataset_status(yyyymmdd(trade_date)), "status") rows = list(response.data or []) if flags.shadow: self._emit_shadow(compare_rows("status", [], rows, response.meta)) if flags.read: return rows return None except Exception as exc: self._log_failure("status", exc) if flags.shadow: self._emit_shadow(compare_rows("status", [], [], {}, self._error_text(exc))) return None def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None: flags = self.settings.flags("status") if not flags.read: return None try: response = self._require_fresh( self.client.batches(yyyymmdd(trade_date), dataset), "status", ) return list(response.data or []) except Exception as exc: self._log_failure("status", exc) return None def try_intraday(self, code: str) -> dict[str, Any] | None: flags = self.settings.flags("intraday") if not flags.read: return None try: response = self.client.intraday_points(code=code) data = response.data if not isinstance(data, dict): raise DatahubError("EMPTY", "datahub intraday payload invalid") points = _usable_intraday_points(data.get("points") or []) if not points: raise DatahubError("EMPTY", "datahub intraday empty") if (response.meta or {}).get("stale"): raise DatahubError("STALE", "datahub intraday stale") self._record_route("intraday", "datahub", str((response.meta or {}).get("source") or "datahub")) return { "entity_type": str(data.get("entity_type") or "stock"), "identifier": str(data.get("identifier") or code), "name": str(data.get("name") or ""), "code": str(data.get("code") or code), "trade_date": str(data.get("trade_date") or points[-1].get("date") or ""), "previous_close": float(data.get("previous_close") or 0), "points": points, "source": "datahub", } except Exception as exc: self._log_failure("intraday", exc) return None def try_market_quotes(self, trade_date: str = "") -> list[dict[str, Any]] | None: return self._try_quote_rows("quotes", {}, expected_date=trade_date, minimum=200) def try_quotes(self, codes: list[str]) -> list[dict[str, Any]] | None: cleaned = [str(item or "").strip() for item in codes if str(item or "").strip()] if not cleaned: return None return self._try_quote_rows("quotes", {"codes": ",".join(cleaned)}, minimum=1) def try_index_quotes(self) -> list[dict[str, Any]] | None: flags = self.settings.flags("index_quotes") if not flags.read: return None try: response = self.client.index_quotes() rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)] if len(rows) < 3: raise DatahubError("EMPTY", "datahub index quotes incomplete") if (response.meta or {}).get("stale"): raise DatahubError("STALE", "datahub index quotes stale") self._record_route( "index_quotes", "datahub", str((response.meta or {}).get("source") or "datahub"), ) return rows except Exception as exc: self._log_failure("index_quotes", exc) return None def try_sector_quote(self, code: str, trade_date: str = "") -> dict[str, Any] | None: flags = self.settings.flags("quotes") if not flags.read: return None try: response = self.client.sector_quote(code, trade_date) data = response.data if not isinstance(data, dict) or not data: raise DatahubError("EMPTY", "datahub sector quote empty") row = dict(data) if (response.meta or {}).get("stale"): row["delayed"] = True row["delay_seconds"] = int((response.meta or {}).get("staleness_seconds") or 0) row["delay_notice"] = str((response.meta or {}).get("delay_notice") or "") self._record_route("quotes", "datahub", str((response.meta or {}).get("source") or "datahub")) return row except Exception as exc: self._log_failure("quotes", exc) return None def try_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]] | None: flags = self.settings.flags("limit_events") if not flags.read: return None try: response = self.client.limit_pool(trade_date) rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)] if not rows: raise DatahubError("EMPTY", "datahub limit pool empty") self._record_route( "limit_events", "datahub", str((response.meta or {}).get("source") or "datahub"), ) return rows except Exception as exc: self._log_failure("limit_events", exc) return None def try_daily_chart( self, code: str, end_date: str, limit: int = 90, dataset: str = "daily", ) -> list[dict[str, Any]] | None: flags = self.settings.flags(dataset) if not flags.read: return None compact_end = yyyymmdd(end_date) if not compact_end: return None try: start = _shift_yyyymmdd(compact_end, -max(190, int(limit) * 3)) if dataset == "index_daily": response = self._paginate( self.client.index_bars, {"code": code, "from": start, "to": compact_end}, ) elif dataset == "sector_daily": response = self._paginate( self.client.sectors, {"code": code, "from": start, "to": compact_end}, ) else: response = self._paginate( self.client.daily_bars, {"code": code, "from": start, "to": compact_end, "adjust": "none"}, ) # Charts can use a partial history window; do not discard usable bars # just because the requested lookback is not fully covered. self._validate_usable( dataset, list(response.data or []), response, require_complete=False, ) rows = _chart_bars(list(response.data or [])) if not rows: raise DatahubError("EMPTY", f"{dataset} chart empty") self._record_route(dataset, "datahub", str((response.meta or {}).get("source") or "datahub")) return rows[-max(1, int(limit)):] except Exception as exc: self._log_failure(dataset, exc) return None def record_legacy(self, dataset: str, source: str = "", error: str = "") -> None: self._record_route(dataset, "legacy", source, error) def route_snapshot(self) -> list[dict[str, Any]]: return LEDGER.snapshot() def _try_quote_rows( self, dataset: str, params: dict[str, Any], expected_date: str = "", minimum: int = 1, ) -> list[dict[str, Any]] | None: flags = self.settings.flags(dataset) if not flags.read: return None try: response = self.client.quotes_latest(**params) rows = [_native_quote(item) for item in (response.data or []) if isinstance(item, dict)] rows = [item for item in rows if item] want = yyyymmdd(expected_date) if want: dated = [item for item in rows if not item.get("quote_date") or item.get("quote_date") == want] if dated: rows = dated if len(rows) < minimum: raise DatahubError("EMPTY", f"datahub {dataset} empty") stale = bool((response.meta or {}).get("stale")) delay = int((response.meta or {}).get("staleness_seconds") or 0) notice = str((response.meta or {}).get("delay_notice") or "") source = str((response.meta or {}).get("source") or "datahub") if stale: for item in rows: item["delayed"] = True item["delay_seconds"] = delay item["delay_notice"] = notice item["source"] = source self._record_route(dataset, "datahub", source) return rows except Exception as exc: self._log_failure(dataset, exc) return None def query( self, api_name: str, params: dict[str, Any] | None = None, fields: str = "", ) -> list[dict[str, Any]]: if api_name == "rt_sw_k": raise TushareError("rt_sw_k is disabled; use published sw_daily or free Shenwan realtime") dataset = API_TO_DATASET.get(api_name) if dataset: flags = self.settings.flags(dataset) if flags.read: try: response = self._fetch_dataset(dataset, params or {}, api_name=api_name) hub_canonical = self._extract_rows(dataset, response, params or {}) hub_rows = to_native_rows(dataset, hub_canonical) self._validate_usable(dataset, hub_rows, response) self._record_route(dataset, "datahub", str(response.meta.get("source") or "datahub")) return project_fields(hub_rows, fields) except Exception as exc: self._log_failure(dataset, exc) try: response = self.client.query_api(api_name, params or {}, fields) rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)] if dataset: self._record_route(dataset, "datahub", str((response.meta or {}).get("source") or "datahub")) else: self._record_route(api_name, "datahub", str((response.meta or {}).get("source") or "datahub")) return rows if not fields else project_fields(rows, fields) except Exception as exc: self._log_failure(dataset or api_name, exc) raise TushareError(self._error_text(exc)) from exc def _fetch_dataset(self, dataset: str, params: dict[str, Any], api_name: str = "") -> DatahubResponse: date = yyyymmdd(params.get("trade_date") or params.get("date")) start = yyyymmdd(params.get("start_date") or params.get("from") or date) end = yyyymmdd(params.get("end_date") or params.get("to") or date) code = str(params.get("ts_code") or params.get("code") or "").strip() if dataset == "calendar": if not start or not end: raise DatahubError("INVALID_ARGUMENT", "calendar requires start_date and end_date") return self.client.calendar(start, end) if dataset == "stocks": return self._paginate(self.client.stocks, {}) fetchers = { "daily": self.client.daily_bars, "index_daily": self.client.index_bars, "valuation": self.client.valuation, "moneyflow": self.client.moneyflow, "auction": self.client.auction, "limit_events": self.client.limit_events, "popularity": self.client.popularity, "dragon_tiger": self.client.dragon_tiger, "sector_daily": self.client.sectors, } fetcher = fetchers[dataset] query: dict[str, Any] = {} if code: query["code"] = code if date and not (params.get("start_date") or params.get("end_date")): query["date"] = date else: if start: query["from"] = start if end: query["to"] = end if dataset == "daily": query["adjust"] = "none" if dataset == "limit_events": limit_type = str(params.get("limit_type") or "").strip().upper() if limit_type: query["limit_type"] = limit_type if dataset == "popularity": if api_name == "ths_hot": query["source"] = "ths" elif api_name == "dc_hot": query["source"] = "dc" if dataset == "sector_daily": family = { "ths_daily": "ths", "dc_index": "dc", "sw_daily": "sw", }.get(api_name, "") if family: query["family"] = family return self._paginate(fetcher, query) def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse: limit = self.settings.page_limit offset = 0 rows: list[Any] = [] meta: dict[str, Any] = {} schema_version = 1 while True: page = fetcher(**{**params, "limit": limit, "offset": offset}) meta = dict(page.meta) schema_version = page.schema_version data = page.data or [] if not isinstance(data, list): raise DatahubError("INTERNAL", "datahub returned a non-list payload") rows.extend(data) if len(data) < limit: break offset += limit if offset > 200_000: break return DatahubResponse(data=rows, meta=meta, schema_version=schema_version) def _extract_rows( self, dataset: str, response: DatahubResponse, params: dict[str, Any], ) -> list[dict[str, Any]]: rows = [dict(item) for item in (response.data or [])] if dataset == "calendar": return filter_calendar_rows(rows, params) if dataset == "stocks": return filter_stock_rows(rows, params) return rows def _validate_usable( self, dataset: str, rows: list[dict[str, Any]], response: DatahubResponse, require_complete: bool = True, ) -> None: meta = response.meta or {} stale_seconds = int(meta.get("staleness_seconds") or 0) if meta.get("stale") or stale_seconds > self.settings.stale_seconds_max: raise DatahubError("STALE", f"{dataset} data is stale") if dataset in EMPTY_FAIL_DATASETS and not rows: raise DatahubError("EMPTY", f"{dataset} returned no rows") coverage = meta.get("coverage") if isinstance(meta.get("coverage"), dict) else {} if require_complete and (meta.get("incomplete") is True or coverage.get("complete") is False): missing = coverage.get("missing_count") raise DatahubError("INCOMPLETE", f"{dataset} range is incomplete missing={missing}") def _require_fresh(self, response: DatahubResponse, dataset: str) -> DatahubResponse: self._validate_usable(dataset, list(response.data or []) if isinstance(response.data, list) else [], response) return response def _emit_shadow(self, report: dict[str, Any]) -> None: safe = redact_value(report, secrets=self.settings.secrets()) LOGGER.info("datahub shadow %s", safe) if self.shadow_sink is not None: self.shadow_sink(report) def _log_failure(self, dataset: str, exc: Exception) -> None: error = redact_text(self._error_text(exc), self.settings.secrets()) LOGGER.warning("datahub unavailable dataset=%s error=%s", dataset, error) self._record_route(dataset, "datahub", "unavailable", error) def _record_route(self, dataset: str, route: str, source: str = "", error: str = "") -> None: LEDGER.record(dataset, route, source, redact_text(error, self.settings.secrets())) def _error_text(self, exc: Exception) -> str: if isinstance(exc, DatahubError): text = f"{exc.code}: {exc.message}" else: text = str(exc) return redact_text(text, self.settings.secrets()) def _native_quote(row: dict[str, Any]) -> dict[str, Any] | None: ts_code = str(row.get("ts_code") or "").strip() close = _finite(row.get("close") if row.get("close") not in (None, "") else row.get("price")) previous = _finite( row.get("pre_close") if row.get("pre_close") not in (None, "") else row.get("previous_close") ) if not ts_code or close <= 0 or previous <= 0: return None volume = _finite(row.get("vol") if row.get("vol") not in (None, "") else row.get("volume")) payload = { "ts_code": ts_code, "name": str(row.get("name") or ts_code).strip(), "pre_close": previous, "open": _finite(row.get("open")), "high": _finite(row.get("high")), "low": _finite(row.get("low")), "close": close, "vol": volume, "amount": _finite(row.get("amount")), "num": 0, "quote_date": yyyymmdd(row.get("quote_date") or row.get("trade_date")), "source": str(row.get("source") or "datahub"), } if row.get("delayed"): payload["delayed"] = True payload["delay_seconds"] = int(row.get("delay_seconds") or 0) payload["delay_notice"] = str(row.get("delay_notice") or "") return payload def _chart_bars(rows: list[Any]) -> list[dict[str, Any]]: normalized: list[dict[str, Any]] = [] for row in rows: if not isinstance(row, dict): continue compact = yyyymmdd(row.get("trade_date")) close = _finite(row.get("close")) if len(compact) != 8 or close <= 0: continue volume = _finite(row.get("volume") if row.get("volume") not in (None, "") else row.get("vol")) amount = _finite(row.get("amount")) if volume and volume < close * 10 and amount > 1000: volume = volume * 100 trade_date = f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" previous = normalized[-1]["close"] if normalized else 0.0 normalized.append( { "trade_date": trade_date, "open": _finite(row.get("open")), "high": _finite(row.get("high")), "low": _finite(row.get("low")), "close": close, "change": round((close / previous - 1) * 100, 4) if previous else _finite(row.get("pct_chg")), "volume": volume, "amount_billion": amount / 100_000_000, } ) return normalized def _shift_yyyymmdd(value: str, days: int) -> str: from datetime import datetime, timedelta stamp = datetime.strptime(value, "%Y%m%d") return (stamp + timedelta(days=days)).strftime("%Y%m%d") def _finite(value: Any) -> float: try: return float(value or 0) except (TypeError, ValueError): return 0.0 class DatahubAwareTushareClient( DashboardMixin, IndexMixin, ShenwanIndustryMixin, SectorMixin, DragonTigerMixin, StockMixin, DailyMarketMixin, ): """Website market facade. Mixins call query(); query talks only to the hub.""" _realtime_reference_cache: ClassVar[dict[str, dict[str, Any]]] = {} _realtime_reference_lock: ClassVar[Lock] = Lock() _capital_cache: ClassVar[dict[str, dict[str, Any]]] = {} _latest_realtime_market: ClassVar[dict[str, dict[str, Any]]] = {} _stock_activity_cache: ClassVar[dict[str, dict[str, Any]]] = {} _stock_listing_cache: ClassVar[dict[str, Any]] = {} _stock_listing_lock: ClassVar[Lock] = Lock() _suspension_cache: ClassVar[dict[str, dict[str, str] | None]] = {} _suspension_lock: ClassVar[Lock] = Lock() _sw_member_cache: ClassVar[dict[str, Any]] = {} _sw_member_lock: ClassVar[Lock] = Lock() def __init__(self, first: Any, second: Any | None = None) -> None: # Production: DatahubAwareTushareClient(bridge) # Older tests: DatahubAwareTushareClient(unused_legacy, bridge) self._bridge = second if second is not None else first self.token = "datahub" self.timeout = 30 self.realtime_aggregator = None def query( self, api_name: str, params: dict[str, Any] | None = None, fields: str = "", ) -> list[dict[str, Any]]: return self._bridge.query(api_name, params, fields) def try_market_quotes(self, trade_date: str = "") -> list[dict[str, Any]] | None: return self._bridge.try_market_quotes(trade_date) def try_quotes(self, codes: list[str]) -> list[dict[str, Any]] | None: return self._bridge.try_quotes(codes) def try_index_quotes(self) -> list[dict[str, Any]] | None: return self._bridge.try_index_quotes() def try_sector_quote(self, code: str, trade_date: str = "") -> dict[str, Any] | None: return self._bridge.try_sector_quote(code, trade_date) def try_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]] | None: return self._bridge.try_limit_pool(trade_date) def record_datahub_legacy(self, dataset: str, source: str = "", error: str = "") -> None: self._bridge.record_legacy(dataset, source, error)