diff --git a/.env.example b/.env.example index faf7291..c89b184 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,11 @@ APP_ENCRYPTION_KEY= # the system settings; all accounts use the same backend market snapshot. TUSHARE_TOKEN=your_tushare_token_here +# Optional xiaobai-datahub client. All DATAHUB_READ_* / DATAHUB_SHADOW_* flags +# default off in config/datahub.config.json, so the website keeps using Tushare. +DATAHUB_BASE_URL=http://127.0.0.1:8766 +DATAHUB_TOKEN= + # Optional iFinD HTTP credential. The backend exchanges it for a short-lived # access token and never exposes either token to browsers. IFIND_REFRESH_TOKEN=your_ifind_refresh_token_here diff --git a/backend/bootstrap/settings.py b/backend/bootstrap/settings.py index 37db457..80e952a 100644 --- a/backend/bootstrap/settings.py +++ b/backend/bootstrap/settings.py @@ -11,6 +11,8 @@ from backend.features.accounts.security import SecretVault def environment_credentials(environment: Mapping[str, str]) -> dict[str, str]: return { "tushare_token": str(environment.get("TUSHARE_TOKEN") or "").strip(), + "datahub_token": str(environment.get("DATAHUB_TOKEN") or "").strip(), + "datahub_base_url": str(environment.get("DATAHUB_BASE_URL") or "").strip(), "ifind_refresh_token": str(environment.get("IFIND_REFRESH_TOKEN") or "").strip(), "ifind_access_token": str(environment.get("IFIND_ACCESS_TOKEN") or "").strip(), "platform_llm_primary_api_key": str( diff --git a/backend/data/datahub/__init__.py b/backend/data/datahub/__init__.py new file mode 100644 index 0000000..71e0fbd --- /dev/null +++ b/backend/data/datahub/__init__.py @@ -0,0 +1,15 @@ +from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge +from backend.data.datahub.client import DatahubClient, DatahubResponse +from backend.data.datahub.errors import DatahubError +from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags + +__all__ = [ + "DATASETS", + "DatahubAwareTushareClient", + "DatahubBridge", + "DatahubClient", + "DatahubError", + "DatahubResponse", + "DatahubSettings", + "DatasetFlags", +] diff --git a/backend/data/datahub/bridge.py b/backend/data/datahub/bridge.py new file mode 100644 index 0000000..068ca30 --- /dev/null +++ b/backend/data/datahub/bridge.py @@ -0,0 +1,249 @@ +from __future__ import annotations + +import logging +import sys +from typing import Any, Callable + +from backend.data.datahub.client import DatahubClient, DatahubResponse +from backend.data.datahub.compare import compare_rows +from backend.data.datahub.errors import DatahubError +from backend.data.datahub.native import ( + API_TO_DATASET, + filter_calendar_rows, + filter_stock_rows, + project_fields, + to_native_rows, + yyyymmdd, +) +from backend.data.datahub.redact import redact_text, redact_value +from backend.data.datahub.settings import DatahubSettings +from backend.data.providers.tushare_client import TushareClient + +LOGGER = logging.getLogger("xiaobai.datahub") +ShadowSink = Callable[[dict[str, Any]], None] +EMPTY_FAIL_DATASETS = {"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction"} + + +def looks_like_heaven(module_name: str, filename: str = "") -> bool: + path = filename.replace("\\", "/") + return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path + + +def caller_is_heaven(depth: int = 24) -> bool: + frame = sys._getframe(1) + for _ in range(depth): + frame = frame.f_back if frame is not None else None + if frame is None: + return False + name = str(frame.f_globals.get("__name__") or "") + filename = str(frame.f_code.co_filename or "") + if looks_like_heaven(name, filename): + return True + return False + + +class DatahubBridge: + def __init__( + self, + settings: DatahubSettings, + client: DatahubClient, + shadow_sink: ShadowSink | None = None, + heaven_guard: Callable[[], bool] | None = None, + ) -> None: + self.settings = settings + self.client = client + self.shadow_sink = shadow_sink + self.heaven_guard = heaven_guard or caller_is_heaven + + def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None: + flags = self.settings.flags("status") + if not flags.read and not flags.shadow: + return None + try: + response = self._require_fresh(self.client.dataset_status(yyyymmdd(trade_date)), "status") + rows = list(response.data or []) + if flags.shadow: + self._emit_shadow(compare_rows("status", [], rows, response.meta)) + if flags.read: + return rows + return None + except Exception as exc: + self._log_failure("status", exc) + if flags.shadow: + self._emit_shadow(compare_rows("status", [], [], {}, self._error_text(exc))) + return None + + def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None: + flags = self.settings.flags("status") + if not flags.read: + return None + try: + response = self._require_fresh( + self.client.batches(yyyymmdd(trade_date), dataset), + "status", + ) + return list(response.data or []) + except Exception as exc: + self._log_failure("status", exc) + return None + + def query( + self, + api_name: str, + params: dict[str, Any] | None, + fields: str, + legacy_query: Callable[..., list[dict[str, Any]]], + ) -> list[dict[str, Any]]: + dataset = API_TO_DATASET.get(api_name) + if not dataset or self.heaven_guard(): + return legacy_query(api_name, params, fields) + flags = self.settings.flags(dataset) + if not flags.read and not flags.shadow: + return legacy_query(api_name, params, fields) + + hub_rows: list[dict[str, Any]] | None = None + hub_meta: dict[str, Any] = {} + hub_error: str | None = None + hub_canonical: list[dict[str, Any]] = [] + try: + response = self._fetch_dataset(dataset, params or {}) + hub_canonical = self._extract_rows(dataset, response, params or {}) + hub_rows = to_native_rows(dataset, hub_canonical) + hub_meta = dict(response.meta) + self._validate_usable(dataset, hub_rows, response) + except Exception as exc: + hub_error = self._error_text(exc) + self._log_failure(dataset, exc) + + if flags.shadow: + try: + legacy_rows = legacy_query(api_name, params, fields) + except Exception as exc: + if flags.read and hub_rows is not None and hub_error is None: + self._emit_shadow(compare_rows(dataset, [], hub_canonical, hub_meta, self._error_text(exc))) + return project_fields(hub_rows, fields) + raise + self._emit_shadow(compare_rows(dataset, legacy_rows, hub_canonical, hub_meta, hub_error)) + if flags.read and hub_rows is not None and hub_error is None: + return project_fields(hub_rows, fields) + return legacy_rows + + if flags.read and hub_rows is not None and hub_error is None: + return project_fields(hub_rows, fields) + return legacy_query(api_name, params, fields) + + def _fetch_dataset(self, dataset: str, params: dict[str, Any]) -> DatahubResponse: + date = yyyymmdd(params.get("trade_date") or params.get("date")) + start = yyyymmdd(params.get("start_date") or params.get("from") or date) + end = yyyymmdd(params.get("end_date") or params.get("to") or date) + code = str(params.get("ts_code") or params.get("code") or "").strip() + if dataset == "calendar": + if not start or not end: + raise DatahubError("INVALID_ARGUMENT", "calendar requires start_date and end_date") + return self.client.calendar(start, end) + if dataset == "stocks": + return self._paginate(self.client.stocks, {}) + fetchers = { + "daily": self.client.daily_bars, + "index_daily": self.client.index_bars, + "valuation": self.client.valuation, + "moneyflow": self.client.moneyflow, + "auction": self.client.auction, + } + fetcher = fetchers[dataset] + query: dict[str, Any] = {} + if code: + query["code"] = code + if date and not (params.get("start_date") or params.get("end_date")): + query["date"] = date + else: + if start: + query["from"] = start + if end: + query["to"] = end + if dataset == "daily": + query["adjust"] = "none" + return self._paginate(fetcher, query) + + def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse: + limit = self.settings.page_limit + offset = 0 + rows: list[Any] = [] + meta: dict[str, Any] = {} + schema_version = 1 + while True: + page = fetcher(**{**params, "limit": limit, "offset": offset}) + meta = dict(page.meta) + schema_version = page.schema_version + data = page.data or [] + if not isinstance(data, list): + raise DatahubError("INTERNAL", "datahub returned a non-list payload") + rows.extend(data) + if len(data) < limit: + break + offset += limit + if offset > 200_000: + break + return DatahubResponse(data=rows, meta=meta, schema_version=schema_version) + + def _extract_rows( + self, + dataset: str, + response: DatahubResponse, + params: dict[str, Any], + ) -> list[dict[str, Any]]: + rows = [dict(item) for item in (response.data or [])] + if dataset == "calendar": + return filter_calendar_rows(rows, params) + if dataset == "stocks": + return filter_stock_rows(rows, params) + return rows + + def _validate_usable(self, dataset: str, rows: list[dict[str, Any]], response: DatahubResponse) -> None: + meta = response.meta or {} + stale_seconds = int(meta.get("staleness_seconds") or 0) + if meta.get("stale") or stale_seconds > self.settings.stale_seconds_max: + raise DatahubError("STALE", f"{dataset} data is stale") + if dataset in EMPTY_FAIL_DATASETS and not rows: + raise DatahubError("EMPTY", f"{dataset} returned no rows") + + def _require_fresh(self, response: DatahubResponse, dataset: str) -> DatahubResponse: + self._validate_usable(dataset, list(response.data or []) if isinstance(response.data, list) else [], response) + return response + + def _emit_shadow(self, report: dict[str, Any]) -> None: + safe = redact_value(report, secrets=self.settings.secrets()) + LOGGER.info("datahub shadow %s", safe) + if self.shadow_sink is not None: + self.shadow_sink(report) + + def _log_failure(self, dataset: str, exc: Exception) -> None: + LOGGER.warning( + "datahub fallback dataset=%s error=%s", + dataset, + redact_text(self._error_text(exc), self.settings.secrets()), + ) + + def _error_text(self, exc: Exception) -> str: + if isinstance(exc, DatahubError): + text = f"{exc.code}: {exc.message}" + else: + text = str(exc) + return redact_text(text, self.settings.secrets()) + + +class DatahubAwareTushareClient: + def __init__(self, legacy: TushareClient, bridge: DatahubBridge) -> None: + self._legacy = legacy + self._bridge = bridge + + def query( + self, + api_name: str, + params: dict[str, Any] | None = None, + fields: str = "", + ) -> list[dict[str, Any]]: + return self._bridge.query(api_name, params, fields, self._legacy.query) + + def __getattr__(self, name: str) -> Any: + return getattr(self._legacy, name) diff --git a/backend/data/datahub/client.py b/backend/data/datahub/client.py new file mode 100644 index 0000000..7f247f2 --- /dev/null +++ b/backend/data/datahub/client.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from typing import Any, Callable + +from backend.data.datahub.errors import DatahubError +from backend.data.datahub.redact import redact_text +from backend.data.datahub.settings import DatahubSettings + +LOGGER = logging.getLogger("xiaobai.datahub") +UrlOpen = Callable[..., Any] + + +@dataclass(frozen=True) +class DatahubResponse: + data: Any + meta: dict[str, Any] = field(default_factory=dict) + schema_version: int = 1 + status: int = 200 + + +class DatahubClient: + def __init__( + self, + settings: DatahubSettings, + urlopen: UrlOpen = urllib.request.urlopen, + ) -> None: + self.settings = settings + self._urlopen = urlopen + + def health(self) -> DatahubResponse: + return self.get("/v1/health") + + def calendar(self, start: str, end: str) -> DatahubResponse: + return self.get("/v1/calendar", {"from": start, "to": end}) + + def stocks(self, updated_since: str = "", limit: int | None = None, offset: int = 0) -> DatahubResponse: + params: dict[str, Any] = {"offset": offset, "limit": limit or self.settings.page_limit} + if updated_since: + params["updated_since"] = updated_since + return self.get("/v1/stocks", params) + + def daily_bars(self, **params: Any) -> DatahubResponse: + return self.get("/v1/bars/daily", params) + + def index_bars(self, **params: Any) -> DatahubResponse: + return self.get("/v1/indexes/bars", params) + + def valuation(self, **params: Any) -> DatahubResponse: + return self.get("/v1/valuation", params) + + def moneyflow(self, **params: Any) -> DatahubResponse: + return self.get("/v1/moneyflow", params) + + def auction(self, **params: Any) -> DatahubResponse: + return self.get("/v1/auction", params) + + def dataset_status(self, date: str) -> DatahubResponse: + return self.get("/v1/datasets/status", {"date": date}) + + def batches(self, date: str, dataset: str = "") -> DatahubResponse: + params: dict[str, Any] = {"date": date} + if dataset: + params["dataset"] = dataset + return self.get("/v1/batches", params) + + def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse: + if not self.settings.token: + raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured") + query = { + key: value + for key, value in (params or {}).items() + if value is not None and value != "" + } + url = self.settings.base_url + path + if query: + url = f"{url}?{urllib.parse.urlencode(query)}" + attempts = 1 + max(0, self.settings.retries) + last_error: DatahubError | None = None + for attempt in range(attempts): + try: + return self._request(url) + except DatahubError as exc: + last_error = exc + if exc.code not in {"TIMEOUT", "UNAVAILABLE"} or attempt + 1 >= attempts: + raise + LOGGER.warning( + "datahub retry %s/%s %s", + attempt + 1, + attempts, + redact_text(str(exc), self.settings.secrets()), + ) + raise last_error or DatahubError("INTERNAL", "datahub request failed") + + def _request(self, url: str) -> DatahubResponse: + request = urllib.request.Request( + url, + headers={ + "Accept": "application/json", + "X-Datahub-Token": self.settings.token, + "User-Agent": "XiaobaiReviewDatahub/1.0", + }, + method="GET", + ) + try: + with self._urlopen(request, timeout=self.settings.timeout_seconds) as response: + status = int(getattr(response, "status", 200) or 200) + raw = response.read().decode("utf-8") + except TimeoutError as exc: + raise DatahubError("TIMEOUT", "datahub request timed out") from exc + except urllib.error.HTTPError as exc: + body = _read_error_body(exc) + raise _http_error(exc.code, body, self.settings.secrets()) from exc + except urllib.error.URLError as exc: + reason = redact_text(str(getattr(exc, "reason", exc)), self.settings.secrets()) + if "timed out" in reason.lower(): + raise DatahubError("TIMEOUT", "datahub request timed out") from exc + raise DatahubError("UNAVAILABLE", f"datahub unavailable: {reason}") from exc + payload = _parse_json(raw, self.settings.secrets()) + return _as_response(payload, status, self.settings.secrets()) + + +def _parse_json(raw: str, secrets: tuple[str, ...]) -> dict[str, Any]: + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise DatahubError("INTERNAL", "datahub returned invalid json") from exc + if not isinstance(payload, dict): + raise DatahubError("INTERNAL", "datahub returned a non-object payload") + return payload + + +def _as_response(payload: dict[str, Any], status: int, secrets: tuple[str, ...]) -> DatahubResponse: + error = payload.get("error") + if isinstance(error, dict): + raise _mapped_error(str(error.get("code") or "INTERNAL"), str(error.get("message") or "datahub error"), status) + if status >= 400: + raise DatahubError("UNAVAILABLE", f"datahub http {status}", status) + return DatahubResponse( + data=payload.get("data"), + meta=dict(payload.get("meta") or {}), + schema_version=int(payload.get("schema_version") or 1), + status=status, + ) + + +def _http_error(status: int, payload: dict[str, Any], secrets: tuple[str, ...]) -> DatahubError: + error = payload.get("error") if isinstance(payload.get("error"), dict) else {} + code = str((error or {}).get("code") or "") + message = str((error or {}).get("message") or payload.get("message") or f"datahub http {status}") + message = redact_text(message, secrets) + if status == 401 or code == "UNAUTHORIZED": + return DatahubError("UNAUTHORIZED", message, status) + if status == 404 or code == "DATASET_NOT_PUBLISHED": + return DatahubError("DATASET_NOT_PUBLISHED", message, status) + if status == 400 or code == "INVALID_ARGUMENT": + return DatahubError("INVALID_ARGUMENT", message, status) + if status in {429, 503} or code in {"RATE_LIMITED", "SOURCE_UNAVAILABLE"}: + return DatahubError("UNAVAILABLE", message, status) + return DatahubError(code or "INTERNAL", message, status) + + +def _mapped_error(code: str, message: str, status: int) -> DatahubError: + if code == "STALE_DATA": + return DatahubError("STALE", message, status) + if code in {"UNAUTHORIZED", "DATASET_NOT_PUBLISHED", "INVALID_ARGUMENT"}: + return DatahubError(code, message, status) + if code in {"RATE_LIMITED", "SOURCE_UNAVAILABLE"}: + return DatahubError("UNAVAILABLE", message, status) + return DatahubError(code or "INTERNAL", message, status) + + +def _read_error_body(exc: urllib.error.HTTPError) -> dict[str, Any]: + try: + raw = exc.read().decode("utf-8") + payload = json.loads(raw) + return payload if isinstance(payload, dict) else {"message": raw} + except Exception: + return {"message": str(exc)} diff --git a/backend/data/datahub/compare.py b/backend/data/datahub/compare.py new file mode 100644 index 0000000..08b45de --- /dev/null +++ b/backend/data/datahub/compare.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +from typing import Any + +from backend.data.datahub.native import SCALE_FIELDS, row_key, to_canonical_row, yyyymmdd + +NUMERIC_TOLERANCE = 1e-4 + + +def compare_rows( + dataset: str, + legacy_rows: list[dict[str, Any]], + hub_rows: list[dict[str, Any]] | None, + hub_meta: dict[str, Any] | None = None, + hub_error: str | None = None, +) -> dict[str, Any]: + hub = hub_rows or [] + legacy_map = {row_key(dataset, row): row for row in legacy_rows} + hub_map = {row_key(dataset, _align_hub_row(row)): row for row in hub} + missing_hub = sorted(key for key in legacy_map if key not in hub_map) + missing_legacy = sorted(key for key in hub_map if key not in legacy_map) + value_diffs: list[dict[str, Any]] = [] + unit_conversion: list[dict[str, Any]] = [] + matched = 0 + for key, legacy in legacy_map.items(): + hub_row = hub_map.get(key) + if hub_row is None: + continue + field_report = _compare_fields(dataset, legacy, hub_row) + if field_report["unit_conversion"]: + unit_conversion.append({"key": list(key), "fields": field_report["unit_conversion"]}) + if field_report["value_diff"]: + value_diffs.append({"key": list(key), "fields": field_report["value_diff"]}) + if not field_report["unit_conversion"] and not field_report["value_diff"]: + matched += 1 + stale_seconds = int((hub_meta or {}).get("staleness_seconds") or 0) + time_skew = bool((hub_meta or {}).get("stale")) or stale_seconds > 0 + return { + "dataset": dataset, + "legacy_rows": len(legacy_rows), + "hub_rows": len(hub), + "matched": matched, + "missing_hub": [list(item) for item in missing_hub[:20]], + "missing_legacy": [list(item) for item in missing_legacy[:20]], + "missing_hub_count": len(missing_hub), + "missing_legacy_count": len(missing_legacy), + "value_diff_count": len(value_diffs), + "unit_conversion_count": len(unit_conversion), + "value_diffs": value_diffs[:20], + "unit_conversion": unit_conversion[:20], + "time_skew": time_skew, + "staleness_seconds": stale_seconds, + "published_at": (hub_meta or {}).get("published_at"), + "trade_date": yyyymmdd((hub_meta or {}).get("trade_date")), + "hub_error": hub_error, + "equal": ( + not hub_error + and not missing_hub + and not missing_legacy + and not value_diffs + and not unit_conversion + and not time_skew + ), + } + + +def _align_hub_row(row: dict[str, Any]) -> dict[str, Any]: + aligned = dict(row) + if "volume" in aligned and "vol" not in aligned: + aligned["vol"] = aligned.get("volume") + return aligned + + +def _compare_fields(dataset: str, legacy: dict[str, Any], hub: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + canonical_legacy = to_canonical_row(dataset, legacy) + hub_canonical = _hub_canonical(dataset, hub) + native_hub = _align_hub_row(hub) + value_diff: list[dict[str, Any]] = [] + unit_conversion: list[dict[str, Any]] = [] + keys = (set(canonical_legacy) | set(hub_canonical)) - {"batch_id", "updated_at", "volume"} + scales = SCALE_FIELDS.get(dataset) or {} + for field in sorted(keys): + left = canonical_legacy.get(field) + right = hub_canonical.get(field) + if _same(left, right): + continue + native_left = legacy.get(field) + hub_raw = native_hub.get(field) + if field in scales and _near(_optional(native_left), _optional(hub_raw)): + unit_conversion.append( + {"field": field, "legacy": native_left, "hub": hub_raw, "reason": "unit_conversion"} + ) + continue + value_diff.append({"field": field, "legacy": left, "hub": right, "reason": "value_diff"}) + return {"value_diff": value_diff, "unit_conversion": unit_conversion} + + +def _hub_canonical(dataset: str, row: dict[str, Any]) -> dict[str, Any]: + """Hub API rows are already canonical; only align field names.""" + aligned = dict(row) + if "volume" in aligned and "vol" not in aligned: + aligned["vol"] = aligned.get("volume") + if dataset == "calendar": + is_open = aligned.get("is_open") + aligned["is_open"] = 1 if is_open in (True, 1, "1", "Y", "y") else 0 + aligned["cal_date"] = yyyymmdd(aligned.get("cal_date")) + aligned["pretrade_date"] = yyyymmdd(aligned.get("pretrade_date")) or None + aligned["exchange"] = str(aligned.get("exchange") or "SSE") + return aligned + + +def _same(left: Any, right: Any) -> bool: + if left in (None, "") and right in (None, ""): + return True + if isinstance(left, (int, float)) or isinstance(right, (int, float)): + return _near(_optional(left), _optional(right)) + return str(left or "") == str(right or "") + + +def _near(left: float | None, right: float | None) -> bool: + if left is None and right is None: + return True + if left is None or right is None: + return False + return abs(left - right) <= max(NUMERIC_TOLERANCE, abs(left) * 1e-9, abs(right) * 1e-9) + + +def _optional(value: Any) -> float | None: + if value in (None, ""): + return None + try: + return float(value) + except (TypeError, ValueError): + return None diff --git a/backend/data/datahub/errors.py b/backend/data/datahub/errors.py new file mode 100644 index 0000000..ce51473 --- /dev/null +++ b/backend/data/datahub/errors.py @@ -0,0 +1,12 @@ +from __future__ import annotations + + +class DatahubError(RuntimeError): + def __init__(self, code: str, message: str, status: int | None = None) -> None: + super().__init__(message) + self.code = code + self.message = message + self.status = status + + def __str__(self) -> str: + return f"{self.code}: {self.message}" diff --git a/backend/data/datahub/native.py b/backend/data/datahub/native.py new file mode 100644 index 0000000..e68b7ef --- /dev/null +++ b/backend/data/datahub/native.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from typing import Any + +from backend.data.numbers import finite_number + + +AMOUNT_THOUSAND_YUAN = 1000.0 +AMOUNT_WAN_YUAN = 10000.0 +VOLUME_LOT = 100.0 + +API_TO_DATASET = { + "trade_cal": "calendar", + "stock_basic": "stocks", + "daily": "daily", + "daily_basic": "valuation", + "index_daily": "index_daily", + "moneyflow": "moneyflow", + "stk_auction": "auction", +} + +SCALE_FIELDS = { + "daily": {"vol": VOLUME_LOT, "amount": AMOUNT_THOUSAND_YUAN}, + "index_daily": {"vol": VOLUME_LOT, "amount": AMOUNT_THOUSAND_YUAN}, + "valuation": {"total_mv": AMOUNT_WAN_YUAN, "circ_mv": AMOUNT_WAN_YUAN}, + "moneyflow": { + "buy_sm_amount": AMOUNT_WAN_YUAN, + "sell_sm_amount": AMOUNT_WAN_YUAN, + "buy_md_amount": AMOUNT_WAN_YUAN, + "sell_md_amount": AMOUNT_WAN_YUAN, + "buy_lg_amount": AMOUNT_WAN_YUAN, + "sell_lg_amount": AMOUNT_WAN_YUAN, + "buy_elg_amount": AMOUNT_WAN_YUAN, + "sell_elg_amount": AMOUNT_WAN_YUAN, + "net_mf_amount": AMOUNT_WAN_YUAN, + }, + "auction": {"vol": VOLUME_LOT, "float_share": AMOUNT_WAN_YUAN}, +} + + +def yyyymmdd(value: Any) -> str: + return str(value or "").replace("-", "")[:8] + + +def to_native_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [to_native_row(dataset, row) for row in rows] + + +def to_native_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]: + if dataset == "calendar": + is_open = row.get("is_open") + return { + "exchange": str(row.get("exchange") or "SSE"), + "cal_date": yyyymmdd(row.get("cal_date")), + "is_open": 1 if is_open in (True, 1, "1", "Y", "y") else 0, + "pretrade_date": yyyymmdd(row.get("pretrade_date")) or None, + } + converted = dict(row) + converted.pop("batch_id", None) + if "volume" in converted and "vol" not in converted: + converted["vol"] = converted.pop("volume") + elif "volume" in converted: + converted.pop("volume", None) + scales = SCALE_FIELDS.get(dataset) or {} + for field, factor in scales.items(): + if field in converted: + converted[field] = _unscale(converted.get(field), factor) + if dataset == "stocks": + converted.pop("updated_at", None) + return converted + + +def to_canonical_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]: + if dataset == "calendar": + is_open = row.get("is_open") + return { + "exchange": str(row.get("exchange") or "SSE"), + "cal_date": yyyymmdd(row.get("cal_date")), + "is_open": 1 if is_open in (True, 1, "1", "Y", "y") else 0, + "pretrade_date": yyyymmdd(row.get("pretrade_date")) or None, + } + converted = dict(row) + if "volume" in converted and "vol" not in converted: + converted["vol"] = converted.pop("volume") + scales = SCALE_FIELDS.get(dataset) or {} + for field, factor in scales.items(): + if field in converted: + converted[field] = _scale(converted.get(field), factor) + return converted + + +def row_key(dataset: str, row: dict[str, Any]) -> tuple[str, ...]: + if dataset == "calendar": + return (yyyymmdd(row.get("cal_date")),) + if dataset == "stocks": + return (str(row.get("ts_code") or "").upper(),) + if dataset == "status": + return (str(row.get("dataset") or ""), yyyymmdd(row.get("trade_date"))) + return (str(row.get("ts_code") or "").upper(), yyyymmdd(row.get("trade_date"))) + + +def project_fields(rows: list[dict[str, Any]], fields: str) -> list[dict[str, Any]]: + keys = [item.strip() for item in str(fields or "").split(",") if item.strip()] + if not keys: + return rows + return [{key: row.get(key) for key in keys} for row in rows] + + +def filter_stock_rows(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]: + payload = params or {} + ts_code = str(payload.get("ts_code") or "").strip().upper() + status = str(payload.get("list_status") or "").strip() + name = str(payload.get("name") or "").strip() + filtered = rows + if ts_code: + filtered = [row for row in filtered if str(row.get("ts_code") or "").upper() == ts_code] + if status: + filtered = [row for row in filtered if str(row.get("list_status") or status) == status] + if name: + filtered = [row for row in filtered if name.casefold() in str(row.get("name") or "").casefold()] + return filtered + + +def filter_calendar_rows(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]: + payload = params or {} + if payload.get("is_open") in (1, "1", True): + return [row for row in rows if int(row.get("is_open") or 0) == 1] + if payload.get("is_open") in (0, "0", False): + return [row for row in rows if int(row.get("is_open") or 0) == 0] + return rows + + +def _scale(value: Any, factor: float) -> float | None: + number = _optional_number(value) + if number is None: + return None + return number * factor + + +def _unscale(value: Any, factor: float) -> float | None: + number = _optional_number(value) + if number is None or factor == 0: + return None + return number / factor + + +def _optional_number(value: Any) -> float | None: + if value in (None, ""): + return None + number = finite_number(value, default=float("nan")) + if number != number: + return None + return number diff --git a/backend/data/datahub/redact.py b/backend/data/datahub/redact.py new file mode 100644 index 0000000..2b1d4ed --- /dev/null +++ b/backend/data/datahub/redact.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Any + + +SECRET_HINTS = ( + "token", + "password", + "secret", + "key", + "authorization", + "credential", + "cookie", +) + + +def redact_value(value: Any, key: str = "", secrets: tuple[str, ...] = ()) -> Any: + lowered = key.lower() + if any(part in lowered for part in SECRET_HINTS): + return "***" + if isinstance(value, dict): + return { + str(item_key): redact_value(item_value, str(item_key), secrets) + for item_key, item_value in value.items() + } + if isinstance(value, list): + return [redact_value(item, key, secrets) for item in value] + text = str(value) if value is not None and not isinstance(value, (int, float, bool)) else value + if isinstance(text, str): + return redact_text(text, secrets) + return value + + +def redact_text(text: str, secrets: tuple[str, ...] = ()) -> str: + redacted = text + for secret in secrets: + if secret: + redacted = redacted.replace(secret, "***") + return redacted diff --git a/backend/data/datahub/settings.py b/backend/data/datahub/settings.py new file mode 100644 index 0000000..419b3c7 --- /dev/null +++ b/backend/data/datahub/settings.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +from backend.bootstrap.config import APP_DIR + + +DATASETS = ( + "calendar", + "stocks", + "daily", + "index_daily", + "valuation", + "moneyflow", + "auction", + "status", +) + +ENV_DATASET = { + "calendar": "CALENDAR", + "stocks": "STOCKS", + "daily": "DAILY", + "index_daily": "INDEX_DAILY", + "valuation": "VALUATION", + "moneyflow": "MONEYFLOW", + "auction": "AUCTION", + "status": "STATUS", +} + +DEFAULT_CONFIG_PATH = APP_DIR / "config" / "datahub.config.json" + + +def _truthy(value: Any) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _int(value: Any, default: int) -> int: + try: + return int(value) + except (TypeError, ValueError): + return default + + +@dataclass(frozen=True) +class DatasetFlags: + name: str + read: bool = False + shadow: bool = False + + +@dataclass(frozen=True) +class DatahubSettings: + base_url: str + token: str + timeout_seconds: int = 8 + retries: int = 1 + page_limit: int = 5000 + stale_seconds_max: int = 86400 + datasets: dict[str, DatasetFlags] | None = None + + def flags(self, dataset: str) -> DatasetFlags: + mapped = self.datasets or {} + return mapped.get(dataset) or DatasetFlags(dataset) + + def any_enabled(self) -> bool: + return any(item.read or item.shadow for item in (self.datasets or {}).values()) + + def secrets(self) -> tuple[str, ...]: + return tuple(item for item in (self.token,) if item) + + @classmethod + def load( + cls, + path: Path | None = None, + environ: Mapping[str, str] | None = None, + credentials: Mapping[str, object] | None = None, + ) -> "DatahubSettings": + config_path = path or DEFAULT_CONFIG_PATH + payload: dict[str, Any] = {} + if config_path.is_file(): + payload = json.loads(config_path.read_text(encoding="utf-8")) + env = dict(os.environ if environ is None else environ) + creds = dict(credentials or {}) + dataset_flags: dict[str, DatasetFlags] = {} + raw_datasets = payload.get("datasets") or {} + for name in DATASETS: + item = raw_datasets.get(name) or {} + env_key = ENV_DATASET[name] + read = _truthy(env.get(f"DATAHUB_READ_{env_key}")) if f"DATAHUB_READ_{env_key}" in env else bool(item.get("read")) + shadow = ( + _truthy(env.get(f"DATAHUB_SHADOW_{env_key}")) + if f"DATAHUB_SHADOW_{env_key}" in env + else bool(item.get("shadow")) + ) + dataset_flags[name] = DatasetFlags(name, read=read, shadow=shadow) + token = str( + env.get("DATAHUB_TOKEN") + or creds.get("datahub_token") + or payload.get("token") + or "" + ).strip() + base_url = str( + env.get("DATAHUB_BASE_URL") + or creds.get("datahub_base_url") + or payload.get("base_url") + or "http://127.0.0.1:8766" + ).strip().rstrip("/") + return cls( + base_url=base_url, + token=token, + timeout_seconds=_int(env.get("DATAHUB_TIMEOUT") or payload.get("timeout_seconds"), 8), + retries=max(0, _int(env.get("DATAHUB_RETRIES") or payload.get("retries"), 1)), + page_limit=max(1, _int(payload.get("page_limit"), 5000)), + stale_seconds_max=max(0, _int(payload.get("stale_seconds_max"), 86400)), + datasets=dataset_flags, + ) diff --git a/backend/data/gateway.py b/backend/data/gateway.py index fcd436c..ac577a2 100644 --- a/backend/data/gateway.py +++ b/backend/data/gateway.py @@ -3,8 +3,10 @@ from __future__ import annotations from collections.abc import Callable from dataclasses import dataclass from datetime import datetime +from typing import Any from backend.data.contracts import DataUsage +from backend.data.datahub import DatahubAwareTushareClient, DatahubBridge, DatahubClient, DatahubSettings from backend.data.policy import DataSourcePolicy from backend.data.providers import IfindProvider, TushareProvider from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport @@ -22,6 +24,7 @@ class DataGateway: ifind_provider: IfindProvider chart_data: MarketChartClient realtime_observer: WebRealtimeAggregator + datahub: DatahubBridge @property def ifind(self) -> IfindHttpClient: @@ -34,7 +37,13 @@ class DataGateway: ) -> TushareClient: if dataset_id: self.policy.assert_allowed(dataset_id, "tushare", usage) - return self.tushare_provider.client() + return DatahubAwareTushareClient(self.tushare_provider.client(), self.datahub) + + def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None: + return self.datahub.dataset_status(trade_date) + + def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None: + return self.datahub.batches(trade_date, dataset) def assert_source(self, dataset_id: str, provider_id: str, usage: DataUsage) -> None: self.policy.assert_allowed(dataset_id, provider_id, usage) @@ -64,6 +73,7 @@ class DataGateway: def build_data_gateway( credentials: dict[str, object], tushare_token_supplier: Callable[[], str] | None = None, + datahub_settings: DatahubSettings | None = None, ) -> DataGateway: ifind = IfindHttpClient( str(credentials.get("ifind_refresh_token") or ""), @@ -73,6 +83,8 @@ def build_data_gateway( lambda: str(credentials.get("tushare_token") or "") ) policy = DataSourcePolicy.load() + settings = datahub_settings or DatahubSettings.load(credentials=credentials) + datahub_client = DatahubClient(settings) return DataGateway( policy=policy, quality=DataQualityGate.load(policy), @@ -80,4 +92,5 @@ def build_data_gateway( ifind_provider=IfindProvider(ifind), chart_data=MarketChartClient(ifind, EastmoneyChartClient()), realtime_observer=WebRealtimeAggregator(), + datahub=DatahubBridge(settings, datahub_client), ) diff --git a/config/README.md b/config/README.md index 5b352f6..9b9d834 100644 --- a/config/README.md +++ b/config/README.md @@ -12,6 +12,9 @@ These registries describe the approved product surface of the standalone applica providers, model entry points, CSS layers, and remaining code hotspots. - `data-fields.config.json`: canonical data products, provider eligibility, intended use, and known blocked datasets. +- `datahub.config.json`: optional read-only client for `xiaobai-datahub`. Each dataset has its + own `read` / `shadow` flag, all default off. Environment variables `DATAHUB_READ_*` and + `DATAHUB_SHADOW_*` can override a single dataset without a master switch. - `data-quality.config.json`: freshness, coverage, units, adjustment, point-in-time, and fail-closed rules for every canonical data product. - `jobs.config.json`: background schedules, dependencies, lock keys, retry policy, timeouts, diff --git a/config/architecture-inventory.json b/config/architecture-inventory.json index 8f33db1..107c544 100644 --- a/config/architecture-inventory.json +++ b/config/architecture-inventory.json @@ -204,6 +204,11 @@ "path": "backend/data/providers/tushare_client.py", "runtime_role": "stable client facade for primary deterministic market data" }, + { + "provider": "datahub", + "path": "backend/data/datahub/client.py", + "runtime_role": "optional official EOD read path behind per-dataset flags" + }, { "provider": "ifind", "path": "backend/data/providers/ifind_client.py", @@ -278,6 +283,18 @@ "owner": "backend/data/providers/tushare.py", "compatibility_fallback": "backend/features/market/service.py" }, + { + "client": "DatahubClient", + "owner": "backend/data/gateway.py" + }, + { + "client": "DatahubBridge", + "owner": "backend/data/gateway.py" + }, + { + "client": "DatahubAwareTushareClient", + "owner": "backend/data/gateway.py" + }, { "client": "IfindHttpClient", "owner": "backend/data/gateway.py" diff --git a/config/data-fields.config.json b/config/data-fields.config.json index 408f6f9..0c1bb1f 100644 --- a/config/data-fields.config.json +++ b/config/data-fields.config.json @@ -2,6 +2,7 @@ "schema_version": 1, "providers": { "tushare": {"class": "licensed", "calculation_allowed": true}, + "datahub": {"class": "licensed", "calculation_allowed": true}, "ifind": {"class": "licensed", "calculation_allowed": true}, "eastmoney": {"class": "public_web", "calculation_allowed": false}, "tencent": {"class": "public_web", "calculation_allowed": false}, diff --git a/config/datahub.config.json b/config/datahub.config.json new file mode 100644 index 0000000..1877fbc --- /dev/null +++ b/config/datahub.config.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "base_url": "http://127.0.0.1:8766", + "timeout_seconds": 8, + "retries": 1, + "page_limit": 5000, + "stale_seconds_max": 86400, + "datasets": { + "calendar": { "read": false, "shadow": false }, + "stocks": { "read": false, "shadow": false }, + "daily": { "read": false, "shadow": false }, + "index_daily": { "read": false, "shadow": false }, + "valuation": { "read": false, "shadow": false }, + "moneyflow": { "read": false, "shadow": false }, + "auction": { "read": false, "shadow": false }, + "status": { "read": false, "shadow": false } + } +} diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index 536a0b5..f3fcbb0 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -67,6 +67,9 @@ class DataGatewayTests(unittest.TestCase): "TushareClient": {"backend/features/market/service.py"}, "TushareProvider": {"backend/data/gateway.py"}, "WebRealtimeAggregator": {"backend/data/gateway.py"}, + "DatahubClient": {"backend/data/gateway.py"}, + "DatahubAwareTushareClient": {"backend/data/gateway.py"}, + "DatahubBridge": {"backend/data/gateway.py"}, } found = {name: set() for name in owners} for path in (root / "backend").rglob("*.py"): diff --git a/tests/test_datahub_bridge.py b/tests/test_datahub_bridge.py new file mode 100644 index 0000000..56f1f75 --- /dev/null +++ b/tests/test_datahub_bridge.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import ast +import json +import unittest +from pathlib import Path +from typing import Any + +from backend.data import build_data_gateway +from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge, looks_like_heaven +from backend.data.datahub.client import DatahubClient, DatahubResponse +from backend.data.datahub.compare import compare_rows +from backend.data.datahub.errors import DatahubError +from backend.data.datahub.native import to_canonical_row, to_native_row +from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags + +ROOT = Path(__file__).resolve().parents[1] +TOKEN = "super-secret-datahub-token" + +LEGACY_DAILY = { + "ts_code": "600000.SH", + "trade_date": "20240902", + "open": 10.11, + "high": 10.25, + "low": 10.01, + "close": 10.20, + "pct_chg": 1.2345, + "vol": 1000.0, + "amount": 2000.0, +} +HUB_DAILY = { + "ts_code": "600000.SH", + "trade_date": "20240902", + "open": 10.11, + "high": 10.25, + "low": 10.01, + "close": 10.20, + "pct_chg": 1.2345, + "volume": 100000.0, + "amount": 2000000.0, +} + + +class FakeLegacy: + def __init__(self, rows: list[dict[str, Any]] | Exception | None = None) -> None: + self.token = "legacy-token" + self.timeout = 30 + self.rows = [] if rows is None else rows + self.calls: list[tuple[str, dict[str, Any] | None, str]] = [] + + def query(self, api_name: str, params: dict[str, Any] | None = None, fields: str = "") -> list[dict[str, Any]]: + self.calls.append((api_name, params, fields)) + if isinstance(self.rows, Exception): + raise self.rows + return [dict(row) for row in self.rows] + + +class FakeClient(DatahubClient): + def __init__(self, error: DatahubError | None = None, response: DatahubResponse | None = None) -> None: + super().__init__(DatahubSettings(base_url="http://127.0.0.1:9", token=TOKEN)) + self.error = error + self.response = response or DatahubResponse( + data=[dict(HUB_DAILY)], + meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0}, + ) + self.paths: list[str] = [] + + def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse: + self.paths.append(path) + if TOKEN in json.dumps(params or {}) or TOKEN in path: + raise AssertionError("token leaked into url") + if self.error: + raise self.error + return self.response + + +def flags(**enabled: tuple[bool, bool]) -> DatahubSettings: + datasets = {name: DatasetFlags(name) for name in DATASETS} + for name, pair in enabled.items(): + datasets[name] = DatasetFlags(name, read=pair[0], shadow=pair[1]) + return DatahubSettings(base_url="http://127.0.0.1:9", token=TOKEN, datasets=datasets) + + +class DatahubBridgeTests(unittest.TestCase): + def test_default_config_keeps_legacy_and_does_not_call_datahub(self) -> None: + settings = DatahubSettings.load(environ={}, credentials={}) + self.assertFalse(settings.any_enabled()) + self.assertTrue(all(not settings.flags(name).read and not settings.flags(name).shadow for name in DATASETS)) + client = FakeClient(error=DatahubError("INTERNAL", "should not be called")) + legacy = FakeLegacy([LEGACY_DAILY]) + wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, client)) + rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,vol,amount") + self.assertEqual(rows[0]["amount"], 2000.0) + self.assertEqual(client.paths, []) + self.assertEqual(len(legacy.calls), 1) + + def test_each_dataset_has_independent_read_flag(self) -> None: + settings = flags(daily=(True, False), auction=(False, False)) + self.assertTrue(settings.flags("daily").read) + self.assertFalse(settings.flags("auction").read) + self.assertFalse(any(settings.flags(name).read for name in DATASETS if name != "daily")) + source = (ROOT / "config" / "datahub.config.json").read_text(encoding="utf-8") + self.assertNotIn("master", source) + self.assertNotIn("DATAHUB_READ_ALL", source) + + def test_read_flag_replaces_only_that_dataset_and_converts_units(self) -> None: + shadows: list[dict[str, Any]] = [] + client = FakeClient() + legacy = FakeLegacy([LEGACY_DAILY]) + wrapped = DatahubAwareTushareClient( + legacy, + DatahubBridge(flags(daily=(True, False)), client, shadow_sink=shadows.append), + ) + rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,vol,amount") + self.assertEqual(rows[0]["vol"], 1000.0) + self.assertEqual(rows[0]["amount"], 2000.0) + self.assertEqual(legacy.calls, []) + self.assertEqual(client.paths, ["/v1/bars/daily"]) + calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}]) + calendar_client = FakeClient(error=DatahubError("INTERNAL", "nope")) + calendar_wrapped = DatahubAwareTushareClient( + calendar_legacy, + DatahubBridge(flags(daily=(True, False)), calendar_client), + ) + calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "") + self.assertEqual(calendar[0]["is_open"], 1) + self.assertEqual(calendar_client.paths, []) + + def test_fallback_on_down_401_timeout_empty_unpublished_and_stale(self) -> None: + cases = [ + DatahubError("UNAVAILABLE", "down"), + DatahubError("UNAUTHORIZED", "401"), + DatahubError("TIMEOUT", "late"), + DatahubError("EMPTY", "no rows"), + DatahubError("DATASET_NOT_PUBLISHED", "not ready"), + DatahubError("STALE", "old"), + ] + for error in cases: + with self.subTest(error=error.code): + if error.code == "EMPTY": + client = FakeClient(response=DatahubResponse(data=[], meta={"stale": False, "staleness_seconds": 0})) + elif error.code == "STALE": + client = FakeClient(response=DatahubResponse( + data=[dict(HUB_DAILY)], + meta={"stale": True, "staleness_seconds": 999999}, + )) + else: + client = FakeClient(error=error) + legacy = FakeLegacy([LEGACY_DAILY]) + wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client)) + rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount") + self.assertEqual(rows[0]["amount"], 2000.0) + self.assertEqual(len(legacy.calls), 1) + + def test_shadow_compares_without_replacing_and_survives_hub_failure(self) -> None: + reports: list[dict[str, Any]] = [] + client = FakeClient() + legacy = FakeLegacy([LEGACY_DAILY]) + wrapped = DatahubAwareTushareClient( + legacy, + DatahubBridge(flags(daily=(False, True)), client, shadow_sink=reports.append), + ) + rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol") + self.assertEqual(rows[0]["amount"], 2000.0) + self.assertEqual(len(legacy.calls), 1) + self.assertEqual(reports[0]["equal"], True) + self.assertEqual(reports[0]["matched"], 1) + + failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN)) + fail_reports: list[dict[str, Any]] = [] + fail_legacy = FakeLegacy([LEGACY_DAILY]) + fail_wrapped = DatahubAwareTushareClient( + fail_legacy, + DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=fail_reports.append), + ) + again = fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount") + self.assertEqual(again[0]["amount"], 2000.0) + self.assertTrue(fail_reports[0]["hub_error"]) + self.assertNotIn(TOKEN, json.dumps(fail_reports[0])) + + def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None: + equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0}) + self.assertTrue(equal["equal"]) + unit = compare_rows("daily", [LEGACY_DAILY], [{**HUB_DAILY, "amount": 2000.0, "volume": 1000.0}]) + self.assertGreater(unit["unit_conversion_count"], 0) + missing = compare_rows("daily", [LEGACY_DAILY], []) + self.assertEqual(missing["missing_hub_count"], 1) + value = compare_rows("daily", [LEGACY_DAILY], [{**HUB_DAILY, "close": 99.0}]) + self.assertEqual(value["value_diff_count"], 1) + skew = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 12}) + self.assertTrue(skew["time_skew"]) + + def test_native_roundtrip_matches_known_scales(self) -> None: + native = to_native_row("daily", HUB_DAILY) + self.assertEqual(native["vol"], 1000.0) + self.assertEqual(native["amount"], 2000.0) + canonical = to_canonical_row("daily", native) + self.assertEqual(canonical["vol"], 100000.0) + self.assertEqual(canonical["amount"], 2000000.0) + + def test_heaven_keeps_legacy_even_when_read_flag_is_on(self) -> None: + self.assertTrue(looks_like_heaven("backend.features.heaven.market_context", "backend/features/heaven/market_context.py")) + self.assertFalse(looks_like_heaven("backend.features.market.service", "backend/features/market/service.py")) + client = FakeClient() + legacy = FakeLegacy([LEGACY_DAILY]) + wrapped = DatahubAwareTushareClient( + legacy, + DatahubBridge(flags(daily=(True, False)), client, heaven_guard=lambda: True), + ) + rows = wrapped.query("daily", {"trade_date": "20240902"}, "amount") + self.assertEqual(rows[0]["amount"], 2000.0) + self.assertEqual(client.paths, []) + + def test_status_flag_does_not_run_when_off_and_falls_back_when_on(self) -> None: + off = DatahubBridge(flags(), FakeClient(error=DatahubError("UNAVAILABLE", "down"))) + self.assertIsNone(off.dataset_status("20240902")) + reports: list[dict[str, Any]] = [] + failed = DatahubBridge( + flags(status=(True, True)), + FakeClient(error=DatahubError("UNAUTHORIZED", "nope")), + shadow_sink=reports.append, + ) + self.assertIsNone(failed.dataset_status("20240902")) + self.assertTrue(reports[0]["hub_error"]) + ok = DatahubBridge( + flags(status=(True, False)), + FakeClient(response=DatahubResponse(data=[{"dataset": "daily", "state": "published"}], meta={"stale": False, "staleness_seconds": 0})), + ) + self.assertEqual(ok.dataset_status("20240902")[0]["state"], "published") + + def test_default_gateway_wraps_tushare_without_calling_datahub(self) -> None: + gateway = build_data_gateway({}, datahub_settings=flags()) + client = gateway.tushare() + self.assertIsInstance(client, DatahubAwareTushareClient) + self.assertFalse(gateway.datahub.settings.any_enabled()) + + def test_features_do_not_import_datahub_client(self) -> None: + violations = [] + for path in (ROOT / "backend" / "features").rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + names = [] + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + names = [node.module] + for name in names: + if "datahub" in name.split("."): + violations.append(f"{path.relative_to(ROOT)} -> {name}") + self.assertEqual(violations, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_datahub_client.py b/tests/test_datahub_client.py new file mode 100644 index 0000000..06493e3 --- /dev/null +++ b/tests/test_datahub_client.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import io +import json +import logging +import threading +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from urllib.parse import parse_qs, urlparse + +from backend.data.datahub.client import DatahubClient +from backend.data.datahub.errors import DatahubError +from backend.data.datahub.redact import redact_text +from backend.data.datahub.settings import DatahubSettings + + +TOKEN = "super-secret-datahub-token" + + +class FakeHubState: + def __init__(self) -> None: + self.mode = "ok" + self.hits = 0 + self.paths: list[str] = [] + + +STATE = FakeHubState() + + +class FakeHubHandler(BaseHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: + return + + def do_GET(self) -> None: # noqa: N802 + STATE.hits += 1 + parsed = urlparse(self.path) + STATE.paths.append(parsed.path) + token = self.headers.get("X-Datahub-Token", "") + if STATE.mode == "timeout": + raise TimeoutError("simulated timeout") + if token != TOKEN: + self._json(401, {"error": {"code": "UNAUTHORIZED", "message": "missing or invalid X-Datahub-Token"}}) + return + if STATE.mode == "unpublished": + self._json(404, {"error": {"code": "DATASET_NOT_PUBLISHED", "message": "daily 19990101 尚未发布", "expected_at": "15:05+08:00"}}) + return + if STATE.mode == "empty": + self._json(200, {"schema_version": 1, "data": [], "meta": {"tier": "official", "stale": False, "staleness_seconds": 0}}) + return + if STATE.mode == "stale": + self._json(200, {"schema_version": 1, "data": [{"ts_code": "600000.SH", "trade_date": "20240902", "close": 10.2, "volume": 100000, "amount": 2000000}], "meta": {"tier": "official", "stale": True, "staleness_seconds": 999999}}) + return + if STATE.mode == "invalid": + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(b"not-json") + return + if parsed.path == "/v1/health": + self._json(200, {"schema_version": 1, "data": {"status": "ok"}, "meta": {"tier": "official", "source": "datahub", "stale": False, "staleness_seconds": 0}}) + return + if parsed.path == "/v1/calendar": + self._json(200, {"schema_version": 1, "data": [{"cal_date": "20240902", "is_open": True, "pretrade_date": "20240830"}], "meta": {"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0}}) + return + if parsed.path == "/v1/bars/daily": + query = {key: values[-1] for key, values in parse_qs(parsed.query).items()} + self._json(200, { + "schema_version": 1, + "data": [{ + "ts_code": "600000.SH", + "trade_date": query.get("date") or "20240902", + "open": 10.11, "high": 10.25, "low": 10.01, "close": 10.20, + "pct_chg": 1.2345, "volume": 100000.0, "amount": 2000000.0, "adj_factor": 1.1, + }], + "meta": {"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0, "source": "tushare:daily"}, + }) + return + if parsed.path == "/v1/datasets/status": + self._json(200, {"schema_version": 1, "data": [{"dataset": "daily", "state": "published", "trade_date": "20240902"}], "meta": {"tier": "official", "stale": False, "staleness_seconds": 0}}) + return + self._json(400, {"error": {"code": "INVALID_ARGUMENT", "message": f"unknown endpoint: {parsed.path}"}}) + + def _json(self, status: int, payload: dict) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + +class DatahubClientTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.server = ThreadingHTTPServer(("127.0.0.1", 0), FakeHubHandler) + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + cls.base = f"http://127.0.0.1:{cls.server.server_address[1]}" + + @classmethod + def tearDownClass(cls) -> None: + cls.server.shutdown() + cls.server.server_close() + + def setUp(self) -> None: + STATE.mode = "ok" + STATE.hits = 0 + STATE.paths = [] + self.client = DatahubClient(DatahubSettings(base_url=self.base, token=TOKEN, retries=1, timeout_seconds=2)) + + def test_health_envelope(self) -> None: + response = self.client.health() + self.assertEqual(response.schema_version, 1) + self.assertEqual(response.data["status"], "ok") + self.assertIn("stale", response.meta) + + def test_missing_and_bad_token_401(self) -> None: + missing = DatahubClient(DatahubSettings(base_url=self.base, token="")) + with self.assertRaises(DatahubError) as raised: + missing.health() + self.assertEqual(raised.exception.code, "NOT_CONFIGURED") + bad = DatahubClient(DatahubSettings(base_url=self.base, token="wrong")) + with self.assertRaises(DatahubError) as raised: + bad.health() + self.assertEqual(raised.exception.code, "UNAUTHORIZED") + self.assertNotIn(TOKEN, str(raised.exception)) + + def test_unpublished_and_empty_and_stale_codes(self) -> None: + STATE.mode = "unpublished" + with self.assertRaises(DatahubError) as raised: + self.client.daily_bars(date="19990101") + self.assertEqual(raised.exception.code, "DATASET_NOT_PUBLISHED") + STATE.mode = "empty" + response = self.client.daily_bars(date="20240902") + self.assertEqual(response.data, []) + STATE.mode = "stale" + stale = self.client.daily_bars(date="20240902") + self.assertTrue(stale.meta["stale"]) + + def test_invalid_json_maps_to_internal(self) -> None: + STATE.mode = "invalid" + with self.assertRaises(DatahubError) as raised: + self.client.health() + self.assertEqual(raised.exception.code, "INTERNAL") + + def test_timeout_maps_and_retries(self) -> None: + hits = {"n": 0} + + def boom(_request, timeout=None): + hits["n"] += 1 + raise TimeoutError("late") + + client = DatahubClient( + DatahubSettings(base_url=self.base, token=TOKEN, retries=1, timeout_seconds=1), + urlopen=boom, + ) + with self.assertRaises(DatahubError) as raised: + client.health() + self.assertEqual(raised.exception.code, "TIMEOUT") + self.assertEqual(hits["n"], 2) + + def test_token_never_appears_in_error_text_or_logs(self) -> None: + stream = io.StringIO() + logger = logging.getLogger("xiaobai.datahub") + handler = logging.StreamHandler(stream) + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + try: + with self.assertRaises(DatahubError): + DatahubClient(DatahubSettings(base_url=self.base, token="wrong")).health() + blob = stream.getvalue() + redact_text("header " + TOKEN, (TOKEN,)) + self.assertNotIn(TOKEN, blob) + self.assertIn("***", redact_text(TOKEN, (TOKEN,))) + finally: + logger.removeHandler(handler) + + def test_calendar_and_status_contract(self) -> None: + calendar = self.client.calendar("20240901", "20240902") + self.assertEqual(calendar.data[0]["cal_date"], "20240902") + status = self.client.dataset_status("20240902") + self.assertEqual(status.data[0]["dataset"], "daily") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_architecture_inventory.py b/tools/build_architecture_inventory.py index 2bd5c75..32c4bee 100644 --- a/tools/build_architecture_inventory.py +++ b/tools/build_architecture_inventory.py @@ -217,6 +217,7 @@ def build() -> dict[str, Any]: ), "external_data_adapters": [ {"provider": "tushare", "path": "backend/data/providers/tushare_client.py", "runtime_role": "stable client facade for primary deterministic market data"}, + {"provider": "datahub", "path": "backend/data/datahub/client.py", "runtime_role": "optional official EOD read path behind per-dataset flags"}, {"provider": "ifind", "path": "backend/data/providers/ifind_client.py", "runtime_role": "realtime, charts, snapshots, enrichment"}, {"provider": "eastmoney", "path": "backend/features/market/charts.py", "runtime_role": "display chart fallback"}, {"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"}, @@ -235,6 +236,9 @@ def build() -> dict[str, Any]: ], "provider_construction": [ {"client": "TushareClient", "owner": "backend/data/providers/tushare.py", "compatibility_fallback": "backend/features/market/service.py"}, + {"client": "DatahubClient", "owner": "backend/data/gateway.py"}, + {"client": "DatahubBridge", "owner": "backend/data/gateway.py"}, + {"client": "DatahubAwareTushareClient", "owner": "backend/data/gateway.py"}, {"client": "IfindHttpClient", "owner": "backend/data/gateway.py"}, {"client": "MarketChartClient", "owner": "backend/data/gateway.py"}, {"client": "WebRealtimeAggregator", "owner": "backend/data/gateway.py"},