feat(HEL-402): 接通网站首批只读 datahub 并建立双路对比
默认全部读取/影子开关关闭，网站继续走旧 Tushare 链路；开启单项时只替换该类原料并在失败时回旧，问天保持旧路径。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
f5dc0f8076
commit
0d13066386
@@ -5,6 +5,11 @@ APP_ENCRYPTION_KEY=
|
||||
# the system settings; all accounts use the same backend market snapshot.
|
||||
TUSHARE_TOKEN=your_tushare_token_here
|
||||
|
||||
# Optional xiaobai-datahub client. All DATAHUB_READ_* / DATAHUB_SHADOW_* flags
|
||||
# default off in config/datahub.config.json, so the website keeps using Tushare.
|
||||
DATAHUB_BASE_URL=http://127.0.0.1:8766
|
||||
DATAHUB_TOKEN=
|
||||
|
||||
# Optional iFinD HTTP credential. The backend exchanges it for a short-lived
|
||||
# access token and never exposes either token to browsers.
|
||||
IFIND_REFRESH_TOKEN=your_ifind_refresh_token_here
|
||||
|
||||
@@ -11,6 +11,8 @@ from backend.features.accounts.security import SecretVault
|
||||
def environment_credentials(environment: Mapping[str, str]) -> dict[str, str]:
|
||||
return {
|
||||
"tushare_token": str(environment.get("TUSHARE_TOKEN") or "").strip(),
|
||||
"datahub_token": str(environment.get("DATAHUB_TOKEN") or "").strip(),
|
||||
"datahub_base_url": str(environment.get("DATAHUB_BASE_URL") or "").strip(),
|
||||
"ifind_refresh_token": str(environment.get("IFIND_REFRESH_TOKEN") or "").strip(),
|
||||
"ifind_access_token": str(environment.get("IFIND_ACCESS_TOKEN") or "").strip(),
|
||||
"platform_llm_primary_api_key": str(
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge
|
||||
from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
|
||||
__all__ = [
|
||||
"DATASETS",
|
||||
"DatahubAwareTushareClient",
|
||||
"DatahubBridge",
|
||||
"DatahubClient",
|
||||
"DatahubError",
|
||||
"DatahubResponse",
|
||||
"DatahubSettings",
|
||||
"DatasetFlags",
|
||||
]
|
||||
@@ -0,0 +1,249 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any, Callable
|
||||
|
||||
from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.compare import compare_rows
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.native import (
|
||||
API_TO_DATASET,
|
||||
filter_calendar_rows,
|
||||
filter_stock_rows,
|
||||
project_fields,
|
||||
to_native_rows,
|
||||
yyyymmdd,
|
||||
)
|
||||
from backend.data.datahub.redact import redact_text, redact_value
|
||||
from backend.data.datahub.settings import DatahubSettings
|
||||
from backend.data.providers.tushare_client import TushareClient
|
||||
|
||||
LOGGER = logging.getLogger("xiaobai.datahub")
|
||||
ShadowSink = Callable[[dict[str, Any]], None]
|
||||
EMPTY_FAIL_DATASETS = {"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction"}
|
||||
|
||||
|
||||
def looks_like_heaven(module_name: str, filename: str = "") -> bool:
|
||||
path = filename.replace("\\", "/")
|
||||
return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path
|
||||
|
||||
|
||||
def caller_is_heaven(depth: int = 24) -> bool:
|
||||
frame = sys._getframe(1)
|
||||
for _ in range(depth):
|
||||
frame = frame.f_back if frame is not None else None
|
||||
if frame is None:
|
||||
return False
|
||||
name = str(frame.f_globals.get("__name__") or "")
|
||||
filename = str(frame.f_code.co_filename or "")
|
||||
if looks_like_heaven(name, filename):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class DatahubBridge:
|
||||
def __init__(
|
||||
self,
|
||||
settings: DatahubSettings,
|
||||
client: DatahubClient,
|
||||
shadow_sink: ShadowSink | None = None,
|
||||
heaven_guard: Callable[[], bool] | None = None,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self.client = client
|
||||
self.shadow_sink = shadow_sink
|
||||
self.heaven_guard = heaven_guard or caller_is_heaven
|
||||
|
||||
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags("status")
|
||||
if not flags.read and not flags.shadow:
|
||||
return None
|
||||
try:
|
||||
response = self._require_fresh(self.client.dataset_status(yyyymmdd(trade_date)), "status")
|
||||
rows = list(response.data or [])
|
||||
if flags.shadow:
|
||||
self._emit_shadow(compare_rows("status", [], rows, response.meta))
|
||||
if flags.read:
|
||||
return rows
|
||||
return None
|
||||
except Exception as exc:
|
||||
self._log_failure("status", exc)
|
||||
if flags.shadow:
|
||||
self._emit_shadow(compare_rows("status", [], [], {}, self._error_text(exc)))
|
||||
return None
|
||||
|
||||
def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags("status")
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self._require_fresh(
|
||||
self.client.batches(yyyymmdd(trade_date), dataset),
|
||||
"status",
|
||||
)
|
||||
return list(response.data or [])
|
||||
except Exception as exc:
|
||||
self._log_failure("status", exc)
|
||||
return None
|
||||
|
||||
def query(
|
||||
self,
|
||||
api_name: str,
|
||||
params: dict[str, Any] | None,
|
||||
fields: str,
|
||||
legacy_query: Callable[..., list[dict[str, Any]]],
|
||||
) -> list[dict[str, Any]]:
|
||||
dataset = API_TO_DATASET.get(api_name)
|
||||
if not dataset or self.heaven_guard():
|
||||
return legacy_query(api_name, params, fields)
|
||||
flags = self.settings.flags(dataset)
|
||||
if not flags.read and not flags.shadow:
|
||||
return legacy_query(api_name, params, fields)
|
||||
|
||||
hub_rows: list[dict[str, Any]] | None = None
|
||||
hub_meta: dict[str, Any] = {}
|
||||
hub_error: str | None = None
|
||||
hub_canonical: list[dict[str, Any]] = []
|
||||
try:
|
||||
response = self._fetch_dataset(dataset, params or {})
|
||||
hub_canonical = self._extract_rows(dataset, response, params or {})
|
||||
hub_rows = to_native_rows(dataset, hub_canonical)
|
||||
hub_meta = dict(response.meta)
|
||||
self._validate_usable(dataset, hub_rows, response)
|
||||
except Exception as exc:
|
||||
hub_error = self._error_text(exc)
|
||||
self._log_failure(dataset, exc)
|
||||
|
||||
if flags.shadow:
|
||||
try:
|
||||
legacy_rows = legacy_query(api_name, params, fields)
|
||||
except Exception as exc:
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
self._emit_shadow(compare_rows(dataset, [], hub_canonical, hub_meta, self._error_text(exc)))
|
||||
return project_fields(hub_rows, fields)
|
||||
raise
|
||||
self._emit_shadow(compare_rows(dataset, legacy_rows, hub_canonical, hub_meta, hub_error))
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
return project_fields(hub_rows, fields)
|
||||
return legacy_rows
|
||||
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
return project_fields(hub_rows, fields)
|
||||
return legacy_query(api_name, params, fields)
|
||||
|
||||
def _fetch_dataset(self, dataset: str, params: dict[str, Any]) -> DatahubResponse:
|
||||
date = yyyymmdd(params.get("trade_date") or params.get("date"))
|
||||
start = yyyymmdd(params.get("start_date") or params.get("from") or date)
|
||||
end = yyyymmdd(params.get("end_date") or params.get("to") or date)
|
||||
code = str(params.get("ts_code") or params.get("code") or "").strip()
|
||||
if dataset == "calendar":
|
||||
if not start or not end:
|
||||
raise DatahubError("INVALID_ARGUMENT", "calendar requires start_date and end_date")
|
||||
return self.client.calendar(start, end)
|
||||
if dataset == "stocks":
|
||||
return self._paginate(self.client.stocks, {})
|
||||
fetchers = {
|
||||
"daily": self.client.daily_bars,
|
||||
"index_daily": self.client.index_bars,
|
||||
"valuation": self.client.valuation,
|
||||
"moneyflow": self.client.moneyflow,
|
||||
"auction": self.client.auction,
|
||||
}
|
||||
fetcher = fetchers[dataset]
|
||||
query: dict[str, Any] = {}
|
||||
if code:
|
||||
query["code"] = code
|
||||
if date and not (params.get("start_date") or params.get("end_date")):
|
||||
query["date"] = date
|
||||
else:
|
||||
if start:
|
||||
query["from"] = start
|
||||
if end:
|
||||
query["to"] = end
|
||||
if dataset == "daily":
|
||||
query["adjust"] = "none"
|
||||
return self._paginate(fetcher, query)
|
||||
|
||||
def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse:
|
||||
limit = self.settings.page_limit
|
||||
offset = 0
|
||||
rows: list[Any] = []
|
||||
meta: dict[str, Any] = {}
|
||||
schema_version = 1
|
||||
while True:
|
||||
page = fetcher(**{**params, "limit": limit, "offset": offset})
|
||||
meta = dict(page.meta)
|
||||
schema_version = page.schema_version
|
||||
data = page.data or []
|
||||
if not isinstance(data, list):
|
||||
raise DatahubError("INTERNAL", "datahub returned a non-list payload")
|
||||
rows.extend(data)
|
||||
if len(data) < limit:
|
||||
break
|
||||
offset += limit
|
||||
if offset > 200_000:
|
||||
break
|
||||
return DatahubResponse(data=rows, meta=meta, schema_version=schema_version)
|
||||
|
||||
def _extract_rows(
|
||||
self,
|
||||
dataset: str,
|
||||
response: DatahubResponse,
|
||||
params: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = [dict(item) for item in (response.data or [])]
|
||||
if dataset == "calendar":
|
||||
return filter_calendar_rows(rows, params)
|
||||
if dataset == "stocks":
|
||||
return filter_stock_rows(rows, params)
|
||||
return rows
|
||||
|
||||
def _validate_usable(self, dataset: str, rows: list[dict[str, Any]], response: DatahubResponse) -> None:
|
||||
meta = response.meta or {}
|
||||
stale_seconds = int(meta.get("staleness_seconds") or 0)
|
||||
if meta.get("stale") or stale_seconds > self.settings.stale_seconds_max:
|
||||
raise DatahubError("STALE", f"{dataset} data is stale")
|
||||
if dataset in EMPTY_FAIL_DATASETS and not rows:
|
||||
raise DatahubError("EMPTY", f"{dataset} returned no rows")
|
||||
|
||||
def _require_fresh(self, response: DatahubResponse, dataset: str) -> DatahubResponse:
|
||||
self._validate_usable(dataset, list(response.data or []) if isinstance(response.data, list) else [], response)
|
||||
return response
|
||||
|
||||
def _emit_shadow(self, report: dict[str, Any]) -> None:
|
||||
safe = redact_value(report, secrets=self.settings.secrets())
|
||||
LOGGER.info("datahub shadow %s", safe)
|
||||
if self.shadow_sink is not None:
|
||||
self.shadow_sink(report)
|
||||
|
||||
def _log_failure(self, dataset: str, exc: Exception) -> None:
|
||||
LOGGER.warning(
|
||||
"datahub fallback dataset=%s error=%s",
|
||||
dataset,
|
||||
redact_text(self._error_text(exc), self.settings.secrets()),
|
||||
)
|
||||
|
||||
def _error_text(self, exc: Exception) -> str:
|
||||
if isinstance(exc, DatahubError):
|
||||
text = f"{exc.code}: {exc.message}"
|
||||
else:
|
||||
text = str(exc)
|
||||
return redact_text(text, self.settings.secrets())
|
||||
|
||||
|
||||
class DatahubAwareTushareClient:
|
||||
def __init__(self, legacy: TushareClient, bridge: DatahubBridge) -> None:
|
||||
self._legacy = legacy
|
||||
self._bridge = bridge
|
||||
|
||||
def query(
|
||||
self,
|
||||
api_name: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
fields: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._bridge.query(api_name, params, fields, self._legacy.query)
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self._legacy, name)
|
||||
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.redact import redact_text
|
||||
from backend.data.datahub.settings import DatahubSettings
|
||||
|
||||
LOGGER = logging.getLogger("xiaobai.datahub")
|
||||
UrlOpen = Callable[..., Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatahubResponse:
|
||||
data: Any
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
schema_version: int = 1
|
||||
status: int = 200
|
||||
|
||||
|
||||
class DatahubClient:
|
||||
def __init__(
|
||||
self,
|
||||
settings: DatahubSettings,
|
||||
urlopen: UrlOpen = urllib.request.urlopen,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self._urlopen = urlopen
|
||||
|
||||
def health(self) -> DatahubResponse:
|
||||
return self.get("/v1/health")
|
||||
|
||||
def calendar(self, start: str, end: str) -> DatahubResponse:
|
||||
return self.get("/v1/calendar", {"from": start, "to": end})
|
||||
|
||||
def stocks(self, updated_since: str = "", limit: int | None = None, offset: int = 0) -> DatahubResponse:
|
||||
params: dict[str, Any] = {"offset": offset, "limit": limit or self.settings.page_limit}
|
||||
if updated_since:
|
||||
params["updated_since"] = updated_since
|
||||
return self.get("/v1/stocks", params)
|
||||
|
||||
def daily_bars(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/bars/daily", params)
|
||||
|
||||
def index_bars(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/indexes/bars", params)
|
||||
|
||||
def valuation(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/valuation", params)
|
||||
|
||||
def moneyflow(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/moneyflow", params)
|
||||
|
||||
def auction(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/auction", params)
|
||||
|
||||
def dataset_status(self, date: str) -> DatahubResponse:
|
||||
return self.get("/v1/datasets/status", {"date": date})
|
||||
|
||||
def batches(self, date: str, dataset: str = "") -> DatahubResponse:
|
||||
params: dict[str, Any] = {"date": date}
|
||||
if dataset:
|
||||
params["dataset"] = dataset
|
||||
return self.get("/v1/batches", params)
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
if not self.settings.token:
|
||||
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
||||
query = {
|
||||
key: value
|
||||
for key, value in (params or {}).items()
|
||||
if value is not None and value != ""
|
||||
}
|
||||
url = self.settings.base_url + path
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
attempts = 1 + max(0, self.settings.retries)
|
||||
last_error: DatahubError | None = None
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return self._request(url)
|
||||
except DatahubError as exc:
|
||||
last_error = exc
|
||||
if exc.code not in {"TIMEOUT", "UNAVAILABLE"} or attempt + 1 >= attempts:
|
||||
raise
|
||||
LOGGER.warning(
|
||||
"datahub retry %s/%s %s",
|
||||
attempt + 1,
|
||||
attempts,
|
||||
redact_text(str(exc), self.settings.secrets()),
|
||||
)
|
||||
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
||||
|
||||
def _request(self, url: str) -> DatahubResponse:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"X-Datahub-Token": self.settings.token,
|
||||
"User-Agent": "XiaobaiReviewDatahub/1.0",
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with self._urlopen(request, timeout=self.settings.timeout_seconds) as response:
|
||||
status = int(getattr(response, "status", 200) or 200)
|
||||
raw = response.read().decode("utf-8")
|
||||
except TimeoutError as exc:
|
||||
raise DatahubError("TIMEOUT", "datahub request timed out") from exc
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = _read_error_body(exc)
|
||||
raise _http_error(exc.code, body, self.settings.secrets()) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
reason = redact_text(str(getattr(exc, "reason", exc)), self.settings.secrets())
|
||||
if "timed out" in reason.lower():
|
||||
raise DatahubError("TIMEOUT", "datahub request timed out") from exc
|
||||
raise DatahubError("UNAVAILABLE", f"datahub unavailable: {reason}") from exc
|
||||
payload = _parse_json(raw, self.settings.secrets())
|
||||
return _as_response(payload, status, self.settings.secrets())
|
||||
|
||||
|
||||
def _parse_json(raw: str, secrets: tuple[str, ...]) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DatahubError("INTERNAL", "datahub returned invalid json") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise DatahubError("INTERNAL", "datahub returned a non-object payload")
|
||||
return payload
|
||||
|
||||
|
||||
def _as_response(payload: dict[str, Any], status: int, secrets: tuple[str, ...]) -> DatahubResponse:
|
||||
error = payload.get("error")
|
||||
if isinstance(error, dict):
|
||||
raise _mapped_error(str(error.get("code") or "INTERNAL"), str(error.get("message") or "datahub error"), status)
|
||||
if status >= 400:
|
||||
raise DatahubError("UNAVAILABLE", f"datahub http {status}", status)
|
||||
return DatahubResponse(
|
||||
data=payload.get("data"),
|
||||
meta=dict(payload.get("meta") or {}),
|
||||
schema_version=int(payload.get("schema_version") or 1),
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def _http_error(status: int, payload: dict[str, Any], secrets: tuple[str, ...]) -> DatahubError:
|
||||
error = payload.get("error") if isinstance(payload.get("error"), dict) else {}
|
||||
code = str((error or {}).get("code") or "")
|
||||
message = str((error or {}).get("message") or payload.get("message") or f"datahub http {status}")
|
||||
message = redact_text(message, secrets)
|
||||
if status == 401 or code == "UNAUTHORIZED":
|
||||
return DatahubError("UNAUTHORIZED", message, status)
|
||||
if status == 404 or code == "DATASET_NOT_PUBLISHED":
|
||||
return DatahubError("DATASET_NOT_PUBLISHED", message, status)
|
||||
if status == 400 or code == "INVALID_ARGUMENT":
|
||||
return DatahubError("INVALID_ARGUMENT", message, status)
|
||||
if status in {429, 503} or code in {"RATE_LIMITED", "SOURCE_UNAVAILABLE"}:
|
||||
return DatahubError("UNAVAILABLE", message, status)
|
||||
return DatahubError(code or "INTERNAL", message, status)
|
||||
|
||||
|
||||
def _mapped_error(code: str, message: str, status: int) -> DatahubError:
|
||||
if code == "STALE_DATA":
|
||||
return DatahubError("STALE", message, status)
|
||||
if code in {"UNAUTHORIZED", "DATASET_NOT_PUBLISHED", "INVALID_ARGUMENT"}:
|
||||
return DatahubError(code, message, status)
|
||||
if code in {"RATE_LIMITED", "SOURCE_UNAVAILABLE"}:
|
||||
return DatahubError("UNAVAILABLE", message, status)
|
||||
return DatahubError(code or "INTERNAL", message, status)
|
||||
|
||||
|
||||
def _read_error_body(exc: urllib.error.HTTPError) -> dict[str, Any]:
|
||||
try:
|
||||
raw = exc.read().decode("utf-8")
|
||||
payload = json.loads(raw)
|
||||
return payload if isinstance(payload, dict) else {"message": raw}
|
||||
except Exception:
|
||||
return {"message": str(exc)}
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend.data.datahub.native import SCALE_FIELDS, row_key, to_canonical_row, yyyymmdd
|
||||
|
||||
NUMERIC_TOLERANCE = 1e-4
|
||||
|
||||
|
||||
def compare_rows(
|
||||
dataset: str,
|
||||
legacy_rows: list[dict[str, Any]],
|
||||
hub_rows: list[dict[str, Any]] | None,
|
||||
hub_meta: dict[str, Any] | None = None,
|
||||
hub_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
hub = hub_rows or []
|
||||
legacy_map = {row_key(dataset, row): row for row in legacy_rows}
|
||||
hub_map = {row_key(dataset, _align_hub_row(row)): row for row in hub}
|
||||
missing_hub = sorted(key for key in legacy_map if key not in hub_map)
|
||||
missing_legacy = sorted(key for key in hub_map if key not in legacy_map)
|
||||
value_diffs: list[dict[str, Any]] = []
|
||||
unit_conversion: list[dict[str, Any]] = []
|
||||
matched = 0
|
||||
for key, legacy in legacy_map.items():
|
||||
hub_row = hub_map.get(key)
|
||||
if hub_row is None:
|
||||
continue
|
||||
field_report = _compare_fields(dataset, legacy, hub_row)
|
||||
if field_report["unit_conversion"]:
|
||||
unit_conversion.append({"key": list(key), "fields": field_report["unit_conversion"]})
|
||||
if field_report["value_diff"]:
|
||||
value_diffs.append({"key": list(key), "fields": field_report["value_diff"]})
|
||||
if not field_report["unit_conversion"] and not field_report["value_diff"]:
|
||||
matched += 1
|
||||
stale_seconds = int((hub_meta or {}).get("staleness_seconds") or 0)
|
||||
time_skew = bool((hub_meta or {}).get("stale")) or stale_seconds > 0
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"legacy_rows": len(legacy_rows),
|
||||
"hub_rows": len(hub),
|
||||
"matched": matched,
|
||||
"missing_hub": [list(item) for item in missing_hub[:20]],
|
||||
"missing_legacy": [list(item) for item in missing_legacy[:20]],
|
||||
"missing_hub_count": len(missing_hub),
|
||||
"missing_legacy_count": len(missing_legacy),
|
||||
"value_diff_count": len(value_diffs),
|
||||
"unit_conversion_count": len(unit_conversion),
|
||||
"value_diffs": value_diffs[:20],
|
||||
"unit_conversion": unit_conversion[:20],
|
||||
"time_skew": time_skew,
|
||||
"staleness_seconds": stale_seconds,
|
||||
"published_at": (hub_meta or {}).get("published_at"),
|
||||
"trade_date": yyyymmdd((hub_meta or {}).get("trade_date")),
|
||||
"hub_error": hub_error,
|
||||
"equal": (
|
||||
not hub_error
|
||||
and not missing_hub
|
||||
and not missing_legacy
|
||||
and not value_diffs
|
||||
and not unit_conversion
|
||||
and not time_skew
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _align_hub_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
aligned = dict(row)
|
||||
if "volume" in aligned and "vol" not in aligned:
|
||||
aligned["vol"] = aligned.get("volume")
|
||||
return aligned
|
||||
|
||||
|
||||
def _compare_fields(dataset: str, legacy: dict[str, Any], hub: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
||||
canonical_legacy = to_canonical_row(dataset, legacy)
|
||||
hub_canonical = _hub_canonical(dataset, hub)
|
||||
native_hub = _align_hub_row(hub)
|
||||
value_diff: list[dict[str, Any]] = []
|
||||
unit_conversion: list[dict[str, Any]] = []
|
||||
keys = (set(canonical_legacy) | set(hub_canonical)) - {"batch_id", "updated_at", "volume"}
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field in sorted(keys):
|
||||
left = canonical_legacy.get(field)
|
||||
right = hub_canonical.get(field)
|
||||
if _same(left, right):
|
||||
continue
|
||||
native_left = legacy.get(field)
|
||||
hub_raw = native_hub.get(field)
|
||||
if field in scales and _near(_optional(native_left), _optional(hub_raw)):
|
||||
unit_conversion.append(
|
||||
{"field": field, "legacy": native_left, "hub": hub_raw, "reason": "unit_conversion"}
|
||||
)
|
||||
continue
|
||||
value_diff.append({"field": field, "legacy": left, "hub": right, "reason": "value_diff"})
|
||||
return {"value_diff": value_diff, "unit_conversion": unit_conversion}
|
||||
|
||||
|
||||
def _hub_canonical(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Hub API rows are already canonical; only align field names."""
|
||||
aligned = dict(row)
|
||||
if "volume" in aligned and "vol" not in aligned:
|
||||
aligned["vol"] = aligned.get("volume")
|
||||
if dataset == "calendar":
|
||||
is_open = aligned.get("is_open")
|
||||
aligned["is_open"] = 1 if is_open in (True, 1, "1", "Y", "y") else 0
|
||||
aligned["cal_date"] = yyyymmdd(aligned.get("cal_date"))
|
||||
aligned["pretrade_date"] = yyyymmdd(aligned.get("pretrade_date")) or None
|
||||
aligned["exchange"] = str(aligned.get("exchange") or "SSE")
|
||||
return aligned
|
||||
|
||||
|
||||
def _same(left: Any, right: Any) -> bool:
|
||||
if left in (None, "") and right in (None, ""):
|
||||
return True
|
||||
if isinstance(left, (int, float)) or isinstance(right, (int, float)):
|
||||
return _near(_optional(left), _optional(right))
|
||||
return str(left or "") == str(right or "")
|
||||
|
||||
|
||||
def _near(left: float | None, right: float | None) -> bool:
|
||||
if left is None and right is None:
|
||||
return True
|
||||
if left is None or right is None:
|
||||
return False
|
||||
return abs(left - right) <= max(NUMERIC_TOLERANCE, abs(left) * 1e-9, abs(right) * 1e-9)
|
||||
|
||||
|
||||
def _optional(value: Any) -> float | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class DatahubError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, status: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status = status
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.code}: {self.message}"
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend.data.numbers import finite_number
|
||||
|
||||
|
||||
AMOUNT_THOUSAND_YUAN = 1000.0
|
||||
AMOUNT_WAN_YUAN = 10000.0
|
||||
VOLUME_LOT = 100.0
|
||||
|
||||
API_TO_DATASET = {
|
||||
"trade_cal": "calendar",
|
||||
"stock_basic": "stocks",
|
||||
"daily": "daily",
|
||||
"daily_basic": "valuation",
|
||||
"index_daily": "index_daily",
|
||||
"moneyflow": "moneyflow",
|
||||
"stk_auction": "auction",
|
||||
}
|
||||
|
||||
SCALE_FIELDS = {
|
||||
"daily": {"vol": VOLUME_LOT, "amount": AMOUNT_THOUSAND_YUAN},
|
||||
"index_daily": {"vol": VOLUME_LOT, "amount": AMOUNT_THOUSAND_YUAN},
|
||||
"valuation": {"total_mv": AMOUNT_WAN_YUAN, "circ_mv": AMOUNT_WAN_YUAN},
|
||||
"moneyflow": {
|
||||
"buy_sm_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_sm_amount": AMOUNT_WAN_YUAN,
|
||||
"buy_md_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_md_amount": AMOUNT_WAN_YUAN,
|
||||
"buy_lg_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_lg_amount": AMOUNT_WAN_YUAN,
|
||||
"buy_elg_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_elg_amount": AMOUNT_WAN_YUAN,
|
||||
"net_mf_amount": AMOUNT_WAN_YUAN,
|
||||
},
|
||||
"auction": {"vol": VOLUME_LOT, "float_share": AMOUNT_WAN_YUAN},
|
||||
}
|
||||
|
||||
|
||||
def yyyymmdd(value: Any) -> str:
|
||||
return str(value or "").replace("-", "")[:8]
|
||||
|
||||
|
||||
def to_native_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [to_native_row(dataset, row) for row in rows]
|
||||
|
||||
|
||||
def to_native_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
if dataset == "calendar":
|
||||
is_open = row.get("is_open")
|
||||
return {
|
||||
"exchange": str(row.get("exchange") or "SSE"),
|
||||
"cal_date": yyyymmdd(row.get("cal_date")),
|
||||
"is_open": 1 if is_open in (True, 1, "1", "Y", "y") else 0,
|
||||
"pretrade_date": yyyymmdd(row.get("pretrade_date")) or None,
|
||||
}
|
||||
converted = dict(row)
|
||||
converted.pop("batch_id", None)
|
||||
if "volume" in converted and "vol" not in converted:
|
||||
converted["vol"] = converted.pop("volume")
|
||||
elif "volume" in converted:
|
||||
converted.pop("volume", None)
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field, factor in scales.items():
|
||||
if field in converted:
|
||||
converted[field] = _unscale(converted.get(field), factor)
|
||||
if dataset == "stocks":
|
||||
converted.pop("updated_at", None)
|
||||
return converted
|
||||
|
||||
|
||||
def to_canonical_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
if dataset == "calendar":
|
||||
is_open = row.get("is_open")
|
||||
return {
|
||||
"exchange": str(row.get("exchange") or "SSE"),
|
||||
"cal_date": yyyymmdd(row.get("cal_date")),
|
||||
"is_open": 1 if is_open in (True, 1, "1", "Y", "y") else 0,
|
||||
"pretrade_date": yyyymmdd(row.get("pretrade_date")) or None,
|
||||
}
|
||||
converted = dict(row)
|
||||
if "volume" in converted and "vol" not in converted:
|
||||
converted["vol"] = converted.pop("volume")
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field, factor in scales.items():
|
||||
if field in converted:
|
||||
converted[field] = _scale(converted.get(field), factor)
|
||||
return converted
|
||||
|
||||
|
||||
def row_key(dataset: str, row: dict[str, Any]) -> tuple[str, ...]:
|
||||
if dataset == "calendar":
|
||||
return (yyyymmdd(row.get("cal_date")),)
|
||||
if dataset == "stocks":
|
||||
return (str(row.get("ts_code") or "").upper(),)
|
||||
if dataset == "status":
|
||||
return (str(row.get("dataset") or ""), yyyymmdd(row.get("trade_date")))
|
||||
return (str(row.get("ts_code") or "").upper(), yyyymmdd(row.get("trade_date")))
|
||||
|
||||
|
||||
def project_fields(rows: list[dict[str, Any]], fields: str) -> list[dict[str, Any]]:
|
||||
keys = [item.strip() for item in str(fields or "").split(",") if item.strip()]
|
||||
if not keys:
|
||||
return rows
|
||||
return [{key: row.get(key) for key in keys} for row in rows]
|
||||
|
||||
|
||||
def filter_stock_rows(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
payload = params or {}
|
||||
ts_code = str(payload.get("ts_code") or "").strip().upper()
|
||||
status = str(payload.get("list_status") or "").strip()
|
||||
name = str(payload.get("name") or "").strip()
|
||||
filtered = rows
|
||||
if ts_code:
|
||||
filtered = [row for row in filtered if str(row.get("ts_code") or "").upper() == ts_code]
|
||||
if status:
|
||||
filtered = [row for row in filtered if str(row.get("list_status") or status) == status]
|
||||
if name:
|
||||
filtered = [row for row in filtered if name.casefold() in str(row.get("name") or "").casefold()]
|
||||
return filtered
|
||||
|
||||
|
||||
def filter_calendar_rows(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
payload = params or {}
|
||||
if payload.get("is_open") in (1, "1", True):
|
||||
return [row for row in rows if int(row.get("is_open") or 0) == 1]
|
||||
if payload.get("is_open") in (0, "0", False):
|
||||
return [row for row in rows if int(row.get("is_open") or 0) == 0]
|
||||
return rows
|
||||
|
||||
|
||||
def _scale(value: Any, factor: float) -> float | None:
|
||||
number = _optional_number(value)
|
||||
if number is None:
|
||||
return None
|
||||
return number * factor
|
||||
|
||||
|
||||
def _unscale(value: Any, factor: float) -> float | None:
|
||||
number = _optional_number(value)
|
||||
if number is None or factor == 0:
|
||||
return None
|
||||
return number / factor
|
||||
|
||||
|
||||
def _optional_number(value: Any) -> float | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
number = finite_number(value, default=float("nan"))
|
||||
if number != number:
|
||||
return None
|
||||
return number
|
||||
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
SECRET_HINTS = (
|
||||
"token",
|
||||
"password",
|
||||
"secret",
|
||||
"key",
|
||||
"authorization",
|
||||
"credential",
|
||||
"cookie",
|
||||
)
|
||||
|
||||
|
||||
def redact_value(value: Any, key: str = "", secrets: tuple[str, ...] = ()) -> Any:
|
||||
lowered = key.lower()
|
||||
if any(part in lowered for part in SECRET_HINTS):
|
||||
return "***"
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
str(item_key): redact_value(item_value, str(item_key), secrets)
|
||||
for item_key, item_value in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_value(item, key, secrets) for item in value]
|
||||
text = str(value) if value is not None and not isinstance(value, (int, float, bool)) else value
|
||||
if isinstance(text, str):
|
||||
return redact_text(text, secrets)
|
||||
return value
|
||||
|
||||
|
||||
def redact_text(text: str, secrets: tuple[str, ...] = ()) -> str:
|
||||
redacted = text
|
||||
for secret in secrets:
|
||||
if secret:
|
||||
redacted = redacted.replace(secret, "***")
|
||||
return redacted
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from backend.bootstrap.config import APP_DIR
|
||||
|
||||
|
||||
DATASETS = (
|
||||
"calendar",
|
||||
"stocks",
|
||||
"daily",
|
||||
"index_daily",
|
||||
"valuation",
|
||||
"moneyflow",
|
||||
"auction",
|
||||
"status",
|
||||
)
|
||||
|
||||
ENV_DATASET = {
|
||||
"calendar": "CALENDAR",
|
||||
"stocks": "STOCKS",
|
||||
"daily": "DAILY",
|
||||
"index_daily": "INDEX_DAILY",
|
||||
"valuation": "VALUATION",
|
||||
"moneyflow": "MONEYFLOW",
|
||||
"auction": "AUCTION",
|
||||
"status": "STATUS",
|
||||
}
|
||||
|
||||
DEFAULT_CONFIG_PATH = APP_DIR / "config" / "datahub.config.json"
|
||||
|
||||
|
||||
def _truthy(value: Any) -> bool:
|
||||
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _int(value: Any, default: int) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetFlags:
|
||||
name: str
|
||||
read: bool = False
|
||||
shadow: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatahubSettings:
|
||||
base_url: str
|
||||
token: str
|
||||
timeout_seconds: int = 8
|
||||
retries: int = 1
|
||||
page_limit: int = 5000
|
||||
stale_seconds_max: int = 86400
|
||||
datasets: dict[str, DatasetFlags] | None = None
|
||||
|
||||
def flags(self, dataset: str) -> DatasetFlags:
|
||||
mapped = self.datasets or {}
|
||||
return mapped.get(dataset) or DatasetFlags(dataset)
|
||||
|
||||
def any_enabled(self) -> bool:
|
||||
return any(item.read or item.shadow for item in (self.datasets or {}).values())
|
||||
|
||||
def secrets(self) -> tuple[str, ...]:
|
||||
return tuple(item for item in (self.token,) if item)
|
||||
|
||||
@classmethod
|
||||
def load(
|
||||
cls,
|
||||
path: Path | None = None,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
credentials: Mapping[str, object] | None = None,
|
||||
) -> "DatahubSettings":
|
||||
config_path = path or DEFAULT_CONFIG_PATH
|
||||
payload: dict[str, Any] = {}
|
||||
if config_path.is_file():
|
||||
payload = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
env = dict(os.environ if environ is None else environ)
|
||||
creds = dict(credentials or {})
|
||||
dataset_flags: dict[str, DatasetFlags] = {}
|
||||
raw_datasets = payload.get("datasets") or {}
|
||||
for name in DATASETS:
|
||||
item = raw_datasets.get(name) or {}
|
||||
env_key = ENV_DATASET[name]
|
||||
read = _truthy(env.get(f"DATAHUB_READ_{env_key}")) if f"DATAHUB_READ_{env_key}" in env else bool(item.get("read"))
|
||||
shadow = (
|
||||
_truthy(env.get(f"DATAHUB_SHADOW_{env_key}"))
|
||||
if f"DATAHUB_SHADOW_{env_key}" in env
|
||||
else bool(item.get("shadow"))
|
||||
)
|
||||
dataset_flags[name] = DatasetFlags(name, read=read, shadow=shadow)
|
||||
token = str(
|
||||
env.get("DATAHUB_TOKEN")
|
||||
or creds.get("datahub_token")
|
||||
or payload.get("token")
|
||||
or ""
|
||||
).strip()
|
||||
base_url = str(
|
||||
env.get("DATAHUB_BASE_URL")
|
||||
or creds.get("datahub_base_url")
|
||||
or payload.get("base_url")
|
||||
or "http://127.0.0.1:8766"
|
||||
).strip().rstrip("/")
|
||||
return cls(
|
||||
base_url=base_url,
|
||||
token=token,
|
||||
timeout_seconds=_int(env.get("DATAHUB_TIMEOUT") or payload.get("timeout_seconds"), 8),
|
||||
retries=max(0, _int(env.get("DATAHUB_RETRIES") or payload.get("retries"), 1)),
|
||||
page_limit=max(1, _int(payload.get("page_limit"), 5000)),
|
||||
stale_seconds_max=max(0, _int(payload.get("stale_seconds_max"), 86400)),
|
||||
datasets=dataset_flags,
|
||||
)
|
||||
+14
-1
@@ -3,8 +3,10 @@ from __future__ import annotations
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from backend.data.contracts import DataUsage
|
||||
from backend.data.datahub import DatahubAwareTushareClient, DatahubBridge, DatahubClient, DatahubSettings
|
||||
from backend.data.policy import DataSourcePolicy
|
||||
from backend.data.providers import IfindProvider, TushareProvider
|
||||
from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
|
||||
@@ -22,6 +24,7 @@ class DataGateway:
|
||||
ifind_provider: IfindProvider
|
||||
chart_data: MarketChartClient
|
||||
realtime_observer: WebRealtimeAggregator
|
||||
datahub: DatahubBridge
|
||||
|
||||
@property
|
||||
def ifind(self) -> IfindHttpClient:
|
||||
@@ -34,7 +37,13 @@ class DataGateway:
|
||||
) -> TushareClient:
|
||||
if dataset_id:
|
||||
self.policy.assert_allowed(dataset_id, "tushare", usage)
|
||||
return self.tushare_provider.client()
|
||||
return DatahubAwareTushareClient(self.tushare_provider.client(), self.datahub)
|
||||
|
||||
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
||||
return self.datahub.dataset_status(trade_date)
|
||||
|
||||
def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None:
|
||||
return self.datahub.batches(trade_date, dataset)
|
||||
|
||||
def assert_source(self, dataset_id: str, provider_id: str, usage: DataUsage) -> None:
|
||||
self.policy.assert_allowed(dataset_id, provider_id, usage)
|
||||
@@ -64,6 +73,7 @@ class DataGateway:
|
||||
def build_data_gateway(
|
||||
credentials: dict[str, object],
|
||||
tushare_token_supplier: Callable[[], str] | None = None,
|
||||
datahub_settings: DatahubSettings | None = None,
|
||||
) -> DataGateway:
|
||||
ifind = IfindHttpClient(
|
||||
str(credentials.get("ifind_refresh_token") or ""),
|
||||
@@ -73,6 +83,8 @@ def build_data_gateway(
|
||||
lambda: str(credentials.get("tushare_token") or "")
|
||||
)
|
||||
policy = DataSourcePolicy.load()
|
||||
settings = datahub_settings or DatahubSettings.load(credentials=credentials)
|
||||
datahub_client = DatahubClient(settings)
|
||||
return DataGateway(
|
||||
policy=policy,
|
||||
quality=DataQualityGate.load(policy),
|
||||
@@ -80,4 +92,5 @@ def build_data_gateway(
|
||||
ifind_provider=IfindProvider(ifind),
|
||||
chart_data=MarketChartClient(ifind, EastmoneyChartClient()),
|
||||
realtime_observer=WebRealtimeAggregator(),
|
||||
datahub=DatahubBridge(settings, datahub_client),
|
||||
)
|
||||
|
||||
@@ -12,6 +12,9 @@ These registries describe the approved product surface of the standalone applica
|
||||
providers, model entry points, CSS layers, and remaining code hotspots.
|
||||
- `data-fields.config.json`: canonical data products, provider eligibility, intended use, and
|
||||
known blocked datasets.
|
||||
- `datahub.config.json`: optional read-only client for `xiaobai-datahub`. Each dataset has its
|
||||
own `read` / `shadow` flag, all default off. Environment variables `DATAHUB_READ_*` and
|
||||
`DATAHUB_SHADOW_*` can override a single dataset without a master switch.
|
||||
- `data-quality.config.json`: freshness, coverage, units, adjustment, point-in-time, and
|
||||
fail-closed rules for every canonical data product.
|
||||
- `jobs.config.json`: background schedules, dependencies, lock keys, retry policy, timeouts,
|
||||
|
||||
@@ -204,6 +204,11 @@
|
||||
"path": "backend/data/providers/tushare_client.py",
|
||||
"runtime_role": "stable client facade for primary deterministic market data"
|
||||
},
|
||||
{
|
||||
"provider": "datahub",
|
||||
"path": "backend/data/datahub/client.py",
|
||||
"runtime_role": "optional official EOD read path behind per-dataset flags"
|
||||
},
|
||||
{
|
||||
"provider": "ifind",
|
||||
"path": "backend/data/providers/ifind_client.py",
|
||||
@@ -278,6 +283,18 @@
|
||||
"owner": "backend/data/providers/tushare.py",
|
||||
"compatibility_fallback": "backend/features/market/service.py"
|
||||
},
|
||||
{
|
||||
"client": "DatahubClient",
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "DatahubBridge",
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "DatahubAwareTushareClient",
|
||||
"owner": "backend/data/gateway.py"
|
||||
},
|
||||
{
|
||||
"client": "IfindHttpClient",
|
||||
"owner": "backend/data/gateway.py"
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
"schema_version": 1,
|
||||
"providers": {
|
||||
"tushare": {"class": "licensed", "calculation_allowed": true},
|
||||
"datahub": {"class": "licensed", "calculation_allowed": true},
|
||||
"ifind": {"class": "licensed", "calculation_allowed": true},
|
||||
"eastmoney": {"class": "public_web", "calculation_allowed": false},
|
||||
"tencent": {"class": "public_web", "calculation_allowed": false},
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"base_url": "http://127.0.0.1:8766",
|
||||
"timeout_seconds": 8,
|
||||
"retries": 1,
|
||||
"page_limit": 5000,
|
||||
"stale_seconds_max": 86400,
|
||||
"datasets": {
|
||||
"calendar": { "read": false, "shadow": false },
|
||||
"stocks": { "read": false, "shadow": false },
|
||||
"daily": { "read": false, "shadow": false },
|
||||
"index_daily": { "read": false, "shadow": false },
|
||||
"valuation": { "read": false, "shadow": false },
|
||||
"moneyflow": { "read": false, "shadow": false },
|
||||
"auction": { "read": false, "shadow": false },
|
||||
"status": { "read": false, "shadow": false }
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,9 @@ class DataGatewayTests(unittest.TestCase):
|
||||
"TushareClient": {"backend/features/market/service.py"},
|
||||
"TushareProvider": {"backend/data/gateway.py"},
|
||||
"WebRealtimeAggregator": {"backend/data/gateway.py"},
|
||||
"DatahubClient": {"backend/data/gateway.py"},
|
||||
"DatahubAwareTushareClient": {"backend/data/gateway.py"},
|
||||
"DatahubBridge": {"backend/data/gateway.py"},
|
||||
}
|
||||
found = {name: set() for name in owners}
|
||||
for path in (root / "backend").rglob("*.py"):
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
import json
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from backend.data import build_data_gateway
|
||||
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge, looks_like_heaven
|
||||
from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.compare import compare_rows
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.native import to_canonical_row, to_native_row
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
TOKEN = "super-secret-datahub-token"
|
||||
|
||||
LEGACY_DAILY = {
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": "20240902",
|
||||
"open": 10.11,
|
||||
"high": 10.25,
|
||||
"low": 10.01,
|
||||
"close": 10.20,
|
||||
"pct_chg": 1.2345,
|
||||
"vol": 1000.0,
|
||||
"amount": 2000.0,
|
||||
}
|
||||
HUB_DAILY = {
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": "20240902",
|
||||
"open": 10.11,
|
||||
"high": 10.25,
|
||||
"low": 10.01,
|
||||
"close": 10.20,
|
||||
"pct_chg": 1.2345,
|
||||
"volume": 100000.0,
|
||||
"amount": 2000000.0,
|
||||
}
|
||||
|
||||
|
||||
class FakeLegacy:
|
||||
def __init__(self, rows: list[dict[str, Any]] | Exception | None = None) -> None:
|
||||
self.token = "legacy-token"
|
||||
self.timeout = 30
|
||||
self.rows = [] if rows is None else rows
|
||||
self.calls: list[tuple[str, dict[str, Any] | None, str]] = []
|
||||
|
||||
def query(self, api_name: str, params: dict[str, Any] | None = None, fields: str = "") -> list[dict[str, Any]]:
|
||||
self.calls.append((api_name, params, fields))
|
||||
if isinstance(self.rows, Exception):
|
||||
raise self.rows
|
||||
return [dict(row) for row in self.rows]
|
||||
|
||||
|
||||
class FakeClient(DatahubClient):
|
||||
def __init__(self, error: DatahubError | None = None, response: DatahubResponse | None = None) -> None:
|
||||
super().__init__(DatahubSettings(base_url="http://127.0.0.1:9", token=TOKEN))
|
||||
self.error = error
|
||||
self.response = response or DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0},
|
||||
)
|
||||
self.paths: list[str] = []
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
self.paths.append(path)
|
||||
if TOKEN in json.dumps(params or {}) or TOKEN in path:
|
||||
raise AssertionError("token leaked into url")
|
||||
if self.error:
|
||||
raise self.error
|
||||
return self.response
|
||||
|
||||
|
||||
def flags(**enabled: tuple[bool, bool]) -> DatahubSettings:
|
||||
datasets = {name: DatasetFlags(name) for name in DATASETS}
|
||||
for name, pair in enabled.items():
|
||||
datasets[name] = DatasetFlags(name, read=pair[0], shadow=pair[1])
|
||||
return DatahubSettings(base_url="http://127.0.0.1:9", token=TOKEN, datasets=datasets)
|
||||
|
||||
|
||||
class DatahubBridgeTests(unittest.TestCase):
|
||||
def test_default_config_keeps_legacy_and_does_not_call_datahub(self) -> None:
|
||||
settings = DatahubSettings.load(environ={}, credentials={})
|
||||
self.assertFalse(settings.any_enabled())
|
||||
self.assertTrue(all(not settings.flags(name).read and not settings.flags(name).shadow for name in DATASETS))
|
||||
client = FakeClient(error=DatahubError("INTERNAL", "should not be called"))
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, client))
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,vol,amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(client.paths, [])
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
|
||||
def test_each_dataset_has_independent_read_flag(self) -> None:
|
||||
settings = flags(daily=(True, False), auction=(False, False))
|
||||
self.assertTrue(settings.flags("daily").read)
|
||||
self.assertFalse(settings.flags("auction").read)
|
||||
self.assertFalse(any(settings.flags(name).read for name in DATASETS if name != "daily"))
|
||||
source = (ROOT / "config" / "datahub.config.json").read_text(encoding="utf-8")
|
||||
self.assertNotIn("master", source)
|
||||
self.assertNotIn("DATAHUB_READ_ALL", source)
|
||||
|
||||
def test_read_flag_replaces_only_that_dataset_and_converts_units(self) -> None:
|
||||
shadows: list[dict[str, Any]] = []
|
||||
client = FakeClient()
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
legacy,
|
||||
DatahubBridge(flags(daily=(True, False)), client, shadow_sink=shadows.append),
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,vol,amount")
|
||||
self.assertEqual(rows[0]["vol"], 1000.0)
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(legacy.calls, [])
|
||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||
calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}])
|
||||
calendar_client = FakeClient(error=DatahubError("INTERNAL", "nope"))
|
||||
calendar_wrapped = DatahubAwareTushareClient(
|
||||
calendar_legacy,
|
||||
DatahubBridge(flags(daily=(True, False)), calendar_client),
|
||||
)
|
||||
calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "")
|
||||
self.assertEqual(calendar[0]["is_open"], 1)
|
||||
self.assertEqual(calendar_client.paths, [])
|
||||
|
||||
def test_fallback_on_down_401_timeout_empty_unpublished_and_stale(self) -> None:
|
||||
cases = [
|
||||
DatahubError("UNAVAILABLE", "down"),
|
||||
DatahubError("UNAUTHORIZED", "401"),
|
||||
DatahubError("TIMEOUT", "late"),
|
||||
DatahubError("EMPTY", "no rows"),
|
||||
DatahubError("DATASET_NOT_PUBLISHED", "not ready"),
|
||||
DatahubError("STALE", "old"),
|
||||
]
|
||||
for error in cases:
|
||||
with self.subTest(error=error.code):
|
||||
if error.code == "EMPTY":
|
||||
client = FakeClient(response=DatahubResponse(data=[], meta={"stale": False, "staleness_seconds": 0}))
|
||||
elif error.code == "STALE":
|
||||
client = FakeClient(response=DatahubResponse(
|
||||
data=[dict(HUB_DAILY)],
|
||||
meta={"stale": True, "staleness_seconds": 999999},
|
||||
))
|
||||
else:
|
||||
client = FakeClient(error=error)
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
|
||||
def test_shadow_compares_without_replacing_and_survives_hub_failure(self) -> None:
|
||||
reports: list[dict[str, Any]] = []
|
||||
client = FakeClient()
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
legacy,
|
||||
DatahubBridge(flags(daily=(False, True)), client, shadow_sink=reports.append),
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(len(legacy.calls), 1)
|
||||
self.assertEqual(reports[0]["equal"], True)
|
||||
self.assertEqual(reports[0]["matched"], 1)
|
||||
|
||||
failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN))
|
||||
fail_reports: list[dict[str, Any]] = []
|
||||
fail_legacy = FakeLegacy([LEGACY_DAILY])
|
||||
fail_wrapped = DatahubAwareTushareClient(
|
||||
fail_legacy,
|
||||
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=fail_reports.append),
|
||||
)
|
||||
again = fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(again[0]["amount"], 2000.0)
|
||||
self.assertTrue(fail_reports[0]["hub_error"])
|
||||
self.assertNotIn(TOKEN, json.dumps(fail_reports[0]))
|
||||
|
||||
def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None:
|
||||
equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0})
|
||||
self.assertTrue(equal["equal"])
|
||||
unit = compare_rows("daily", [LEGACY_DAILY], [{**HUB_DAILY, "amount": 2000.0, "volume": 1000.0}])
|
||||
self.assertGreater(unit["unit_conversion_count"], 0)
|
||||
missing = compare_rows("daily", [LEGACY_DAILY], [])
|
||||
self.assertEqual(missing["missing_hub_count"], 1)
|
||||
value = compare_rows("daily", [LEGACY_DAILY], [{**HUB_DAILY, "close": 99.0}])
|
||||
self.assertEqual(value["value_diff_count"], 1)
|
||||
skew = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 12})
|
||||
self.assertTrue(skew["time_skew"])
|
||||
|
||||
def test_native_roundtrip_matches_known_scales(self) -> None:
|
||||
native = to_native_row("daily", HUB_DAILY)
|
||||
self.assertEqual(native["vol"], 1000.0)
|
||||
self.assertEqual(native["amount"], 2000.0)
|
||||
canonical = to_canonical_row("daily", native)
|
||||
self.assertEqual(canonical["vol"], 100000.0)
|
||||
self.assertEqual(canonical["amount"], 2000000.0)
|
||||
|
||||
def test_heaven_keeps_legacy_even_when_read_flag_is_on(self) -> None:
|
||||
self.assertTrue(looks_like_heaven("backend.features.heaven.market_context", "backend/features/heaven/market_context.py"))
|
||||
self.assertFalse(looks_like_heaven("backend.features.market.service", "backend/features/market/service.py"))
|
||||
client = FakeClient()
|
||||
legacy = FakeLegacy([LEGACY_DAILY])
|
||||
wrapped = DatahubAwareTushareClient(
|
||||
legacy,
|
||||
DatahubBridge(flags(daily=(True, False)), client, heaven_guard=lambda: True),
|
||||
)
|
||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||
self.assertEqual(client.paths, [])
|
||||
|
||||
def test_status_flag_does_not_run_when_off_and_falls_back_when_on(self) -> None:
|
||||
off = DatahubBridge(flags(), FakeClient(error=DatahubError("UNAVAILABLE", "down")))
|
||||
self.assertIsNone(off.dataset_status("20240902"))
|
||||
reports: list[dict[str, Any]] = []
|
||||
failed = DatahubBridge(
|
||||
flags(status=(True, True)),
|
||||
FakeClient(error=DatahubError("UNAUTHORIZED", "nope")),
|
||||
shadow_sink=reports.append,
|
||||
)
|
||||
self.assertIsNone(failed.dataset_status("20240902"))
|
||||
self.assertTrue(reports[0]["hub_error"])
|
||||
ok = DatahubBridge(
|
||||
flags(status=(True, False)),
|
||||
FakeClient(response=DatahubResponse(data=[{"dataset": "daily", "state": "published"}], meta={"stale": False, "staleness_seconds": 0})),
|
||||
)
|
||||
self.assertEqual(ok.dataset_status("20240902")[0]["state"], "published")
|
||||
|
||||
def test_default_gateway_wraps_tushare_without_calling_datahub(self) -> None:
|
||||
gateway = build_data_gateway({}, datahub_settings=flags())
|
||||
client = gateway.tushare()
|
||||
self.assertIsInstance(client, DatahubAwareTushareClient)
|
||||
self.assertFalse(gateway.datahub.settings.any_enabled())
|
||||
|
||||
def test_features_do_not_import_datahub_client(self) -> None:
|
||||
violations = []
|
||||
for path in (ROOT / "backend" / "features").rglob("*.py"):
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
names = []
|
||||
if isinstance(node, ast.Import):
|
||||
names = [alias.name for alias in node.names]
|
||||
elif isinstance(node, ast.ImportFrom) and node.module:
|
||||
names = [node.module]
|
||||
for name in names:
|
||||
if "datahub" in name.split("."):
|
||||
violations.append(f"{path.relative_to(ROOT)} -> {name}")
|
||||
self.assertEqual(violations, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,185 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
from backend.data.datahub.client import DatahubClient
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.redact import redact_text
|
||||
from backend.data.datahub.settings import DatahubSettings
|
||||
|
||||
|
||||
TOKEN = "super-secret-datahub-token"
|
||||
|
||||
|
||||
class FakeHubState:
|
||||
def __init__(self) -> None:
|
||||
self.mode = "ok"
|
||||
self.hits = 0
|
||||
self.paths: list[str] = []
|
||||
|
||||
|
||||
STATE = FakeHubState()
|
||||
|
||||
|
||||
class FakeHubHandler(BaseHTTPRequestHandler):
|
||||
def log_message(self, format: str, *args: object) -> None:
|
||||
return
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
STATE.hits += 1
|
||||
parsed = urlparse(self.path)
|
||||
STATE.paths.append(parsed.path)
|
||||
token = self.headers.get("X-Datahub-Token", "")
|
||||
if STATE.mode == "timeout":
|
||||
raise TimeoutError("simulated timeout")
|
||||
if token != TOKEN:
|
||||
self._json(401, {"error": {"code": "UNAUTHORIZED", "message": "missing or invalid X-Datahub-Token"}})
|
||||
return
|
||||
if STATE.mode == "unpublished":
|
||||
self._json(404, {"error": {"code": "DATASET_NOT_PUBLISHED", "message": "daily 19990101 尚未发布", "expected_at": "15:05+08:00"}})
|
||||
return
|
||||
if STATE.mode == "empty":
|
||||
self._json(200, {"schema_version": 1, "data": [], "meta": {"tier": "official", "stale": False, "staleness_seconds": 0}})
|
||||
return
|
||||
if STATE.mode == "stale":
|
||||
self._json(200, {"schema_version": 1, "data": [{"ts_code": "600000.SH", "trade_date": "20240902", "close": 10.2, "volume": 100000, "amount": 2000000}], "meta": {"tier": "official", "stale": True, "staleness_seconds": 999999}})
|
||||
return
|
||||
if STATE.mode == "invalid":
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"not-json")
|
||||
return
|
||||
if parsed.path == "/v1/health":
|
||||
self._json(200, {"schema_version": 1, "data": {"status": "ok"}, "meta": {"tier": "official", "source": "datahub", "stale": False, "staleness_seconds": 0}})
|
||||
return
|
||||
if parsed.path == "/v1/calendar":
|
||||
self._json(200, {"schema_version": 1, "data": [{"cal_date": "20240902", "is_open": True, "pretrade_date": "20240830"}], "meta": {"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0}})
|
||||
return
|
||||
if parsed.path == "/v1/bars/daily":
|
||||
query = {key: values[-1] for key, values in parse_qs(parsed.query).items()}
|
||||
self._json(200, {
|
||||
"schema_version": 1,
|
||||
"data": [{
|
||||
"ts_code": "600000.SH",
|
||||
"trade_date": query.get("date") or "20240902",
|
||||
"open": 10.11, "high": 10.25, "low": 10.01, "close": 10.20,
|
||||
"pct_chg": 1.2345, "volume": 100000.0, "amount": 2000000.0, "adj_factor": 1.1,
|
||||
}],
|
||||
"meta": {"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0, "source": "tushare:daily"},
|
||||
})
|
||||
return
|
||||
if parsed.path == "/v1/datasets/status":
|
||||
self._json(200, {"schema_version": 1, "data": [{"dataset": "daily", "state": "published", "trade_date": "20240902"}], "meta": {"tier": "official", "stale": False, "staleness_seconds": 0}})
|
||||
return
|
||||
self._json(400, {"error": {"code": "INVALID_ARGUMENT", "message": f"unknown endpoint: {parsed.path}"}})
|
||||
|
||||
def _json(self, status: int, payload: dict) -> None:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
|
||||
class DatahubClientTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
cls.server = ThreadingHTTPServer(("127.0.0.1", 0), FakeHubHandler)
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
||||
cls.thread.start()
|
||||
cls.base = f"http://127.0.0.1:{cls.server.server_address[1]}"
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
cls.server.shutdown()
|
||||
cls.server.server_close()
|
||||
|
||||
def setUp(self) -> None:
|
||||
STATE.mode = "ok"
|
||||
STATE.hits = 0
|
||||
STATE.paths = []
|
||||
self.client = DatahubClient(DatahubSettings(base_url=self.base, token=TOKEN, retries=1, timeout_seconds=2))
|
||||
|
||||
def test_health_envelope(self) -> None:
|
||||
response = self.client.health()
|
||||
self.assertEqual(response.schema_version, 1)
|
||||
self.assertEqual(response.data["status"], "ok")
|
||||
self.assertIn("stale", response.meta)
|
||||
|
||||
def test_missing_and_bad_token_401(self) -> None:
|
||||
missing = DatahubClient(DatahubSettings(base_url=self.base, token=""))
|
||||
with self.assertRaises(DatahubError) as raised:
|
||||
missing.health()
|
||||
self.assertEqual(raised.exception.code, "NOT_CONFIGURED")
|
||||
bad = DatahubClient(DatahubSettings(base_url=self.base, token="wrong"))
|
||||
with self.assertRaises(DatahubError) as raised:
|
||||
bad.health()
|
||||
self.assertEqual(raised.exception.code, "UNAUTHORIZED")
|
||||
self.assertNotIn(TOKEN, str(raised.exception))
|
||||
|
||||
def test_unpublished_and_empty_and_stale_codes(self) -> None:
|
||||
STATE.mode = "unpublished"
|
||||
with self.assertRaises(DatahubError) as raised:
|
||||
self.client.daily_bars(date="19990101")
|
||||
self.assertEqual(raised.exception.code, "DATASET_NOT_PUBLISHED")
|
||||
STATE.mode = "empty"
|
||||
response = self.client.daily_bars(date="20240902")
|
||||
self.assertEqual(response.data, [])
|
||||
STATE.mode = "stale"
|
||||
stale = self.client.daily_bars(date="20240902")
|
||||
self.assertTrue(stale.meta["stale"])
|
||||
|
||||
def test_invalid_json_maps_to_internal(self) -> None:
|
||||
STATE.mode = "invalid"
|
||||
with self.assertRaises(DatahubError) as raised:
|
||||
self.client.health()
|
||||
self.assertEqual(raised.exception.code, "INTERNAL")
|
||||
|
||||
def test_timeout_maps_and_retries(self) -> None:
|
||||
hits = {"n": 0}
|
||||
|
||||
def boom(_request, timeout=None):
|
||||
hits["n"] += 1
|
||||
raise TimeoutError("late")
|
||||
|
||||
client = DatahubClient(
|
||||
DatahubSettings(base_url=self.base, token=TOKEN, retries=1, timeout_seconds=1),
|
||||
urlopen=boom,
|
||||
)
|
||||
with self.assertRaises(DatahubError) as raised:
|
||||
client.health()
|
||||
self.assertEqual(raised.exception.code, "TIMEOUT")
|
||||
self.assertEqual(hits["n"], 2)
|
||||
|
||||
def test_token_never_appears_in_error_text_or_logs(self) -> None:
|
||||
stream = io.StringIO()
|
||||
logger = logging.getLogger("xiaobai.datahub")
|
||||
handler = logging.StreamHandler(stream)
|
||||
logger.addHandler(handler)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
try:
|
||||
with self.assertRaises(DatahubError):
|
||||
DatahubClient(DatahubSettings(base_url=self.base, token="wrong")).health()
|
||||
blob = stream.getvalue() + redact_text("header " + TOKEN, (TOKEN,))
|
||||
self.assertNotIn(TOKEN, blob)
|
||||
self.assertIn("***", redact_text(TOKEN, (TOKEN,)))
|
||||
finally:
|
||||
logger.removeHandler(handler)
|
||||
|
||||
def test_calendar_and_status_contract(self) -> None:
|
||||
calendar = self.client.calendar("20240901", "20240902")
|
||||
self.assertEqual(calendar.data[0]["cal_date"], "20240902")
|
||||
status = self.client.dataset_status("20240902")
|
||||
self.assertEqual(status.data[0]["dataset"], "daily")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -217,6 +217,7 @@ def build() -> dict[str, Any]:
|
||||
),
|
||||
"external_data_adapters": [
|
||||
{"provider": "tushare", "path": "backend/data/providers/tushare_client.py", "runtime_role": "stable client facade for primary deterministic market data"},
|
||||
{"provider": "datahub", "path": "backend/data/datahub/client.py", "runtime_role": "optional official EOD read path behind per-dataset flags"},
|
||||
{"provider": "ifind", "path": "backend/data/providers/ifind_client.py", "runtime_role": "realtime, charts, snapshots, enrichment"},
|
||||
{"provider": "eastmoney", "path": "backend/features/market/charts.py", "runtime_role": "display chart fallback"},
|
||||
{"provider": "eastmoney", "path": "backend/data/realtime.py", "runtime_role": "isolated realtime observation"},
|
||||
@@ -235,6 +236,9 @@ def build() -> dict[str, Any]:
|
||||
],
|
||||
"provider_construction": [
|
||||
{"client": "TushareClient", "owner": "backend/data/providers/tushare.py", "compatibility_fallback": "backend/features/market/service.py"},
|
||||
{"client": "DatahubClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "DatahubBridge", "owner": "backend/data/gateway.py"},
|
||||
{"client": "DatahubAwareTushareClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "IfindHttpClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "MarketChartClient", "owner": "backend/data/gateway.py"},
|
||||
{"client": "WebRealtimeAggregator", "owner": "backend/data/gateway.py"},
|
||||
|
||||
Reference in New Issue
Block a user