feat(HEL-402): 接通网站首批只读 datahub 并建立双路对比
默认全部读取/影子开关关闭，网站继续走旧 Tushare 链路；开启单项时只替换该类原料并在失败时回旧，问天保持旧路径。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
f5dc0f8076
commit
0d13066386
@@ -11,6 +11,8 @@ from backend.features.accounts.security import SecretVault
|
||||
def environment_credentials(environment: Mapping[str, str]) -> dict[str, str]:
|
||||
return {
|
||||
"tushare_token": str(environment.get("TUSHARE_TOKEN") or "").strip(),
|
||||
"datahub_token": str(environment.get("DATAHUB_TOKEN") or "").strip(),
|
||||
"datahub_base_url": str(environment.get("DATAHUB_BASE_URL") or "").strip(),
|
||||
"ifind_refresh_token": str(environment.get("IFIND_REFRESH_TOKEN") or "").strip(),
|
||||
"ifind_access_token": str(environment.get("IFIND_ACCESS_TOKEN") or "").strip(),
|
||||
"platform_llm_primary_api_key": str(
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge
|
||||
from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||
|
||||
__all__ = [
|
||||
"DATASETS",
|
||||
"DatahubAwareTushareClient",
|
||||
"DatahubBridge",
|
||||
"DatahubClient",
|
||||
"DatahubError",
|
||||
"DatahubResponse",
|
||||
"DatahubSettings",
|
||||
"DatasetFlags",
|
||||
]
|
||||
@@ -0,0 +1,249 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from typing import Any, Callable
|
||||
|
||||
from backend.data.datahub.client import DatahubClient, DatahubResponse
|
||||
from backend.data.datahub.compare import compare_rows
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.native import (
|
||||
API_TO_DATASET,
|
||||
filter_calendar_rows,
|
||||
filter_stock_rows,
|
||||
project_fields,
|
||||
to_native_rows,
|
||||
yyyymmdd,
|
||||
)
|
||||
from backend.data.datahub.redact import redact_text, redact_value
|
||||
from backend.data.datahub.settings import DatahubSettings
|
||||
from backend.data.providers.tushare_client import TushareClient
|
||||
|
||||
LOGGER = logging.getLogger("xiaobai.datahub")
|
||||
ShadowSink = Callable[[dict[str, Any]], None]
|
||||
EMPTY_FAIL_DATASETS = {"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction"}
|
||||
|
||||
|
||||
def looks_like_heaven(module_name: str, filename: str = "") -> bool:
|
||||
path = filename.replace("\\", "/")
|
||||
return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path
|
||||
|
||||
|
||||
def caller_is_heaven(depth: int = 24) -> bool:
|
||||
frame = sys._getframe(1)
|
||||
for _ in range(depth):
|
||||
frame = frame.f_back if frame is not None else None
|
||||
if frame is None:
|
||||
return False
|
||||
name = str(frame.f_globals.get("__name__") or "")
|
||||
filename = str(frame.f_code.co_filename or "")
|
||||
if looks_like_heaven(name, filename):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class DatahubBridge:
|
||||
def __init__(
|
||||
self,
|
||||
settings: DatahubSettings,
|
||||
client: DatahubClient,
|
||||
shadow_sink: ShadowSink | None = None,
|
||||
heaven_guard: Callable[[], bool] | None = None,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self.client = client
|
||||
self.shadow_sink = shadow_sink
|
||||
self.heaven_guard = heaven_guard or caller_is_heaven
|
||||
|
||||
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags("status")
|
||||
if not flags.read and not flags.shadow:
|
||||
return None
|
||||
try:
|
||||
response = self._require_fresh(self.client.dataset_status(yyyymmdd(trade_date)), "status")
|
||||
rows = list(response.data or [])
|
||||
if flags.shadow:
|
||||
self._emit_shadow(compare_rows("status", [], rows, response.meta))
|
||||
if flags.read:
|
||||
return rows
|
||||
return None
|
||||
except Exception as exc:
|
||||
self._log_failure("status", exc)
|
||||
if flags.shadow:
|
||||
self._emit_shadow(compare_rows("status", [], [], {}, self._error_text(exc)))
|
||||
return None
|
||||
|
||||
def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None:
|
||||
flags = self.settings.flags("status")
|
||||
if not flags.read:
|
||||
return None
|
||||
try:
|
||||
response = self._require_fresh(
|
||||
self.client.batches(yyyymmdd(trade_date), dataset),
|
||||
"status",
|
||||
)
|
||||
return list(response.data or [])
|
||||
except Exception as exc:
|
||||
self._log_failure("status", exc)
|
||||
return None
|
||||
|
||||
def query(
|
||||
self,
|
||||
api_name: str,
|
||||
params: dict[str, Any] | None,
|
||||
fields: str,
|
||||
legacy_query: Callable[..., list[dict[str, Any]]],
|
||||
) -> list[dict[str, Any]]:
|
||||
dataset = API_TO_DATASET.get(api_name)
|
||||
if not dataset or self.heaven_guard():
|
||||
return legacy_query(api_name, params, fields)
|
||||
flags = self.settings.flags(dataset)
|
||||
if not flags.read and not flags.shadow:
|
||||
return legacy_query(api_name, params, fields)
|
||||
|
||||
hub_rows: list[dict[str, Any]] | None = None
|
||||
hub_meta: dict[str, Any] = {}
|
||||
hub_error: str | None = None
|
||||
hub_canonical: list[dict[str, Any]] = []
|
||||
try:
|
||||
response = self._fetch_dataset(dataset, params or {})
|
||||
hub_canonical = self._extract_rows(dataset, response, params or {})
|
||||
hub_rows = to_native_rows(dataset, hub_canonical)
|
||||
hub_meta = dict(response.meta)
|
||||
self._validate_usable(dataset, hub_rows, response)
|
||||
except Exception as exc:
|
||||
hub_error = self._error_text(exc)
|
||||
self._log_failure(dataset, exc)
|
||||
|
||||
if flags.shadow:
|
||||
try:
|
||||
legacy_rows = legacy_query(api_name, params, fields)
|
||||
except Exception as exc:
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
self._emit_shadow(compare_rows(dataset, [], hub_canonical, hub_meta, self._error_text(exc)))
|
||||
return project_fields(hub_rows, fields)
|
||||
raise
|
||||
self._emit_shadow(compare_rows(dataset, legacy_rows, hub_canonical, hub_meta, hub_error))
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
return project_fields(hub_rows, fields)
|
||||
return legacy_rows
|
||||
|
||||
if flags.read and hub_rows is not None and hub_error is None:
|
||||
return project_fields(hub_rows, fields)
|
||||
return legacy_query(api_name, params, fields)
|
||||
|
||||
def _fetch_dataset(self, dataset: str, params: dict[str, Any]) -> DatahubResponse:
|
||||
date = yyyymmdd(params.get("trade_date") or params.get("date"))
|
||||
start = yyyymmdd(params.get("start_date") or params.get("from") or date)
|
||||
end = yyyymmdd(params.get("end_date") or params.get("to") or date)
|
||||
code = str(params.get("ts_code") or params.get("code") or "").strip()
|
||||
if dataset == "calendar":
|
||||
if not start or not end:
|
||||
raise DatahubError("INVALID_ARGUMENT", "calendar requires start_date and end_date")
|
||||
return self.client.calendar(start, end)
|
||||
if dataset == "stocks":
|
||||
return self._paginate(self.client.stocks, {})
|
||||
fetchers = {
|
||||
"daily": self.client.daily_bars,
|
||||
"index_daily": self.client.index_bars,
|
||||
"valuation": self.client.valuation,
|
||||
"moneyflow": self.client.moneyflow,
|
||||
"auction": self.client.auction,
|
||||
}
|
||||
fetcher = fetchers[dataset]
|
||||
query: dict[str, Any] = {}
|
||||
if code:
|
||||
query["code"] = code
|
||||
if date and not (params.get("start_date") or params.get("end_date")):
|
||||
query["date"] = date
|
||||
else:
|
||||
if start:
|
||||
query["from"] = start
|
||||
if end:
|
||||
query["to"] = end
|
||||
if dataset == "daily":
|
||||
query["adjust"] = "none"
|
||||
return self._paginate(fetcher, query)
|
||||
|
||||
def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse:
|
||||
limit = self.settings.page_limit
|
||||
offset = 0
|
||||
rows: list[Any] = []
|
||||
meta: dict[str, Any] = {}
|
||||
schema_version = 1
|
||||
while True:
|
||||
page = fetcher(**{**params, "limit": limit, "offset": offset})
|
||||
meta = dict(page.meta)
|
||||
schema_version = page.schema_version
|
||||
data = page.data or []
|
||||
if not isinstance(data, list):
|
||||
raise DatahubError("INTERNAL", "datahub returned a non-list payload")
|
||||
rows.extend(data)
|
||||
if len(data) < limit:
|
||||
break
|
||||
offset += limit
|
||||
if offset > 200_000:
|
||||
break
|
||||
return DatahubResponse(data=rows, meta=meta, schema_version=schema_version)
|
||||
|
||||
def _extract_rows(
|
||||
self,
|
||||
dataset: str,
|
||||
response: DatahubResponse,
|
||||
params: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = [dict(item) for item in (response.data or [])]
|
||||
if dataset == "calendar":
|
||||
return filter_calendar_rows(rows, params)
|
||||
if dataset == "stocks":
|
||||
return filter_stock_rows(rows, params)
|
||||
return rows
|
||||
|
||||
def _validate_usable(self, dataset: str, rows: list[dict[str, Any]], response: DatahubResponse) -> None:
|
||||
meta = response.meta or {}
|
||||
stale_seconds = int(meta.get("staleness_seconds") or 0)
|
||||
if meta.get("stale") or stale_seconds > self.settings.stale_seconds_max:
|
||||
raise DatahubError("STALE", f"{dataset} data is stale")
|
||||
if dataset in EMPTY_FAIL_DATASETS and not rows:
|
||||
raise DatahubError("EMPTY", f"{dataset} returned no rows")
|
||||
|
||||
def _require_fresh(self, response: DatahubResponse, dataset: str) -> DatahubResponse:
|
||||
self._validate_usable(dataset, list(response.data or []) if isinstance(response.data, list) else [], response)
|
||||
return response
|
||||
|
||||
def _emit_shadow(self, report: dict[str, Any]) -> None:
|
||||
safe = redact_value(report, secrets=self.settings.secrets())
|
||||
LOGGER.info("datahub shadow %s", safe)
|
||||
if self.shadow_sink is not None:
|
||||
self.shadow_sink(report)
|
||||
|
||||
def _log_failure(self, dataset: str, exc: Exception) -> None:
|
||||
LOGGER.warning(
|
||||
"datahub fallback dataset=%s error=%s",
|
||||
dataset,
|
||||
redact_text(self._error_text(exc), self.settings.secrets()),
|
||||
)
|
||||
|
||||
def _error_text(self, exc: Exception) -> str:
|
||||
if isinstance(exc, DatahubError):
|
||||
text = f"{exc.code}: {exc.message}"
|
||||
else:
|
||||
text = str(exc)
|
||||
return redact_text(text, self.settings.secrets())
|
||||
|
||||
|
||||
class DatahubAwareTushareClient:
|
||||
def __init__(self, legacy: TushareClient, bridge: DatahubBridge) -> None:
|
||||
self._legacy = legacy
|
||||
self._bridge = bridge
|
||||
|
||||
def query(
|
||||
self,
|
||||
api_name: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
fields: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
return self._bridge.query(api_name, params, fields, self._legacy.query)
|
||||
|
||||
def __getattr__(self, name: str) -> Any:
|
||||
return getattr(self._legacy, name)
|
||||
@@ -0,0 +1,184 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
|
||||
from backend.data.datahub.errors import DatahubError
|
||||
from backend.data.datahub.redact import redact_text
|
||||
from backend.data.datahub.settings import DatahubSettings
|
||||
|
||||
LOGGER = logging.getLogger("xiaobai.datahub")
|
||||
UrlOpen = Callable[..., Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatahubResponse:
|
||||
data: Any
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
schema_version: int = 1
|
||||
status: int = 200
|
||||
|
||||
|
||||
class DatahubClient:
|
||||
def __init__(
|
||||
self,
|
||||
settings: DatahubSettings,
|
||||
urlopen: UrlOpen = urllib.request.urlopen,
|
||||
) -> None:
|
||||
self.settings = settings
|
||||
self._urlopen = urlopen
|
||||
|
||||
def health(self) -> DatahubResponse:
|
||||
return self.get("/v1/health")
|
||||
|
||||
def calendar(self, start: str, end: str) -> DatahubResponse:
|
||||
return self.get("/v1/calendar", {"from": start, "to": end})
|
||||
|
||||
def stocks(self, updated_since: str = "", limit: int | None = None, offset: int = 0) -> DatahubResponse:
|
||||
params: dict[str, Any] = {"offset": offset, "limit": limit or self.settings.page_limit}
|
||||
if updated_since:
|
||||
params["updated_since"] = updated_since
|
||||
return self.get("/v1/stocks", params)
|
||||
|
||||
def daily_bars(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/bars/daily", params)
|
||||
|
||||
def index_bars(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/indexes/bars", params)
|
||||
|
||||
def valuation(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/valuation", params)
|
||||
|
||||
def moneyflow(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/moneyflow", params)
|
||||
|
||||
def auction(self, **params: Any) -> DatahubResponse:
|
||||
return self.get("/v1/auction", params)
|
||||
|
||||
def dataset_status(self, date: str) -> DatahubResponse:
|
||||
return self.get("/v1/datasets/status", {"date": date})
|
||||
|
||||
def batches(self, date: str, dataset: str = "") -> DatahubResponse:
|
||||
params: dict[str, Any] = {"date": date}
|
||||
if dataset:
|
||||
params["dataset"] = dataset
|
||||
return self.get("/v1/batches", params)
|
||||
|
||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||
if not self.settings.token:
|
||||
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
||||
query = {
|
||||
key: value
|
||||
for key, value in (params or {}).items()
|
||||
if value is not None and value != ""
|
||||
}
|
||||
url = self.settings.base_url + path
|
||||
if query:
|
||||
url = f"{url}?{urllib.parse.urlencode(query)}"
|
||||
attempts = 1 + max(0, self.settings.retries)
|
||||
last_error: DatahubError | None = None
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return self._request(url)
|
||||
except DatahubError as exc:
|
||||
last_error = exc
|
||||
if exc.code not in {"TIMEOUT", "UNAVAILABLE"} or attempt + 1 >= attempts:
|
||||
raise
|
||||
LOGGER.warning(
|
||||
"datahub retry %s/%s %s",
|
||||
attempt + 1,
|
||||
attempts,
|
||||
redact_text(str(exc), self.settings.secrets()),
|
||||
)
|
||||
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
||||
|
||||
def _request(self, url: str) -> DatahubResponse:
|
||||
request = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"Accept": "application/json",
|
||||
"X-Datahub-Token": self.settings.token,
|
||||
"User-Agent": "XiaobaiReviewDatahub/1.0",
|
||||
},
|
||||
method="GET",
|
||||
)
|
||||
try:
|
||||
with self._urlopen(request, timeout=self.settings.timeout_seconds) as response:
|
||||
status = int(getattr(response, "status", 200) or 200)
|
||||
raw = response.read().decode("utf-8")
|
||||
except TimeoutError as exc:
|
||||
raise DatahubError("TIMEOUT", "datahub request timed out") from exc
|
||||
except urllib.error.HTTPError as exc:
|
||||
body = _read_error_body(exc)
|
||||
raise _http_error(exc.code, body, self.settings.secrets()) from exc
|
||||
except urllib.error.URLError as exc:
|
||||
reason = redact_text(str(getattr(exc, "reason", exc)), self.settings.secrets())
|
||||
if "timed out" in reason.lower():
|
||||
raise DatahubError("TIMEOUT", "datahub request timed out") from exc
|
||||
raise DatahubError("UNAVAILABLE", f"datahub unavailable: {reason}") from exc
|
||||
payload = _parse_json(raw, self.settings.secrets())
|
||||
return _as_response(payload, status, self.settings.secrets())
|
||||
|
||||
|
||||
def _parse_json(raw: str, secrets: tuple[str, ...]) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise DatahubError("INTERNAL", "datahub returned invalid json") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise DatahubError("INTERNAL", "datahub returned a non-object payload")
|
||||
return payload
|
||||
|
||||
|
||||
def _as_response(payload: dict[str, Any], status: int, secrets: tuple[str, ...]) -> DatahubResponse:
|
||||
error = payload.get("error")
|
||||
if isinstance(error, dict):
|
||||
raise _mapped_error(str(error.get("code") or "INTERNAL"), str(error.get("message") or "datahub error"), status)
|
||||
if status >= 400:
|
||||
raise DatahubError("UNAVAILABLE", f"datahub http {status}", status)
|
||||
return DatahubResponse(
|
||||
data=payload.get("data"),
|
||||
meta=dict(payload.get("meta") or {}),
|
||||
schema_version=int(payload.get("schema_version") or 1),
|
||||
status=status,
|
||||
)
|
||||
|
||||
|
||||
def _http_error(status: int, payload: dict[str, Any], secrets: tuple[str, ...]) -> DatahubError:
|
||||
error = payload.get("error") if isinstance(payload.get("error"), dict) else {}
|
||||
code = str((error or {}).get("code") or "")
|
||||
message = str((error or {}).get("message") or payload.get("message") or f"datahub http {status}")
|
||||
message = redact_text(message, secrets)
|
||||
if status == 401 or code == "UNAUTHORIZED":
|
||||
return DatahubError("UNAUTHORIZED", message, status)
|
||||
if status == 404 or code == "DATASET_NOT_PUBLISHED":
|
||||
return DatahubError("DATASET_NOT_PUBLISHED", message, status)
|
||||
if status == 400 or code == "INVALID_ARGUMENT":
|
||||
return DatahubError("INVALID_ARGUMENT", message, status)
|
||||
if status in {429, 503} or code in {"RATE_LIMITED", "SOURCE_UNAVAILABLE"}:
|
||||
return DatahubError("UNAVAILABLE", message, status)
|
||||
return DatahubError(code or "INTERNAL", message, status)
|
||||
|
||||
|
||||
def _mapped_error(code: str, message: str, status: int) -> DatahubError:
|
||||
if code == "STALE_DATA":
|
||||
return DatahubError("STALE", message, status)
|
||||
if code in {"UNAUTHORIZED", "DATASET_NOT_PUBLISHED", "INVALID_ARGUMENT"}:
|
||||
return DatahubError(code, message, status)
|
||||
if code in {"RATE_LIMITED", "SOURCE_UNAVAILABLE"}:
|
||||
return DatahubError("UNAVAILABLE", message, status)
|
||||
return DatahubError(code or "INTERNAL", message, status)
|
||||
|
||||
|
||||
def _read_error_body(exc: urllib.error.HTTPError) -> dict[str, Any]:
|
||||
try:
|
||||
raw = exc.read().decode("utf-8")
|
||||
payload = json.loads(raw)
|
||||
return payload if isinstance(payload, dict) else {"message": raw}
|
||||
except Exception:
|
||||
return {"message": str(exc)}
|
||||
@@ -0,0 +1,134 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend.data.datahub.native import SCALE_FIELDS, row_key, to_canonical_row, yyyymmdd
|
||||
|
||||
NUMERIC_TOLERANCE = 1e-4
|
||||
|
||||
|
||||
def compare_rows(
|
||||
dataset: str,
|
||||
legacy_rows: list[dict[str, Any]],
|
||||
hub_rows: list[dict[str, Any]] | None,
|
||||
hub_meta: dict[str, Any] | None = None,
|
||||
hub_error: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
hub = hub_rows or []
|
||||
legacy_map = {row_key(dataset, row): row for row in legacy_rows}
|
||||
hub_map = {row_key(dataset, _align_hub_row(row)): row for row in hub}
|
||||
missing_hub = sorted(key for key in legacy_map if key not in hub_map)
|
||||
missing_legacy = sorted(key for key in hub_map if key not in legacy_map)
|
||||
value_diffs: list[dict[str, Any]] = []
|
||||
unit_conversion: list[dict[str, Any]] = []
|
||||
matched = 0
|
||||
for key, legacy in legacy_map.items():
|
||||
hub_row = hub_map.get(key)
|
||||
if hub_row is None:
|
||||
continue
|
||||
field_report = _compare_fields(dataset, legacy, hub_row)
|
||||
if field_report["unit_conversion"]:
|
||||
unit_conversion.append({"key": list(key), "fields": field_report["unit_conversion"]})
|
||||
if field_report["value_diff"]:
|
||||
value_diffs.append({"key": list(key), "fields": field_report["value_diff"]})
|
||||
if not field_report["unit_conversion"] and not field_report["value_diff"]:
|
||||
matched += 1
|
||||
stale_seconds = int((hub_meta or {}).get("staleness_seconds") or 0)
|
||||
time_skew = bool((hub_meta or {}).get("stale")) or stale_seconds > 0
|
||||
return {
|
||||
"dataset": dataset,
|
||||
"legacy_rows": len(legacy_rows),
|
||||
"hub_rows": len(hub),
|
||||
"matched": matched,
|
||||
"missing_hub": [list(item) for item in missing_hub[:20]],
|
||||
"missing_legacy": [list(item) for item in missing_legacy[:20]],
|
||||
"missing_hub_count": len(missing_hub),
|
||||
"missing_legacy_count": len(missing_legacy),
|
||||
"value_diff_count": len(value_diffs),
|
||||
"unit_conversion_count": len(unit_conversion),
|
||||
"value_diffs": value_diffs[:20],
|
||||
"unit_conversion": unit_conversion[:20],
|
||||
"time_skew": time_skew,
|
||||
"staleness_seconds": stale_seconds,
|
||||
"published_at": (hub_meta or {}).get("published_at"),
|
||||
"trade_date": yyyymmdd((hub_meta or {}).get("trade_date")),
|
||||
"hub_error": hub_error,
|
||||
"equal": (
|
||||
not hub_error
|
||||
and not missing_hub
|
||||
and not missing_legacy
|
||||
and not value_diffs
|
||||
and not unit_conversion
|
||||
and not time_skew
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _align_hub_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
aligned = dict(row)
|
||||
if "volume" in aligned and "vol" not in aligned:
|
||||
aligned["vol"] = aligned.get("volume")
|
||||
return aligned
|
||||
|
||||
|
||||
def _compare_fields(dataset: str, legacy: dict[str, Any], hub: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
|
||||
canonical_legacy = to_canonical_row(dataset, legacy)
|
||||
hub_canonical = _hub_canonical(dataset, hub)
|
||||
native_hub = _align_hub_row(hub)
|
||||
value_diff: list[dict[str, Any]] = []
|
||||
unit_conversion: list[dict[str, Any]] = []
|
||||
keys = (set(canonical_legacy) | set(hub_canonical)) - {"batch_id", "updated_at", "volume"}
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field in sorted(keys):
|
||||
left = canonical_legacy.get(field)
|
||||
right = hub_canonical.get(field)
|
||||
if _same(left, right):
|
||||
continue
|
||||
native_left = legacy.get(field)
|
||||
hub_raw = native_hub.get(field)
|
||||
if field in scales and _near(_optional(native_left), _optional(hub_raw)):
|
||||
unit_conversion.append(
|
||||
{"field": field, "legacy": native_left, "hub": hub_raw, "reason": "unit_conversion"}
|
||||
)
|
||||
continue
|
||||
value_diff.append({"field": field, "legacy": left, "hub": right, "reason": "value_diff"})
|
||||
return {"value_diff": value_diff, "unit_conversion": unit_conversion}
|
||||
|
||||
|
||||
def _hub_canonical(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Hub API rows are already canonical; only align field names."""
|
||||
aligned = dict(row)
|
||||
if "volume" in aligned and "vol" not in aligned:
|
||||
aligned["vol"] = aligned.get("volume")
|
||||
if dataset == "calendar":
|
||||
is_open = aligned.get("is_open")
|
||||
aligned["is_open"] = 1 if is_open in (True, 1, "1", "Y", "y") else 0
|
||||
aligned["cal_date"] = yyyymmdd(aligned.get("cal_date"))
|
||||
aligned["pretrade_date"] = yyyymmdd(aligned.get("pretrade_date")) or None
|
||||
aligned["exchange"] = str(aligned.get("exchange") or "SSE")
|
||||
return aligned
|
||||
|
||||
|
||||
def _same(left: Any, right: Any) -> bool:
|
||||
if left in (None, "") and right in (None, ""):
|
||||
return True
|
||||
if isinstance(left, (int, float)) or isinstance(right, (int, float)):
|
||||
return _near(_optional(left), _optional(right))
|
||||
return str(left or "") == str(right or "")
|
||||
|
||||
|
||||
def _near(left: float | None, right: float | None) -> bool:
|
||||
if left is None and right is None:
|
||||
return True
|
||||
if left is None or right is None:
|
||||
return False
|
||||
return abs(left - right) <= max(NUMERIC_TOLERANCE, abs(left) * 1e-9, abs(right) * 1e-9)
|
||||
|
||||
|
||||
def _optional(value: Any) -> float | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
@@ -0,0 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
class DatahubError(RuntimeError):
|
||||
def __init__(self, code: str, message: str, status: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.status = status
|
||||
|
||||
def __str__(self) -> str:
|
||||
return f"{self.code}: {self.message}"
|
||||
@@ -0,0 +1,153 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from backend.data.numbers import finite_number
|
||||
|
||||
|
||||
AMOUNT_THOUSAND_YUAN = 1000.0
|
||||
AMOUNT_WAN_YUAN = 10000.0
|
||||
VOLUME_LOT = 100.0
|
||||
|
||||
API_TO_DATASET = {
|
||||
"trade_cal": "calendar",
|
||||
"stock_basic": "stocks",
|
||||
"daily": "daily",
|
||||
"daily_basic": "valuation",
|
||||
"index_daily": "index_daily",
|
||||
"moneyflow": "moneyflow",
|
||||
"stk_auction": "auction",
|
||||
}
|
||||
|
||||
SCALE_FIELDS = {
|
||||
"daily": {"vol": VOLUME_LOT, "amount": AMOUNT_THOUSAND_YUAN},
|
||||
"index_daily": {"vol": VOLUME_LOT, "amount": AMOUNT_THOUSAND_YUAN},
|
||||
"valuation": {"total_mv": AMOUNT_WAN_YUAN, "circ_mv": AMOUNT_WAN_YUAN},
|
||||
"moneyflow": {
|
||||
"buy_sm_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_sm_amount": AMOUNT_WAN_YUAN,
|
||||
"buy_md_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_md_amount": AMOUNT_WAN_YUAN,
|
||||
"buy_lg_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_lg_amount": AMOUNT_WAN_YUAN,
|
||||
"buy_elg_amount": AMOUNT_WAN_YUAN,
|
||||
"sell_elg_amount": AMOUNT_WAN_YUAN,
|
||||
"net_mf_amount": AMOUNT_WAN_YUAN,
|
||||
},
|
||||
"auction": {"vol": VOLUME_LOT, "float_share": AMOUNT_WAN_YUAN},
|
||||
}
|
||||
|
||||
|
||||
def yyyymmdd(value: Any) -> str:
|
||||
return str(value or "").replace("-", "")[:8]
|
||||
|
||||
|
||||
def to_native_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
return [to_native_row(dataset, row) for row in rows]
|
||||
|
||||
|
||||
def to_native_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
if dataset == "calendar":
|
||||
is_open = row.get("is_open")
|
||||
return {
|
||||
"exchange": str(row.get("exchange") or "SSE"),
|
||||
"cal_date": yyyymmdd(row.get("cal_date")),
|
||||
"is_open": 1 if is_open in (True, 1, "1", "Y", "y") else 0,
|
||||
"pretrade_date": yyyymmdd(row.get("pretrade_date")) or None,
|
||||
}
|
||||
converted = dict(row)
|
||||
converted.pop("batch_id", None)
|
||||
if "volume" in converted and "vol" not in converted:
|
||||
converted["vol"] = converted.pop("volume")
|
||||
elif "volume" in converted:
|
||||
converted.pop("volume", None)
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field, factor in scales.items():
|
||||
if field in converted:
|
||||
converted[field] = _unscale(converted.get(field), factor)
|
||||
if dataset == "stocks":
|
||||
converted.pop("updated_at", None)
|
||||
return converted
|
||||
|
||||
|
||||
def to_canonical_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
if dataset == "calendar":
|
||||
is_open = row.get("is_open")
|
||||
return {
|
||||
"exchange": str(row.get("exchange") or "SSE"),
|
||||
"cal_date": yyyymmdd(row.get("cal_date")),
|
||||
"is_open": 1 if is_open in (True, 1, "1", "Y", "y") else 0,
|
||||
"pretrade_date": yyyymmdd(row.get("pretrade_date")) or None,
|
||||
}
|
||||
converted = dict(row)
|
||||
if "volume" in converted and "vol" not in converted:
|
||||
converted["vol"] = converted.pop("volume")
|
||||
scales = SCALE_FIELDS.get(dataset) or {}
|
||||
for field, factor in scales.items():
|
||||
if field in converted:
|
||||
converted[field] = _scale(converted.get(field), factor)
|
||||
return converted
|
||||
|
||||
|
||||
def row_key(dataset: str, row: dict[str, Any]) -> tuple[str, ...]:
|
||||
if dataset == "calendar":
|
||||
return (yyyymmdd(row.get("cal_date")),)
|
||||
if dataset == "stocks":
|
||||
return (str(row.get("ts_code") or "").upper(),)
|
||||
if dataset == "status":
|
||||
return (str(row.get("dataset") or ""), yyyymmdd(row.get("trade_date")))
|
||||
return (str(row.get("ts_code") or "").upper(), yyyymmdd(row.get("trade_date")))
|
||||
|
||||
|
||||
def project_fields(rows: list[dict[str, Any]], fields: str) -> list[dict[str, Any]]:
|
||||
keys = [item.strip() for item in str(fields or "").split(",") if item.strip()]
|
||||
if not keys:
|
||||
return rows
|
||||
return [{key: row.get(key) for key in keys} for row in rows]
|
||||
|
||||
|
||||
def filter_stock_rows(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
payload = params or {}
|
||||
ts_code = str(payload.get("ts_code") or "").strip().upper()
|
||||
status = str(payload.get("list_status") or "").strip()
|
||||
name = str(payload.get("name") or "").strip()
|
||||
filtered = rows
|
||||
if ts_code:
|
||||
filtered = [row for row in filtered if str(row.get("ts_code") or "").upper() == ts_code]
|
||||
if status:
|
||||
filtered = [row for row in filtered if str(row.get("list_status") or status) == status]
|
||||
if name:
|
||||
filtered = [row for row in filtered if name.casefold() in str(row.get("name") or "").casefold()]
|
||||
return filtered
|
||||
|
||||
|
||||
def filter_calendar_rows(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
payload = params or {}
|
||||
if payload.get("is_open") in (1, "1", True):
|
||||
return [row for row in rows if int(row.get("is_open") or 0) == 1]
|
||||
if payload.get("is_open") in (0, "0", False):
|
||||
return [row for row in rows if int(row.get("is_open") or 0) == 0]
|
||||
return rows
|
||||
|
||||
|
||||
def _scale(value: Any, factor: float) -> float | None:
|
||||
number = _optional_number(value)
|
||||
if number is None:
|
||||
return None
|
||||
return number * factor
|
||||
|
||||
|
||||
def _unscale(value: Any, factor: float) -> float | None:
|
||||
number = _optional_number(value)
|
||||
if number is None or factor == 0:
|
||||
return None
|
||||
return number / factor
|
||||
|
||||
|
||||
def _optional_number(value: Any) -> float | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
number = finite_number(value, default=float("nan"))
|
||||
if number != number:
|
||||
return None
|
||||
return number
|
||||
@@ -0,0 +1,39 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
SECRET_HINTS = (
|
||||
"token",
|
||||
"password",
|
||||
"secret",
|
||||
"key",
|
||||
"authorization",
|
||||
"credential",
|
||||
"cookie",
|
||||
)
|
||||
|
||||
|
||||
def redact_value(value: Any, key: str = "", secrets: tuple[str, ...] = ()) -> Any:
|
||||
lowered = key.lower()
|
||||
if any(part in lowered for part in SECRET_HINTS):
|
||||
return "***"
|
||||
if isinstance(value, dict):
|
||||
return {
|
||||
str(item_key): redact_value(item_value, str(item_key), secrets)
|
||||
for item_key, item_value in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [redact_value(item, key, secrets) for item in value]
|
||||
text = str(value) if value is not None and not isinstance(value, (int, float, bool)) else value
|
||||
if isinstance(text, str):
|
||||
return redact_text(text, secrets)
|
||||
return value
|
||||
|
||||
|
||||
def redact_text(text: str, secrets: tuple[str, ...] = ()) -> str:
|
||||
redacted = text
|
||||
for secret in secrets:
|
||||
if secret:
|
||||
redacted = redacted.replace(secret, "***")
|
||||
return redacted
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from backend.bootstrap.config import APP_DIR
|
||||
|
||||
|
||||
DATASETS = (
|
||||
"calendar",
|
||||
"stocks",
|
||||
"daily",
|
||||
"index_daily",
|
||||
"valuation",
|
||||
"moneyflow",
|
||||
"auction",
|
||||
"status",
|
||||
)
|
||||
|
||||
ENV_DATASET = {
|
||||
"calendar": "CALENDAR",
|
||||
"stocks": "STOCKS",
|
||||
"daily": "DAILY",
|
||||
"index_daily": "INDEX_DAILY",
|
||||
"valuation": "VALUATION",
|
||||
"moneyflow": "MONEYFLOW",
|
||||
"auction": "AUCTION",
|
||||
"status": "STATUS",
|
||||
}
|
||||
|
||||
DEFAULT_CONFIG_PATH = APP_DIR / "config" / "datahub.config.json"
|
||||
|
||||
|
||||
def _truthy(value: Any) -> bool:
|
||||
return str(value or "").strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _int(value: Any, default: int) -> int:
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetFlags:
|
||||
name: str
|
||||
read: bool = False
|
||||
shadow: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatahubSettings:
|
||||
base_url: str
|
||||
token: str
|
||||
timeout_seconds: int = 8
|
||||
retries: int = 1
|
||||
page_limit: int = 5000
|
||||
stale_seconds_max: int = 86400
|
||||
datasets: dict[str, DatasetFlags] | None = None
|
||||
|
||||
def flags(self, dataset: str) -> DatasetFlags:
|
||||
mapped = self.datasets or {}
|
||||
return mapped.get(dataset) or DatasetFlags(dataset)
|
||||
|
||||
def any_enabled(self) -> bool:
|
||||
return any(item.read or item.shadow for item in (self.datasets or {}).values())
|
||||
|
||||
def secrets(self) -> tuple[str, ...]:
|
||||
return tuple(item for item in (self.token,) if item)
|
||||
|
||||
@classmethod
|
||||
def load(
|
||||
cls,
|
||||
path: Path | None = None,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
credentials: Mapping[str, object] | None = None,
|
||||
) -> "DatahubSettings":
|
||||
config_path = path or DEFAULT_CONFIG_PATH
|
||||
payload: dict[str, Any] = {}
|
||||
if config_path.is_file():
|
||||
payload = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
env = dict(os.environ if environ is None else environ)
|
||||
creds = dict(credentials or {})
|
||||
dataset_flags: dict[str, DatasetFlags] = {}
|
||||
raw_datasets = payload.get("datasets") or {}
|
||||
for name in DATASETS:
|
||||
item = raw_datasets.get(name) or {}
|
||||
env_key = ENV_DATASET[name]
|
||||
read = _truthy(env.get(f"DATAHUB_READ_{env_key}")) if f"DATAHUB_READ_{env_key}" in env else bool(item.get("read"))
|
||||
shadow = (
|
||||
_truthy(env.get(f"DATAHUB_SHADOW_{env_key}"))
|
||||
if f"DATAHUB_SHADOW_{env_key}" in env
|
||||
else bool(item.get("shadow"))
|
||||
)
|
||||
dataset_flags[name] = DatasetFlags(name, read=read, shadow=shadow)
|
||||
token = str(
|
||||
env.get("DATAHUB_TOKEN")
|
||||
or creds.get("datahub_token")
|
||||
or payload.get("token")
|
||||
or ""
|
||||
).strip()
|
||||
base_url = str(
|
||||
env.get("DATAHUB_BASE_URL")
|
||||
or creds.get("datahub_base_url")
|
||||
or payload.get("base_url")
|
||||
or "http://127.0.0.1:8766"
|
||||
).strip().rstrip("/")
|
||||
return cls(
|
||||
base_url=base_url,
|
||||
token=token,
|
||||
timeout_seconds=_int(env.get("DATAHUB_TIMEOUT") or payload.get("timeout_seconds"), 8),
|
||||
retries=max(0, _int(env.get("DATAHUB_RETRIES") or payload.get("retries"), 1)),
|
||||
page_limit=max(1, _int(payload.get("page_limit"), 5000)),
|
||||
stale_seconds_max=max(0, _int(payload.get("stale_seconds_max"), 86400)),
|
||||
datasets=dataset_flags,
|
||||
)
|
||||
+14
-1
@@ -3,8 +3,10 @@ from __future__ import annotations
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from backend.data.contracts import DataUsage
|
||||
from backend.data.datahub import DatahubAwareTushareClient, DatahubBridge, DatahubClient, DatahubSettings
|
||||
from backend.data.policy import DataSourcePolicy
|
||||
from backend.data.providers import IfindProvider, TushareProvider
|
||||
from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport
|
||||
@@ -22,6 +24,7 @@ class DataGateway:
|
||||
ifind_provider: IfindProvider
|
||||
chart_data: MarketChartClient
|
||||
realtime_observer: WebRealtimeAggregator
|
||||
datahub: DatahubBridge
|
||||
|
||||
@property
|
||||
def ifind(self) -> IfindHttpClient:
|
||||
@@ -34,7 +37,13 @@ class DataGateway:
|
||||
) -> TushareClient:
|
||||
if dataset_id:
|
||||
self.policy.assert_allowed(dataset_id, "tushare", usage)
|
||||
return self.tushare_provider.client()
|
||||
return DatahubAwareTushareClient(self.tushare_provider.client(), self.datahub)
|
||||
|
||||
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
||||
return self.datahub.dataset_status(trade_date)
|
||||
|
||||
def batches(self, trade_date: str, dataset: str = "") -> list[dict[str, Any]] | None:
|
||||
return self.datahub.batches(trade_date, dataset)
|
||||
|
||||
def assert_source(self, dataset_id: str, provider_id: str, usage: DataUsage) -> None:
|
||||
self.policy.assert_allowed(dataset_id, provider_id, usage)
|
||||
@@ -64,6 +73,7 @@ class DataGateway:
|
||||
def build_data_gateway(
|
||||
credentials: dict[str, object],
|
||||
tushare_token_supplier: Callable[[], str] | None = None,
|
||||
datahub_settings: DatahubSettings | None = None,
|
||||
) -> DataGateway:
|
||||
ifind = IfindHttpClient(
|
||||
str(credentials.get("ifind_refresh_token") or ""),
|
||||
@@ -73,6 +83,8 @@ def build_data_gateway(
|
||||
lambda: str(credentials.get("tushare_token") or "")
|
||||
)
|
||||
policy = DataSourcePolicy.load()
|
||||
settings = datahub_settings or DatahubSettings.load(credentials=credentials)
|
||||
datahub_client = DatahubClient(settings)
|
||||
return DataGateway(
|
||||
policy=policy,
|
||||
quality=DataQualityGate.load(policy),
|
||||
@@ -80,4 +92,5 @@ def build_data_gateway(
|
||||
ifind_provider=IfindProvider(ifind),
|
||||
chart_data=MarketChartClient(ifind, EastmoneyChartClient()),
|
||||
realtime_observer=WebRealtimeAggregator(),
|
||||
datahub=DatahubBridge(settings, datahub_client),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user