fix(HEL-494): 数据中枢独占调度,主网站不再回退旧接口
主网站只向中枢要业务数据;来源选择、切源、补数全部在中枢内部完成,失败不再走东财/腾讯/Tushare 保底。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
co-authored by
Cursor
multica-agent
parent
ef13d6feb5
commit
0b8419abca
@@ -19,6 +19,7 @@ from backend.data.datahub.redact import redact_text, redact_value
|
|||||||
from backend.data.datahub.route_state import LEDGER
|
from backend.data.datahub.route_state import LEDGER
|
||||||
from backend.data.datahub.settings import DatahubSettings
|
from backend.data.datahub.settings import DatahubSettings
|
||||||
from backend.data.providers.tushare_client import TushareClient
|
from backend.data.providers.tushare_client import TushareClient
|
||||||
|
from backend.data.providers.tushare_transport import TushareError
|
||||||
|
|
||||||
LOGGER = logging.getLogger("xiaobai.datahub")
|
LOGGER = logging.getLogger("xiaobai.datahub")
|
||||||
ShadowSink = Callable[[dict[str, Any]], None]
|
ShadowSink = Callable[[dict[str, Any]], None]
|
||||||
@@ -171,6 +172,45 @@ class DatahubBridge:
|
|||||||
self._log_failure("index_quotes", exc)
|
self._log_failure("index_quotes", exc)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def try_sector_quote(self, code: str, trade_date: str = "") -> dict[str, Any] | None:
|
||||||
|
flags = self.settings.flags("quotes")
|
||||||
|
if not flags.read:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
response = self.client.sector_quote(code, trade_date)
|
||||||
|
data = response.data
|
||||||
|
if not isinstance(data, dict) or not data:
|
||||||
|
raise DatahubError("EMPTY", "datahub sector quote empty")
|
||||||
|
row = dict(data)
|
||||||
|
if (response.meta or {}).get("stale"):
|
||||||
|
row["delayed"] = True
|
||||||
|
row["delay_seconds"] = int((response.meta or {}).get("staleness_seconds") or 0)
|
||||||
|
row["delay_notice"] = str((response.meta or {}).get("delay_notice") or "")
|
||||||
|
self._record_route("quotes", "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||||
|
return row
|
||||||
|
except Exception as exc:
|
||||||
|
self._log_failure("quotes", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
|
def try_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]] | None:
|
||||||
|
flags = self.settings.flags("limit_events")
|
||||||
|
if not flags.read:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
response = self.client.limit_pool(trade_date)
|
||||||
|
rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)]
|
||||||
|
if not rows:
|
||||||
|
raise DatahubError("EMPTY", "datahub limit pool empty")
|
||||||
|
self._record_route(
|
||||||
|
"limit_events",
|
||||||
|
"datahub",
|
||||||
|
str((response.meta or {}).get("source") or "datahub"),
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
except Exception as exc:
|
||||||
|
self._log_failure("limit_events", exc)
|
||||||
|
return None
|
||||||
|
|
||||||
def try_daily_chart(
|
def try_daily_chart(
|
||||||
self,
|
self,
|
||||||
code: str,
|
code: str,
|
||||||
@@ -191,6 +231,11 @@ class DatahubBridge:
|
|||||||
self.client.index_bars,
|
self.client.index_bars,
|
||||||
{"code": code, "from": start, "to": compact_end},
|
{"code": code, "from": start, "to": compact_end},
|
||||||
)
|
)
|
||||||
|
elif dataset == "sector_daily":
|
||||||
|
response = self._paginate(
|
||||||
|
self.client.sectors,
|
||||||
|
{"code": code, "from": start, "to": compact_end},
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
response = self._paginate(
|
response = self._paginate(
|
||||||
self.client.daily_bars,
|
self.client.daily_bars,
|
||||||
@@ -263,53 +308,33 @@ class DatahubBridge:
|
|||||||
fields: str,
|
fields: str,
|
||||||
legacy_query: Callable[..., list[dict[str, Any]]],
|
legacy_query: Callable[..., list[dict[str, Any]]],
|
||||||
) -> list[dict[str, Any]]:
|
) -> list[dict[str, Any]]:
|
||||||
|
del legacy_query # 主网站不再直连 Tushare;调度全部由数据中枢完成。
|
||||||
|
if api_name == "rt_sw_k":
|
||||||
|
raise TushareError("rt_sw_k is disabled; use published sw_daily or free Shenwan realtime")
|
||||||
dataset = API_TO_DATASET.get(api_name)
|
dataset = API_TO_DATASET.get(api_name)
|
||||||
# 问天按实际数据依赖接入:已映射到 hub 的 API 跟随开关;未映射的继续旧链路。
|
if dataset:
|
||||||
if not dataset:
|
flags = self.settings.flags(dataset)
|
||||||
return legacy_query(api_name, params, fields)
|
|
||||||
flags = self.settings.flags(dataset)
|
|
||||||
if not flags.read and not flags.shadow:
|
|
||||||
return legacy_query(api_name, params, fields)
|
|
||||||
|
|
||||||
hub_rows: list[dict[str, Any]] | None = None
|
|
||||||
hub_meta: dict[str, Any] = {}
|
|
||||||
hub_error: str | None = None
|
|
||||||
hub_canonical: list[dict[str, Any]] = []
|
|
||||||
try:
|
|
||||||
response = self._fetch_dataset(dataset, params or {}, api_name=api_name)
|
|
||||||
hub_canonical = self._extract_rows(dataset, response, params or {})
|
|
||||||
hub_rows = to_native_rows(dataset, hub_canonical)
|
|
||||||
hub_meta = dict(response.meta)
|
|
||||||
self._validate_usable(dataset, hub_rows, response)
|
|
||||||
except Exception as exc:
|
|
||||||
hub_error = self._error_text(exc)
|
|
||||||
self._log_failure(dataset, exc)
|
|
||||||
|
|
||||||
if flags.shadow:
|
|
||||||
try:
|
|
||||||
legacy_rows = legacy_query(api_name, params, fields)
|
|
||||||
except Exception as exc:
|
|
||||||
if flags.read and hub_rows is not None and hub_error is None:
|
|
||||||
self._emit_shadow(
|
|
||||||
compare_rows(dataset, [], hub_canonical, hub_meta, self._error_text(exc), fields)
|
|
||||||
)
|
|
||||||
return project_fields(hub_rows, fields)
|
|
||||||
raise
|
|
||||||
self._emit_shadow(compare_rows(dataset, legacy_rows, hub_canonical, hub_meta, hub_error, fields))
|
|
||||||
if flags.read and hub_rows is not None and hub_error is None:
|
|
||||||
self._record_route(dataset, "datahub", str(hub_meta.get("source") or "datahub"))
|
|
||||||
return project_fields(hub_rows, fields)
|
|
||||||
if flags.read:
|
if flags.read:
|
||||||
self._record_route(dataset, "legacy", "tushare", hub_error or "")
|
try:
|
||||||
return legacy_rows
|
response = self._fetch_dataset(dataset, params or {}, api_name=api_name)
|
||||||
|
hub_canonical = self._extract_rows(dataset, response, params or {})
|
||||||
if flags.read and hub_rows is not None and hub_error is None:
|
hub_rows = to_native_rows(dataset, hub_canonical)
|
||||||
self._record_route(dataset, "datahub", str(hub_meta.get("source") or "datahub"))
|
self._validate_usable(dataset, hub_rows, response)
|
||||||
return project_fields(hub_rows, fields)
|
self._record_route(dataset, "datahub", str(response.meta.get("source") or "datahub"))
|
||||||
result = legacy_query(api_name, params, fields)
|
return project_fields(hub_rows, fields)
|
||||||
if flags.read:
|
except Exception as exc:
|
||||||
self._record_route(dataset, "legacy", "tushare", hub_error or "")
|
self._log_failure(dataset, exc)
|
||||||
return result
|
try:
|
||||||
|
response = self.client.query_api(api_name, params or {}, fields)
|
||||||
|
rows = [dict(item) for item in (response.data or []) if isinstance(item, dict)]
|
||||||
|
if dataset:
|
||||||
|
self._record_route(dataset, "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||||
|
else:
|
||||||
|
self._record_route(api_name, "datahub", str((response.meta or {}).get("source") or "datahub"))
|
||||||
|
return rows if not fields else project_fields(rows, fields)
|
||||||
|
except Exception as exc:
|
||||||
|
self._log_failure(dataset or api_name, exc)
|
||||||
|
raise TushareError(self._error_text(exc)) from exc
|
||||||
|
|
||||||
def _fetch_dataset(self, dataset: str, params: dict[str, Any], api_name: str = "") -> DatahubResponse:
|
def _fetch_dataset(self, dataset: str, params: dict[str, Any], api_name: str = "") -> DatahubResponse:
|
||||||
date = yyyymmdd(params.get("trade_date") or params.get("date"))
|
date = yyyymmdd(params.get("trade_date") or params.get("date"))
|
||||||
@@ -429,8 +454,8 @@ class DatahubBridge:
|
|||||||
|
|
||||||
def _log_failure(self, dataset: str, exc: Exception) -> None:
|
def _log_failure(self, dataset: str, exc: Exception) -> None:
|
||||||
error = redact_text(self._error_text(exc), self.settings.secrets())
|
error = redact_text(self._error_text(exc), self.settings.secrets())
|
||||||
LOGGER.warning("datahub fallback dataset=%s error=%s", dataset, error)
|
LOGGER.warning("datahub unavailable dataset=%s error=%s", dataset, error)
|
||||||
self._record_route(dataset, "legacy", "pending-legacy", error)
|
self._record_route(dataset, "datahub", "unavailable", error)
|
||||||
|
|
||||||
def _record_route(self, dataset: str, route: str, source: str = "", error: str = "") -> None:
|
def _record_route(self, dataset: str, route: str, source: str = "", error: str = "") -> None:
|
||||||
LEDGER.record(dataset, route, source, redact_text(error, self.settings.secrets()))
|
LEDGER.record(dataset, route, source, redact_text(error, self.settings.secrets()))
|
||||||
@@ -529,6 +554,8 @@ class DatahubAwareTushareClient:
|
|||||||
legacy.try_market_quotes = self.try_market_quotes
|
legacy.try_market_quotes = self.try_market_quotes
|
||||||
legacy.try_quotes = self.try_quotes
|
legacy.try_quotes = self.try_quotes
|
||||||
legacy.try_index_quotes = self.try_index_quotes
|
legacy.try_index_quotes = self.try_index_quotes
|
||||||
|
legacy.try_sector_quote = self.try_sector_quote
|
||||||
|
legacy.try_limit_pool = self.try_limit_pool
|
||||||
legacy.record_datahub_legacy = self.record_datahub_legacy
|
legacy.record_datahub_legacy = self.record_datahub_legacy
|
||||||
|
|
||||||
def query(
|
def query(
|
||||||
@@ -548,6 +575,12 @@ class DatahubAwareTushareClient:
|
|||||||
def try_index_quotes(self) -> list[dict[str, Any]] | None:
|
def try_index_quotes(self) -> list[dict[str, Any]] | None:
|
||||||
return self._bridge.try_index_quotes()
|
return self._bridge.try_index_quotes()
|
||||||
|
|
||||||
|
def try_sector_quote(self, code: str, trade_date: str = "") -> dict[str, Any] | None:
|
||||||
|
return self._bridge.try_sector_quote(code, trade_date)
|
||||||
|
|
||||||
|
def try_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]] | None:
|
||||||
|
return self._bridge.try_limit_pool(trade_date)
|
||||||
|
|
||||||
def record_datahub_legacy(self, dataset: str, source: str = "", error: str = "") -> None:
|
def record_datahub_legacy(self, dataset: str, source: str = "", error: str = "") -> None:
|
||||||
self._bridge.record_legacy(dataset, source, error)
|
self._bridge.record_legacy(dataset, source, error)
|
||||||
|
|
||||||
|
|||||||
@@ -90,6 +90,24 @@ class DatahubClient:
|
|||||||
params["dataset"] = dataset
|
params["dataset"] = dataset
|
||||||
return self.get("/v1/batches", params)
|
return self.get("/v1/batches", params)
|
||||||
|
|
||||||
|
def query_api(self, api_name: str, params: dict[str, Any] | None = None, fields: str = "") -> DatahubResponse:
|
||||||
|
return self.post(
|
||||||
|
"/v1/query",
|
||||||
|
{"api_name": api_name, "params": params or {}, "fields": fields},
|
||||||
|
)
|
||||||
|
|
||||||
|
def sector_quote(self, code: str, date: str = "") -> DatahubResponse:
|
||||||
|
payload: dict[str, Any] = {"code": code}
|
||||||
|
if date:
|
||||||
|
payload["date"] = date
|
||||||
|
return self.get("/v1/sectors/quote", payload)
|
||||||
|
|
||||||
|
def limit_pool(self, trade_date: str = "") -> DatahubResponse:
|
||||||
|
params: dict[str, Any] = {}
|
||||||
|
if trade_date:
|
||||||
|
params["date"] = trade_date
|
||||||
|
return self.get("/v1/limit-pool", params)
|
||||||
|
|
||||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||||
if not self.settings.token:
|
if not self.settings.token:
|
||||||
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
||||||
@@ -118,15 +136,41 @@ class DatahubClient:
|
|||||||
)
|
)
|
||||||
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
||||||
|
|
||||||
def _request(self, url: str) -> DatahubResponse:
|
def post(self, path: str, body: dict[str, Any] | None = None) -> DatahubResponse:
|
||||||
|
if not self.settings.token:
|
||||||
|
raise DatahubError("NOT_CONFIGURED", "DATAHUB_TOKEN is not configured")
|
||||||
|
url = self.settings.base_url + path
|
||||||
|
attempts = 1 + max(0, self.settings.retries)
|
||||||
|
last_error: DatahubError | None = None
|
||||||
|
payload = json.dumps(body or {}, ensure_ascii=False).encode("utf-8")
|
||||||
|
for attempt in range(attempts):
|
||||||
|
try:
|
||||||
|
return self._request(url, method="POST", data=payload)
|
||||||
|
except DatahubError as exc:
|
||||||
|
last_error = exc
|
||||||
|
if exc.code not in {"TIMEOUT", "UNAVAILABLE"} or attempt + 1 >= attempts:
|
||||||
|
raise
|
||||||
|
LOGGER.warning(
|
||||||
|
"datahub retry %s/%s %s",
|
||||||
|
attempt + 1,
|
||||||
|
attempts,
|
||||||
|
redact_text(str(exc), self.settings.secrets()),
|
||||||
|
)
|
||||||
|
raise last_error or DatahubError("INTERNAL", "datahub request failed")
|
||||||
|
|
||||||
|
def _request(self, url: str, method: str = "GET", data: bytes | None = None) -> DatahubResponse:
|
||||||
|
headers = {
|
||||||
|
"Accept": "application/json",
|
||||||
|
"X-Datahub-Token": self.settings.token,
|
||||||
|
"User-Agent": "XiaobaiReviewDatahub/1.0",
|
||||||
|
}
|
||||||
|
if data is not None:
|
||||||
|
headers["Content-Type"] = "application/json"
|
||||||
request = urllib.request.Request(
|
request = urllib.request.Request(
|
||||||
url,
|
url,
|
||||||
headers={
|
data=data,
|
||||||
"Accept": "application/json",
|
headers=headers,
|
||||||
"X-Datahub-Token": self.settings.token,
|
method=method,
|
||||||
"User-Agent": "XiaobaiReviewDatahub/1.0",
|
|
||||||
},
|
|
||||||
method="GET",
|
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
with self._urlopen(request, timeout=self.settings.timeout_seconds) as response:
|
with self._urlopen(request, timeout=self.settings.timeout_seconds) as response:
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ class DataGateway:
|
|||||||
if dataset_id:
|
if dataset_id:
|
||||||
self.policy.assert_allowed(dataset_id, "tushare", usage)
|
self.policy.assert_allowed(dataset_id, "tushare", usage)
|
||||||
legacy = self.tushare_provider.client()
|
legacy = self.tushare_provider.client()
|
||||||
legacy.realtime_aggregator = self.realtime_observer
|
legacy.realtime_aggregator = None
|
||||||
return DatahubAwareTushareClient(legacy, self.datahub)
|
return DatahubAwareTushareClient(legacy, self.datahub)
|
||||||
|
|
||||||
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
def dataset_status(self, trade_date: str) -> list[dict[str, Any]] | None:
|
||||||
|
|||||||
@@ -186,12 +186,11 @@ class DailyMarketMixin:
|
|||||||
return mapped
|
return mapped
|
||||||
|
|
||||||
def _free_board_map(self, trade_date: str) -> dict[str, dict[str, Any]]:
|
def _free_board_map(self, trade_date: str) -> dict[str, dict[str, Any]]:
|
||||||
aggregator = getattr(self, "realtime_aggregator", None)
|
loader = getattr(self, "try_limit_pool", None)
|
||||||
loader = getattr(aggregator, "eastmoney_limit_pool", None) if aggregator else None
|
|
||||||
if not callable(loader):
|
if not callable(loader):
|
||||||
return {}
|
return {}
|
||||||
try:
|
try:
|
||||||
rows = loader(trade_date)
|
rows = loader(trade_date) or []
|
||||||
except Exception:
|
except Exception:
|
||||||
return {}
|
return {}
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -251,27 +251,23 @@ class DashboardMixin:
|
|||||||
quotes = hub(trade_date)
|
quotes = hub(trade_date)
|
||||||
if quotes:
|
if quotes:
|
||||||
return list(quotes), "datahub"
|
return list(quotes), "datahub"
|
||||||
rt_error = ""
|
named = getattr(self, "try_quotes", None)
|
||||||
|
code_list = [item for item in str(codes or "").split(",") if item]
|
||||||
|
if callable(named) and code_list:
|
||||||
|
collected: list[dict[str, Any]] = []
|
||||||
|
for index in range(0, len(code_list), 60):
|
||||||
|
collected.extend(named(code_list[index:index + 60]) or [])
|
||||||
|
if collected:
|
||||||
|
delayed = any(item.get("delayed") for item in collected)
|
||||||
|
return collected, "datahub_delayed" if delayed else "datahub"
|
||||||
try:
|
try:
|
||||||
quotes = self.query("rt_k", {"ts_code": codes})
|
quotes = self.query("rt_k", {"ts_code": codes})
|
||||||
if quotes:
|
if quotes:
|
||||||
self._mark_quote_legacy("tushare_rt_k", rt_error)
|
delayed = any(item.get("delayed") for item in quotes)
|
||||||
return list(quotes), "tushare_rt_k"
|
return list(quotes), "datahub_delayed" if delayed else "datahub"
|
||||||
rt_error = f"No realtime data returned for {trade_date}"
|
|
||||||
except TushareError as exc:
|
except TushareError as exc:
|
||||||
rt_error = str(exc)
|
raise TushareError(f"当天盘中实时行情不可用:{exc}") from exc
|
||||||
try:
|
raise TushareError("当天盘中实时行情不可用:数据中枢未返回可用行情")
|
||||||
quotes, quote_source = self._free_realtime_quotes(trade_date, codes)
|
|
||||||
except Exception as exc:
|
|
||||||
raise TushareError(
|
|
||||||
f"当天盘中实时行情不可用:rt_k={rt_error};免费源={exc}"
|
|
||||||
) from exc
|
|
||||||
if not quotes:
|
|
||||||
raise TushareError(
|
|
||||||
f"当天盘中实时行情不可用:rt_k={rt_error};免费源=empty"
|
|
||||||
)
|
|
||||||
self._mark_quote_legacy(quote_source, rt_error)
|
|
||||||
return quotes, quote_source
|
|
||||||
|
|
||||||
def _mark_quote_legacy(self, source: str, error: str = "") -> None:
|
def _mark_quote_legacy(self, source: str, error: str = "") -> None:
|
||||||
marker = getattr(self, "record_datahub_legacy", None)
|
marker = getattr(self, "record_datahub_legacy", None)
|
||||||
@@ -283,27 +279,8 @@ class DashboardMixin:
|
|||||||
trade_date: str,
|
trade_date: str,
|
||||||
codes: str = "",
|
codes: str = "",
|
||||||
) -> tuple[list[dict[str, Any]], str]:
|
) -> tuple[list[dict[str, Any]], str]:
|
||||||
aggregator = self._realtime_aggregator()
|
del trade_date, codes
|
||||||
last_error = ""
|
raise TushareError("主网站不再直连免费行情源,请走数据中枢")
|
||||||
try:
|
|
||||||
quotes = aggregator.eastmoney_market_quotes(expected_date=trade_date)
|
|
||||||
if quotes:
|
|
||||||
return quotes, "eastmoney_clist"
|
|
||||||
except Exception as exc:
|
|
||||||
last_error = str(exc)
|
|
||||||
code_list = [item for item in str(codes or "").split(",") if item]
|
|
||||||
try:
|
|
||||||
if code_list:
|
|
||||||
quotes = aggregator.tencent_stock_quotes(code_list, expected_date=trade_date)
|
|
||||||
else:
|
|
||||||
quotes = aggregator.tencent_market_quotes(code_list, expected_date=trade_date)
|
|
||||||
except Exception as exc:
|
|
||||||
raise TushareError(
|
|
||||||
f"eastmoney={last_error or 'empty'};tencent={exc}"
|
|
||||||
) from exc
|
|
||||||
if not quotes:
|
|
||||||
raise TushareError(f"eastmoney={last_error or 'empty'};tencent=empty")
|
|
||||||
return quotes, "tencent_qt"
|
|
||||||
|
|
||||||
def _free_realtime_indices(self) -> list[dict[str, Any]]:
|
def _free_realtime_indices(self) -> list[dict[str, Any]]:
|
||||||
hub = getattr(self, "try_index_quotes", None)
|
hub = getattr(self, "try_index_quotes", None)
|
||||||
@@ -312,14 +289,7 @@ class DashboardMixin:
|
|||||||
converted = [item for item in (_hub_index_quote(row) for row in rows or []) if item]
|
converted = [item for item in (_hub_index_quote(row) for row in rows or []) if item]
|
||||||
if converted:
|
if converted:
|
||||||
return converted
|
return converted
|
||||||
try:
|
return []
|
||||||
rows = self._realtime_aggregator().eastmoney_indices()
|
|
||||||
marker = getattr(self, "record_datahub_legacy", None)
|
|
||||||
if callable(marker):
|
|
||||||
marker("index_quotes", "eastmoney_push2")
|
|
||||||
return rows
|
|
||||||
except Exception:
|
|
||||||
return []
|
|
||||||
|
|
||||||
def _load_realtime_reference(
|
def _load_realtime_reference(
|
||||||
self,
|
self,
|
||||||
@@ -470,21 +440,6 @@ class DashboardMixin:
|
|||||||
return dict(rows[0])
|
return dict(rows[0])
|
||||||
except TushareError:
|
except TushareError:
|
||||||
pass
|
pass
|
||||||
aggregator = getattr(self, "realtime_aggregator", None)
|
|
||||||
if aggregator is None:
|
|
||||||
return {}
|
|
||||||
for loader in (
|
|
||||||
getattr(aggregator, "eastmoney_stock_quote", None),
|
|
||||||
getattr(aggregator, "tencent_stock_quote", None),
|
|
||||||
):
|
|
||||||
if not callable(loader):
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
quote = loader(ts_code, expected_date=reference_date)
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if quote:
|
|
||||||
return dict(quote)
|
|
||||||
return {}
|
return {}
|
||||||
|
|
||||||
def _stock_activity_metrics(
|
def _stock_activity_metrics(
|
||||||
|
|||||||
@@ -63,22 +63,8 @@ class IndexMixin:
|
|||||||
if callable(hub):
|
if callable(hub):
|
||||||
rows = hub()
|
rows = hub()
|
||||||
if rows:
|
if rows:
|
||||||
try:
|
return self._hub_realtime_market_indices(requested_date, rows)
|
||||||
return self._hub_realtime_market_indices(requested_date, rows)
|
raise TushareError("Realtime index quotes are incomplete")
|
||||||
except TushareError:
|
|
||||||
pass
|
|
||||||
try:
|
|
||||||
payload = self._tushare_realtime_market_indices(requested_date)
|
|
||||||
marker = getattr(self, "record_datahub_legacy", None)
|
|
||||||
if callable(marker):
|
|
||||||
marker("index_quotes", "tushare_rt_idx_k")
|
|
||||||
return payload
|
|
||||||
except TushareError:
|
|
||||||
payload = self._free_realtime_market_indices(requested_date)
|
|
||||||
marker = getattr(self, "record_datahub_legacy", None)
|
|
||||||
if callable(marker):
|
|
||||||
marker("index_quotes", str(payload.get("source") or "eastmoney_push2"))
|
|
||||||
return payload
|
|
||||||
|
|
||||||
def _hub_realtime_market_indices(
|
def _hub_realtime_market_indices(
|
||||||
self,
|
self,
|
||||||
@@ -199,50 +185,5 @@ class IndexMixin:
|
|||||||
}
|
}
|
||||||
|
|
||||||
def _free_realtime_market_indices(self, requested_date: str) -> dict[str, Any]:
|
def _free_realtime_market_indices(self, requested_date: str) -> dict[str, Any]:
|
||||||
trade_date, _ = self.resolve_trade_context(requested_date)
|
del requested_date
|
||||||
aggregator = getattr(self, "realtime_aggregator", None)
|
raise TushareError("主网站不再直连免费行情源,请走数据中枢")
|
||||||
if aggregator is None:
|
|
||||||
raise TushareError("免费实时源未配置")
|
|
||||||
quotes = aggregator.eastmoney_indices()
|
|
||||||
index_names = {
|
|
||||||
"000001": ("000001.SH", "上证指数"),
|
|
||||||
"399001": ("399001.SZ", "深证成指"),
|
|
||||||
"399006": ("399006.SZ", "创业板指"),
|
|
||||||
}
|
|
||||||
indices = []
|
|
||||||
for quote in quotes:
|
|
||||||
mapped = index_names.get(str(quote.get("code") or ""))
|
|
||||||
if not mapped:
|
|
||||||
continue
|
|
||||||
ts_code, name = mapped
|
|
||||||
close = _number(quote.get("price"))
|
|
||||||
previous_close = _number(quote.get("previous_close"))
|
|
||||||
if close <= 0 or previous_close <= 0:
|
|
||||||
continue
|
|
||||||
indices.append(
|
|
||||||
{
|
|
||||||
"ts_code": ts_code,
|
|
||||||
"name": str(quote.get("name") or name).strip(),
|
|
||||||
"trade_date": trade_date,
|
|
||||||
"close": close,
|
|
||||||
"pct_chg": round(_number(quote.get("change")) or (close / previous_close - 1) * 100, 3),
|
|
||||||
"return_5d": 0,
|
|
||||||
"amount_billion": round(_number(quote.get("amount_billion")), 2),
|
|
||||||
"quote_time": quote.get("quote_time") or "",
|
|
||||||
"source": quote.get("source") or "eastmoney_push2",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
if len(indices) != 3:
|
|
||||||
raise TushareError("Realtime index quotes are incomplete")
|
|
||||||
return {
|
|
||||||
"trade_date": trade_date,
|
|
||||||
"source": "eastmoney_push2",
|
|
||||||
"realtime": True,
|
|
||||||
"precise": True,
|
|
||||||
"indices": indices,
|
|
||||||
"aggregate": {
|
|
||||||
"average_pct_chg": round(sum(item["pct_chg"] for item in indices) / len(indices), 3),
|
|
||||||
"average_return_5d": 0,
|
|
||||||
"average_return_20d": 0,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -618,27 +618,19 @@ class ShenwanIndustryMixin:
|
|||||||
trade_date: str,
|
trade_date: str,
|
||||||
finalized: bool = False,
|
finalized: bool = False,
|
||||||
) -> tuple[dict[str, Any], str, str]:
|
) -> tuple[dict[str, Any], str, str]:
|
||||||
aggregator = getattr(self, "realtime_aggregator", None)
|
hub = getattr(self, "try_sector_quote", None)
|
||||||
loader = getattr(aggregator, "eastmoney_shenwan_quote", None) if aggregator else None
|
if callable(hub):
|
||||||
if callable(loader):
|
|
||||||
try:
|
try:
|
||||||
row = loader(sector_code, expected_date="" if finalized else trade_date)
|
row = hub(sector_code, "" if finalized else trade_date)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
message = str(exc)
|
message = str(exc)
|
||||||
if finalized:
|
if finalized:
|
||||||
return {}, "", f"申万行业 {sector_code} 盘后正式数据待入库"
|
return {}, "", f"申万行业 {sector_code} 盘后正式数据待入库"
|
||||||
return {}, "", f"免费申万实时暂不可用:{message[:180]}"
|
return {}, "", f"数据中枢申万实时暂不可用:{message[:180]}"
|
||||||
if row:
|
if row:
|
||||||
return dict(row), str(row.get("source") or "eastmoney_sw"), ""
|
return dict(row), str(row.get("source") or "datahub"), ""
|
||||||
if finalized:
|
if finalized:
|
||||||
return {}, "", f"申万行业 {sector_code} 当日盘后正式数据尚未入库"
|
return {}, "", f"申万行业 {sector_code} 当日盘后正式数据尚未入库"
|
||||||
if aggregator and sector_name:
|
|
||||||
try:
|
|
||||||
row = aggregator.eastmoney_sector(sector_name)
|
|
||||||
except Exception as exc:
|
|
||||||
return {}, "", f"免费行业实时暂不可用:{str(exc)[:180]}"
|
|
||||||
if row:
|
|
||||||
return dict(row), str(row.get("source") or "eastmoney_sector"), ""
|
|
||||||
return {}, "", f"申万行业 {sector_code} 当日外显待补充"
|
return {}, "", f"申万行业 {sector_code} 当日外显待补充"
|
||||||
|
|
||||||
def _load_member_realtime_quotes(
|
def _load_member_realtime_quotes(
|
||||||
@@ -677,31 +669,6 @@ class ShenwanIndustryMixin:
|
|||||||
delayed = any(item.get("delayed") for item in filtered)
|
delayed = any(item.get("delayed") for item in filtered)
|
||||||
return filtered, "datahub_delayed" if delayed else "datahub"
|
return filtered, "datahub_delayed" if delayed else "datahub"
|
||||||
|
|
||||||
aggregator = getattr(self, "realtime_aggregator", None)
|
|
||||||
eastmoney_loader = getattr(aggregator, "eastmoney_stock_quotes", None) if aggregator else None
|
|
||||||
if callable(eastmoney_loader):
|
|
||||||
try:
|
|
||||||
filtered = consider(eastmoney_loader(wanted, expected_date=trade_date) or [], "eastmoney_ulist")
|
|
||||||
if len(filtered) >= max(1, int(len(wanted) * 0.9)):
|
|
||||||
return filtered, "eastmoney_ulist"
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
tencent_loader = getattr(aggregator, "tencent_stock_quotes", None) if aggregator else None
|
|
||||||
if callable(tencent_loader):
|
|
||||||
try:
|
|
||||||
filtered = consider(tencent_loader(wanted, expected_date=trade_date) or [], "tencent_qt")
|
|
||||||
if len(filtered) >= max(1, int(len(wanted) * 0.9)):
|
|
||||||
return filtered, "tencent_qt"
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
quotes, source = self._free_realtime_quotes(trade_date, ",".join(wanted))
|
|
||||||
consider(quotes, source)
|
|
||||||
except TushareError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if best_rows:
|
if best_rows:
|
||||||
delayed = any(item.get("delayed") for item in best_rows)
|
delayed = any(item.get("delayed") for item in best_rows)
|
||||||
if delayed and not str(best_source).endswith("_delayed"):
|
if delayed and not str(best_source).endswith("_delayed"):
|
||||||
|
|||||||
@@ -61,11 +61,7 @@ class MarketChartClient:
|
|||||||
hub_chart = self._datahub_intraday(normalized)
|
hub_chart = self._datahub_intraday(normalized)
|
||||||
if hub_chart is not None:
|
if hub_chart is not None:
|
||||||
return hub_chart
|
return hub_chart
|
||||||
ifind_code = _stock_market_code(normalized)
|
raise ChartDataError("分时图数据中枢暂不可用")
|
||||||
try:
|
|
||||||
return self._ifind_intraday(ifind_code, "stock", normalized)
|
|
||||||
except (IfindError, ChartDataError):
|
|
||||||
return self.fallback.stock_intraday(normalized)
|
|
||||||
|
|
||||||
def stock_daily(self, code: str, end_date: str, limit: int = DAILY_CHART_LIMIT) -> list[dict[str, Any]]:
|
def stock_daily(self, code: str, end_date: str, limit: int = DAILY_CHART_LIMIT) -> list[dict[str, Any]]:
|
||||||
normalized = str(code or "").strip()
|
normalized = str(code or "").strip()
|
||||||
@@ -74,7 +70,7 @@ class MarketChartClient:
|
|||||||
hub_rows = self._datahub_daily(normalized, end_date, limit, "daily")
|
hub_rows = self._datahub_daily(normalized, end_date, limit, "daily")
|
||||||
if hub_rows:
|
if hub_rows:
|
||||||
return hub_rows
|
return hub_rows
|
||||||
return self._ifind_daily(_stock_market_code(normalized), end_date, limit)
|
raise ChartDataError("日K数据中枢暂不可用")
|
||||||
|
|
||||||
def index_daily(self, identifier: str, end_date: str, limit: int = DAILY_CHART_LIMIT) -> list[dict[str, Any]]:
|
def index_daily(self, identifier: str, end_date: str, limit: int = DAILY_CHART_LIMIT) -> list[dict[str, Any]]:
|
||||||
normalized = str(identifier or "").strip().upper()
|
normalized = str(identifier or "").strip().upper()
|
||||||
@@ -83,13 +79,16 @@ class MarketChartClient:
|
|||||||
hub_rows = self._datahub_daily(normalized, end_date, limit, "index_daily")
|
hub_rows = self._datahub_daily(normalized, end_date, limit, "index_daily")
|
||||||
if hub_rows:
|
if hub_rows:
|
||||||
return hub_rows
|
return hub_rows
|
||||||
return self._ifind_daily(normalized, end_date, limit)
|
raise ChartDataError("指数日K数据中枢暂不可用")
|
||||||
|
|
||||||
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
|
def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]:
|
||||||
normalized = str(identifier or "").strip().upper()
|
normalized = str(identifier or "").strip().upper()
|
||||||
if not normalized:
|
if not normalized:
|
||||||
raise ChartDataError("Invalid board code")
|
raise ChartDataError("Invalid board code")
|
||||||
return self._ifind_daily(normalized, end_date, limit)
|
hub_rows = self._datahub_daily(normalized, end_date, limit, "sector_daily")
|
||||||
|
if hub_rows:
|
||||||
|
return hub_rows
|
||||||
|
raise ChartDataError("板块日K数据中枢暂不可用")
|
||||||
|
|
||||||
def index_intraday(self, identifier: str) -> dict[str, Any]:
|
def index_intraday(self, identifier: str) -> dict[str, Any]:
|
||||||
normalized = str(identifier or "").strip().upper()
|
normalized = str(identifier or "").strip().upper()
|
||||||
@@ -98,10 +97,7 @@ class MarketChartClient:
|
|||||||
hub_chart = self._datahub_intraday(normalized)
|
hub_chart = self._datahub_intraday(normalized)
|
||||||
if hub_chart is not None:
|
if hub_chart is not None:
|
||||||
return hub_chart
|
return hub_chart
|
||||||
try:
|
raise ChartDataError("指数分时数据中枢暂不可用")
|
||||||
return self._ifind_intraday(normalized, "index", normalized)
|
|
||||||
except (IfindError, ChartDataError):
|
|
||||||
return self.fallback.index_intraday(normalized)
|
|
||||||
|
|
||||||
def _datahub_intraday(self, code: str) -> dict[str, Any] | None:
|
def _datahub_intraday(self, code: str) -> dict[str, Any] | None:
|
||||||
if self.datahub is None:
|
if self.datahub is None:
|
||||||
@@ -133,8 +129,6 @@ class MarketChartClient:
|
|||||||
LOGGER.warning("datahub daily unexpected error: %s", exc)
|
LOGGER.warning("datahub daily unexpected error: %s", exc)
|
||||||
rows = None
|
rows = None
|
||||||
if not rows:
|
if not rows:
|
||||||
if hasattr(self.datahub, "record_legacy"):
|
|
||||||
self.datahub.record_legacy(dataset, "ifind")
|
|
||||||
return None
|
return None
|
||||||
compact_end = str(end_date or "").replace("-", "")
|
compact_end = str(end_date or "").replace("-", "")
|
||||||
market_now = datetime.now().astimezone()
|
market_now = datetime.now().astimezone()
|
||||||
@@ -226,10 +220,13 @@ class MarketChartClient:
|
|||||||
|
|
||||||
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
|
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
|
||||||
normalized = str(identifier or "").strip().upper()
|
normalized = str(identifier or "").strip().upper()
|
||||||
try:
|
hub_chart = self._datahub_intraday(normalized)
|
||||||
return self._ifind_intraday(normalized, "board", normalized, name)
|
if hub_chart is not None:
|
||||||
except (IfindError, ChartDataError):
|
if name:
|
||||||
return self.fallback.board_intraday(normalized, name)
|
hub_chart = dict(hub_chart)
|
||||||
|
hub_chart["name"] = name
|
||||||
|
return hub_chart
|
||||||
|
raise ChartDataError("板块分时数据中枢暂不可用")
|
||||||
|
|
||||||
def _ifind_intraday(
|
def _ifind_intraday(
|
||||||
self,
|
self,
|
||||||
|
|||||||
@@ -488,8 +488,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/data/providers/tushare_industries.py",
|
"path": "backend/data/providers/tushare_industries.py",
|
||||||
"bytes": 38757,
|
"bytes": 37168,
|
||||||
"lines": 892
|
"lines": 859
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/features/screener/catalog.py",
|
"path": "backend/features/screener/catalog.py",
|
||||||
@@ -503,8 +503,8 @@
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/data/providers/tushare_dashboard.py",
|
"path": "backend/data/providers/tushare_dashboard.py",
|
||||||
"bytes": 34773,
|
"bytes": 33230,
|
||||||
"lines": 815
|
"lines": 770
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"path": "database.py",
|
"path": "database.py",
|
||||||
@@ -591,11 +591,6 @@
|
|||||||
"bytes": 12829,
|
"bytes": 12829,
|
||||||
"lines": 318
|
"lines": 318
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"path": "backend/data/providers/tushare_indices.py",
|
|
||||||
"bytes": 10956,
|
|
||||||
"lines": 248
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"path": "backend/features/market/insights_auction.py",
|
"path": "backend/features/market/insights_auction.py",
|
||||||
"bytes": 10717,
|
"bytes": 10717,
|
||||||
@@ -616,16 +611,16 @@
|
|||||||
"bytes": 9348,
|
"bytes": 9348,
|
||||||
"lines": 222
|
"lines": 222
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"path": "backend/data/providers/tushare_daily.py",
|
|
||||||
"bytes": 9170,
|
|
||||||
"lines": 233
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"path": "frontend/pages/market/entity-detail.js",
|
"path": "frontend/pages/market/entity-detail.js",
|
||||||
"bytes": 9139,
|
"bytes": 9139,
|
||||||
"lines": 199
|
"lines": 199
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/data/providers/tushare_daily.py",
|
||||||
|
"bytes": 9076,
|
||||||
|
"lines": 232
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "backend/data/providers/tushare_dragon_tiger.py",
|
"path": "backend/data/providers/tushare_dragon_tiger.py",
|
||||||
"bytes": 9059,
|
"bytes": 9059,
|
||||||
@@ -636,6 +631,11 @@
|
|||||||
"bytes": 8562,
|
"bytes": 8562,
|
||||||
"lines": 238
|
"lines": 238
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
"path": "backend/data/providers/tushare_indices.py",
|
||||||
|
"bytes": 8447,
|
||||||
|
"lines": 189
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"path": "frontend/pages/mentor/page.html",
|
"path": "frontend/pages/mentor/page.html",
|
||||||
"bytes": 8357,
|
"bytes": 8357,
|
||||||
|
|||||||
@@ -202,7 +202,7 @@ class DatahubChartFallbackTests(unittest.TestCase):
|
|||||||
self.assertEqual(hub.calls, ["601318"])
|
self.assertEqual(hub.calls, ["601318"])
|
||||||
self.assertEqual(fallback.requests, [])
|
self.assertEqual(fallback.requests, [])
|
||||||
|
|
||||||
def test_datahub_timeout_or_empty_falls_back_to_eastmoney(self):
|
def test_datahub_timeout_or_empty_does_not_use_old_channel(self):
|
||||||
fallback = LookbackChartClient()
|
fallback = LookbackChartClient()
|
||||||
for hub in (
|
for hub in (
|
||||||
FakeHub(chart=None),
|
FakeHub(chart=None),
|
||||||
@@ -213,10 +213,9 @@ class DatahubChartFallbackTests(unittest.TestCase):
|
|||||||
EastmoneyChartClient._cache.clear()
|
EastmoneyChartClient._cache.clear()
|
||||||
fallback.requests.clear()
|
fallback.requests.clear()
|
||||||
client = MarketChartClient(IfindHttpClient(), fallback, hub)
|
client = MarketChartClient(IfindHttpClient(), fallback, hub)
|
||||||
payload = client.stock_intraday("000001")
|
with self.assertRaises(ChartDataError):
|
||||||
self.assertEqual(payload["trade_date"], "2026-09-07")
|
client.stock_intraday("000001")
|
||||||
self.assertGreaterEqual(len(payload["points"]), 1)
|
self.assertEqual(fallback.requests, [])
|
||||||
self.assertTrue(fallback.requests)
|
|
||||||
|
|
||||||
def test_datahub_daily_skips_ifind(self):
|
def test_datahub_daily_skips_ifind(self):
|
||||||
hub = FakeHub(
|
hub = FakeHub(
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from backend.data.datahub.compare import compare_rows
|
|||||||
from backend.data.datahub.errors import DatahubError
|
from backend.data.datahub.errors import DatahubError
|
||||||
from backend.data.datahub.native import to_canonical_row, to_native_row
|
from backend.data.datahub.native import to_canonical_row, to_native_row
|
||||||
from backend.data.datahub.route_state import LEDGER
|
from backend.data.datahub.route_state import LEDGER
|
||||||
|
from backend.data.providers.tushare_transport import TushareError
|
||||||
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
@@ -68,9 +69,16 @@ class FakeClient(DatahubClient):
|
|||||||
self.calls: list[tuple[str, dict[str, Any]]] = []
|
self.calls: list[tuple[str, dict[str, Any]]] = []
|
||||||
|
|
||||||
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
|
||||||
|
return self._record(path, params)
|
||||||
|
|
||||||
|
def post(self, path: str, body: dict[str, Any] | None = None) -> DatahubResponse:
|
||||||
|
return self._record(path, body)
|
||||||
|
|
||||||
|
def _record(self, path: str, payload: dict[str, Any] | None) -> DatahubResponse:
|
||||||
self.paths.append(path)
|
self.paths.append(path)
|
||||||
self.calls.append((path, {key: value for key, value in (params or {}).items()}))
|
self.calls.append((path, {key: value for key, value in (payload or {}).items()}))
|
||||||
if TOKEN in json.dumps(params or {}) or TOKEN in path:
|
packed = json.dumps(payload or {})
|
||||||
|
if TOKEN in packed or TOKEN in path:
|
||||||
raise AssertionError("token leaked into url")
|
raise AssertionError("token leaked into url")
|
||||||
if self.error:
|
if self.error:
|
||||||
raise self.error
|
raise self.error
|
||||||
@@ -131,16 +139,22 @@ class DatahubBridgeTests(unittest.TestCase):
|
|||||||
self.assertEqual(legacy.calls, [])
|
self.assertEqual(legacy.calls, [])
|
||||||
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||||
calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}])
|
calendar_legacy = FakeLegacy([{"cal_date": "20240902", "is_open": 1}])
|
||||||
calendar_client = FakeClient(error=DatahubError("INTERNAL", "nope"))
|
calendar_client = FakeClient(
|
||||||
|
response=DatahubResponse(
|
||||||
|
data=[{"cal_date": "20240902", "is_open": 1, "pretrade_date": "20240830"}],
|
||||||
|
meta={"source": "datahub", "stale": False, "staleness_seconds": 0},
|
||||||
|
)
|
||||||
|
)
|
||||||
calendar_wrapped = DatahubAwareTushareClient(
|
calendar_wrapped = DatahubAwareTushareClient(
|
||||||
calendar_legacy,
|
calendar_legacy,
|
||||||
DatahubBridge(flags(daily=(True, False)), calendar_client),
|
DatahubBridge(flags(daily=(True, False)), calendar_client),
|
||||||
)
|
)
|
||||||
calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "")
|
calendar = calendar_wrapped.query("trade_cal", {"start_date": "20240902", "end_date": "20240902"}, "")
|
||||||
self.assertEqual(calendar[0]["is_open"], 1)
|
self.assertEqual(calendar[0]["is_open"], 1)
|
||||||
self.assertEqual(calendar_client.paths, [])
|
self.assertEqual(calendar_legacy.calls, [])
|
||||||
|
self.assertEqual(calendar_client.paths, ["/v1/query"])
|
||||||
|
|
||||||
def test_fallback_on_down_401_timeout_empty_unpublished_stale_and_incomplete(self) -> None:
|
def test_hub_failure_does_not_call_website_legacy(self) -> None:
|
||||||
cases = [
|
cases = [
|
||||||
DatahubError("UNAVAILABLE", "down"),
|
DatahubError("UNAVAILABLE", "down"),
|
||||||
DatahubError("UNAUTHORIZED", "401"),
|
DatahubError("UNAUTHORIZED", "401"),
|
||||||
@@ -152,34 +166,21 @@ class DatahubBridgeTests(unittest.TestCase):
|
|||||||
]
|
]
|
||||||
for error in cases:
|
for error in cases:
|
||||||
with self.subTest(error=error.code):
|
with self.subTest(error=error.code):
|
||||||
if error.code == "EMPTY":
|
client = FakeClient(error=error)
|
||||||
client = FakeClient(response=DatahubResponse(data=[], meta={"stale": False, "staleness_seconds": 0}))
|
|
||||||
elif error.code == "STALE":
|
|
||||||
client = FakeClient(response=DatahubResponse(
|
|
||||||
data=[dict(HUB_DAILY)],
|
|
||||||
meta={"stale": True, "staleness_seconds": 999999},
|
|
||||||
))
|
|
||||||
elif error.code == "INCOMPLETE":
|
|
||||||
client = FakeClient(response=DatahubResponse(
|
|
||||||
data=[dict(HUB_DAILY)],
|
|
||||||
meta={
|
|
||||||
"stale": False,
|
|
||||||
"staleness_seconds": 0,
|
|
||||||
"incomplete": True,
|
|
||||||
"coverage": {"complete": False, "missing_count": 80},
|
|
||||||
},
|
|
||||||
))
|
|
||||||
else:
|
|
||||||
client = FakeClient(error=error)
|
|
||||||
legacy = FakeLegacy([LEGACY_DAILY])
|
legacy = FakeLegacy([LEGACY_DAILY])
|
||||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
with self.assertRaises(TushareError):
|
||||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||||
self.assertEqual(len(legacy.calls), 1)
|
self.assertEqual(legacy.calls, [])
|
||||||
|
|
||||||
def test_shadow_compares_without_replacing_and_survives_hub_failure(self) -> None:
|
def test_shadow_mode_no_longer_calls_website_tushare(self) -> None:
|
||||||
reports: list[dict[str, Any]] = []
|
reports: list[dict[str, Any]] = []
|
||||||
client = FakeClient()
|
client = FakeClient(
|
||||||
|
response=DatahubResponse(
|
||||||
|
data=[dict(LEGACY_DAILY)],
|
||||||
|
meta={"source": "tushare", "stale": False, "staleness_seconds": 0, "row_shape": "tushare"},
|
||||||
|
)
|
||||||
|
)
|
||||||
legacy = FakeLegacy([LEGACY_DAILY])
|
legacy = FakeLegacy([LEGACY_DAILY])
|
||||||
wrapped = DatahubAwareTushareClient(
|
wrapped = DatahubAwareTushareClient(
|
||||||
legacy,
|
legacy,
|
||||||
@@ -187,21 +188,19 @@ class DatahubBridgeTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol")
|
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount,vol")
|
||||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||||
self.assertEqual(len(legacy.calls), 1)
|
self.assertEqual(legacy.calls, [])
|
||||||
self.assertEqual(reports[0]["equal"], True)
|
self.assertEqual(client.paths, ["/v1/query"])
|
||||||
self.assertEqual(reports[0]["matched"], 1)
|
|
||||||
|
|
||||||
failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN))
|
failed = FakeClient(error=DatahubError("UNAVAILABLE", TOKEN))
|
||||||
fail_reports: list[dict[str, Any]] = []
|
|
||||||
fail_legacy = FakeLegacy([LEGACY_DAILY])
|
fail_legacy = FakeLegacy([LEGACY_DAILY])
|
||||||
fail_wrapped = DatahubAwareTushareClient(
|
fail_wrapped = DatahubAwareTushareClient(
|
||||||
fail_legacy,
|
fail_legacy,
|
||||||
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=fail_reports.append),
|
DatahubBridge(flags(daily=(False, True)), failed, shadow_sink=reports.append),
|
||||||
)
|
)
|
||||||
again = fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
with self.assertRaises(TushareError):
|
||||||
self.assertEqual(again[0]["amount"], 2000.0)
|
fail_wrapped.query("daily", {"trade_date": "20240902"}, "amount")
|
||||||
self.assertTrue(fail_reports[0]["hub_error"])
|
self.assertEqual(fail_legacy.calls, [])
|
||||||
self.assertNotIn(TOKEN, json.dumps(fail_reports[0]))
|
self.assertNotIn(TOKEN, str(failed.calls))
|
||||||
|
|
||||||
def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None:
|
def test_compare_classifies_unit_conversion_missing_row_and_value_diff(self) -> None:
|
||||||
equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0})
|
equal = compare_rows("daily", [LEGACY_DAILY], [HUB_DAILY], {"stale": False, "staleness_seconds": 0})
|
||||||
@@ -288,13 +287,12 @@ class DatahubBridgeTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
wrapped = DatahubAwareTushareClient(
|
wrapped = DatahubAwareTushareClient(
|
||||||
FakeLegacy([legacy_close_only]),
|
FakeLegacy([legacy_close_only]),
|
||||||
DatahubBridge(flags(daily=(False, True)), client, shadow_sink=reports.append),
|
DatahubBridge(flags(daily=(True, False)), client, shadow_sink=reports.append),
|
||||||
)
|
)
|
||||||
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,trade_date,close,vol,amount")
|
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,trade_date,close,vol,amount")
|
||||||
self.assertEqual(rows[0]["close"], 10.20)
|
self.assertEqual(rows[0]["close"], 10.20)
|
||||||
self.assertEqual(rows[0]["vol"], 1000.0)
|
self.assertEqual(rows[0]["vol"], 1000.0)
|
||||||
self.assertTrue(reports[0]["equal"])
|
self.assertEqual(client.paths, ["/v1/bars/daily"])
|
||||||
self.assertEqual(reports[0]["matched"], 1)
|
|
||||||
|
|
||||||
def test_native_roundtrip_matches_known_scales(self) -> None:
|
def test_native_roundtrip_matches_known_scales(self) -> None:
|
||||||
native = to_native_row("daily", HUB_DAILY)
|
native = to_native_row("daily", HUB_DAILY)
|
||||||
@@ -347,21 +345,17 @@ class DatahubBridgeTests(unittest.TestCase):
|
|||||||
self.assertIn('"daily"', source)
|
self.assertIn('"daily"', source)
|
||||||
self.assertIn("start_date", source)
|
self.assertIn("start_date", source)
|
||||||
self.assertIn("end_date", source)
|
self.assertIn("end_date", source)
|
||||||
client = FakeClient(
|
client = FakeClient(error=DatahubError("INCOMPLETE", "truncated"))
|
||||||
response=DatahubResponse(
|
|
||||||
data=[dict(HUB_DAILY)],
|
|
||||||
meta={"stale": False, "staleness_seconds": 0, "incomplete": True, "coverage": {"complete": False, "missing_count": 89}},
|
|
||||||
)
|
|
||||||
)
|
|
||||||
legacy = FakeLegacy([LEGACY_DAILY])
|
legacy = FakeLegacy([LEGACY_DAILY])
|
||||||
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(flags(daily=(True, False)), client))
|
||||||
rows = wrapped.query(
|
with self.assertRaises(TushareError):
|
||||||
"daily",
|
wrapped.query(
|
||||||
{"ts_code": "600000.SH", "start_date": "20240301", "end_date": "20240902"},
|
"daily",
|
||||||
"ts_code,amount",
|
{"ts_code": "600000.SH", "start_date": "20240301", "end_date": "20240902"},
|
||||||
)
|
"ts_code,amount",
|
||||||
self.assertEqual(rows[0]["amount"], 2000.0)
|
)
|
||||||
self.assertEqual(len(legacy.calls), 1)
|
self.assertEqual(legacy.calls, [])
|
||||||
|
self.assertIn("/v1/query", client.paths)
|
||||||
|
|
||||||
def test_try_intraday_respects_switch_and_falls_back_on_bad_payload(self) -> None:
|
def test_try_intraday_respects_switch_and_falls_back_on_bad_payload(self) -> None:
|
||||||
closed = DatahubBridge(flags(), FakeClient(error=DatahubError("INTERNAL", "should not run")))
|
closed = DatahubBridge(flags(), FakeClient(error=DatahubError("INTERNAL", "should not run")))
|
||||||
@@ -462,17 +456,15 @@ class DatahubBridgeTests(unittest.TestCase):
|
|||||||
FakeClient(error=DatahubError("UNAVAILABLE", "down")),
|
FakeClient(error=DatahubError("UNAVAILABLE", "down")),
|
||||||
)
|
)
|
||||||
self.assertIsNone(failed.try_market_quotes("20240902"))
|
self.assertIsNone(failed.try_market_quotes("20240902"))
|
||||||
failed.record_legacy("quotes", "tencent_qt", "down")
|
|
||||||
snap = next(item for item in LEDGER.snapshot() if item["dataset"] == "quotes")
|
snap = next(item for item in LEDGER.snapshot() if item["dataset"] == "quotes")
|
||||||
self.assertEqual(snap["route"], "legacy")
|
self.assertEqual(snap["route"], "datahub")
|
||||||
self.assertEqual(snap["source"], "tencent_qt")
|
self.assertEqual(snap["source"], "unavailable")
|
||||||
self.assertIn("备用", "备用")
|
|
||||||
|
|
||||||
gateway = build_data_gateway({}, datahub_settings=flags(quotes=(True, False)))
|
gateway = build_data_gateway({}, datahub_settings=flags(quotes=(True, False)))
|
||||||
status = gateway.datahub_status()
|
status = gateway.datahub_status()
|
||||||
self.assertEqual(status["enabled_reads"], 1)
|
self.assertEqual(status["enabled_reads"], 1)
|
||||||
self.assertEqual(status["total_reads"], len(DATASETS))
|
self.assertEqual(status["total_reads"], len(DATASETS))
|
||||||
self.assertGreaterEqual(status["fallback_count"], 1)
|
self.assertEqual(status["fallback_count"], 0)
|
||||||
|
|
||||||
def test_try_daily_chart_converts_hub_bars(self) -> None:
|
def test_try_daily_chart_converts_hub_bars(self) -> None:
|
||||||
rows = [
|
rows = [
|
||||||
|
|||||||
@@ -87,11 +87,10 @@ class ShenwanRealtimeSourceTests(unittest.TestCase):
|
|||||||
with self.assertRaisesRegex(TushareError, "rt_sw_k is disabled"):
|
with self.assertRaisesRegex(TushareError, "rt_sw_k is disabled"):
|
||||||
client.query("rt_sw_k", {"ts_code": "801074.SI"})
|
client.query("rt_sw_k", {"ts_code": "801074.SI"})
|
||||||
|
|
||||||
def test_outer_realtime_uses_eastmoney_shenwan_not_rt_sw_k(self) -> None:
|
def test_outer_realtime_uses_hub_sector_quote_not_rt_sw_k(self) -> None:
|
||||||
client = TushareClient(token="demo")
|
client = TushareClient(token="demo")
|
||||||
client.query = MagicMock(side_effect=AssertionError("should not call tushare"))
|
client.query = MagicMock(side_effect=AssertionError("should not call tushare"))
|
||||||
client.realtime_aggregator = MagicMock()
|
client.try_sector_quote = MagicMock(return_value={
|
||||||
client.realtime_aggregator.eastmoney_shenwan_quote.return_value = {
|
|
||||||
"code": "801074.SI",
|
"code": "801074.SI",
|
||||||
"name": "工业金属",
|
"name": "工业金属",
|
||||||
"close": 1234.5,
|
"close": 1234.5,
|
||||||
@@ -101,7 +100,7 @@ class ShenwanRealtimeSourceTests(unittest.TestCase):
|
|||||||
"quote_date": "20260908",
|
"quote_date": "20260908",
|
||||||
"quote_time": "2026-09-08T14:50:00+08:00",
|
"quote_time": "2026-09-08T14:50:00+08:00",
|
||||||
"source": "eastmoney_sw",
|
"source": "eastmoney_sw",
|
||||||
}
|
})
|
||||||
row, source, error = client._sw_outer_realtime("801074.SI", "工业金属", "20260908")
|
row, source, error = client._sw_outer_realtime("801074.SI", "工业金属", "20260908")
|
||||||
self.assertEqual(source, "eastmoney_sw")
|
self.assertEqual(source, "eastmoney_sw")
|
||||||
self.assertEqual(error, "")
|
self.assertEqual(error, "")
|
||||||
@@ -216,23 +215,21 @@ class MemberQuoteCoverageTests(unittest.TestCase):
|
|||||||
self.assertEqual(source, "datahub")
|
self.assertEqual(source, "datahub")
|
||||||
client.try_quotes.assert_not_called()
|
client.try_quotes.assert_not_called()
|
||||||
|
|
||||||
def test_eastmoney_failure_uses_tencent_member_quotes(self) -> None:
|
def test_hub_named_quotes_cover_members_when_market_missing(self) -> None:
|
||||||
client = TushareClient(token="demo")
|
client = TushareClient(token="demo")
|
||||||
wanted = ["000737.SZ", "000630.SZ"]
|
wanted = ["000737.SZ", "000630.SZ"]
|
||||||
client.try_market_quotes = MagicMock(return_value=None)
|
client.try_market_quotes = MagicMock(return_value=None)
|
||||||
client.try_quotes = MagicMock(return_value=None)
|
client.try_quotes = MagicMock(return_value=[
|
||||||
aggregator = MagicMock()
|
|
||||||
aggregator.eastmoney_stock_quotes.side_effect = RuntimeError("HTTP 503")
|
|
||||||
aggregator.tencent_stock_quotes.return_value = [
|
|
||||||
{"ts_code": "000737.SZ", "close": 12.3, "pre_close": 11.2},
|
{"ts_code": "000737.SZ", "close": 12.3, "pre_close": 11.2},
|
||||||
{"ts_code": "000630.SZ", "close": 4.5, "pre_close": 4.4},
|
{"ts_code": "000630.SZ", "close": 4.5, "pre_close": 4.4},
|
||||||
]
|
])
|
||||||
client.realtime_aggregator = aggregator
|
client.realtime_aggregator = MagicMock()
|
||||||
client._free_realtime_quotes = MagicMock(side_effect=AssertionError("tencent already won"))
|
|
||||||
rows, source = client._load_member_realtime_quotes(wanted, "20260908")
|
rows, source = client._load_member_realtime_quotes(wanted, "20260908")
|
||||||
self.assertEqual(len(rows), 2)
|
self.assertEqual(len(rows), 2)
|
||||||
self.assertEqual(source, "tencent_qt")
|
self.assertEqual(source, "datahub")
|
||||||
aggregator.tencent_stock_quotes.assert_called_once()
|
client.try_quotes.assert_called()
|
||||||
|
client.realtime_aggregator.eastmoney_stock_quotes.assert_not_called()
|
||||||
|
client.realtime_aggregator.tencent_stock_quotes.assert_not_called()
|
||||||
|
|
||||||
def test_delayed_hub_quotes_are_kept_not_zeroed(self) -> None:
|
def test_delayed_hub_quotes_are_kept_not_zeroed(self) -> None:
|
||||||
client = TushareClient(token="demo")
|
client = TushareClient(token="demo")
|
||||||
@@ -323,8 +320,7 @@ class MemberQuoteCoverageTests(unittest.TestCase):
|
|||||||
client._sw_realtime_sector_snapshot = MagicMock(
|
client._sw_realtime_sector_snapshot = MagicMock(
|
||||||
side_effect=AssertionError("daily inner should be kept")
|
side_effect=AssertionError("daily inner should be kept")
|
||||||
)
|
)
|
||||||
client.realtime_aggregator = MagicMock()
|
client.try_sector_quote = MagicMock(return_value={
|
||||||
client.realtime_aggregator.eastmoney_shenwan_quote.return_value = {
|
|
||||||
"code": "801074.SI",
|
"code": "801074.SI",
|
||||||
"name": "工业金属",
|
"name": "工业金属",
|
||||||
"change": 1.5,
|
"change": 1.5,
|
||||||
@@ -332,7 +328,7 @@ class MemberQuoteCoverageTests(unittest.TestCase):
|
|||||||
"quote_date": "20260908",
|
"quote_date": "20260908",
|
||||||
"quote_time": "2026-09-08T15:00:00+08:00",
|
"quote_time": "2026-09-08T15:00:00+08:00",
|
||||||
"source": "eastmoney_sw",
|
"source": "eastmoney_sw",
|
||||||
}
|
})
|
||||||
snapshot = client.sw_sector_snapshot(
|
snapshot = client.sw_sector_snapshot(
|
||||||
"000737.SZ", "20260908", allow_realtime_close=True
|
"000737.SZ", "20260908", allow_realtime_close=True
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import ast
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from backend.data.datahub.bridge import DatahubAwareTushareClient, DatahubBridge
|
||||||
|
from backend.data.datahub.client import DatahubClient
|
||||||
|
from backend.data.datahub.settings import DATASETS, DatahubSettings, DatasetFlags
|
||||||
|
from backend.data.providers.tushare_transport import TushareError
|
||||||
|
from tests.test_datahub_bridge import FakeClient, FakeLegacy, flags
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BLOCKED_HOSTS = (
|
||||||
|
"api.tushare.pro",
|
||||||
|
"push2.eastmoney.com",
|
||||||
|
"push2delay.eastmoney.com",
|
||||||
|
"push2his.eastmoney.com",
|
||||||
|
"push2ex.eastmoney.com",
|
||||||
|
"qt.gtimg.cn",
|
||||||
|
"hq.sinajs.cn",
|
||||||
|
"10jqka.com.cn",
|
||||||
|
"xuangubao.cn",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class HubExclusiveWebsiteTests(unittest.TestCase):
|
||||||
|
def test_query_never_calls_website_tushare_transport(self) -> None:
|
||||||
|
client = FakeClient()
|
||||||
|
legacy = FakeLegacy(TushareError("website tushare must stay dark"))
|
||||||
|
wrapped = DatahubAwareTushareClient(
|
||||||
|
legacy,
|
||||||
|
DatahubBridge(flags(daily=(True, False)), client),
|
||||||
|
)
|
||||||
|
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,amount")
|
||||||
|
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||||
|
self.assertEqual(legacy.calls, [])
|
||||||
|
|
||||||
|
def test_blocked_external_hosts_still_read_hub(self) -> None:
|
||||||
|
settings = DatahubSettings(
|
||||||
|
base_url="http://127.0.0.1:8766",
|
||||||
|
token="hub-token",
|
||||||
|
datasets={name: DatasetFlags(name, read=True) for name in DATASETS},
|
||||||
|
)
|
||||||
|
|
||||||
|
def blocked_urlopen(request, timeout=None):
|
||||||
|
url = str(getattr(request, "full_url", None) or request)
|
||||||
|
if any(host in url for host in BLOCKED_HOSTS):
|
||||||
|
raise AssertionError(f"website opened blocked host: {url}")
|
||||||
|
if "127.0.0.1:8766" in url or "v1/bars/daily" in url:
|
||||||
|
class _Resp:
|
||||||
|
status = 200
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
return (
|
||||||
|
b'{"schema_version":1,"data":[{"ts_code":"600000.SH","trade_date":"20240902",'
|
||||||
|
b'"close":10.2,"volume":100000,"amount":2000000}],'
|
||||||
|
b'"meta":{"stale":false,"staleness_seconds":0,"source":"datahub"}}'
|
||||||
|
)
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
return _Resp()
|
||||||
|
raise AssertionError(f"unexpected url: {url}")
|
||||||
|
|
||||||
|
hub_client = DatahubClient(settings, urlopen=blocked_urlopen)
|
||||||
|
legacy = FakeLegacy(TushareError("blocked"))
|
||||||
|
wrapped = DatahubAwareTushareClient(legacy, DatahubBridge(settings, hub_client))
|
||||||
|
with patch("urllib.request.urlopen", blocked_urlopen):
|
||||||
|
rows = wrapped.query("daily", {"trade_date": "20240902"}, "ts_code,close,amount")
|
||||||
|
self.assertEqual(rows[0]["close"], 10.2)
|
||||||
|
self.assertEqual(rows[0]["amount"], 2000.0)
|
||||||
|
self.assertEqual(legacy.calls, [])
|
||||||
|
|
||||||
|
def test_website_runtime_does_not_call_blocked_hosts_from_gateway(self) -> None:
|
||||||
|
gateway_src = (ROOT / "backend" / "data" / "gateway.py").read_text(encoding="utf-8")
|
||||||
|
self.assertIn("legacy.realtime_aggregator = None", gateway_src)
|
||||||
|
self.assertIn("DatahubAwareTushareClient", gateway_src)
|
||||||
|
|
||||||
|
def test_bridge_query_has_no_legacy_call(self) -> None:
|
||||||
|
source = (ROOT / "backend" / "data" / "datahub" / "bridge.py").read_text(encoding="utf-8")
|
||||||
|
tree = ast.parse(source)
|
||||||
|
query_fn = next(
|
||||||
|
node
|
||||||
|
for node in tree.body
|
||||||
|
if isinstance(node, ast.ClassDef) and node.name == "DatahubBridge"
|
||||||
|
for item in node.body
|
||||||
|
if isinstance(item, ast.FunctionDef) and item.name == "query"
|
||||||
|
)
|
||||||
|
called = [
|
||||||
|
ast.unparse(item.func) if hasattr(ast, "unparse") else ""
|
||||||
|
for item in ast.walk(query_fn)
|
||||||
|
if isinstance(item, ast.Call)
|
||||||
|
]
|
||||||
|
self.assertTrue(any("query_api" in text for text in called))
|
||||||
|
self.assertFalse(any("legacy_query" in text for text in called))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -12,6 +12,20 @@ from backend.features.market.insights import MarketInsightsService
|
|||||||
from server import DashboardService
|
from server import DashboardService
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeDailyHub:
|
||||||
|
def __init__(self, rows: list) -> None:
|
||||||
|
self.rows = rows
|
||||||
|
|
||||||
|
def try_daily_chart(self, code, end_date, limit, dataset="daily"):
|
||||||
|
return list(self.rows)
|
||||||
|
|
||||||
|
def try_quotes(self, codes):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def try_index_quotes(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class FakeIfind:
|
class FakeIfind:
|
||||||
configured = True
|
configured = True
|
||||||
|
|
||||||
@@ -128,13 +142,50 @@ class IfindFeatureTests(unittest.TestCase):
|
|||||||
self.assertEqual(database.list_wencai_saved_queries(second["id"]), [])
|
self.assertEqual(database.list_wencai_saved_queries(second["id"]), [])
|
||||||
|
|
||||||
def test_ifind_daily_chart_normalizes_change(self):
|
def test_ifind_daily_chart_normalizes_change(self):
|
||||||
client = MarketChartClient(FakeIfind(), EastmoneyChartClient())
|
hub = _FakeDailyHub(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"trade_date": "2026-07-27",
|
||||||
|
"open": 10,
|
||||||
|
"high": 10.5,
|
||||||
|
"low": 9.8,
|
||||||
|
"close": 10.2,
|
||||||
|
"volume": 100,
|
||||||
|
"amount_billion": 0.01,
|
||||||
|
"change": 0,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"trade_date": "2026-07-28",
|
||||||
|
"open": 10.2,
|
||||||
|
"high": 10.8,
|
||||||
|
"low": 10.1,
|
||||||
|
"close": 10.5,
|
||||||
|
"volume": 120,
|
||||||
|
"amount_billion": 0.012,
|
||||||
|
"change": 2.9412,
|
||||||
|
},
|
||||||
|
]
|
||||||
|
)
|
||||||
|
client = MarketChartClient(FakeIfind(), EastmoneyChartClient(), hub)
|
||||||
rows = client.stock_daily("000001", "20260728")
|
rows = client.stock_daily("000001", "20260728")
|
||||||
self.assertEqual(rows[-1]["trade_date"], "2026-07-28")
|
self.assertEqual(rows[-1]["trade_date"], "2026-07-28")
|
||||||
self.assertAlmostEqual(rows[-1]["change"], 2.9412, places=4)
|
self.assertAlmostEqual(rows[-1]["change"], 2.9412, places=4)
|
||||||
|
|
||||||
def test_ifind_daily_chart_keeps_last_traded_bar_before_market_open(self):
|
def test_ifind_daily_chart_keeps_last_traded_bar_before_market_open(self):
|
||||||
client = MarketChartClient(FakeIfindStalePreopen(), EastmoneyChartClient())
|
hub = _FakeDailyHub(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"trade_date": "2026-07-28",
|
||||||
|
"open": 10.2,
|
||||||
|
"high": 10.8,
|
||||||
|
"low": 10.1,
|
||||||
|
"close": 10.5,
|
||||||
|
"volume": 120,
|
||||||
|
"amount_billion": 0.012,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
client = MarketChartClient(FakeIfindStalePreopen(), EastmoneyChartClient(), hub)
|
||||||
with patch("backend.features.market.charts.datetime", FixedPreopenDatetime):
|
with patch("backend.features.market.charts.datetime", FixedPreopenDatetime):
|
||||||
rows = client.stock_daily("000001", "20260729")
|
rows = client.stock_daily("000001", "20260729")
|
||||||
|
|
||||||
|
|||||||
@@ -256,7 +256,7 @@ class RealtimeDashboardTests(unittest.TestCase):
|
|||||||
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
||||||
self.assertEqual(dashboard["overview"]["limit_up_count"], 0)
|
self.assertEqual(dashboard["overview"]["limit_up_count"], 0)
|
||||||
|
|
||||||
def test_rt_k_permission_error_falls_back_to_free_quotes(self):
|
def test_hub_quotes_used_when_rt_k_denied(self):
|
||||||
original_query = self.client.query
|
original_query = self.client.query
|
||||||
|
|
||||||
def query(api_name, params=None, fields=""):
|
def query(api_name, params=None, fields=""):
|
||||||
@@ -265,21 +265,20 @@ class RealtimeDashboardTests(unittest.TestCase):
|
|||||||
return original_query(api_name, params, fields)
|
return original_query(api_name, params, fields)
|
||||||
|
|
||||||
self.client.query = query
|
self.client.query = query
|
||||||
self.client.realtime_aggregator = FakeFreeAggregator()
|
self.client.try_market_quotes = lambda trade_date: list(FREE_QUOTES)
|
||||||
TushareClient._realtime_reference_cache.clear()
|
TushareClient._realtime_reference_cache.clear()
|
||||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||||
|
|
||||||
self.assertTrue(dashboard["meta"]["realtime"])
|
self.assertTrue(dashboard["meta"]["realtime"])
|
||||||
self.assertEqual(dashboard["meta"]["quote_source"], "eastmoney_clist")
|
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||||
self.assertEqual(dashboard["meta"]["trade_date"], "2026-07-20")
|
self.assertEqual(dashboard["meta"]["trade_date"], "2026-07-20")
|
||||||
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
self.assertEqual(dashboard["meta"]["quote_count"], 3)
|
||||||
self.assertEqual(dashboard["overview"]["limit_up_count"], 1)
|
self.assertEqual(dashboard["overview"]["limit_up_count"], 1)
|
||||||
self.assertEqual(dashboard["overview"]["limit_down_count"], 1)
|
self.assertEqual(dashboard["overview"]["limit_down_count"], 1)
|
||||||
self.assertEqual(dashboard["overview"]["amount_billion"], 6.0)
|
self.assertEqual(dashboard["overview"]["amount_billion"], 6.0)
|
||||||
self.assertIn("东财免费实时", dashboard["meta"]["notice"])
|
self.assertIn("数据中枢", dashboard["meta"]["notice"])
|
||||||
self.assertEqual(dashboard["meta"]["indices"][0]["price"], 3800.12)
|
|
||||||
|
|
||||||
def test_rt_k_empty_result_falls_back_to_free_quotes(self):
|
def test_hub_quotes_used_when_rt_k_empty(self):
|
||||||
original_query = self.client.query
|
original_query = self.client.query
|
||||||
|
|
||||||
def query(api_name, params=None, fields=""):
|
def query(api_name, params=None, fields=""):
|
||||||
@@ -288,29 +287,27 @@ class RealtimeDashboardTests(unittest.TestCase):
|
|||||||
return original_query(api_name, params, fields)
|
return original_query(api_name, params, fields)
|
||||||
|
|
||||||
self.client.query = query
|
self.client.query = query
|
||||||
self.client.realtime_aggregator = FakeFreeAggregator()
|
self.client.try_market_quotes = lambda trade_date: list(FREE_QUOTES)
|
||||||
TushareClient._realtime_reference_cache.clear()
|
TushareClient._realtime_reference_cache.clear()
|
||||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||||
self.assertEqual(dashboard["meta"]["quote_source"], "eastmoney_clist")
|
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||||
self.assertEqual(str(dashboard["meta"]["trade_date"]).replace("-", ""), "20260720")
|
self.assertEqual(str(dashboard["meta"]["trade_date"]).replace("-", ""), "20260720")
|
||||||
|
|
||||||
def test_rt_k_and_free_source_failure_keeps_today_error(self):
|
def test_hub_failure_keeps_today_error(self):
|
||||||
original_query = self.client.query
|
original_query = self.client.query
|
||||||
|
|
||||||
def query(api_name, params=None, fields=""):
|
def query(api_name, params=None, fields=""):
|
||||||
if api_name == "rt_k":
|
if api_name == "rt_k":
|
||||||
raise TushareError("没有接口访问权限")
|
raise TushareError("数据中枢行情暂不可用")
|
||||||
return original_query(api_name, params, fields)
|
return original_query(api_name, params, fields)
|
||||||
|
|
||||||
self.client.query = query
|
self.client.query = query
|
||||||
self.client.realtime_aggregator = FakeFreeAggregator(fail=True)
|
|
||||||
TushareClient._realtime_reference_cache.clear()
|
TushareClient._realtime_reference_cache.clear()
|
||||||
with self.assertRaises(TushareError) as ctx:
|
with self.assertRaises(TushareError) as ctx:
|
||||||
self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||||
self.assertIn("当天盘中实时行情不可用", str(ctx.exception))
|
self.assertIn("当天盘中实时行情不可用", str(ctx.exception))
|
||||||
self.assertIn("没有接口访问权限", str(ctx.exception))
|
|
||||||
|
|
||||||
def test_rt_k_and_eastmoney_failure_falls_back_to_tencent(self):
|
def test_hub_failover_is_invisible_to_website(self):
|
||||||
original_query = self.client.query
|
original_query = self.client.query
|
||||||
|
|
||||||
def query(api_name, params=None, fields=""):
|
def query(api_name, params=None, fields=""):
|
||||||
@@ -318,23 +315,13 @@ class RealtimeDashboardTests(unittest.TestCase):
|
|||||||
raise TushareError("没有接口访问权限")
|
raise TushareError("没有接口访问权限")
|
||||||
return original_query(api_name, params, fields)
|
return original_query(api_name, params, fields)
|
||||||
|
|
||||||
class TencentOnlyAggregator(FakeFreeAggregator):
|
|
||||||
def eastmoney_market_quotes(self, expected_date=""):
|
|
||||||
raise RealtimeAggregateError("eastmoney blocked")
|
|
||||||
|
|
||||||
def tencent_market_quotes(self, codes, expected_date=""):
|
|
||||||
return list(FREE_QUOTES)
|
|
||||||
|
|
||||||
def tencent_stock_quotes(self, codes, expected_date="", minimum=None):
|
|
||||||
return list(FREE_QUOTES)
|
|
||||||
|
|
||||||
self.client.query = query
|
self.client.query = query
|
||||||
self.client.realtime_aggregator = TencentOnlyAggregator()
|
self.client.try_market_quotes = lambda trade_date: list(FREE_QUOTES)
|
||||||
TushareClient._realtime_reference_cache.clear()
|
TushareClient._realtime_reference_cache.clear()
|
||||||
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
|
||||||
self.assertEqual(dashboard["meta"]["quote_source"], "tencent_qt")
|
self.assertEqual(dashboard["meta"]["quote_source"], "datahub")
|
||||||
self.assertEqual(str(dashboard["meta"]["trade_date"]).replace("-", ""), "20260720")
|
self.assertEqual(str(dashboard["meta"]["trade_date"]).replace("-", ""), "20260720")
|
||||||
self.assertIn("腾讯免费实时", dashboard["meta"]["notice"])
|
self.assertIn("数据中枢", dashboard["meta"]["notice"])
|
||||||
self.assertEqual(dashboard["overview"]["amount_billion"], 6.0)
|
self.assertEqual(dashboard["overview"]["amount_billion"], 6.0)
|
||||||
|
|
||||||
def test_normalize_eastmoney_quote_maps_units_and_exchange(self):
|
def test_normalize_eastmoney_quote_maps_units_and_exchange(self):
|
||||||
@@ -433,6 +420,13 @@ class RealtimeDashboardTests(unittest.TestCase):
|
|||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.calls = []
|
self.calls = []
|
||||||
|
|
||||||
|
def query_api(self, api_name, params=None, fields=""):
|
||||||
|
rows = FakeRealtimeClient("tok").query(api_name, params or {}, fields)
|
||||||
|
return DatahubResponse(
|
||||||
|
data=rows,
|
||||||
|
meta={"source": "datahub", "stale": False, "staleness_seconds": 0, "row_shape": "tushare"},
|
||||||
|
)
|
||||||
|
|
||||||
def quotes_latest(self, **params):
|
def quotes_latest(self, **params):
|
||||||
return self.get("/v1/quotes/latest", params)
|
return self.get("/v1/quotes/latest", params)
|
||||||
|
|
||||||
|
|||||||
@@ -250,6 +250,11 @@ class EastmoneyAdapter(MarketAdapter):
|
|||||||
secid = INDEX_SECIDS[code]
|
secid = INDEX_SECIDS[code]
|
||||||
entity = "index"
|
entity = "index"
|
||||||
identifier = code
|
identifier = code
|
||||||
|
elif code.startswith("BK") or code.endswith((".TI", ".SI")):
|
||||||
|
symbol = code.split(".")[0]
|
||||||
|
secid = f"90.{symbol}"
|
||||||
|
entity = "board"
|
||||||
|
identifier = symbol
|
||||||
else:
|
else:
|
||||||
symbol = code.split(".")[0]
|
symbol = code.split(".")[0]
|
||||||
market = "1" if symbol.startswith(("5", "6", "9")) else "0"
|
market = "1" if symbol.startswith(("5", "6", "9")) else "0"
|
||||||
@@ -294,6 +299,108 @@ class EastmoneyAdapter(MarketAdapter):
|
|||||||
"source": "eastmoney_trends2",
|
"source": "eastmoney_trends2",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def fetch_shenwan_quote(self, ts_code: str) -> dict[str, Any]:
|
||||||
|
code = str(ts_code or "").split(".")[0]
|
||||||
|
if not code:
|
||||||
|
raise AdapterError("Invalid Shenwan code")
|
||||||
|
payload = self._get_json(
|
||||||
|
EASTMONEY_INDEX_URL,
|
||||||
|
{
|
||||||
|
"secids": f"90.{code}",
|
||||||
|
"fltt": "2",
|
||||||
|
"invt": "2",
|
||||||
|
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f8,f104,f105,f128,f136,f140,f124",
|
||||||
|
},
|
||||||
|
referer="https://quote.eastmoney.com/",
|
||||||
|
)
|
||||||
|
rows = list((payload.get("data") or {}).get("diff") or [])
|
||||||
|
row = next((item for item in rows if item), None)
|
||||||
|
if not row:
|
||||||
|
raise AdapterError(f"Eastmoney Shenwan quote missing for {code}")
|
||||||
|
epoch = int(finite_number(row.get("f124")) or 0)
|
||||||
|
close = round4(finite_number(row.get("f2")))
|
||||||
|
previous = round4(finite_number(row.get("f18")))
|
||||||
|
if close <= 0 or previous <= 0:
|
||||||
|
raise AdapterError(f"Eastmoney Shenwan quote empty for {code}")
|
||||||
|
quote_time = (
|
||||||
|
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||||
|
if epoch
|
||||||
|
else ""
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"ts_code": f"{code}.SI",
|
||||||
|
"code": f"{code}.SI",
|
||||||
|
"name": row.get("f14") or code,
|
||||||
|
"price": close,
|
||||||
|
"close": close,
|
||||||
|
"pre_close": previous,
|
||||||
|
"previous_close": previous,
|
||||||
|
"open": round4(finite_number(row.get("f17"))),
|
||||||
|
"high": round4(finite_number(row.get("f15"))),
|
||||||
|
"low": round4(finite_number(row.get("f16"))),
|
||||||
|
"change": round4(finite_number(row.get("f3"))),
|
||||||
|
"pct_change": round4(finite_number(row.get("f3"))),
|
||||||
|
"pct_chg": round4(finite_number(row.get("f3"))),
|
||||||
|
"amount": round4(finite_number(row.get("f6"))),
|
||||||
|
"leader": row.get("f128") or "--",
|
||||||
|
"leader_code": row.get("f140") or "",
|
||||||
|
"leading_pct": round4(finite_number(row.get("f136"))),
|
||||||
|
"up_count": int(finite_number(row.get("f104")) or 0),
|
||||||
|
"down_count": int(finite_number(row.get("f105")) or 0),
|
||||||
|
"quote_time": quote_time,
|
||||||
|
"trade_time": quote_time,
|
||||||
|
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||||||
|
"quote_time_epoch": epoch,
|
||||||
|
"source": "eastmoney_sw",
|
||||||
|
}
|
||||||
|
|
||||||
|
def fetch_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]]:
|
||||||
|
day = str(trade_date or "").replace("-", "")
|
||||||
|
rows: list[dict[str, Any]] = []
|
||||||
|
for url, limit_type in (
|
||||||
|
("https://push2ex.eastmoney.com/getTopicZTPool", "U"),
|
||||||
|
("https://push2ex.eastmoney.com/getTopicZBPool", "Z"),
|
||||||
|
):
|
||||||
|
params = {
|
||||||
|
"ut": "7eea3edcaed734bea9cbfc24409ed989",
|
||||||
|
"dpt": "wz.ztzt",
|
||||||
|
"PageIndex": "0",
|
||||||
|
"PageSize": "200",
|
||||||
|
"sort": "fbt:asc",
|
||||||
|
"stat": "1",
|
||||||
|
}
|
||||||
|
if day:
|
||||||
|
params["date"] = day
|
||||||
|
try:
|
||||||
|
payload = self._get_json(url, params, referer="https://quote.eastmoney.com/")
|
||||||
|
except AdapterError:
|
||||||
|
continue
|
||||||
|
pool = ((payload.get("data") or {}).get("pool") or []) if isinstance(payload.get("data"), dict) else []
|
||||||
|
for item in pool:
|
||||||
|
code = str(item.get("c") or item.get("code") or "")
|
||||||
|
if not code:
|
||||||
|
continue
|
||||||
|
market = str(item.get("m") or item.get("market") or "")
|
||||||
|
suffix = "SH" if market in {"1", "SH"} or code.startswith(("5", "6", "9")) else "SZ"
|
||||||
|
first = str(item.get("fbt") or item.get("first_time") or "")
|
||||||
|
last = str(item.get("lbt") or item.get("last_time") or "")
|
||||||
|
rows.append(
|
||||||
|
{
|
||||||
|
"ts_code": f"{code}.{suffix}",
|
||||||
|
"limit_type": limit_type,
|
||||||
|
"first_time": first,
|
||||||
|
"last_time": last,
|
||||||
|
"fd_amount": item.get("fund") or item.get("fd_amount"),
|
||||||
|
"open_times": item.get("zbc") or item.get("open_times"),
|
||||||
|
"limit_times": item.get("lbc") or item.get("limit_times"),
|
||||||
|
"turnover_ratio": item.get("hs") or item.get("turnover_ratio"),
|
||||||
|
"source": "eastmoney_zt_pool",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
if not rows:
|
||||||
|
raise AdapterError("Eastmoney limit pool empty")
|
||||||
|
return rows
|
||||||
|
|
||||||
def _get_json(self, url: str, params: dict[str, str], referer: str) -> dict[str, Any]:
|
def _get_json(self, url: str, params: dict[str, str], referer: str) -> dict[str, Any]:
|
||||||
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
||||||
request = urllib.request.Request(
|
request = urllib.request.Request(
|
||||||
|
|||||||
@@ -50,6 +50,14 @@ TUSHARE_FIELDS = {
|
|||||||
"ths_daily": "ts_code,trade_date,open,high,low,close,pre_close,pct_change,vol,turnover_rate",
|
"ths_daily": "ts_code,trade_date,open,high,low,close,pre_close,pct_change,vol,turnover_rate",
|
||||||
"dc_index": "ts_code,trade_date,name,open,high,low,close,pre_close,pct_change,vol,amount,turnover_rate",
|
"dc_index": "ts_code,trade_date,name,open,high,low,close,pre_close,pct_change,vol,amount,turnover_rate",
|
||||||
"sw_daily": "ts_code,trade_date,name,open,high,low,close,pct_change,vol,amount",
|
"sw_daily": "ts_code,trade_date,name,open,high,low,close,pct_change,vol,amount",
|
||||||
|
"index_member_all": (
|
||||||
|
"l1_code,l1_name,l2_code,l2_name,l3_code,l3_name,"
|
||||||
|
"ts_code,name,in_date,out_date,is_new"
|
||||||
|
),
|
||||||
|
"stk_limit": "ts_code,trade_date,up_limit,down_limit",
|
||||||
|
"suspend_d": "ts_code,suspend_date,resume_date,ann_date,suspend_reason,reason_type",
|
||||||
|
"ths_member": "ts_code,con_code,con_name,in_date,out_date,is_new",
|
||||||
|
"stk_mins": "ts_code,trade_time,open,close,high,low,vol,amount",
|
||||||
}
|
}
|
||||||
|
|
||||||
DATASET_API = {
|
DATASET_API = {
|
||||||
@@ -254,3 +262,6 @@ class TushareAdapter(MarketAdapter):
|
|||||||
items = data.get("items") or []
|
items = data.get("items") or []
|
||||||
fields_list = data.get("fields") or (fields.split(",") if fields else [])
|
fields_list = data.get("fields") or (fields.split(",") if fields else [])
|
||||||
return [dict(zip(fields_list, item)) for item in items]
|
return [dict(zip(fields_list, item)) for item in items]
|
||||||
|
|
||||||
|
def query_raw(self, api_name: str, params: dict[str, Any], fields: str = "") -> list[dict[str, Any]]:
|
||||||
|
return self._query(api_name, params, fields or TUSHARE_FIELDS.get(api_name, ""))
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
|||||||
self._json({"status": "ok"}, HTTPStatus.OK)
|
self._json({"status": "ok"}, HTTPStatus.OK)
|
||||||
return
|
return
|
||||||
if path.startswith("/v1/"):
|
if path.startswith("/v1/"):
|
||||||
self._v1(path, parsed.query)
|
self._v1(path, parsed.query, method)
|
||||||
return
|
return
|
||||||
if path.startswith("/admin/api/"):
|
if path.startswith("/admin/api/"):
|
||||||
self._admin_api(method, path)
|
self._admin_api(method, path)
|
||||||
@@ -66,11 +66,16 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
|||||||
LOGGER.exception("internal error")
|
LOGGER.exception("internal error")
|
||||||
self._json({"error": {"code": "INTERNAL", "message": "internal error"}}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
self._json({"error": {"code": "INTERNAL", "message": "internal error"}}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||||
|
|
||||||
def _v1(self, path: str, query: str) -> None:
|
def _v1(self, path: str, query: str, method: str = "GET") -> None:
|
||||||
token = self.headers.get("X-Datahub-Token", "")
|
token = self.headers.get("X-Datahub-Token", "")
|
||||||
if not self.hub.auth.check_api_token(token):
|
if not self.hub.auth.check_api_token(token):
|
||||||
self.hub.pipeline.audit("anonymous", "unauthorized", path, "")
|
self.hub.pipeline.audit("anonymous", "unauthorized", path, "")
|
||||||
raise ApiError("UNAUTHORIZED", "missing or invalid X-Datahub-Token")
|
raise ApiError("UNAUTHORIZED", "missing or invalid X-Datahub-Token")
|
||||||
|
if path == "/v1/query" and method == "POST":
|
||||||
|
body = self._read_json(max_bytes=1_000_000)
|
||||||
|
payload = self.hub.api.query_api(body)
|
||||||
|
self._json(payload, HTTPStatus.OK)
|
||||||
|
return
|
||||||
payload = self.hub.api.handle(path, parse_query(query))
|
payload = self.hub.api.handle(path, parse_query(query))
|
||||||
self._json(payload, HTTPStatus.OK)
|
self._json(payload, HTTPStatus.OK)
|
||||||
|
|
||||||
@@ -184,11 +189,11 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
|||||||
self.end_headers()
|
self.end_headers()
|
||||||
self.wfile.write(content)
|
self.wfile.write(content)
|
||||||
|
|
||||||
def _read_json(self, allow_empty: bool = False) -> dict[str, Any]:
|
def _read_json(self, allow_empty: bool = False, max_bytes: int = 65536) -> dict[str, Any]:
|
||||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||||
if length == 0 and allow_empty:
|
if length == 0 and allow_empty:
|
||||||
return {}
|
return {}
|
||||||
if length <= 0 or length > 65536:
|
if length <= 0 or length > max_bytes:
|
||||||
raise ValueError("请求内容为空或过大")
|
raise ValueError("请求内容为空或过大")
|
||||||
raw = self.rfile.read(length)
|
raw = self.rfile.read(length)
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -149,6 +149,78 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
|
|||||||
return payload
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_sector_quote(db: HubDB, code: str, expected_date: str = "") -> dict[str, Any]:
|
||||||
|
ts_code = str(code or "").strip().upper()
|
||||||
|
if ts_code.isdigit():
|
||||||
|
ts_code = f"{ts_code}.SI"
|
||||||
|
cache_key = f"sector:{ts_code}"
|
||||||
|
cached = _read_cache(db, cache_key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
errors: list[str] = []
|
||||||
|
try:
|
||||||
|
row = EastmoneyAdapter().fetch_shenwan_quote(ts_code)
|
||||||
|
source = str(row.get("source") or "eastmoney_sw")
|
||||||
|
except Exception as exc:
|
||||||
|
errors.append(f"eastmoney:{exc}")
|
||||||
|
recovered = _load_quotes_lkg(db, cache_key)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise RealtimeApiError(
|
||||||
|
"SOURCE_UNAVAILABLE",
|
||||||
|
"sector quote unavailable: " + ";".join(errors),
|
||||||
|
) from exc
|
||||||
|
want = str(expected_date or "").replace("-", "")[:8]
|
||||||
|
quote_date = str(row.get("quote_date") or "")
|
||||||
|
if want and quote_date and quote_date != want:
|
||||||
|
recovered = _load_quotes_lkg(db, cache_key)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise RealtimeApiError("SOURCE_UNAVAILABLE", f"sector quote date {quote_date} != {want}")
|
||||||
|
payload = _envelope(
|
||||||
|
row,
|
||||||
|
{
|
||||||
|
"tier": "provisional",
|
||||||
|
"trade_date": quote_date or yyyymmdd(now_shanghai()),
|
||||||
|
"source": source,
|
||||||
|
"stale": False,
|
||||||
|
"staleness_seconds": 0,
|
||||||
|
"published_at": isoformat(now_shanghai()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_write_cache(db, cache_key, payload, INDEX_TTL, source)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_limit_pool(db: HubDB, trade_date: str = "") -> dict[str, Any]:
|
||||||
|
day = yyyymmdd(trade_date or now_shanghai())
|
||||||
|
cache_key = f"limit-pool:{day}"
|
||||||
|
cached = _read_cache(db, cache_key)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
try:
|
||||||
|
rows = EastmoneyAdapter().fetch_limit_pool(day)
|
||||||
|
source = "eastmoney:zt_pool"
|
||||||
|
except Exception as exc:
|
||||||
|
recovered = _load_quotes_lkg(db, cache_key)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise RealtimeApiError("SOURCE_UNAVAILABLE", f"limit pool unavailable: {exc}") from exc
|
||||||
|
payload = _envelope(
|
||||||
|
rows,
|
||||||
|
{
|
||||||
|
"tier": "provisional",
|
||||||
|
"trade_date": day,
|
||||||
|
"source": source,
|
||||||
|
"stale": False,
|
||||||
|
"staleness_seconds": 0,
|
||||||
|
"published_at": isoformat(now_shanghai()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_write_cache(db, cache_key, payload, QUOTE_TTL, source)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
def _eastmoney_named_quotes(codes: list[str]) -> list[dict[str, Any]]:
|
def _eastmoney_named_quotes(codes: list[str]) -> list[dict[str, Any]]:
|
||||||
adapter = EastmoneyAdapter()
|
adapter = EastmoneyAdapter()
|
||||||
rows: list[dict[str, Any]] = []
|
rows: list[dict[str, Any]] = []
|
||||||
|
|||||||
@@ -87,12 +87,46 @@ class V1API:
|
|||||||
return self.index_quotes(q)
|
return self.index_quotes(q)
|
||||||
if path == "/v1/intraday/points":
|
if path == "/v1/intraday/points":
|
||||||
return self.intraday_points(q)
|
return self.intraday_points(q)
|
||||||
|
if path == "/v1/sectors/quote":
|
||||||
|
return self.sector_quote(q)
|
||||||
|
if path == "/v1/limit-pool":
|
||||||
|
return self.limit_pool(q)
|
||||||
|
if path == "/v1/query":
|
||||||
|
return self.query_api(q)
|
||||||
if path == "/v1/datasets/status":
|
if path == "/v1/datasets/status":
|
||||||
return self.dataset_status(q.get("date") or "")
|
return self.dataset_status(q.get("date") or "")
|
||||||
if path == "/v1/batches":
|
if path == "/v1/batches":
|
||||||
return self.batches(q.get("date") or "", q.get("dataset") or "")
|
return self.batches(q.get("date") or "", q.get("dataset") or "")
|
||||||
raise ApiError("INVALID_ARGUMENT", f"unknown endpoint: {path}")
|
raise ApiError("INVALID_ARGUMENT", f"unknown endpoint: {path}")
|
||||||
|
|
||||||
|
def query_api(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
from datahub.steward import steward_query
|
||||||
|
|
||||||
|
payload = dict(body or {})
|
||||||
|
raw_params = payload.get("params")
|
||||||
|
if isinstance(raw_params, str):
|
||||||
|
payload["params"] = _parse_json(raw_params) or {}
|
||||||
|
return steward_query(self, payload)
|
||||||
|
|
||||||
|
def sector_quote(self, q: dict[str, str]) -> dict[str, Any]:
|
||||||
|
from datahub.realtime_serve import RealtimeApiError, fetch_sector_quote
|
||||||
|
|
||||||
|
code = str(q.get("code") or q.get("ts_code") or "").strip()
|
||||||
|
if not code:
|
||||||
|
raise ApiError("INVALID_ARGUMENT", "code is required")
|
||||||
|
try:
|
||||||
|
return fetch_sector_quote(self.db, code, str(q.get("date") or ""))
|
||||||
|
except RealtimeApiError as exc:
|
||||||
|
raise ApiError(exc.code, exc.message) from exc
|
||||||
|
|
||||||
|
def limit_pool(self, q: dict[str, str]) -> dict[str, Any]:
|
||||||
|
from datahub.realtime_serve import RealtimeApiError, fetch_limit_pool
|
||||||
|
|
||||||
|
try:
|
||||||
|
return fetch_limit_pool(self.db, str(q.get("date") or q.get("trade_date") or ""))
|
||||||
|
except RealtimeApiError as exc:
|
||||||
|
raise ApiError(exc.code, exc.message) from exc
|
||||||
|
|
||||||
def health(self) -> dict[str, Any]:
|
def health(self) -> dict[str, Any]:
|
||||||
today = yyyymmdd(now_shanghai())
|
today = yyyymmdd(now_shanghai())
|
||||||
cal = self.db.fetchone(
|
cal = self.db.fetchone(
|
||||||
|
|||||||
@@ -0,0 +1,360 @@
|
|||||||
|
"""Website-facing data steward: pick source, fail over, cache, never fake zeros.
|
||||||
|
|
||||||
|
The main site asks for a business/Tushare-shaped API. This module decides whether
|
||||||
|
to serve a published EOD table, live free quotes, or an internal Tushare pull.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from datahub.adapters.base import AdapterError
|
||||||
|
from datahub.adapters.tushare import TUSHARE_FIELDS
|
||||||
|
from datahub.numbers import finite_number
|
||||||
|
from datahub.realtime_serve import (
|
||||||
|
RealtimeApiError,
|
||||||
|
_read_cache,
|
||||||
|
_write_cache,
|
||||||
|
fetch_index_quotes,
|
||||||
|
fetch_market_quotes,
|
||||||
|
fetch_quotes,
|
||||||
|
)
|
||||||
|
from datahub.serving import ApiError, envelope
|
||||||
|
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
|
||||||
|
|
||||||
|
API_TO_DATASET = {
|
||||||
|
"trade_cal": "calendar",
|
||||||
|
"stock_basic": "stocks",
|
||||||
|
"daily": "daily",
|
||||||
|
"daily_basic": "valuation",
|
||||||
|
"index_daily": "index_daily",
|
||||||
|
"moneyflow": "moneyflow",
|
||||||
|
"stk_auction": "auction",
|
||||||
|
"limit_list_d": "limit_events",
|
||||||
|
"ths_hot": "popularity",
|
||||||
|
"dc_hot": "popularity",
|
||||||
|
"hm_detail": "dragon_tiger",
|
||||||
|
"ths_daily": "sector_daily",
|
||||||
|
"dc_index": "sector_daily",
|
||||||
|
"sw_daily": "sector_daily",
|
||||||
|
}
|
||||||
|
|
||||||
|
DATASET_FETCHER = {
|
||||||
|
"calendar": lambda api, q: api.calendar(q.get("from") or q.get("start_date") or "", q.get("to") or q.get("end_date") or ""),
|
||||||
|
"stocks": lambda api, q: api.stocks(q.get("updated_since") or "", q),
|
||||||
|
"daily": lambda api, q: api.daily_bars(_hub_query(q, adjust="none")),
|
||||||
|
"valuation": lambda api, q: api.valuation(_hub_query(q)),
|
||||||
|
"index_daily": lambda api, q: api.index_bars(_hub_query(q)),
|
||||||
|
"moneyflow": lambda api, q: api.moneyflow(_hub_query(q)),
|
||||||
|
"auction": lambda api, q: api.auction(_hub_query(q)),
|
||||||
|
"limit_events": lambda api, q: api.limit_events(_hub_query(q)),
|
||||||
|
"popularity": lambda api, q: api.popularity(_hub_query(q)),
|
||||||
|
"dragon_tiger": lambda api, q: api.dragon_tiger(_hub_query(q)),
|
||||||
|
"sector_daily": lambda api, q: api.sectors(_hub_query(q)),
|
||||||
|
}
|
||||||
|
|
||||||
|
SCALE_TO_TUSHARE = {
|
||||||
|
"daily": {"vol": 100.0, "amount": 1000.0},
|
||||||
|
"index_daily": {"vol": 100.0, "amount": 1000.0},
|
||||||
|
"valuation": {"total_mv": 10000.0, "circ_mv": 10000.0},
|
||||||
|
"moneyflow": {
|
||||||
|
"buy_sm_amount": 10000.0,
|
||||||
|
"sell_sm_amount": 10000.0,
|
||||||
|
"buy_md_amount": 10000.0,
|
||||||
|
"sell_md_amount": 10000.0,
|
||||||
|
"buy_lg_amount": 10000.0,
|
||||||
|
"sell_lg_amount": 10000.0,
|
||||||
|
"buy_elg_amount": 10000.0,
|
||||||
|
"sell_elg_amount": 10000.0,
|
||||||
|
"net_mf_amount": 10000.0,
|
||||||
|
},
|
||||||
|
"auction": {"vol": 100.0, "float_share": 10000.0},
|
||||||
|
"limit_events": {"limit_amount": 10000.0, "float_mv": 10000.0, "total_mv": 10000.0},
|
||||||
|
"dragon_tiger": {"buy_amount": 10000.0, "sell_amount": 10000.0, "net_amount": 10000.0},
|
||||||
|
}
|
||||||
|
|
||||||
|
LIVE_TTL = {
|
||||||
|
"index_member_all": 6 * 3600,
|
||||||
|
"stk_limit": 3600,
|
||||||
|
"suspend_d": 6 * 3600,
|
||||||
|
"adj_factor": 3600,
|
||||||
|
"hm_list": 24 * 3600,
|
||||||
|
"ths_index": 24 * 3600,
|
||||||
|
"ths_member": 6 * 3600,
|
||||||
|
"stk_mins": 20,
|
||||||
|
"top_list": 3600,
|
||||||
|
"top_inst": 3600,
|
||||||
|
}
|
||||||
|
|
||||||
|
BLOCKED_LIVE_APIS = {"rt_sw_k"}
|
||||||
|
|
||||||
|
|
||||||
|
def steward_query(api, body: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
api_name = str(body.get("api_name") or "").strip()
|
||||||
|
params = body.get("params") if isinstance(body.get("params"), dict) else {}
|
||||||
|
fields = str(body.get("fields") or "")
|
||||||
|
if not api_name:
|
||||||
|
raise ApiError("INVALID_ARGUMENT", "api_name is required")
|
||||||
|
if api_name in BLOCKED_LIVE_APIS:
|
||||||
|
raise ApiError("INVALID_ARGUMENT", "rt_sw_k is disabled; use published sw_daily or free Shenwan realtime")
|
||||||
|
if api_name == "rt_k":
|
||||||
|
return _realtime_quotes(api, params, fields)
|
||||||
|
if api_name == "rt_idx_k":
|
||||||
|
return _realtime_index_quotes(api, params, fields)
|
||||||
|
dataset = API_TO_DATASET.get(api_name)
|
||||||
|
if dataset:
|
||||||
|
published = _try_published(api, api_name, dataset, params, fields)
|
||||||
|
if published is not None:
|
||||||
|
return published
|
||||||
|
rows = _live_tushare(api, api_name, params, fields)
|
||||||
|
return envelope(
|
||||||
|
_project(rows, fields),
|
||||||
|
{
|
||||||
|
"tier": "live",
|
||||||
|
"source": "tushare",
|
||||||
|
"stale": False,
|
||||||
|
"staleness_seconds": 0,
|
||||||
|
"row_shape": "tushare",
|
||||||
|
"published_at": isoformat(now_shanghai()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _try_published(api, api_name: str, dataset: str, params: dict[str, Any], fields: str) -> dict[str, Any] | None:
|
||||||
|
fetcher = DATASET_FETCHER.get(dataset)
|
||||||
|
if fetcher is None:
|
||||||
|
return None
|
||||||
|
query = _hub_query(params)
|
||||||
|
if dataset == "popularity":
|
||||||
|
query["source"] = "ths" if api_name == "ths_hot" else "dc"
|
||||||
|
if dataset == "sector_daily":
|
||||||
|
query["family"] = {"ths_daily": "ths", "dc_index": "dc", "sw_daily": "sw"}.get(api_name, "")
|
||||||
|
if dataset == "limit_events":
|
||||||
|
limit_type = str(params.get("limit_type") or "").strip().upper()
|
||||||
|
if limit_type:
|
||||||
|
query["limit_type"] = limit_type
|
||||||
|
if dataset == "calendar" and not (query.get("from") and query.get("to")):
|
||||||
|
start = str(params.get("start_date") or params.get("from") or "")
|
||||||
|
end = str(params.get("end_date") or params.get("to") or start)
|
||||||
|
if not start or not end:
|
||||||
|
return None
|
||||||
|
query = {"from": start, "to": end}
|
||||||
|
try:
|
||||||
|
payload = fetcher(api, query)
|
||||||
|
except ApiError as exc:
|
||||||
|
if exc.code in {"DATASET_NOT_PUBLISHED", "STALE_DATA", "INVALID_ARGUMENT"}:
|
||||||
|
return None
|
||||||
|
raise
|
||||||
|
rows = list(payload.get("data") or [])
|
||||||
|
if dataset == "stocks":
|
||||||
|
rows = _filter_stocks(rows, params)
|
||||||
|
if dataset == "calendar":
|
||||||
|
rows = _filter_calendar(rows, params)
|
||||||
|
native = _to_tushare_native(dataset, rows)
|
||||||
|
meta = dict(payload.get("meta") or {})
|
||||||
|
meta["row_shape"] = "tushare"
|
||||||
|
meta["source"] = str(meta.get("source") or "datahub")
|
||||||
|
return envelope(_project(native, fields), meta)
|
||||||
|
|
||||||
|
|
||||||
|
def _realtime_quotes(api, params: dict[str, Any], fields: str) -> dict[str, Any]:
|
||||||
|
codes = [item.strip() for item in str(params.get("ts_code") or params.get("codes") or "").split(",") if item.strip()]
|
||||||
|
try:
|
||||||
|
payload = fetch_quotes(api.db, codes) if codes else fetch_market_quotes(api.db)
|
||||||
|
except RealtimeApiError as exc:
|
||||||
|
raise ApiError(exc.code, exc.message) from exc
|
||||||
|
rows = [_quote_to_rt_k(item) for item in (payload.get("data") or []) if isinstance(item, dict)]
|
||||||
|
rows = [item for item in rows if item]
|
||||||
|
meta = dict(payload.get("meta") or {})
|
||||||
|
meta["row_shape"] = "tushare"
|
||||||
|
return envelope(_project(rows, fields), meta)
|
||||||
|
|
||||||
|
|
||||||
|
def _realtime_index_quotes(api, params: dict[str, Any], fields: str) -> dict[str, Any]:
|
||||||
|
try:
|
||||||
|
payload = fetch_index_quotes(api.db)
|
||||||
|
except RealtimeApiError as exc:
|
||||||
|
raise ApiError(exc.code, exc.message) from exc
|
||||||
|
wanted = {
|
||||||
|
item.strip()
|
||||||
|
for item in str(params.get("ts_code") or "").split(",")
|
||||||
|
if item.strip()
|
||||||
|
}
|
||||||
|
rows = []
|
||||||
|
for item in payload.get("data") or []:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
converted = _quote_to_rt_k(item)
|
||||||
|
if not converted:
|
||||||
|
continue
|
||||||
|
if wanted and converted.get("ts_code") not in wanted and str(item.get("code") or "") not in {
|
||||||
|
code.split(".")[0] for code in wanted
|
||||||
|
}:
|
||||||
|
continue
|
||||||
|
rows.append(converted)
|
||||||
|
meta = dict(payload.get("meta") or {})
|
||||||
|
meta["row_shape"] = "tushare"
|
||||||
|
return envelope(_project(rows, fields), meta)
|
||||||
|
|
||||||
|
|
||||||
|
def _live_tushare(api, api_name: str, params: dict[str, Any], fields: str) -> list[dict[str, Any]]:
|
||||||
|
wanted_fields = fields or TUSHARE_FIELDS.get(api_name, "")
|
||||||
|
cache_key = _live_cache_key(api_name, params, wanted_fields)
|
||||||
|
ttl = LIVE_TTL.get(api_name, 1800)
|
||||||
|
cached = _read_cache(api.db, cache_key)
|
||||||
|
if cached is not None:
|
||||||
|
data = cached.get("data")
|
||||||
|
if isinstance(data, list):
|
||||||
|
return [dict(item) for item in data if isinstance(item, dict)]
|
||||||
|
pipeline = api.pipeline
|
||||||
|
if not pipeline.breaker.allow():
|
||||||
|
recovered = _live_lkg(api.db, cache_key)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise ApiError("SOURCE_UNAVAILABLE", "Tushare circuit open")
|
||||||
|
pipeline.bucket.acquire()
|
||||||
|
try:
|
||||||
|
rows = pipeline.adapter.query_raw(api_name, dict(params), wanted_fields)
|
||||||
|
pipeline.breaker.record_success()
|
||||||
|
except Exception as exc:
|
||||||
|
pipeline.breaker.record_failure(str(exc))
|
||||||
|
recovered = _live_lkg(api.db, cache_key)
|
||||||
|
if recovered is not None:
|
||||||
|
return recovered
|
||||||
|
raise ApiError("SOURCE_UNAVAILABLE", f"Tushare {api_name} unavailable: {exc}") from exc
|
||||||
|
payload = envelope(
|
||||||
|
rows,
|
||||||
|
{
|
||||||
|
"tier": "live",
|
||||||
|
"source": "tushare",
|
||||||
|
"stale": False,
|
||||||
|
"staleness_seconds": 0,
|
||||||
|
"row_shape": "tushare",
|
||||||
|
"published_at": isoformat(now_shanghai()),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
_write_cache(api.db, cache_key, payload, ttl, "tushare")
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _live_lkg(db, cache_key: str) -> list[dict[str, Any]] | None:
|
||||||
|
from datahub.governance.lkg import LastKnownGood
|
||||||
|
|
||||||
|
item = LastKnownGood(db).load(cache_key)
|
||||||
|
payload = item.get("payload") if item else None
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None
|
||||||
|
data = payload.get("data")
|
||||||
|
if not isinstance(data, list) or not data:
|
||||||
|
return None
|
||||||
|
return [dict(row) for row in data if isinstance(row, dict)]
|
||||||
|
|
||||||
|
|
||||||
|
def _live_cache_key(api_name: str, params: dict[str, Any], fields: str) -> str:
|
||||||
|
packed = json.dumps({"api": api_name, "params": params, "fields": fields}, sort_keys=True, ensure_ascii=False)
|
||||||
|
digest = hashlib.sha1(packed.encode("utf-8")).hexdigest()
|
||||||
|
return f"steward:{api_name}:{digest}"
|
||||||
|
|
||||||
|
|
||||||
|
def _hub_query(params: dict[str, Any], **extra: Any) -> dict[str, str]:
|
||||||
|
query = {key: str(value) for key, value in extra.items() if value not in (None, "")}
|
||||||
|
date = yyyymmdd(params.get("trade_date") or params.get("date") or "")
|
||||||
|
start = yyyymmdd(params.get("start_date") or params.get("from") or date)
|
||||||
|
end = yyyymmdd(params.get("end_date") or params.get("to") or date)
|
||||||
|
code = str(params.get("ts_code") or params.get("code") or "").strip()
|
||||||
|
if code:
|
||||||
|
query["code"] = code
|
||||||
|
if date and not (params.get("start_date") or params.get("end_date")):
|
||||||
|
query["date"] = date
|
||||||
|
else:
|
||||||
|
if start:
|
||||||
|
query["from"] = start
|
||||||
|
if end:
|
||||||
|
query["to"] = end
|
||||||
|
return query
|
||||||
|
|
||||||
|
|
||||||
|
def _to_tushare_native(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
scales = SCALE_TO_TUSHARE.get(dataset) or {}
|
||||||
|
converted: list[dict[str, Any]] = []
|
||||||
|
for row in rows:
|
||||||
|
item = dict(row)
|
||||||
|
if item.get("vol") in (None, ""):
|
||||||
|
item["vol"] = item.get("volume")
|
||||||
|
item.pop("volume", None)
|
||||||
|
for field, factor in scales.items():
|
||||||
|
if field in item and item[field] not in (None, ""):
|
||||||
|
number = finite_number(item.get(field))
|
||||||
|
item[field] = number / factor if factor else number
|
||||||
|
if dataset == "popularity" and item.get("ts_name") and not item.get("name"):
|
||||||
|
item["name"] = item.get("ts_name")
|
||||||
|
if dataset == "dragon_tiger" and item.get("ts_name") and not item.get("name"):
|
||||||
|
item["name"] = item.get("ts_name")
|
||||||
|
if dataset == "sector_daily" and item.get("pct_change") is not None and item.get("pct_chg") is None:
|
||||||
|
item["pct_chg"] = item.get("pct_change")
|
||||||
|
if dataset == "calendar":
|
||||||
|
item["is_open"] = 1 if item.get("is_open") in (True, 1, "1", "Y", "y") else 0
|
||||||
|
converted.append(item)
|
||||||
|
return converted
|
||||||
|
|
||||||
|
|
||||||
|
def _quote_to_rt_k(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
|
ts_code = str(row.get("ts_code") or "").strip()
|
||||||
|
close = finite_number(row.get("close") if row.get("close") not in (None, "") else row.get("price"))
|
||||||
|
previous = finite_number(
|
||||||
|
row.get("pre_close") if row.get("pre_close") not in (None, "") else row.get("previous_close")
|
||||||
|
)
|
||||||
|
if not ts_code or close <= 0:
|
||||||
|
return None
|
||||||
|
item = {
|
||||||
|
"ts_code": ts_code,
|
||||||
|
"name": row.get("name") or "",
|
||||||
|
"open": row.get("open"),
|
||||||
|
"high": row.get("high"),
|
||||||
|
"low": row.get("low"),
|
||||||
|
"close": close,
|
||||||
|
"pre_close": previous,
|
||||||
|
"vol": row.get("vol") if row.get("vol") not in (None, "") else row.get("volume"),
|
||||||
|
"amount": row.get("amount"),
|
||||||
|
"pct_chg": row.get("pct_chg") if row.get("pct_chg") not in (None, "") else row.get("change"),
|
||||||
|
"trade_time": row.get("quote_time") or row.get("trade_time") or "",
|
||||||
|
"quote_date": row.get("quote_date") or "",
|
||||||
|
"source": row.get("source") or "",
|
||||||
|
"delayed": bool(row.get("delayed")),
|
||||||
|
"delay_seconds": row.get("delay_seconds") or 0,
|
||||||
|
"delay_notice": row.get("delay_notice") or "",
|
||||||
|
}
|
||||||
|
return item
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_stocks(rows: list[dict[str, Any]], params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
ts_code = str(params.get("ts_code") or "").strip().upper()
|
||||||
|
status = str(params.get("list_status") or "").strip()
|
||||||
|
name = str(params.get("name") or "").strip()
|
||||||
|
filtered = rows
|
||||||
|
if ts_code:
|
||||||
|
filtered = [row for row in filtered if str(row.get("ts_code") or "").upper() == ts_code]
|
||||||
|
if status:
|
||||||
|
filtered = [row for row in filtered if str(row.get("list_status") or status) == status]
|
||||||
|
if name:
|
||||||
|
filtered = [row for row in filtered if name.casefold() in str(row.get("name") or "").casefold()]
|
||||||
|
return filtered
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_calendar(rows: list[dict[str, Any]], params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
start = yyyymmdd(params.get("start_date") or params.get("from") or "")
|
||||||
|
end = yyyymmdd(params.get("end_date") or params.get("to") or start)
|
||||||
|
if start and end:
|
||||||
|
rows = [row for row in rows if start <= yyyymmdd(row.get("cal_date")) <= end]
|
||||||
|
if params.get("is_open") in (1, "1", True):
|
||||||
|
rows = [row for row in rows if int(row.get("is_open") or 0) == 1]
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def _project(rows: list[dict[str, Any]], fields: str) -> list[dict[str, Any]]:
|
||||||
|
keys = [item.strip() for item in str(fields or "").split(",") if item.strip()]
|
||||||
|
if not keys:
|
||||||
|
return rows
|
||||||
|
return [{key: row.get(key) for key in keys} for row in rows]
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -69,13 +70,29 @@ RAW = {
|
|||||||
"sw_daily": [
|
"sw_daily": [
|
||||||
{"ts_code": "801780.SI", "trade_date": "20240902", "name": "银行", "open": 2000, "high": 2010, "low": 1990, "close": 2005, "pct_change": 0.8, "vol": 50, "amount": 2e8},
|
{"ts_code": "801780.SI", "trade_date": "20240902", "name": "银行", "open": 2000, "high": 2010, "low": 1990, "close": 2005, "pct_change": 0.8, "vol": 50, "amount": 2e8},
|
||||||
],
|
],
|
||||||
|
"stk_limit": [
|
||||||
|
{"ts_code": "600000.SH", "trade_date": "20240902", "up_limit": 11.22, "down_limit": 9.18},
|
||||||
|
{"ts_code": "000001.SZ", "trade_date": "20240902", "up_limit": 12.21, "down_limit": 9.99},
|
||||||
|
],
|
||||||
|
"index_member_all": [
|
||||||
|
{
|
||||||
|
"l2_code": "801780.SI",
|
||||||
|
"l2_name": "银行",
|
||||||
|
"ts_code": "600000.SH",
|
||||||
|
"name": "浦发银行",
|
||||||
|
"in_date": "20140101",
|
||||||
|
"out_date": "",
|
||||||
|
"is_new": "Y",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"hm_list": [{"name": "测试游资", "desc": "测试", "orgs": "某某营业部"}],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def fake_transport(api_name: str, params: dict, fields: str):
|
def fake_transport(api_name: str, params: dict, fields: str):
|
||||||
if api_name == "index_daily":
|
if api_name == "index_daily":
|
||||||
code = params.get("ts_code")
|
code = params.get("ts_code")
|
||||||
rows = [row for row in RAW["index_daily"] if row["ts_code"] == code]
|
rows = [copy.deepcopy(row) for row in RAW["index_daily"] if row["ts_code"] == code]
|
||||||
trade_date = str(params.get("trade_date") or "")
|
trade_date = str(params.get("trade_date") or "")
|
||||||
start = str(params.get("start_date") or "")
|
start = str(params.get("start_date") or "")
|
||||||
end = str(params.get("end_date") or "")
|
end = str(params.get("end_date") or "")
|
||||||
@@ -89,8 +106,8 @@ def fake_transport(api_name: str, params: dict, fields: str):
|
|||||||
if api_name == "trade_cal":
|
if api_name == "trade_cal":
|
||||||
start = str(params.get("start_date") or "")
|
start = str(params.get("start_date") or "")
|
||||||
end = str(params.get("end_date") or "99999999")
|
end = str(params.get("end_date") or "99999999")
|
||||||
return [row for row in RAW["trade_cal"] if start <= row["cal_date"] <= end]
|
return [copy.deepcopy(row) for row in RAW["trade_cal"] if start <= row["cal_date"] <= end]
|
||||||
rows = list(RAW.get(api_name) or [])
|
rows = copy.deepcopy(list(RAW.get(api_name) or []))
|
||||||
if api_name == "limit_list_d":
|
if api_name == "limit_list_d":
|
||||||
limit_type = str(params.get("limit_type") or "")
|
limit_type = str(params.get("limit_type") or "")
|
||||||
if limit_type:
|
if limit_type:
|
||||||
|
|||||||
@@ -0,0 +1,111 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from datahub.adapters.tushare import TushareAdapter
|
||||||
|
from datahub.crypto import SecretVault
|
||||||
|
from datahub.hub import Hub
|
||||||
|
from datahub.settings import Settings
|
||||||
|
from tests.fixtures import TRADE_DATE, fake_transport
|
||||||
|
|
||||||
|
|
||||||
|
class StewardQueryTests(unittest.TestCase):
|
||||||
|
def setUp(self) -> None:
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
settings = Settings(
|
||||||
|
host="127.0.0.1",
|
||||||
|
port=0,
|
||||||
|
encryption_key=SecretVault.generate_key(),
|
||||||
|
api_token="k" * 32,
|
||||||
|
admin_password="StartPass1",
|
||||||
|
tushare_token="tushare-secret-token-xyz",
|
||||||
|
db_path=Path(self.tmp.name) / "hub.db",
|
||||||
|
backup_dir=Path(self.tmp.name) / "backups",
|
||||||
|
scheduler_enabled=False,
|
||||||
|
quality={"daily_row_ratio": 0.5, "null_rate_max": 0.5, "list_limit_default": 5000, "list_limit_max": 5000},
|
||||||
|
)
|
||||||
|
adapter = TushareAdapter("tushare-secret-token-xyz", transport=fake_transport)
|
||||||
|
self.hub = Hub(settings, adapter=adapter)
|
||||||
|
self.hub.pipeline.ingest_reference(TRADE_DATE)
|
||||||
|
for dataset in ("daily", "valuation", "moneyflow", "auction", "index_daily"):
|
||||||
|
self.hub.pipeline.run_dataset(dataset, TRADE_DATE)
|
||||||
|
|
||||||
|
def tearDown(self) -> None:
|
||||||
|
self.hub.stop()
|
||||||
|
self.tmp.cleanup()
|
||||||
|
|
||||||
|
def test_published_daily_is_tushare_native(self) -> None:
|
||||||
|
payload = self.hub.api.query_api(
|
||||||
|
{"api_name": "daily", "params": {"trade_date": TRADE_DATE}, "fields": "ts_code,close,vol,amount"}
|
||||||
|
)
|
||||||
|
rows = payload["data"]
|
||||||
|
by_code = {row["ts_code"]: row for row in rows}
|
||||||
|
self.assertEqual(by_code["600000.SH"]["vol"], 1000.0)
|
||||||
|
self.assertEqual(by_code["600000.SH"]["amount"], 2000.0)
|
||||||
|
self.assertEqual(payload["meta"]["row_shape"], "tushare")
|
||||||
|
|
||||||
|
def test_live_stk_limit_uses_internal_tushare(self) -> None:
|
||||||
|
payload = self.hub.api.query_api(
|
||||||
|
{"api_name": "stk_limit", "params": {"trade_date": TRADE_DATE}, "fields": "ts_code,up_limit,down_limit"}
|
||||||
|
)
|
||||||
|
self.assertEqual(payload["meta"]["source"], "tushare")
|
||||||
|
self.assertEqual(payload["data"][0]["ts_code"], "600000.SH")
|
||||||
|
|
||||||
|
def test_rt_sw_k_is_blocked(self) -> None:
|
||||||
|
from datahub.serving import ApiError
|
||||||
|
|
||||||
|
with self.assertRaises(ApiError):
|
||||||
|
self.hub.api.query_api({"api_name": "rt_sw_k", "params": {"ts_code": "801074.SI"}})
|
||||||
|
|
||||||
|
def test_rt_k_uses_free_quotes_not_tushare(self) -> None:
|
||||||
|
quotes = [
|
||||||
|
{
|
||||||
|
"ts_code": "600000.SH",
|
||||||
|
"name": "浦发银行",
|
||||||
|
"close": 10.2,
|
||||||
|
"pre_close": 10.0,
|
||||||
|
"open": 10.1,
|
||||||
|
"high": 10.3,
|
||||||
|
"low": 9.9,
|
||||||
|
"vol": 1000,
|
||||||
|
"amount": 2000000,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
with patch("datahub.steward.fetch_quotes", return_value={"data": quotes, "meta": {"source": "eastmoney:ulist", "stale": False}}):
|
||||||
|
payload = self.hub.api.query_api({"api_name": "rt_k", "params": {"ts_code": "600000.SH"}})
|
||||||
|
self.assertEqual(payload["data"][0]["close"], 10.2)
|
||||||
|
self.assertEqual(payload["meta"]["source"], "eastmoney:ulist")
|
||||||
|
|
||||||
|
def test_shenwan_quote_uses_eastmoney_90_prefix(self) -> None:
|
||||||
|
from datahub.adapters.eastmoney import EastmoneyAdapter
|
||||||
|
|
||||||
|
with patch.object(EastmoneyAdapter, "_get_json") as get_json:
|
||||||
|
get_json.return_value = {
|
||||||
|
"data": {
|
||||||
|
"diff": [
|
||||||
|
{
|
||||||
|
"f12": "801074",
|
||||||
|
"f14": "工业金属",
|
||||||
|
"f2": 1234.5,
|
||||||
|
"f3": 2.88,
|
||||||
|
"f18": 1200,
|
||||||
|
"f17": 1205,
|
||||||
|
"f15": 1240,
|
||||||
|
"f16": 1198,
|
||||||
|
"f6": 1,
|
||||||
|
"f124": 1757319000,
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
quote = EastmoneyAdapter().fetch_shenwan_quote("801074.SI")
|
||||||
|
self.assertEqual(quote["source"], "eastmoney_sw")
|
||||||
|
self.assertAlmostEqual(quote["change"], 2.88)
|
||||||
|
self.assertEqual(get_json.call_args.args[1]["secids"], "90.801074")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user