fix(HEL-494): 数据中枢独占调度，主网站不再回退旧接口
主网站只向中枢要业务数据；来源选择、切源、补数全部在中枢内部完成，失败不再走东财/腾讯/Tushare 保底。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
ef13d6feb5
commit
0b8419abca
@@ -250,6 +250,11 @@ class EastmoneyAdapter(MarketAdapter):
|
||||
secid = INDEX_SECIDS[code]
|
||||
entity = "index"
|
||||
identifier = code
|
||||
elif code.startswith("BK") or code.endswith((".TI", ".SI")):
|
||||
symbol = code.split(".")[0]
|
||||
secid = f"90.{symbol}"
|
||||
entity = "board"
|
||||
identifier = symbol
|
||||
else:
|
||||
symbol = code.split(".")[0]
|
||||
market = "1" if symbol.startswith(("5", "6", "9")) else "0"
|
||||
@@ -294,6 +299,108 @@ class EastmoneyAdapter(MarketAdapter):
|
||||
"source": "eastmoney_trends2",
|
||||
}
|
||||
|
||||
def fetch_shenwan_quote(self, ts_code: str) -> dict[str, Any]:
|
||||
code = str(ts_code or "").split(".")[0]
|
||||
if not code:
|
||||
raise AdapterError("Invalid Shenwan code")
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": f"90.{code}",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f8,f104,f105,f128,f136,f140,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||||
row = next((item for item in rows if item), None)
|
||||
if not row:
|
||||
raise AdapterError(f"Eastmoney Shenwan quote missing for {code}")
|
||||
epoch = int(finite_number(row.get("f124")) or 0)
|
||||
close = round4(finite_number(row.get("f2")))
|
||||
previous = round4(finite_number(row.get("f18")))
|
||||
if close <= 0 or previous <= 0:
|
||||
raise AdapterError(f"Eastmoney Shenwan quote empty for {code}")
|
||||
quote_time = (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch
|
||||
else ""
|
||||
)
|
||||
return {
|
||||
"ts_code": f"{code}.SI",
|
||||
"code": f"{code}.SI",
|
||||
"name": row.get("f14") or code,
|
||||
"price": close,
|
||||
"close": close,
|
||||
"pre_close": previous,
|
||||
"previous_close": previous,
|
||||
"open": round4(finite_number(row.get("f17"))),
|
||||
"high": round4(finite_number(row.get("f15"))),
|
||||
"low": round4(finite_number(row.get("f16"))),
|
||||
"change": round4(finite_number(row.get("f3"))),
|
||||
"pct_change": round4(finite_number(row.get("f3"))),
|
||||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||||
"amount": round4(finite_number(row.get("f6"))),
|
||||
"leader": row.get("f128") or "--",
|
||||
"leader_code": row.get("f140") or "",
|
||||
"leading_pct": round4(finite_number(row.get("f136"))),
|
||||
"up_count": int(finite_number(row.get("f104")) or 0),
|
||||
"down_count": int(finite_number(row.get("f105")) or 0),
|
||||
"quote_time": quote_time,
|
||||
"trade_time": quote_time,
|
||||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||||
"quote_time_epoch": epoch,
|
||||
"source": "eastmoney_sw",
|
||||
}
|
||||
|
||||
def fetch_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]]:
|
||||
day = str(trade_date or "").replace("-", "")
|
||||
rows: list[dict[str, Any]] = []
|
||||
for url, limit_type in (
|
||||
("https://push2ex.eastmoney.com/getTopicZTPool", "U"),
|
||||
("https://push2ex.eastmoney.com/getTopicZBPool", "Z"),
|
||||
):
|
||||
params = {
|
||||
"ut": "7eea3edcaed734bea9cbfc24409ed989",
|
||||
"dpt": "wz.ztzt",
|
||||
"PageIndex": "0",
|
||||
"PageSize": "200",
|
||||
"sort": "fbt:asc",
|
||||
"stat": "1",
|
||||
}
|
||||
if day:
|
||||
params["date"] = day
|
||||
try:
|
||||
payload = self._get_json(url, params, referer="https://quote.eastmoney.com/")
|
||||
except AdapterError:
|
||||
continue
|
||||
pool = ((payload.get("data") or {}).get("pool") or []) if isinstance(payload.get("data"), dict) else []
|
||||
for item in pool:
|
||||
code = str(item.get("c") or item.get("code") or "")
|
||||
if not code:
|
||||
continue
|
||||
market = str(item.get("m") or item.get("market") or "")
|
||||
suffix = "SH" if market in {"1", "SH"} or code.startswith(("5", "6", "9")) else "SZ"
|
||||
first = str(item.get("fbt") or item.get("first_time") or "")
|
||||
last = str(item.get("lbt") or item.get("last_time") or "")
|
||||
rows.append(
|
||||
{
|
||||
"ts_code": f"{code}.{suffix}",
|
||||
"limit_type": limit_type,
|
||||
"first_time": first,
|
||||
"last_time": last,
|
||||
"fd_amount": item.get("fund") or item.get("fd_amount"),
|
||||
"open_times": item.get("zbc") or item.get("open_times"),
|
||||
"limit_times": item.get("lbc") or item.get("limit_times"),
|
||||
"turnover_ratio": item.get("hs") or item.get("turnover_ratio"),
|
||||
"source": "eastmoney_zt_pool",
|
||||
}
|
||||
)
|
||||
if not rows:
|
||||
raise AdapterError("Eastmoney limit pool empty")
|
||||
return rows
|
||||
|
||||
def _get_json(self, url: str, params: dict[str, str], referer: str) -> dict[str, Any]:
|
||||
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
||||
request = urllib.request.Request(
|
||||
|
||||
@@ -50,6 +50,14 @@ TUSHARE_FIELDS = {
|
||||
"ths_daily": "ts_code,trade_date,open,high,low,close,pre_close,pct_change,vol,turnover_rate",
|
||||
"dc_index": "ts_code,trade_date,name,open,high,low,close,pre_close,pct_change,vol,amount,turnover_rate",
|
||||
"sw_daily": "ts_code,trade_date,name,open,high,low,close,pct_change,vol,amount",
|
||||
"index_member_all": (
|
||||
"l1_code,l1_name,l2_code,l2_name,l3_code,l3_name,"
|
||||
"ts_code,name,in_date,out_date,is_new"
|
||||
),
|
||||
"stk_limit": "ts_code,trade_date,up_limit,down_limit",
|
||||
"suspend_d": "ts_code,suspend_date,resume_date,ann_date,suspend_reason,reason_type",
|
||||
"ths_member": "ts_code,con_code,con_name,in_date,out_date,is_new",
|
||||
"stk_mins": "ts_code,trade_time,open,close,high,low,vol,amount",
|
||||
}
|
||||
|
||||
DATASET_API = {
|
||||
@@ -254,3 +262,6 @@ class TushareAdapter(MarketAdapter):
|
||||
items = data.get("items") or []
|
||||
fields_list = data.get("fields") or (fields.split(",") if fields else [])
|
||||
return [dict(zip(fields_list, item)) for item in items]
|
||||
|
||||
def query_raw(self, api_name: str, params: dict[str, Any], fields: str = "") -> list[dict[str, Any]]:
|
||||
return self._query(api_name, params, fields or TUSHARE_FIELDS.get(api_name, ""))
|
||||
|
||||
@@ -42,7 +42,7 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
self._json({"status": "ok"}, HTTPStatus.OK)
|
||||
return
|
||||
if path.startswith("/v1/"):
|
||||
self._v1(path, parsed.query)
|
||||
self._v1(path, parsed.query, method)
|
||||
return
|
||||
if path.startswith("/admin/api/"):
|
||||
self._admin_api(method, path)
|
||||
@@ -66,11 +66,16 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
LOGGER.exception("internal error")
|
||||
self._json({"error": {"code": "INTERNAL", "message": "internal error"}}, HTTPStatus.INTERNAL_SERVER_ERROR)
|
||||
|
||||
def _v1(self, path: str, query: str) -> None:
|
||||
def _v1(self, path: str, query: str, method: str = "GET") -> None:
|
||||
token = self.headers.get("X-Datahub-Token", "")
|
||||
if not self.hub.auth.check_api_token(token):
|
||||
self.hub.pipeline.audit("anonymous", "unauthorized", path, "")
|
||||
raise ApiError("UNAUTHORIZED", "missing or invalid X-Datahub-Token")
|
||||
if path == "/v1/query" and method == "POST":
|
||||
body = self._read_json(max_bytes=1_000_000)
|
||||
payload = self.hub.api.query_api(body)
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
return
|
||||
payload = self.hub.api.handle(path, parse_query(query))
|
||||
self._json(payload, HTTPStatus.OK)
|
||||
|
||||
@@ -184,11 +189,11 @@ class HubRequestHandler(BaseHTTPRequestHandler):
|
||||
self.end_headers()
|
||||
self.wfile.write(content)
|
||||
|
||||
def _read_json(self, allow_empty: bool = False) -> dict[str, Any]:
|
||||
def _read_json(self, allow_empty: bool = False, max_bytes: int = 65536) -> dict[str, Any]:
|
||||
length = int(self.headers.get("Content-Length", "0") or 0)
|
||||
if length == 0 and allow_empty:
|
||||
return {}
|
||||
if length <= 0 or length > 65536:
|
||||
if length <= 0 or length > max_bytes:
|
||||
raise ValueError("请求内容为空或过大")
|
||||
raw = self.rfile.read(length)
|
||||
try:
|
||||
|
||||
@@ -149,6 +149,78 @@ def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
|
||||
return payload
|
||||
|
||||
|
||||
def fetch_sector_quote(db: HubDB, code: str, expected_date: str = "") -> dict[str, Any]:
|
||||
ts_code = str(code or "").strip().upper()
|
||||
if ts_code.isdigit():
|
||||
ts_code = f"{ts_code}.SI"
|
||||
cache_key = f"sector:{ts_code}"
|
||||
cached = _read_cache(db, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
errors: list[str] = []
|
||||
try:
|
||||
row = EastmoneyAdapter().fetch_shenwan_quote(ts_code)
|
||||
source = str(row.get("source") or "eastmoney_sw")
|
||||
except Exception as exc:
|
||||
errors.append(f"eastmoney:{exc}")
|
||||
recovered = _load_quotes_lkg(db, cache_key)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
raise RealtimeApiError(
|
||||
"SOURCE_UNAVAILABLE",
|
||||
"sector quote unavailable: " + "；".join(errors),
|
||||
) from exc
|
||||
want = str(expected_date or "").replace("-", "")[:8]
|
||||
quote_date = str(row.get("quote_date") or "")
|
||||
if want and quote_date and quote_date != want:
|
||||
recovered = _load_quotes_lkg(db, cache_key)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
raise RealtimeApiError("SOURCE_UNAVAILABLE", f"sector quote date {quote_date} != {want}")
|
||||
payload = _envelope(
|
||||
row,
|
||||
{
|
||||
"tier": "provisional",
|
||||
"trade_date": quote_date or yyyymmdd(now_shanghai()),
|
||||
"source": source,
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"published_at": isoformat(now_shanghai()),
|
||||
},
|
||||
)
|
||||
_write_cache(db, cache_key, payload, INDEX_TTL, source)
|
||||
return payload
|
||||
|
||||
|
||||
def fetch_limit_pool(db: HubDB, trade_date: str = "") -> dict[str, Any]:
|
||||
day = yyyymmdd(trade_date or now_shanghai())
|
||||
cache_key = f"limit-pool:{day}"
|
||||
cached = _read_cache(db, cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
rows = EastmoneyAdapter().fetch_limit_pool(day)
|
||||
source = "eastmoney:zt_pool"
|
||||
except Exception as exc:
|
||||
recovered = _load_quotes_lkg(db, cache_key)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
raise RealtimeApiError("SOURCE_UNAVAILABLE", f"limit pool unavailable: {exc}") from exc
|
||||
payload = _envelope(
|
||||
rows,
|
||||
{
|
||||
"tier": "provisional",
|
||||
"trade_date": day,
|
||||
"source": source,
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"published_at": isoformat(now_shanghai()),
|
||||
},
|
||||
)
|
||||
_write_cache(db, cache_key, payload, QUOTE_TTL, source)
|
||||
return payload
|
||||
|
||||
|
||||
def _eastmoney_named_quotes(codes: list[str]) -> list[dict[str, Any]]:
|
||||
adapter = EastmoneyAdapter()
|
||||
rows: list[dict[str, Any]] = []
|
||||
|
||||
@@ -87,12 +87,46 @@ class V1API:
|
||||
return self.index_quotes(q)
|
||||
if path == "/v1/intraday/points":
|
||||
return self.intraday_points(q)
|
||||
if path == "/v1/sectors/quote":
|
||||
return self.sector_quote(q)
|
||||
if path == "/v1/limit-pool":
|
||||
return self.limit_pool(q)
|
||||
if path == "/v1/query":
|
||||
return self.query_api(q)
|
||||
if path == "/v1/datasets/status":
|
||||
return self.dataset_status(q.get("date") or "")
|
||||
if path == "/v1/batches":
|
||||
return self.batches(q.get("date") or "", q.get("dataset") or "")
|
||||
raise ApiError("INVALID_ARGUMENT", f"unknown endpoint: {path}")
|
||||
|
||||
def query_api(self, body: dict[str, Any]) -> dict[str, Any]:
|
||||
from datahub.steward import steward_query
|
||||
|
||||
payload = dict(body or {})
|
||||
raw_params = payload.get("params")
|
||||
if isinstance(raw_params, str):
|
||||
payload["params"] = _parse_json(raw_params) or {}
|
||||
return steward_query(self, payload)
|
||||
|
||||
def sector_quote(self, q: dict[str, str]) -> dict[str, Any]:
|
||||
from datahub.realtime_serve import RealtimeApiError, fetch_sector_quote
|
||||
|
||||
code = str(q.get("code") or q.get("ts_code") or "").strip()
|
||||
if not code:
|
||||
raise ApiError("INVALID_ARGUMENT", "code is required")
|
||||
try:
|
||||
return fetch_sector_quote(self.db, code, str(q.get("date") or ""))
|
||||
except RealtimeApiError as exc:
|
||||
raise ApiError(exc.code, exc.message) from exc
|
||||
|
||||
def limit_pool(self, q: dict[str, str]) -> dict[str, Any]:
|
||||
from datahub.realtime_serve import RealtimeApiError, fetch_limit_pool
|
||||
|
||||
try:
|
||||
return fetch_limit_pool(self.db, str(q.get("date") or q.get("trade_date") or ""))
|
||||
except RealtimeApiError as exc:
|
||||
raise ApiError(exc.code, exc.message) from exc
|
||||
|
||||
def health(self) -> dict[str, Any]:
|
||||
today = yyyymmdd(now_shanghai())
|
||||
cal = self.db.fetchone(
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
"""Website-facing data steward: pick source, fail over, cache, never fake zeros.
|
||||
|
||||
The main site asks for a business/Tushare-shaped API. This module decides whether
|
||||
to serve a published EOD table, live free quotes, or an internal Tushare pull.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.adapters.tushare import TUSHARE_FIELDS
|
||||
from datahub.numbers import finite_number
|
||||
from datahub.realtime_serve import (
|
||||
RealtimeApiError,
|
||||
_read_cache,
|
||||
_write_cache,
|
||||
fetch_index_quotes,
|
||||
fetch_market_quotes,
|
||||
fetch_quotes,
|
||||
)
|
||||
from datahub.serving import ApiError, envelope
|
||||
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
|
||||
|
||||
API_TO_DATASET = {
|
||||
"trade_cal": "calendar",
|
||||
"stock_basic": "stocks",
|
||||
"daily": "daily",
|
||||
"daily_basic": "valuation",
|
||||
"index_daily": "index_daily",
|
||||
"moneyflow": "moneyflow",
|
||||
"stk_auction": "auction",
|
||||
"limit_list_d": "limit_events",
|
||||
"ths_hot": "popularity",
|
||||
"dc_hot": "popularity",
|
||||
"hm_detail": "dragon_tiger",
|
||||
"ths_daily": "sector_daily",
|
||||
"dc_index": "sector_daily",
|
||||
"sw_daily": "sector_daily",
|
||||
}
|
||||
|
||||
DATASET_FETCHER = {
|
||||
"calendar": lambda api, q: api.calendar(q.get("from") or q.get("start_date") or "", q.get("to") or q.get("end_date") or ""),
|
||||
"stocks": lambda api, q: api.stocks(q.get("updated_since") or "", q),
|
||||
"daily": lambda api, q: api.daily_bars(_hub_query(q, adjust="none")),
|
||||
"valuation": lambda api, q: api.valuation(_hub_query(q)),
|
||||
"index_daily": lambda api, q: api.index_bars(_hub_query(q)),
|
||||
"moneyflow": lambda api, q: api.moneyflow(_hub_query(q)),
|
||||
"auction": lambda api, q: api.auction(_hub_query(q)),
|
||||
"limit_events": lambda api, q: api.limit_events(_hub_query(q)),
|
||||
"popularity": lambda api, q: api.popularity(_hub_query(q)),
|
||||
"dragon_tiger": lambda api, q: api.dragon_tiger(_hub_query(q)),
|
||||
"sector_daily": lambda api, q: api.sectors(_hub_query(q)),
|
||||
}
|
||||
|
||||
SCALE_TO_TUSHARE = {
|
||||
"daily": {"vol": 100.0, "amount": 1000.0},
|
||||
"index_daily": {"vol": 100.0, "amount": 1000.0},
|
||||
"valuation": {"total_mv": 10000.0, "circ_mv": 10000.0},
|
||||
"moneyflow": {
|
||||
"buy_sm_amount": 10000.0,
|
||||
"sell_sm_amount": 10000.0,
|
||||
"buy_md_amount": 10000.0,
|
||||
"sell_md_amount": 10000.0,
|
||||
"buy_lg_amount": 10000.0,
|
||||
"sell_lg_amount": 10000.0,
|
||||
"buy_elg_amount": 10000.0,
|
||||
"sell_elg_amount": 10000.0,
|
||||
"net_mf_amount": 10000.0,
|
||||
},
|
||||
"auction": {"vol": 100.0, "float_share": 10000.0},
|
||||
"limit_events": {"limit_amount": 10000.0, "float_mv": 10000.0, "total_mv": 10000.0},
|
||||
"dragon_tiger": {"buy_amount": 10000.0, "sell_amount": 10000.0, "net_amount": 10000.0},
|
||||
}
|
||||
|
||||
LIVE_TTL = {
|
||||
"index_member_all": 6 * 3600,
|
||||
"stk_limit": 3600,
|
||||
"suspend_d": 6 * 3600,
|
||||
"adj_factor": 3600,
|
||||
"hm_list": 24 * 3600,
|
||||
"ths_index": 24 * 3600,
|
||||
"ths_member": 6 * 3600,
|
||||
"stk_mins": 20,
|
||||
"top_list": 3600,
|
||||
"top_inst": 3600,
|
||||
}
|
||||
|
||||
BLOCKED_LIVE_APIS = {"rt_sw_k"}
|
||||
|
||||
|
||||
def steward_query(api, body: dict[str, Any]) -> dict[str, Any]:
|
||||
api_name = str(body.get("api_name") or "").strip()
|
||||
params = body.get("params") if isinstance(body.get("params"), dict) else {}
|
||||
fields = str(body.get("fields") or "")
|
||||
if not api_name:
|
||||
raise ApiError("INVALID_ARGUMENT", "api_name is required")
|
||||
if api_name in BLOCKED_LIVE_APIS:
|
||||
raise ApiError("INVALID_ARGUMENT", "rt_sw_k is disabled; use published sw_daily or free Shenwan realtime")
|
||||
if api_name == "rt_k":
|
||||
return _realtime_quotes(api, params, fields)
|
||||
if api_name == "rt_idx_k":
|
||||
return _realtime_index_quotes(api, params, fields)
|
||||
dataset = API_TO_DATASET.get(api_name)
|
||||
if dataset:
|
||||
published = _try_published(api, api_name, dataset, params, fields)
|
||||
if published is not None:
|
||||
return published
|
||||
rows = _live_tushare(api, api_name, params, fields)
|
||||
return envelope(
|
||||
_project(rows, fields),
|
||||
{
|
||||
"tier": "live",
|
||||
"source": "tushare",
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"row_shape": "tushare",
|
||||
"published_at": isoformat(now_shanghai()),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _try_published(api, api_name: str, dataset: str, params: dict[str, Any], fields: str) -> dict[str, Any] | None:
|
||||
fetcher = DATASET_FETCHER.get(dataset)
|
||||
if fetcher is None:
|
||||
return None
|
||||
query = _hub_query(params)
|
||||
if dataset == "popularity":
|
||||
query["source"] = "ths" if api_name == "ths_hot" else "dc"
|
||||
if dataset == "sector_daily":
|
||||
query["family"] = {"ths_daily": "ths", "dc_index": "dc", "sw_daily": "sw"}.get(api_name, "")
|
||||
if dataset == "limit_events":
|
||||
limit_type = str(params.get("limit_type") or "").strip().upper()
|
||||
if limit_type:
|
||||
query["limit_type"] = limit_type
|
||||
if dataset == "calendar" and not (query.get("from") and query.get("to")):
|
||||
start = str(params.get("start_date") or params.get("from") or "")
|
||||
end = str(params.get("end_date") or params.get("to") or start)
|
||||
if not start or not end:
|
||||
return None
|
||||
query = {"from": start, "to": end}
|
||||
try:
|
||||
payload = fetcher(api, query)
|
||||
except ApiError as exc:
|
||||
if exc.code in {"DATASET_NOT_PUBLISHED", "STALE_DATA", "INVALID_ARGUMENT"}:
|
||||
return None
|
||||
raise
|
||||
rows = list(payload.get("data") or [])
|
||||
if dataset == "stocks":
|
||||
rows = _filter_stocks(rows, params)
|
||||
if dataset == "calendar":
|
||||
rows = _filter_calendar(rows, params)
|
||||
native = _to_tushare_native(dataset, rows)
|
||||
meta = dict(payload.get("meta") or {})
|
||||
meta["row_shape"] = "tushare"
|
||||
meta["source"] = str(meta.get("source") or "datahub")
|
||||
return envelope(_project(native, fields), meta)
|
||||
|
||||
|
||||
def _realtime_quotes(api, params: dict[str, Any], fields: str) -> dict[str, Any]:
|
||||
codes = [item.strip() for item in str(params.get("ts_code") or params.get("codes") or "").split(",") if item.strip()]
|
||||
try:
|
||||
payload = fetch_quotes(api.db, codes) if codes else fetch_market_quotes(api.db)
|
||||
except RealtimeApiError as exc:
|
||||
raise ApiError(exc.code, exc.message) from exc
|
||||
rows = [_quote_to_rt_k(item) for item in (payload.get("data") or []) if isinstance(item, dict)]
|
||||
rows = [item for item in rows if item]
|
||||
meta = dict(payload.get("meta") or {})
|
||||
meta["row_shape"] = "tushare"
|
||||
return envelope(_project(rows, fields), meta)
|
||||
|
||||
|
||||
def _realtime_index_quotes(api, params: dict[str, Any], fields: str) -> dict[str, Any]:
|
||||
try:
|
||||
payload = fetch_index_quotes(api.db)
|
||||
except RealtimeApiError as exc:
|
||||
raise ApiError(exc.code, exc.message) from exc
|
||||
wanted = {
|
||||
item.strip()
|
||||
for item in str(params.get("ts_code") or "").split(",")
|
||||
if item.strip()
|
||||
}
|
||||
rows = []
|
||||
for item in payload.get("data") or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
converted = _quote_to_rt_k(item)
|
||||
if not converted:
|
||||
continue
|
||||
if wanted and converted.get("ts_code") not in wanted and str(item.get("code") or "") not in {
|
||||
code.split(".")[0] for code in wanted
|
||||
}:
|
||||
continue
|
||||
rows.append(converted)
|
||||
meta = dict(payload.get("meta") or {})
|
||||
meta["row_shape"] = "tushare"
|
||||
return envelope(_project(rows, fields), meta)
|
||||
|
||||
|
||||
def _live_tushare(api, api_name: str, params: dict[str, Any], fields: str) -> list[dict[str, Any]]:
|
||||
wanted_fields = fields or TUSHARE_FIELDS.get(api_name, "")
|
||||
cache_key = _live_cache_key(api_name, params, wanted_fields)
|
||||
ttl = LIVE_TTL.get(api_name, 1800)
|
||||
cached = _read_cache(api.db, cache_key)
|
||||
if cached is not None:
|
||||
data = cached.get("data")
|
||||
if isinstance(data, list):
|
||||
return [dict(item) for item in data if isinstance(item, dict)]
|
||||
pipeline = api.pipeline
|
||||
if not pipeline.breaker.allow():
|
||||
recovered = _live_lkg(api.db, cache_key)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
raise ApiError("SOURCE_UNAVAILABLE", "Tushare circuit open")
|
||||
pipeline.bucket.acquire()
|
||||
try:
|
||||
rows = pipeline.adapter.query_raw(api_name, dict(params), wanted_fields)
|
||||
pipeline.breaker.record_success()
|
||||
except Exception as exc:
|
||||
pipeline.breaker.record_failure(str(exc))
|
||||
recovered = _live_lkg(api.db, cache_key)
|
||||
if recovered is not None:
|
||||
return recovered
|
||||
raise ApiError("SOURCE_UNAVAILABLE", f"Tushare {api_name} unavailable: {exc}") from exc
|
||||
payload = envelope(
|
||||
rows,
|
||||
{
|
||||
"tier": "live",
|
||||
"source": "tushare",
|
||||
"stale": False,
|
||||
"staleness_seconds": 0,
|
||||
"row_shape": "tushare",
|
||||
"published_at": isoformat(now_shanghai()),
|
||||
},
|
||||
)
|
||||
_write_cache(api.db, cache_key, payload, ttl, "tushare")
|
||||
return rows
|
||||
|
||||
|
||||
def _live_lkg(db, cache_key: str) -> list[dict[str, Any]] | None:
|
||||
from datahub.governance.lkg import LastKnownGood
|
||||
|
||||
item = LastKnownGood(db).load(cache_key)
|
||||
payload = item.get("payload") if item else None
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, list) or not data:
|
||||
return None
|
||||
return [dict(row) for row in data if isinstance(row, dict)]
|
||||
|
||||
|
||||
def _live_cache_key(api_name: str, params: dict[str, Any], fields: str) -> str:
|
||||
packed = json.dumps({"api": api_name, "params": params, "fields": fields}, sort_keys=True, ensure_ascii=False)
|
||||
digest = hashlib.sha1(packed.encode("utf-8")).hexdigest()
|
||||
return f"steward:{api_name}:{digest}"
|
||||
|
||||
|
||||
def _hub_query(params: dict[str, Any], **extra: Any) -> dict[str, str]:
|
||||
query = {key: str(value) for key, value in extra.items() if value not in (None, "")}
|
||||
date = yyyymmdd(params.get("trade_date") or params.get("date") or "")
|
||||
start = yyyymmdd(params.get("start_date") or params.get("from") or date)
|
||||
end = yyyymmdd(params.get("end_date") or params.get("to") or date)
|
||||
code = str(params.get("ts_code") or params.get("code") or "").strip()
|
||||
if code:
|
||||
query["code"] = code
|
||||
if date and not (params.get("start_date") or params.get("end_date")):
|
||||
query["date"] = date
|
||||
else:
|
||||
if start:
|
||||
query["from"] = start
|
||||
if end:
|
||||
query["to"] = end
|
||||
return query
|
||||
|
||||
|
||||
def _to_tushare_native(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
scales = SCALE_TO_TUSHARE.get(dataset) or {}
|
||||
converted: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
item = dict(row)
|
||||
if item.get("vol") in (None, ""):
|
||||
item["vol"] = item.get("volume")
|
||||
item.pop("volume", None)
|
||||
for field, factor in scales.items():
|
||||
if field in item and item[field] not in (None, ""):
|
||||
number = finite_number(item.get(field))
|
||||
item[field] = number / factor if factor else number
|
||||
if dataset == "popularity" and item.get("ts_name") and not item.get("name"):
|
||||
item["name"] = item.get("ts_name")
|
||||
if dataset == "dragon_tiger" and item.get("ts_name") and not item.get("name"):
|
||||
item["name"] = item.get("ts_name")
|
||||
if dataset == "sector_daily" and item.get("pct_change") is not None and item.get("pct_chg") is None:
|
||||
item["pct_chg"] = item.get("pct_change")
|
||||
if dataset == "calendar":
|
||||
item["is_open"] = 1 if item.get("is_open") in (True, 1, "1", "Y", "y") else 0
|
||||
converted.append(item)
|
||||
return converted
|
||||
|
||||
|
||||
def _quote_to_rt_k(row: dict[str, Any]) -> dict[str, Any] | None:
|
||||
ts_code = str(row.get("ts_code") or "").strip()
|
||||
close = finite_number(row.get("close") if row.get("close") not in (None, "") else row.get("price"))
|
||||
previous = finite_number(
|
||||
row.get("pre_close") if row.get("pre_close") not in (None, "") else row.get("previous_close")
|
||||
)
|
||||
if not ts_code or close <= 0:
|
||||
return None
|
||||
item = {
|
||||
"ts_code": ts_code,
|
||||
"name": row.get("name") or "",
|
||||
"open": row.get("open"),
|
||||
"high": row.get("high"),
|
||||
"low": row.get("low"),
|
||||
"close": close,
|
||||
"pre_close": previous,
|
||||
"vol": row.get("vol") if row.get("vol") not in (None, "") else row.get("volume"),
|
||||
"amount": row.get("amount"),
|
||||
"pct_chg": row.get("pct_chg") if row.get("pct_chg") not in (None, "") else row.get("change"),
|
||||
"trade_time": row.get("quote_time") or row.get("trade_time") or "",
|
||||
"quote_date": row.get("quote_date") or "",
|
||||
"source": row.get("source") or "",
|
||||
"delayed": bool(row.get("delayed")),
|
||||
"delay_seconds": row.get("delay_seconds") or 0,
|
||||
"delay_notice": row.get("delay_notice") or "",
|
||||
}
|
||||
return item
|
||||
|
||||
|
||||
def _filter_stocks(rows: list[dict[str, Any]], params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
ts_code = str(params.get("ts_code") or "").strip().upper()
|
||||
status = str(params.get("list_status") or "").strip()
|
||||
name = str(params.get("name") or "").strip()
|
||||
filtered = rows
|
||||
if ts_code:
|
||||
filtered = [row for row in filtered if str(row.get("ts_code") or "").upper() == ts_code]
|
||||
if status:
|
||||
filtered = [row for row in filtered if str(row.get("list_status") or status) == status]
|
||||
if name:
|
||||
filtered = [row for row in filtered if name.casefold() in str(row.get("name") or "").casefold()]
|
||||
return filtered
|
||||
|
||||
|
||||
def _filter_calendar(rows: list[dict[str, Any]], params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
start = yyyymmdd(params.get("start_date") or params.get("from") or "")
|
||||
end = yyyymmdd(params.get("end_date") or params.get("to") or start)
|
||||
if start and end:
|
||||
rows = [row for row in rows if start <= yyyymmdd(row.get("cal_date")) <= end]
|
||||
if params.get("is_open") in (1, "1", True):
|
||||
rows = [row for row in rows if int(row.get("is_open") or 0) == 1]
|
||||
return rows
|
||||
|
||||
|
||||
def _project(rows: list[dict[str, Any]], fields: str) -> list[dict[str, Any]]:
|
||||
keys = [item.strip() for item in str(fields or "").split(",") if item.strip()]
|
||||
if not keys:
|
||||
return rows
|
||||
return [{key: row.get(key) for key in keys} for row in rows]
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -69,13 +70,29 @@ RAW = {
|
||||
"sw_daily": [
|
||||
{"ts_code": "801780.SI", "trade_date": "20240902", "name": "银行", "open": 2000, "high": 2010, "low": 1990, "close": 2005, "pct_change": 0.8, "vol": 50, "amount": 2e8},
|
||||
],
|
||||
"stk_limit": [
|
||||
{"ts_code": "600000.SH", "trade_date": "20240902", "up_limit": 11.22, "down_limit": 9.18},
|
||||
{"ts_code": "000001.SZ", "trade_date": "20240902", "up_limit": 12.21, "down_limit": 9.99},
|
||||
],
|
||||
"index_member_all": [
|
||||
{
|
||||
"l2_code": "801780.SI",
|
||||
"l2_name": "银行",
|
||||
"ts_code": "600000.SH",
|
||||
"name": "浦发银行",
|
||||
"in_date": "20140101",
|
||||
"out_date": "",
|
||||
"is_new": "Y",
|
||||
}
|
||||
],
|
||||
"hm_list": [{"name": "测试游资", "desc": "测试", "orgs": "某某营业部"}],
|
||||
}
|
||||
|
||||
|
||||
def fake_transport(api_name: str, params: dict, fields: str):
|
||||
if api_name == "index_daily":
|
||||
code = params.get("ts_code")
|
||||
rows = [row for row in RAW["index_daily"] if row["ts_code"] == code]
|
||||
rows = [copy.deepcopy(row) for row in RAW["index_daily"] if row["ts_code"] == code]
|
||||
trade_date = str(params.get("trade_date") or "")
|
||||
start = str(params.get("start_date") or "")
|
||||
end = str(params.get("end_date") or "")
|
||||
@@ -89,8 +106,8 @@ def fake_transport(api_name: str, params: dict, fields: str):
|
||||
if api_name == "trade_cal":
|
||||
start = str(params.get("start_date") or "")
|
||||
end = str(params.get("end_date") or "99999999")
|
||||
return [row for row in RAW["trade_cal"] if start <= row["cal_date"] <= end]
|
||||
rows = list(RAW.get(api_name) or [])
|
||||
return [copy.deepcopy(row) for row in RAW["trade_cal"] if start <= row["cal_date"] <= end]
|
||||
rows = copy.deepcopy(list(RAW.get(api_name) or []))
|
||||
if api_name == "limit_list_d":
|
||||
limit_type = str(params.get("limit_type") or "")
|
||||
if limit_type:
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from datahub.adapters.tushare import TushareAdapter
|
||||
from datahub.crypto import SecretVault
|
||||
from datahub.hub import Hub
|
||||
from datahub.settings import Settings
|
||||
from tests.fixtures import TRADE_DATE, fake_transport
|
||||
|
||||
|
||||
class StewardQueryTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
settings = Settings(
|
||||
host="127.0.0.1",
|
||||
port=0,
|
||||
encryption_key=SecretVault.generate_key(),
|
||||
api_token="k" * 32,
|
||||
admin_password="StartPass1",
|
||||
tushare_token="tushare-secret-token-xyz",
|
||||
db_path=Path(self.tmp.name) / "hub.db",
|
||||
backup_dir=Path(self.tmp.name) / "backups",
|
||||
scheduler_enabled=False,
|
||||
quality={"daily_row_ratio": 0.5, "null_rate_max": 0.5, "list_limit_default": 5000, "list_limit_max": 5000},
|
||||
)
|
||||
adapter = TushareAdapter("tushare-secret-token-xyz", transport=fake_transport)
|
||||
self.hub = Hub(settings, adapter=adapter)
|
||||
self.hub.pipeline.ingest_reference(TRADE_DATE)
|
||||
for dataset in ("daily", "valuation", "moneyflow", "auction", "index_daily"):
|
||||
self.hub.pipeline.run_dataset(dataset, TRADE_DATE)
|
||||
|
||||
def tearDown(self) -> None:
|
||||
self.hub.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_published_daily_is_tushare_native(self) -> None:
|
||||
payload = self.hub.api.query_api(
|
||||
{"api_name": "daily", "params": {"trade_date": TRADE_DATE}, "fields": "ts_code,close,vol,amount"}
|
||||
)
|
||||
rows = payload["data"]
|
||||
by_code = {row["ts_code"]: row for row in rows}
|
||||
self.assertEqual(by_code["600000.SH"]["vol"], 1000.0)
|
||||
self.assertEqual(by_code["600000.SH"]["amount"], 2000.0)
|
||||
self.assertEqual(payload["meta"]["row_shape"], "tushare")
|
||||
|
||||
def test_live_stk_limit_uses_internal_tushare(self) -> None:
|
||||
payload = self.hub.api.query_api(
|
||||
{"api_name": "stk_limit", "params": {"trade_date": TRADE_DATE}, "fields": "ts_code,up_limit,down_limit"}
|
||||
)
|
||||
self.assertEqual(payload["meta"]["source"], "tushare")
|
||||
self.assertEqual(payload["data"][0]["ts_code"], "600000.SH")
|
||||
|
||||
def test_rt_sw_k_is_blocked(self) -> None:
|
||||
from datahub.serving import ApiError
|
||||
|
||||
with self.assertRaises(ApiError):
|
||||
self.hub.api.query_api({"api_name": "rt_sw_k", "params": {"ts_code": "801074.SI"}})
|
||||
|
||||
def test_rt_k_uses_free_quotes_not_tushare(self) -> None:
|
||||
quotes = [
|
||||
{
|
||||
"ts_code": "600000.SH",
|
||||
"name": "浦发银行",
|
||||
"close": 10.2,
|
||||
"pre_close": 10.0,
|
||||
"open": 10.1,
|
||||
"high": 10.3,
|
||||
"low": 9.9,
|
||||
"vol": 1000,
|
||||
"amount": 2000000,
|
||||
}
|
||||
]
|
||||
with patch("datahub.steward.fetch_quotes", return_value={"data": quotes, "meta": {"source": "eastmoney:ulist", "stale": False}}):
|
||||
payload = self.hub.api.query_api({"api_name": "rt_k", "params": {"ts_code": "600000.SH"}})
|
||||
self.assertEqual(payload["data"][0]["close"], 10.2)
|
||||
self.assertEqual(payload["meta"]["source"], "eastmoney:ulist")
|
||||
|
||||
def test_shenwan_quote_uses_eastmoney_90_prefix(self) -> None:
|
||||
from datahub.adapters.eastmoney import EastmoneyAdapter
|
||||
|
||||
with patch.object(EastmoneyAdapter, "_get_json") as get_json:
|
||||
get_json.return_value = {
|
||||
"data": {
|
||||
"diff": [
|
||||
{
|
||||
"f12": "801074",
|
||||
"f14": "工业金属",
|
||||
"f2": 1234.5,
|
||||
"f3": 2.88,
|
||||
"f18": 1200,
|
||||
"f17": 1205,
|
||||
"f15": 1240,
|
||||
"f16": 1198,
|
||||
"f6": 1,
|
||||
"f124": 1757319000,
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
quote = EastmoneyAdapter().fetch_shenwan_quote("801074.SI")
|
||||
self.assertEqual(quote["source"], "eastmoney_sw")
|
||||
self.assertAlmostEqual(quote["change"], 2.88)
|
||||
self.assertEqual(get_json.call_args.args[1]["secids"], "90.801074")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user