Compare commits

...
Author SHA1 Message Date
a043bc9eb1 fix(HEL-485): 盘中选择当天不再整页退回昨天
交易时段缺少盘后正式数据时继续展示当天盘中行情,只有开盘前、周末和历史日期才沿用最近收盘结果。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-08 10:16:52 +08:00
acde4de40d fix(HEL-484): 中枢分时接口空 date 按当天查询
缺少或为空的 date 不再 400,按当天处理;显式历史日期保持原行为。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-08 10:07:44 +08:00
3d2c1252f1 fix(HEL-482): 开盘前分时回退最近交易日,并接通中枢失败回旧通道
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-08 09:44:00 +08:00
605f97e5df feat(HEL-463): 接入剩余行情数据到 datahub
扩展盘后正式集(涨跌停/人气/龙虎榜/板块日线)与盘中观察 API(报价/指数/分时),网站 bridge 按开关接入并回退旧链路;问天改为按数据依赖跟随开关,不再整栈强制旧路径。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 17:30:58 +08:00
16ba83ec01 fix(HEL-461): 切换事务失败写入 release-group 审计日志
整组切换中断时除回滚与废弃批次外,同步记录
action=release-group 的失败审计,便于后台追踪。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 16:02:11 +08:00
1c740a9d48 fix(HEL-461): 后台整组切换异常统一为 FAILED_PRECONDITION
管理后台补数在切换事务中断时不再抛出原始异常,
统一映射为 ApiError FAILED_PRECONDITION,并保留旧完整版本。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 15:56:28 +08:00
75c2e33b68 fix(HEL-461): CLI/后台强制重发改为整组边界切换
eod-refresh --force 与管理后台补数不再单数据集发布,
统一走 force_republish_boundary,避免绕过 A/B 完整边界。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 11:26:27 +08:00
32f565ecb9 fix(HEL-461): 整批发布按完整边界重暂存,主档与快照同事务
边界内任有缺失则整组重暂存后统一切换,避免旧新批次混发;
refresh_stocks 失败时主档保持旧值,并补齐回归测试。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
2026-09-05 08:52:50 +08:00
40 changed files with 2673 additions and 201 deletions
+3
View File
@@ -7,6 +7,9 @@ TUSHARE_TOKEN=your_tushare_token_here
# Optional xiaobai-datahub client. All DATAHUB_READ_* / DATAHUB_SHADOW_* flags # Optional xiaobai-datahub client. All DATAHUB_READ_* / DATAHUB_SHADOW_* flags
# default off in config/datahub.config.json, so the website keeps using Tushare. # default off in config/datahub.config.json, so the website keeps using Tushare.
# Extended datasets (HEL-463): LIMIT_EVENTS POPULARITY DRAGON_TIGER SECTOR_DAILY
# QUOTES INDEX_QUOTES INTRADAY — plus first-batch CALENDAR STOCKS DAILY INDEX_DAILY
# VALUATION MONEYFLOW AUCTION STATUS.
DATAHUB_BASE_URL=http://127.0.0.1:8766 DATAHUB_BASE_URL=http://127.0.0.1:8766
DATAHUB_TOKEN= DATAHUB_TOKEN=
+78 -6
View File
@@ -21,11 +21,34 @@ from backend.data.providers.tushare_client import TushareClient
LOGGER = logging.getLogger("xiaobai.datahub") LOGGER = logging.getLogger("xiaobai.datahub")
ShadowSink = Callable[[dict[str, Any]], None] ShadowSink = Callable[[dict[str, Any]], None]
EMPTY_FAIL_DATASETS = {"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction"}
def _usable_intraday_points(rows: list[Any]) -> list[dict[str, Any]]:
points: list[dict[str, Any]] = []
for row in rows:
if not isinstance(row, dict):
continue
try:
close = float(row.get("close") or 0)
except (TypeError, ValueError):
close = 0.0
if close <= 0:
continue
point = dict(row)
if "average" not in point and point.get("avg_price") is not None:
point["average"] = point.get("avg_price")
points.append(point)
return points
EMPTY_FAIL_DATASETS = {
"stocks", "daily", "index_daily", "valuation", "moneyflow", "auction",
"limit_events", "sector_daily",
}
def looks_like_heaven(module_name: str, filename: str = "") -> bool: def looks_like_heaven(module_name: str, filename: str = "") -> bool:
"""问天调用栈识别。问天未永久冻结,只是本阶段仍走旧 Tushare 链路。""" """问天调用栈识别(诊断用)。问天按数据集依赖接入,不再整栈强制旧链路。"""
path = filename.replace("\\", "/") path = filename.replace("\\", "/")
return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path return module_name.startswith("backend.features.heaven") or "/features/heaven/" in path
@@ -88,6 +111,34 @@ class DatahubBridge:
self._log_failure("status", exc) self._log_failure("status", exc)
return None return None
def try_intraday(self, code: str) -> dict[str, Any] | None:
flags = self.settings.flags("intraday")
if not flags.read:
return None
try:
response = self.client.intraday_points(code=code)
data = response.data
if not isinstance(data, dict):
raise DatahubError("EMPTY", "datahub intraday payload invalid")
points = _usable_intraday_points(data.get("points") or [])
if not points:
raise DatahubError("EMPTY", "datahub intraday empty")
if (response.meta or {}).get("stale"):
raise DatahubError("STALE", "datahub intraday stale")
return {
"entity_type": str(data.get("entity_type") or "stock"),
"identifier": str(data.get("identifier") or code),
"name": str(data.get("name") or ""),
"code": str(data.get("code") or code),
"trade_date": str(data.get("trade_date") or points[-1].get("date") or ""),
"previous_close": float(data.get("previous_close") or 0),
"points": points,
"source": "datahub",
}
except Exception as exc:
self._log_failure("intraday", exc)
return None
def query( def query(
self, self,
api_name: str, api_name: str,
@@ -96,8 +147,8 @@ class DatahubBridge:
legacy_query: Callable[..., list[dict[str, Any]]], legacy_query: Callable[..., list[dict[str, Any]]],
) -> list[dict[str, Any]]: ) -> list[dict[str, Any]]:
dataset = API_TO_DATASET.get(api_name) dataset = API_TO_DATASET.get(api_name)
# 问天允许后续纳入 datahub;首批只读接入仍保持旧链路,避免误切 # 问天按实际数据依赖接入:已映射到 hub 的 API 跟随开关;未映射的继续旧链路
if not dataset or self.heaven_guard(): if not dataset:
return legacy_query(api_name, params, fields) return legacy_query(api_name, params, fields)
flags = self.settings.flags(dataset) flags = self.settings.flags(dataset)
if not flags.read and not flags.shadow: if not flags.read and not flags.shadow:
@@ -108,7 +159,7 @@ class DatahubBridge:
hub_error: str | None = None hub_error: str | None = None
hub_canonical: list[dict[str, Any]] = [] hub_canonical: list[dict[str, Any]] = []
try: try:
response = self._fetch_dataset(dataset, params or {}) response = self._fetch_dataset(dataset, params or {}, api_name=api_name)
hub_canonical = self._extract_rows(dataset, response, params or {}) hub_canonical = self._extract_rows(dataset, response, params or {})
hub_rows = to_native_rows(dataset, hub_canonical) hub_rows = to_native_rows(dataset, hub_canonical)
hub_meta = dict(response.meta) hub_meta = dict(response.meta)
@@ -136,7 +187,7 @@ class DatahubBridge:
return project_fields(hub_rows, fields) return project_fields(hub_rows, fields)
return legacy_query(api_name, params, fields) return legacy_query(api_name, params, fields)
def _fetch_dataset(self, dataset: str, params: dict[str, Any]) -> DatahubResponse: def _fetch_dataset(self, dataset: str, params: dict[str, Any], api_name: str = "") -> DatahubResponse:
date = yyyymmdd(params.get("trade_date") or params.get("date")) date = yyyymmdd(params.get("trade_date") or params.get("date"))
start = yyyymmdd(params.get("start_date") or params.get("from") or date) start = yyyymmdd(params.get("start_date") or params.get("from") or date)
end = yyyymmdd(params.get("end_date") or params.get("to") or date) end = yyyymmdd(params.get("end_date") or params.get("to") or date)
@@ -153,6 +204,10 @@ class DatahubBridge:
"valuation": self.client.valuation, "valuation": self.client.valuation,
"moneyflow": self.client.moneyflow, "moneyflow": self.client.moneyflow,
"auction": self.client.auction, "auction": self.client.auction,
"limit_events": self.client.limit_events,
"popularity": self.client.popularity,
"dragon_tiger": self.client.dragon_tiger,
"sector_daily": self.client.sectors,
} }
fetcher = fetchers[dataset] fetcher = fetchers[dataset]
query: dict[str, Any] = {} query: dict[str, Any] = {}
@@ -167,6 +222,23 @@ class DatahubBridge:
query["to"] = end query["to"] = end
if dataset == "daily": if dataset == "daily":
query["adjust"] = "none" query["adjust"] = "none"
if dataset == "limit_events":
limit_type = str(params.get("limit_type") or "").strip().upper()
if limit_type:
query["limit_type"] = limit_type
if dataset == "popularity":
if api_name == "ths_hot":
query["source"] = "ths"
elif api_name == "dc_hot":
query["source"] = "dc"
if dataset == "sector_daily":
family = {
"ths_daily": "ths",
"dc_index": "dc",
"sw_daily": "sw",
}.get(api_name, "")
if family:
query["family"] = family
return self._paginate(fetcher, query) return self._paginate(fetcher, query)
def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse: def _paginate(self, fetcher: Callable[..., DatahubResponse], params: dict[str, Any]) -> DatahubResponse:
+21
View File
@@ -60,6 +60,27 @@ class DatahubClient:
def auction(self, **params: Any) -> DatahubResponse: def auction(self, **params: Any) -> DatahubResponse:
return self.get("/v1/auction", params) return self.get("/v1/auction", params)
def limit_events(self, **params: Any) -> DatahubResponse:
return self.get("/v1/limit-events", params)
def popularity(self, **params: Any) -> DatahubResponse:
return self.get("/v1/popularity", params)
def dragon_tiger(self, **params: Any) -> DatahubResponse:
return self.get("/v1/dragon-tiger", params)
def sectors(self, **params: Any) -> DatahubResponse:
return self.get("/v1/sectors", params)
def quotes_latest(self, **params: Any) -> DatahubResponse:
return self.get("/v1/quotes/latest", params)
def index_quotes(self, **params: Any) -> DatahubResponse:
return self.get("/v1/indexes/quotes", params)
def intraday_points(self, **params: Any) -> DatahubResponse:
return self.get("/v1/intraday/points", params)
def dataset_status(self, date: str) -> DatahubResponse: def dataset_status(self, date: str) -> DatahubResponse:
return self.get("/v1/datasets/status", {"date": date}) return self.get("/v1/datasets/status", {"date": date})
+51
View File
@@ -17,6 +17,13 @@ API_TO_DATASET = {
"index_daily": "index_daily", "index_daily": "index_daily",
"moneyflow": "moneyflow", "moneyflow": "moneyflow",
"stk_auction": "auction", "stk_auction": "auction",
"limit_list_d": "limit_events",
"ths_hot": "popularity",
"dc_hot": "popularity",
"hm_detail": "dragon_tiger",
"ths_daily": "sector_daily",
"dc_index": "sector_daily",
"sw_daily": "sector_daily",
} }
SCALE_FIELDS = { SCALE_FIELDS = {
@@ -35,6 +42,16 @@ SCALE_FIELDS = {
"net_mf_amount": AMOUNT_WAN_YUAN, "net_mf_amount": AMOUNT_WAN_YUAN,
}, },
"auction": {"vol": VOLUME_LOT, "float_share": AMOUNT_WAN_YUAN}, "auction": {"vol": VOLUME_LOT, "float_share": AMOUNT_WAN_YUAN},
"limit_events": {
"limit_amount": AMOUNT_WAN_YUAN,
"float_mv": AMOUNT_WAN_YUAN,
"total_mv": AMOUNT_WAN_YUAN,
},
"dragon_tiger": {
"buy_amount": AMOUNT_WAN_YUAN,
"sell_amount": AMOUNT_WAN_YUAN,
"net_amount": AMOUNT_WAN_YUAN,
},
} }
@@ -67,6 +84,16 @@ def to_native_row(dataset: str, row: dict[str, Any]) -> dict[str, Any]:
converted[field] = _unscale(converted.get(field), factor) converted[field] = _unscale(converted.get(field), factor)
if dataset == "stocks": if dataset == "stocks":
converted.pop("updated_at", None) converted.pop("updated_at", None)
if dataset == "popularity":
# keep hub source; callers filter ths/dc themselves when needed
if converted.get("ts_name") and not converted.get("name"):
converted["name"] = converted.get("ts_name")
if dataset == "dragon_tiger":
if converted.get("ts_name") and not converted.get("name"):
converted["name"] = converted.get("ts_name")
if dataset == "sector_daily":
if converted.get("pct_change") is not None and converted.get("pct_chg") is None:
converted["pct_chg"] = converted.get("pct_change")
return converted return converted
@@ -96,6 +123,30 @@ def row_key(dataset: str, row: dict[str, Any]) -> tuple[str, ...]:
return (str(row.get("ts_code") or "").upper(),) return (str(row.get("ts_code") or "").upper(),)
if dataset == "status": if dataset == "status":
return (str(row.get("dataset") or ""), yyyymmdd(row.get("trade_date"))) return (str(row.get("dataset") or ""), yyyymmdd(row.get("trade_date")))
if dataset == "limit_events":
return (
str(row.get("ts_code") or "").upper(),
yyyymmdd(row.get("trade_date")),
str(row.get("limit_type") or ""),
)
if dataset == "popularity":
return (
str(row.get("ts_code") or "").upper(),
yyyymmdd(row.get("trade_date")),
str(row.get("source") or ""),
)
if dataset == "dragon_tiger":
return (
str(row.get("ts_code") or "").upper(),
yyyymmdd(row.get("trade_date")),
str(row.get("hm_name") or ""),
)
if dataset == "sector_daily":
return (
str(row.get("ts_code") or "").upper(),
yyyymmdd(row.get("trade_date")),
str(row.get("family") or ""),
)
return (str(row.get("ts_code") or "").upper(), yyyymmdd(row.get("trade_date"))) return (str(row.get("ts_code") or "").upper(), yyyymmdd(row.get("trade_date")))
+14
View File
@@ -17,6 +17,13 @@ DATASETS = (
"valuation", "valuation",
"moneyflow", "moneyflow",
"auction", "auction",
"limit_events",
"popularity",
"dragon_tiger",
"sector_daily",
"quotes",
"index_quotes",
"intraday",
"status", "status",
) )
@@ -28,6 +35,13 @@ ENV_DATASET = {
"valuation": "VALUATION", "valuation": "VALUATION",
"moneyflow": "MONEYFLOW", "moneyflow": "MONEYFLOW",
"auction": "AUCTION", "auction": "AUCTION",
"limit_events": "LIMIT_EVENTS",
"popularity": "POPULARITY",
"dragon_tiger": "DRAGON_TIGER",
"sector_daily": "SECTOR_DAILY",
"quotes": "QUOTES",
"index_quotes": "INDEX_QUOTES",
"intraday": "INTRADAY",
"status": "STATUS", "status": "STATUS",
} }
+3 -2
View File
@@ -85,12 +85,13 @@ def build_data_gateway(
policy = DataSourcePolicy.load() policy = DataSourcePolicy.load()
settings = datahub_settings or DatahubSettings.load(credentials=credentials) settings = datahub_settings or DatahubSettings.load(credentials=credentials)
datahub_client = DatahubClient(settings) datahub_client = DatahubClient(settings)
datahub = DatahubBridge(settings, datahub_client)
return DataGateway( return DataGateway(
policy=policy, policy=policy,
quality=DataQualityGate.load(policy), quality=DataQualityGate.load(policy),
tushare_provider=TushareProvider(token_supplier), tushare_provider=TushareProvider(token_supplier),
ifind_provider=IfindProvider(ifind), ifind_provider=IfindProvider(ifind),
chart_data=MarketChartClient(ifind, EastmoneyChartClient()), chart_data=MarketChartClient(ifind, EastmoneyChartClient(), datahub),
realtime_observer=WebRealtimeAggregator(), realtime_observer=WebRealtimeAggregator(),
datahub=DatahubBridge(settings, datahub_client), datahub=datahub,
) )
+10 -2
View File
@@ -3,7 +3,11 @@ from __future__ import annotations
from typing import Any from typing import Any
from backend.data.numbers import finite_number as _number from backend.data.numbers import finite_number as _number
from backend.data.providers.tushare_helpers import _display_time, _prices_equal from backend.data.providers.tushare_helpers import (
_display_time,
_prices_equal,
calendar_is_open,
)
class DailyMarketMixin: class DailyMarketMixin:
@@ -17,7 +21,11 @@ class DailyMarketMixin:
trade_date = requested trade_date = requested
else: else:
row = requested_rows[0] row = requested_rows[0]
trade_date = row["cal_date"] if row.get("is_open") == 1 else row.get("pretrade_date", requested) trade_date = (
row["cal_date"]
if calendar_is_open(row.get("is_open"))
else row.get("pretrade_date", requested)
)
resolved_rows = self.query( resolved_rows = self.query(
"trade_cal", "trade_cal",
+15 -9
View File
@@ -16,6 +16,12 @@ from backend.data.providers.tushare_transport import TushareError
class DashboardMixin: class DashboardMixin:
def _now(self) -> datetime:
clock = getattr(self, "clock", None)
if callable(clock):
return clock()
return datetime.now().astimezone()
def dashboard(self, requested_date: str) -> dict[str, Any]: def dashboard(self, requested_date: str) -> dict[str, Any]:
trade_date, previous_trade_date = self.resolve_trade_context(requested_date) trade_date, previous_trade_date = self.resolve_trade_context(requested_date)
if self.should_use_realtime(requested_date, trade_date): if self.should_use_realtime(requested_date, trade_date):
@@ -26,11 +32,12 @@ class DashboardMixin:
) )
daily = self._load_daily(trade_date) daily = self._load_daily(trade_date)
now = self._now()
if ( if (
not daily not daily
and requested_date == datetime.now().astimezone().strftime("%Y%m%d") and requested_date == now.strftime("%Y%m%d")
and trade_date == requested_date and trade_date == requested_date
and datetime.now().astimezone().time().replace(tzinfo=None) >= dt_time(9, 15) and now.time().replace(tzinfo=None) >= dt_time(9, 15)
): ):
return self._realtime_dashboard( return self._realtime_dashboard(
requested_date, requested_date,
@@ -98,15 +105,14 @@ class DashboardMixin:
} }
return apply_sentiment_to_dashboard(dashboard) return apply_sentiment_to_dashboard(dashboard)
@staticmethod def should_use_realtime(self, requested_date: str, trade_date: str) -> bool:
def should_use_realtime(requested_date: str, trade_date: str) -> bool: """Use live quotes for today's open session until official daily settles."""
"""Use rt_k for today's open market until end-of-day datasets settle.""" now = self._now()
now = datetime.now().astimezone()
today = now.strftime("%Y%m%d") today = now.strftime("%Y%m%d")
return ( return (
requested_date == today requested_date == today
and trade_date == today and trade_date == today
and dt_time(9, 15) <= now.time().replace(tzinfo=None) < dt_time(16, 30) and dt_time(9, 15) <= now.time().replace(tzinfo=None) < dt_time(15, 5)
) )
def _realtime_dashboard( def _realtime_dashboard(
@@ -178,7 +184,7 @@ class DashboardMixin:
) )
sectors = _build_sectors(limits) sectors = _build_sectors(limits)
previous_sectors = _build_sectors(previous_limits) previous_sectors = _build_sectors(previous_limits)
now = datetime.now().astimezone() now = self._now()
market_status = _realtime_market_status(now.time().replace(tzinfo=None)) market_status = _realtime_market_status(now.time().replace(tzinfo=None))
dashboard = { dashboard = {
"meta": { "meta": {
@@ -234,7 +240,7 @@ class DashboardMixin:
{"trade_date": previous_trade_date}, {"trade_date": previous_trade_date},
"ts_code,trade_date,total_share,float_share,free_share,total_mv,circ_mv", "ts_code,trade_date,total_share,float_share,free_share,total_mv,circ_mv",
) )
if not basic_rows or not price_limits: if not basic_rows:
raise TushareError(f"Realtime reference data is incomplete for {trade_date}") raise TushareError(f"Realtime reference data is incomplete for {trade_date}")
result = { result = {
"basic_rows": basic_rows, "basic_rows": basic_rows,
+11
View File
@@ -6,6 +6,17 @@ from typing import Any
from backend.data.numbers import finite_number as _number from backend.data.numbers import finite_number as _number
def calendar_is_open(value: Any) -> bool:
if value in (True, 1, "1", "Y", "y"):
return True
if value in (False, 0, "0", "N", "n", None, ""):
return False
try:
return int(value) == 1
except (TypeError, ValueError):
return False
def _text(value: Any) -> str: def _text(value: Any) -> str:
if isinstance(value, (list, tuple, set)): if isinstance(value, (list, tuple, set)):
return "".join(str(item).strip() for item in value if str(item).strip()) return "".join(str(item).strip() for item in value if str(item).strip())
+63 -15
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import http.client import http.client
import json import json
import logging
import re import re
import time import time
import urllib.error import urllib.error
@@ -15,12 +16,15 @@ from typing import Any, ClassVar
from backend.bootstrap.config import tushare_code as _stock_market_code from backend.bootstrap.config import tushare_code as _stock_market_code
from backend.data.providers.ifind_client import IfindError, IfindHttpClient from backend.data.providers.ifind_client import IfindError, IfindHttpClient
LOGGER = logging.getLogger("xiaobai.charts")
class ChartDataError(RuntimeError): class ChartDataError(RuntimeError):
pass pass
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get" TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
HIS_TRENDS_URL = "https://push2his.eastmoney.com/api/qt/stock/trends2/get"
BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get" BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get"
BROWSER_USER_AGENT = ( BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
@@ -37,14 +41,23 @@ INDEX_SECIDS = {
class MarketChartClient: class MarketChartClient:
"""Prefer iFinD for display charts and retain Eastmoney as a last resort.""" """Prefer iFinD for display charts and retain Eastmoney as a last resort."""
def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None: def __init__(
self,
ifind: IfindHttpClient,
fallback: "EastmoneyChartClient",
datahub: Any = None,
) -> None:
self.ifind = ifind self.ifind = ifind
self.fallback = fallback self.fallback = fallback
self.datahub = datahub
def stock_intraday(self, code: str) -> dict[str, Any]: def stock_intraday(self, code: str) -> dict[str, Any]:
normalized = str(code or "").strip() normalized = str(code or "").strip()
if not re.fullmatch(r"\d{6}", normalized): if not re.fullmatch(r"\d{6}", normalized):
raise ChartDataError("Invalid stock code") raise ChartDataError("Invalid stock code")
hub_chart = self._datahub_intraday(normalized)
if hub_chart is not None:
return hub_chart
ifind_code = _stock_market_code(normalized) ifind_code = _stock_market_code(normalized)
try: try:
return self._ifind_intraday(ifind_code, "stock", normalized) return self._ifind_intraday(ifind_code, "stock", normalized)
@@ -73,11 +86,29 @@ class MarketChartClient:
normalized = str(identifier or "").strip().upper() normalized = str(identifier or "").strip().upper()
if normalized not in INDEX_SECIDS: if normalized not in INDEX_SECIDS:
raise ChartDataError("Unsupported index") raise ChartDataError("Unsupported index")
hub_chart = self._datahub_intraday(normalized)
if hub_chart is not None:
return hub_chart
try: try:
return self._ifind_intraday(normalized, "index", normalized) return self._ifind_intraday(normalized, "index", normalized)
except (IfindError, ChartDataError): except (IfindError, ChartDataError):
return self.fallback.index_intraday(normalized) return self.fallback.index_intraday(normalized)
def _datahub_intraday(self, code: str) -> dict[str, Any] | None:
if self.datahub is None:
return None
try:
chart = self.datahub.try_intraday(code)
except Exception as exc:
LOGGER.warning("datahub intraday unexpected error: %s", exc)
return None
if not chart:
return None
points = list(chart.get("points") or [])
if not points:
return None
return chart
def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]: def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]:
normalized = str(identifier or "").strip().upper() normalized = str(identifier or "").strip().upper()
try: try:
@@ -305,21 +336,29 @@ class EastmoneyChartClient:
if cached is not None: if cached is not None:
return cached return cached
payload = self._request_json( params = {
TRENDS_URL, "secid": secid,
{ "fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
"secid": secid, "fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13", "iscr": "0",
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58", }
"iscr": "0", last_error: Exception | None = None
"ndays": "1", data: dict[str, Any] = {}
}, points: list[dict[str, Any]] = []
"https://quote.eastmoney.com/", for url, ndays in ((TRENDS_URL, "1"), (TRENDS_URL, "5"), (HIS_TRENDS_URL, "5")):
) request_params = {**params, "ndays": ndays}
data = payload.get("data") or {} try:
points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))] payload = self._request_json(url, request_params, "https://quote.eastmoney.com/")
except ChartDataError as exc:
last_error = exc
continue
data = payload.get("data") or {}
parsed = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
points = _latest_session(parsed)
if points:
break
if not points: if not points:
raise ChartDataError("No intraday chart data returned") raise ChartDataError("No intraday chart data returned") from last_error
result = { result = {
"entity_type": entity_type, "entity_type": entity_type,
@@ -433,6 +472,15 @@ class EastmoneyChartClient:
raise ChartDataError("Intraday chart request failed") from last_error raise ChartDataError("Intraday chart request failed") from last_error
def _latest_session(points: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not points:
return []
latest = max(str(point.get("date") or "") for point in points)
if not latest:
return points
return [point for point in points if str(point.get("date") or "") == latest]
def _parse_trend(raw: Any) -> dict[str, Any] | None: def _parse_trend(raw: Any) -> dict[str, Any] | None:
fields = str(raw or "").split(",") fields = str(raw or "").split(",")
if len(fields) < 8 or " " not in fields[0]: if len(fields) < 8 or " " not in fields[0]:
+64 -9
View File
@@ -65,9 +65,31 @@ class MarketServiceMixin:
# Compatibility for isolated legacy unit-test service stubs. # Compatibility for isolated legacy unit-test service stubs.
return TushareClient(self.token) return TushareClient(self.token)
def _now(self) -> datetime:
clock = getattr(self, "clock", None)
if callable(clock):
return clock()
return datetime.now().astimezone()
def _is_requested_open_session(self, requested_date: str) -> bool:
now = self._now()
if requested_date != now.strftime("%Y%m%d"):
return False
if now.time().replace(tzinfo=None) < dt_time(9, 15):
return False
client = self._tushare_client() if self.configured else None
resolve = getattr(client, "resolve_trade_context", None) if client else None
if resolve is None:
return now.weekday() < 5
try:
trade_date, _ = resolve(requested_date)
except Exception:
return now.weekday() < 5
return str(trade_date or "") == requested_date
def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]: def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]:
normalized_date = normalize_date(trade_date) normalized_date = normalize_date(trade_date)
now = datetime.now().astimezone() now = self._now()
if ( if (
normalized_date == now.strftime("%Y%m%d") normalized_date == now.strftime("%Y%m%d")
and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time() and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time()
@@ -174,14 +196,14 @@ class MarketServiceMixin:
def _should_retry_incomplete_snapshot( def _should_retry_incomplete_snapshot(
self, snapshot: dict[str, Any], requested_date: str self, snapshot: dict[str, Any], requested_date: str
) -> bool: ) -> bool:
if requested_date != date.today().strftime("%Y%m%d"): if requested_date != self._now().strftime("%Y%m%d"):
return False return False
meta = snapshot.get("meta") or {} meta = snapshot.get("meta") or {}
incomplete = ( actual = str(meta.get("trade_date") or "").replace("-", "")
meta.get("limit_data_source") == "derived" stale_carry = bool(meta.get("carried_forward") or actual != requested_date)
or bool(meta.get("carried_forward")) if stale_carry and self._is_requested_open_session(requested_date):
or str(meta.get("trade_date") or "").replace("-", "") != requested_date return True
) incomplete = meta.get("limit_data_source") == "derived" or stale_carry
return incomplete and self._snapshot_age_seconds(meta) >= 60 return incomplete and self._snapshot_age_seconds(meta) >= 60
def _annotate_data_status(self, dashboard: dict[str, Any]) -> dict[str, Any]: def _annotate_data_status(self, dashboard: dict[str, Any]) -> dict[str, Any]:
@@ -199,6 +221,9 @@ class MarketServiceMixin:
else: else:
meta["data_status"] = "preparing" meta["data_status"] = "preparing"
meta["display_notice"] = self._preparing_display_notice(actual, requested) meta["display_notice"] = self._preparing_display_notice(actual, requested)
elif meta.get("realtime"):
meta["data_status"] = "intraday"
meta.setdefault("display_notice", "")
else: else:
meta["data_status"] = "official" meta["data_status"] = "official"
meta.setdefault("display_notice", "") meta.setdefault("display_notice", "")
@@ -225,9 +250,9 @@ class MarketServiceMixin:
normalized_date: str, normalized_date: str,
snapshot: dict[str, Any], snapshot: dict[str, Any],
) -> bool: ) -> bool:
if not self.configured or normalized_date != date.today().strftime("%Y%m%d"): if not self.configured or normalized_date != self._now().strftime("%Y%m%d"):
return False return False
now = datetime.now().astimezone() now = self._now()
local_time = now.time().replace(tzinfo=None) local_time = now.time().replace(tzinfo=None)
realtime_start = datetime.strptime("09:15", "%H:%M").time() realtime_start = datetime.strptime("09:15", "%H:%M").time()
morning_end = datetime.strptime("11:35", "%H:%M").time() morning_end = datetime.strptime("11:35", "%H:%M").time()
@@ -276,6 +301,12 @@ class MarketServiceMixin:
actual_date = normalize_date( actual_date = normalize_date(
str(dashboard.get("meta", {}).get("trade_date") or normalized_date) str(dashboard.get("meta", {}).get("trade_date") or normalized_date)
) )
if actual_date != normalized_date and self._is_requested_open_session(
normalized_date
):
raise TushareError(
f"Intraday dashboard resolved {actual_date} instead of {normalized_date}"
)
self.database.save_snapshot(actual_date, source, dashboard) self.database.save_snapshot(actual_date, source, dashboard)
if actual_date != normalized_date: if actual_date != normalized_date:
dashboard.setdefault("meta", {}).update( dashboard.setdefault("meta", {}).update(
@@ -297,6 +328,30 @@ class MarketServiceMixin:
) )
return self._apply_reason_overrides(self._with_storage(dashboard, cached=False)) return self._apply_reason_overrides(self._with_storage(dashboard, cached=False))
except TushareError as exc: except TushareError as exc:
if self._is_requested_open_session(normalized_date):
existing = self.database.get_snapshot(normalized_date)
existing_date = str(
((existing or {}).get("meta") or {}).get("trade_date") or ""
).replace("-", "")
if existing and existing_date == normalized_date:
kept = copy.deepcopy(existing)
kept.setdefault("meta", {}).update(
{
"requested_date": self._display_compact_date(normalized_date),
}
)
self.database.finish_sync(
sync_id,
"fallback",
self._record_count(kept),
str(exc),
"tushare",
)
return self._apply_reason_overrides(
self._with_storage(kept, cached=True)
)
self.database.finish_sync(sync_id, "failed", message=str(exc))
raise ValueError("当天盘中行情暂时不可用,请稍后重试。") from exc
fallback = self.database.get_latest_real_snapshot(normalized_date) fallback = self.database.get_latest_real_snapshot(normalized_date)
if fallback: if fallback:
actual = str((fallback.get("meta") or {}).get("trade_date") or "") actual = str((fallback.get("meta") or {}).get("trade_date") or "")
+2
View File
@@ -41,6 +41,8 @@ def official_catchup_due(today: str, snapshot: dict[str, object]) -> bool:
actual == today actual == today
and meta.get("limit_data_source") != "derived" and meta.get("limit_data_source") != "derived"
and not meta.get("carried_forward") and not meta.get("carried_forward")
and not meta.get("realtime")
and meta.get("mode") != "realtime"
): ):
return False return False
return True return True
+19 -19
View File
@@ -508,8 +508,8 @@
}, },
{ {
"path": "backend/data/providers/tushare_dashboard.py", "path": "backend/data/providers/tushare_dashboard.py",
"bytes": 28234, "bytes": 28327,
"lines": 648 "lines": 654
}, },
{ {
"path": "backend/data/providers/tushare_industries.py", "path": "backend/data/providers/tushare_industries.py",
@@ -551,6 +551,11 @@
"bytes": 15311, "bytes": 15311,
"lines": 387 "lines": 387
}, },
{
"path": "frontend/shared/dashboard.js",
"bytes": 15063,
"lines": 321
},
{ {
"path": "frontend/pages/pools/page.html", "path": "frontend/pages/pools/page.html",
"bytes": 14942, "bytes": 14942,
@@ -561,11 +566,6 @@
"bytes": 14743, "bytes": 14743,
"lines": 342 "lines": 342
}, },
{
"path": "frontend/shared/dashboard.js",
"bytes": 14740,
"lines": 316
},
{ {
"path": "frontend/shared/admin.js", "path": "frontend/shared/admin.js",
"bytes": 14410, "bytes": 14410,
@@ -638,8 +638,8 @@
}, },
{ {
"path": "backend/data/providers/tushare_daily.py", "path": "backend/data/providers/tushare_daily.py",
"bytes": 6837, "bytes": 6949,
"lines": 160 "lines": 168
}, },
{ {
"path": "backend/application.py", "path": "backend/application.py",
@@ -786,6 +786,11 @@
"bytes": 2514, "bytes": 2514,
"lines": 63 "lines": 63
}, },
{
"path": "backend/data/providers/tushare_helpers.py",
"bytes": 2360,
"lines": 75
},
{ {
"path": "backend/jobs/service.py", "path": "backend/jobs/service.py",
"bytes": 2337, "bytes": 2337,
@@ -811,11 +816,6 @@
"bytes": 2165, "bytes": 2165,
"lines": 35 "lines": 35
}, },
{
"path": "backend/data/providers/tushare_helpers.py",
"bytes": 2083,
"lines": 64
},
{ {
"path": "frontend/pages/market/breadth.js", "path": "frontend/pages/market/breadth.js",
"bytes": 2071, "bytes": 2071,
@@ -827,13 +827,13 @@
"lines": 45 "lines": 45
}, },
{ {
"path": "backend/features/system/routes.py", "path": "backend/jobs/refresh.py",
"bytes": 1791, "bytes": 1808,
"lines": 46 "lines": 48
}, },
{ {
"path": "backend/jobs/refresh.py", "path": "backend/features/system/routes.py",
"bytes": 1728, "bytes": 1791,
"lines": 46 "lines": 46
}, },
{ {
+7
View File
@@ -13,6 +13,13 @@
"valuation": { "read": false, "shadow": false }, "valuation": { "read": false, "shadow": false },
"moneyflow": { "read": false, "shadow": false }, "moneyflow": { "read": false, "shadow": false },
"auction": { "read": false, "shadow": false }, "auction": { "read": false, "shadow": false },
"limit_events": { "read": false, "shadow": false },
"popularity": { "read": false, "shadow": false },
"dragon_tiger": { "read": false, "shadow": false },
"sector_daily": { "read": false, "shadow": false },
"quotes": { "read": false, "shadow": false },
"index_quotes": { "read": false, "shadow": false },
"intraday": { "read": false, "shadow": false },
"status": { "read": false, "shadow": false } "status": { "read": false, "shadow": false }
} }
} }
+5
View File
@@ -67,6 +67,11 @@ async function startAdminRefresh() {
const actualCompact = actualDate.replaceAll("-", ""); const actualCompact = actualDate.replaceAll("-", "");
const updated = formatTimestamp(meta.updated_at); const updated = formatTimestamp(meta.updated_at);
const freshness = dashboardFreshnessMessage(meta); const freshness = dashboardFreshnessMessage(meta);
if (meta.realtime && actualCompact === requestedCompact && !meta.carried_forward) {
setAdminRefreshStatus("success", `刷新成功:已获取 ${actualDate} 的盘中行情,更新时间 ${updated}`, "circle-check");
showToast(`刷新成功:已获取 ${actualDate} 的盘中行情`);
return;
}
if (freshness || actualCompact !== requestedCompact || meta.carried_forward || meta.limit_data_source === "derived") { if (freshness || actualCompact !== requestedCompact || meta.carried_forward || meta.limit_data_source === "derived") {
setAdminRefreshStatus("warning", freshness || `部分正式数据尚未到齐,当前展示 ${actualDate || "最近可用数据"}`, "triangle-alert"); setAdminRefreshStatus("warning", freshness || `部分正式数据尚未到齐,当前展示 ${actualDate || "最近可用数据"}`, "triangle-alert");
setStatus(freshness || "部分正式数据尚未到齐,当前展示最近可用数据"); setStatus(freshness || "部分正式数据尚未到齐,当前展示最近可用数据");
+196 -15
View File
@@ -3,7 +3,8 @@ from __future__ import annotations
import copy import copy
import threading import threading
import unittest import unittest
from datetime import date, datetime, timedelta, timezone from datetime import date, datetime, timedelta, timezone, time as dt_time
from unittest.mock import patch
from pathlib import Path from pathlib import Path
from backend.features.market.service import MarketServiceMixin from backend.features.market.service import MarketServiceMixin
@@ -105,18 +106,58 @@ class FakeDerivedClient:
} }
SHANGHAI = timezone(timedelta(hours=8))
TRADE_DAY = date(2026, 9, 8)
def at_clock(hour: int, minute: int, day: date = TRADE_DAY) -> datetime:
return datetime(day.year, day.month, day.day, hour, minute, tzinfo=SHANGHAI)
class FakeMissingDailyClient: class FakeMissingDailyClient:
def __init__(self, open_today: bool = True):
self.open_today = open_today
def dashboard(self, trade_date: str): def dashboard(self, trade_date: str):
raise TushareError(f"No daily data returned for {trade_date}") raise TushareError(f"No daily data returned for {trade_date}")
def resolve_trade_context(self, requested: str):
if self.open_today:
return requested, "20260907"
return "20260907", "20260904"
class FakeRealtimeTodayClient:
def dashboard(self, trade_date: str):
return {
"meta": {
"trade_date": f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}",
"requested_date": f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:8]}",
"realtime": True,
"mode": "realtime",
"market_status": "trading",
"notice": "盘中行情由 Tushare rt_k 实时计算;涨停原因、封板时间和开板次数以盘后榜单校正为准。",
"updated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
},
"overview": {"limit_up_count": 15},
"limits": [{"code": "000001"}],
"broken": [],
"down_limits": [],
"yesterday_limits": [],
}
def resolve_trade_context(self, requested: str):
return requested, "20260907"
class SyncHarness(MarketServiceMixin): class SyncHarness(MarketServiceMixin):
def __init__(self, client, latest=None): def __init__(self, client, latest=None, clock=None):
self.configured = True self.configured = True
self.sync_lock = threading.Lock() self.sync_lock = threading.Lock()
self.database = FakeSyncDatabase(latest) self.database = FakeSyncDatabase(latest)
self._client = client self._client = client
self.current_user_id = 1 self.current_user_id = 1
self.clock = clock
def _tushare_client(self): def _tushare_client(self):
return self._client return self._client
@@ -142,23 +183,139 @@ class DashboardFreshnessTests(unittest.TestCase):
self.assertEqual(harness.database.finished[0][0][1], "success") self.assertEqual(harness.database.finished[0][0][1], "success")
self.assertEqual(verified_dashboard_result(payload), payload) self.assertEqual(verified_dashboard_result(payload), payload)
def test_missing_official_data_keeps_previous_day_with_preparing_notice(self): def test_intraday_refresh_keeps_today_and_does_not_fall_back_to_yesterday(self):
today = date.today() today = TRADE_DAY.strftime("%Y%m%d")
previous = (today - timedelta(days=1)).strftime("%Y-%m-%d")
latest = { latest = {
"meta": {"trade_date": previous, "source": "tushare"}, "meta": {"trade_date": "2026-09-07", "source": "tushare"},
"overview": {"limit_up_count": 20}, "overview": {"limit_up_count": 20},
} }
harness = SyncHarness(FakeMissingDailyClient(), latest) harness = SyncHarness(
payload = harness.sync_dashboard(today.strftime("%Y%m%d")) FakeRealtimeTodayClient(),
latest,
clock=lambda: at_clock(10, 5),
)
payload = harness.sync_dashboard(today)
meta = payload["meta"] meta = payload["meta"]
self.assertTrue(meta["carried_forward"]) self.assertFalse(meta.get("carried_forward"))
self.assertEqual(meta["data_status"], "preparing") self.assertTrue(meta["realtime"])
self.assertIn("今日数据正在准备,当前展示", meta["display_notice"]) self.assertEqual(meta["data_status"], "intraday")
self.assertIn("", meta["display_notice"]) self.assertEqual(str(meta["trade_date"]).replace("-", ""), today)
self.assertNotIn("No daily data", meta["display_notice"]) self.assertNotIn("今日数据正在准备", meta.get("display_notice") or "")
self.assertNotEqual(verified_dashboard_result(payload).get("status"), "failed") self.assertEqual(harness.database.saved[0][0], today)
def test_intraday_missing_quotes_do_not_carry_yesterday(self):
today = TRADE_DAY.strftime("%Y%m%d")
latest = {
"meta": {"trade_date": "2026-09-07", "source": "tushare"},
"overview": {"limit_up_count": 20},
}
harness = SyncHarness(
FakeMissingDailyClient(),
latest,
clock=lambda: at_clock(10, 5),
)
with self.assertRaises(ValueError) as ctx:
harness.sync_dashboard(today)
self.assertIn("当天盘中行情", str(ctx.exception))
self.assertFalse(harness.database.saved)
def test_intraday_keeps_existing_today_snapshot_when_refresh_fails(self):
today = TRADE_DAY.strftime("%Y%m%d")
existing = {
"meta": {
"trade_date": "2026-09-08",
"realtime": True,
"mode": "realtime",
"source": "tushare",
},
"overview": {"limit_up_count": 11},
"limits": [{"code": "600000"}],
"broken": [],
"down_limits": [],
"yesterday_limits": [],
}
harness = SyncHarness(
FakeMissingDailyClient(),
clock=lambda: at_clock(10, 5),
)
harness.database.get_snapshot = lambda *_args, **_kwargs: copy.deepcopy(existing)
payload = harness.sync_dashboard(today)
meta = payload["meta"]
self.assertEqual(str(meta["trade_date"]).replace("-", ""), today)
self.assertTrue(meta["realtime"])
self.assertEqual(meta["data_status"], "intraday")
self.assertFalse(meta.get("carried_forward"))
def test_lunch_and_after_hours_keep_today_until_official_arrives(self):
today = TRADE_DAY.strftime("%Y%m%d")
for clock in (lambda: at_clock(12, 0), lambda: at_clock(16, 10)):
harness = SyncHarness(
FakeRealtimeTodayClient(),
clock=clock,
)
payload = harness.sync_dashboard(today)
self.assertEqual(str(payload["meta"]["trade_date"]).replace("-", ""), today)
self.assertFalse(payload["meta"].get("carried_forward"))
def test_preopen_and_weekend_still_carry_last_session(self):
latest = {
"meta": {"trade_date": "2026-09-07", "source": "tushare"},
"overview": {"limit_up_count": 20},
}
preopen = SyncHarness(
FakeMissingDailyClient(),
latest,
clock=lambda: at_clock(8, 30),
)
preopen_payload = preopen.sync_dashboard(TRADE_DAY.strftime("%Y%m%d"))
self.assertTrue(preopen_payload["meta"]["carried_forward"])
self.assertEqual(preopen_payload["meta"]["data_status"], "preparing")
self.assertIn("今日数据正在准备,当前展示", preopen_payload["meta"]["display_notice"])
weekend = SyncHarness(
FakeMissingDailyClient(open_today=False),
latest,
clock=lambda: at_clock(10, 5, date(2026, 9, 5)),
)
weekend_payload = weekend.sync_dashboard("20260905")
self.assertTrue(weekend_payload["meta"]["carried_forward"])
def test_history_date_still_uses_official_or_preparing_notice(self):
latest = {
"meta": {"trade_date": "2026-09-01", "source": "tushare"},
"overview": {"limit_up_count": 8},
}
harness = SyncHarness(
FakeMissingDailyClient(),
latest,
clock=lambda: at_clock(10, 5),
)
payload = harness.sync_dashboard("20260902")
self.assertTrue(payload["meta"]["carried_forward"])
self.assertIn("所选日期数据尚未到齐", payload["meta"]["display_notice"])
def test_carried_today_snapshot_is_retried_immediately_in_session(self):
today = TRADE_DAY.strftime("%Y%m%d")
snapshot = {
"meta": {
"source": "tushare",
"trade_date": "2026-09-07",
"carried_forward": True,
"requested_date": "2026-09-08",
"updated_at": at_clock(10, 0).isoformat(),
},
"overview": {"limit_up_count": 1},
}
harness = SyncHarness(
FakeRealtimeTodayClient(),
clock=lambda: at_clock(10, 5),
)
harness.database.get_snapshot = lambda *_args, **_kwargs: copy.deepcopy(snapshot)
payload = harness.get_dashboard(today)
self.assertEqual(str(payload["meta"]["trade_date"]).replace("-", ""), today)
self.assertEqual(payload["meta"]["data_status"], "intraday")
self.assertTrue(harness.database.saved)
def test_weekend_carry_is_not_labeled_as_preparing(self): def test_weekend_carry_is_not_labeled_as_preparing(self):
snapshot = { snapshot = {
@@ -200,19 +357,43 @@ class DashboardFreshnessTests(unittest.TestCase):
{"meta": {"trade_date": iso, "limit_data_source": "derived"}}, {"meta": {"trade_date": iso, "limit_data_source": "derived"}},
) )
now = datetime.now().astimezone().time().replace(tzinfo=None) now = datetime.now().astimezone().time().replace(tzinfo=None)
if datetime.strptime("15:05", "%H:%M").time() <= now < datetime.strptime("22:00", "%H:%M").time(): if dt_time(15, 5) <= now < dt_time(22, 0):
self.assertFalse(due) self.assertFalse(due)
self.assertTrue(derived_due) self.assertTrue(derived_due)
else: else:
self.assertFalse(due) self.assertFalse(due)
self.assertFalse(derived_due) self.assertFalse(derived_due)
def test_official_catchup_is_due_for_intraday_snapshot_after_close(self):
today = TRADE_DAY.strftime("%Y%m%d")
snapshot = {
"meta": {
"trade_date": "2026-09-08",
"realtime": True,
"mode": "realtime",
}
}
with patch("backend.jobs.refresh.datetime") as mocked:
mocked.now.return_value = at_clock(16, 10)
mocked.strptime = datetime.strptime
self.assertTrue(official_catchup_due(today, snapshot))
official = {
"meta": {
"trade_date": "2026-09-08",
"limit_data_source": "official",
"realtime": False,
}
}
self.assertFalse(official_catchup_due(today, official))
class FrontendRefreshCopyTests(unittest.TestCase): class FrontendRefreshCopyTests(unittest.TestCase):
def test_dashboard_script_distinguishes_partial_from_failure(self): def test_dashboard_script_distinguishes_partial_from_failure(self):
script = (Path(__file__).resolve().parents[1] / "frontend" / "shared" / "dashboard.js").read_text(encoding="utf-8") script = (Path(__file__).resolve().parents[1] / "frontend" / "shared" / "dashboard.js").read_text(encoding="utf-8")
self.assertIn("今日数据正在准备,当前展示", script) self.assertIn("今日数据正在准备,当前展示", script)
self.assertIn("部分正式数据尚未到齐", script) self.assertIn("部分正式数据尚未到齐", script)
self.assertIn("盘中行情", script)
self.assertIn("meta.realtime && actualCompact === requestedCompact", script)
self.assertIn('job.status === "failed"', script) self.assertIn('job.status === "failed"', script)
failed_block = script.split("if (job.status === \"failed\")", 1)[1].split("const query", 1)[0] failed_block = script.split("if (job.status === \"failed\")", 1)[1].split("const query", 1)[0]
self.assertIn("后台刷新失败", failed_block) self.assertIn("后台刷新失败", failed_block)
+137 -1
View File
@@ -2,7 +2,8 @@ from __future__ import annotations
import unittest import unittest
from backend.features.market.charts import ChartDataError, EastmoneyChartClient from backend.data.providers.ifind_client import IfindHttpClient
from backend.features.market.charts import ChartDataError, EastmoneyChartClient, HIS_TRENDS_URL, MarketChartClient, TRENDS_URL
from server import DashboardService from server import DashboardService
@@ -72,6 +73,141 @@ class ChartDataProviderTests(unittest.TestCase):
self.client.stock_intraday("abc") self.client.stock_intraday("abc")
class LookbackChartClient(EastmoneyChartClient):
def __init__(self) -> None:
super().__init__(cache_ttl_seconds=20)
self.requests: list[tuple[str, dict[str, str]]] = []
def _request_json(self, url, params, referer):
self.requests.append((url, params))
if url == TRENDS_URL and params.get("ndays") == "1":
return {"data": {"code": "601318", "name": "中国平安", "preClose": 56.0, "trends": []}}
if url == TRENDS_URL and params.get("ndays") == "5":
return {"data": {"code": "601318", "name": "中国平安", "preClose": 56.0, "trends": []}}
if url == HIS_TRENDS_URL:
return {
"data": {
"code": "601318",
"name": "中国平安",
"preClose": 55.8,
"trends": [
"2026-09-07 09:30,55.80,55.90,56.00,55.70,100,5580.00,55.900",
"2026-09-07 15:00,56.10,56.20,56.30,56.00,200,11240.00,56.150",
"2026-09-08 09:30,0,0,0,0,0,0.00,0",
],
}
}
raise ChartDataError("unexpected url")
class ChartLookbackTests(unittest.TestCase):
def setUp(self) -> None:
EastmoneyChartClient._cache.clear()
self.client = LookbackChartClient()
def test_empty_today_falls_back_to_latest_available_session(self):
payload = self.client.stock_intraday("601318")
urls = [url for url, _ in self.client.requests]
self.assertEqual(urls[0], TRENDS_URL)
self.assertEqual(self.client.requests[0][1]["ndays"], "1")
self.assertEqual(urls[1], TRENDS_URL)
self.assertEqual(self.client.requests[1][1]["ndays"], "5")
self.assertEqual(urls[2], HIS_TRENDS_URL)
self.assertEqual(payload["trade_date"], "2026-09-07")
self.assertEqual([point["time"] for point in payload["points"]], ["09:30", "15:00"])
self.assertEqual(payload["points"][0]["close"], 55.9)
def test_delay_multiday_can_recover_without_his(self):
class DelayFive(EastmoneyChartClient):
def __init__(self):
super().__init__(cache_ttl_seconds=20)
self.requests = []
def _request_json(self, url, params, referer):
self.requests.append((url, params))
if params.get("ndays") == "1":
return {"data": {"code": "000001", "name": "平安银行", "preClose": 11.7, "trends": []}}
return {
"data": {
"code": "000001",
"name": "平安银行",
"preClose": 11.5,
"trends": [
"2026-09-07 09:30,11.50,11.60,11.70,11.40,100,1160.00,11.600",
"2026-09-07 15:00,11.70,11.80,11.90,11.60,200,2360.00,11.750",
],
}
}
EastmoneyChartClient._cache.clear()
client = DelayFive()
payload = client.stock_intraday("000001")
self.assertEqual(payload["trade_date"], "2026-09-07")
self.assertEqual(len(payload["points"]), 2)
self.assertEqual([url for url, _ in client.requests], [TRENDS_URL, TRENDS_URL])
def test_sh_sz_cyb_codes_use_correct_secid(self):
for code, secid in (("601318", "1.601318"), ("000001", "0.000001"), ("300750", "0.300750")):
EastmoneyChartClient._cache.clear()
client = LookbackChartClient()
client.stock_intraday(code)
self.assertEqual(client.requests[0][1]["secid"], secid)
class FakeHub:
def __init__(self, chart=None, error=None):
self.chart = chart
self.error = error
self.calls: list[str] = []
def try_intraday(self, code):
self.calls.append(code)
if self.error:
raise self.error
return self.chart
class DatahubChartFallbackTests(unittest.TestCase):
def setUp(self) -> None:
EastmoneyChartClient._cache.clear()
def test_datahub_success_skips_old_channel(self):
hub = FakeHub(
{
"entity_type": "stock",
"identifier": "601318",
"name": "中国平安",
"code": "601318",
"trade_date": "2026-09-08",
"previous_close": 56.36,
"points": [{"date": "2026-09-08", "time": "09:30", "close": 56.5, "average": 56.4}],
"source": "datahub",
}
)
fallback = LookbackChartClient()
client = MarketChartClient(IfindHttpClient(), fallback, hub)
payload = client.stock_intraday("601318")
self.assertEqual(payload["source"], "datahub")
self.assertEqual(hub.calls, ["601318"])
self.assertEqual(fallback.requests, [])
def test_datahub_timeout_or_empty_falls_back_to_eastmoney(self):
fallback = LookbackChartClient()
for hub in (
FakeHub(chart=None),
FakeHub(error=RuntimeError("timeout")),
FakeHub(error=RuntimeError("datahub exploded")),
FakeHub(chart={"points": []}),
):
EastmoneyChartClient._cache.clear()
fallback.requests.clear()
client = MarketChartClient(IfindHttpClient(), fallback, hub)
payload = client.stock_intraday("000001")
self.assertEqual(payload["trade_date"], "2026-09-07")
self.assertGreaterEqual(len(payload["points"]), 1)
self.assertTrue(fallback.requests)
class ChartServiceStub: class ChartServiceStub:
@staticmethod @staticmethod
def _payload(code: str, name: str): def _payload(code: str, name: str):
+69 -3
View File
@@ -64,9 +64,11 @@ class FakeClient(DatahubClient):
meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0}, meta={"tier": "official", "trade_date": "20240902", "stale": False, "staleness_seconds": 0},
) )
self.paths: list[str] = [] self.paths: list[str] = []
self.calls: list[tuple[str, dict[str, Any]]] = []
def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse: def get(self, path: str, params: dict[str, Any] | None = None) -> DatahubResponse:
self.paths.append(path) self.paths.append(path)
self.calls.append((path, {key: value for key, value in (params or {}).items()}))
if TOKEN in json.dumps(params or {}) or TOKEN in path: if TOKEN in json.dumps(params or {}) or TOKEN in path:
raise AssertionError("token leaked into url") raise AssertionError("token leaked into url")
if self.error: if self.error:
@@ -290,8 +292,8 @@ class DatahubBridgeTests(unittest.TestCase):
self.assertEqual(canonical["vol"], 100000.0) self.assertEqual(canonical["vol"], 100000.0)
self.assertEqual(canonical["amount"], 2000000.0) self.assertEqual(canonical["amount"], 2000000.0)
def test_heaven_keeps_legacy_on_first_batch_even_when_read_flag_is_on(self) -> None: def test_heaven_can_use_hub_when_dataset_flag_is_on(self) -> None:
"""问天未永久冻结;首批只读接入仍走旧链路,后续迁移可以纳入""" """问天按数据依赖接入:已映射 API 跟随开关,不再整栈强制旧链路"""
self.assertTrue(looks_like_heaven("backend.features.heaven.market_context", "backend/features/heaven/market_context.py")) self.assertTrue(looks_like_heaven("backend.features.heaven.market_context", "backend/features/heaven/market_context.py"))
self.assertFalse(looks_like_heaven("backend.features.market.service", "backend/features/market/service.py")) self.assertFalse(looks_like_heaven("backend.features.market.service", "backend/features/market/service.py"))
client = FakeClient() client = FakeClient()
@@ -302,7 +304,8 @@ class DatahubBridgeTests(unittest.TestCase):
) )
rows = wrapped.query("daily", {"trade_date": "20240902"}, "amount") rows = wrapped.query("daily", {"trade_date": "20240902"}, "amount")
self.assertEqual(rows[0]["amount"], 2000.0) self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(client.paths, []) self.assertEqual(client.paths, ["/v1/bars/daily"])
self.assertEqual(legacy.calls, [])
def test_status_flag_does_not_run_when_off_and_falls_back_when_on(self) -> None: def test_status_flag_does_not_run_when_off_and_falls_back_when_on(self) -> None:
off = DatahubBridge(flags(), FakeClient(error=DatahubError("UNAVAILABLE", "down"))) off = DatahubBridge(flags(), FakeClient(error=DatahubError("UNAVAILABLE", "down")))
@@ -348,6 +351,69 @@ class DatahubBridgeTests(unittest.TestCase):
self.assertEqual(rows[0]["amount"], 2000.0) self.assertEqual(rows[0]["amount"], 2000.0)
self.assertEqual(len(legacy.calls), 1) self.assertEqual(len(legacy.calls), 1)
def test_try_intraday_respects_switch_and_falls_back_on_bad_payload(self) -> None:
closed = DatahubBridge(flags(), FakeClient(error=DatahubError("INTERNAL", "should not run")))
self.assertIsNone(closed.try_intraday("601318"))
empty = DatahubBridge(
flags(intraday=(True, False)),
FakeClient(response=DatahubResponse(data={"points": []}, meta={"stale": False})),
)
self.assertIsNone(empty.try_intraday("601318"))
stale = DatahubBridge(
flags(intraday=(True, False)),
FakeClient(response=DatahubResponse(
data={
"entity_type": "stock",
"code": "601318",
"trade_date": "2026-09-07",
"previous_close": 55.8,
"points": [{"date": "2026-09-07", "time": "09:30", "close": 55.9, "avg_price": 55.85}],
},
meta={"stale": True},
)),
)
self.assertIsNone(stale.try_intraday("601318"))
ok = DatahubBridge(
flags(intraday=(True, False)),
FakeClient(response=DatahubResponse(
data={
"entity_type": "stock",
"identifier": "601318",
"name": "中国平安",
"code": "601318",
"trade_date": "2026-09-08",
"previous_close": 56.36,
"points": [
{"date": "2026-09-08", "time": "09:30", "close": 0},
{"date": "2026-09-08", "time": "09:31", "close": 56.5, "avg_price": 56.4},
],
},
meta={"stale": False},
)),
)
chart = ok.try_intraday("601318")
self.assertEqual(chart["source"], "datahub")
self.assertEqual(len(chart["points"]), 1)
self.assertEqual(chart["points"][0]["average"], 56.4)
self.assertEqual(ok.client.paths, ["/v1/intraday/points"])
self.assertEqual(ok.client.calls, [("/v1/intraday/points", {"code": "601318"})])
self.assertNotIn("date", ok.client.calls[0][1])
timeout = DatahubBridge(
flags(intraday=(True, False)),
FakeClient(error=DatahubError("TIMEOUT", "datahub request timed out")),
)
self.assertIsNone(timeout.try_intraday("601318"))
broken = DatahubBridge(
flags(intraday=(True, False)),
FakeClient(error=DatahubError("INTERNAL", "datahub exploded")),
)
self.assertIsNone(broken.try_intraday("601318"))
self.assertFalse(DatahubSettings.load(environ={}, credentials={}).flags("intraday").read)
def test_features_do_not_import_datahub_client(self) -> None: def test_features_do_not_import_datahub_client(self) -> None:
violations = [] violations = []
for path in (ROOT / "backend" / "features").rglob("*.py"): for path in (ROOT / "backend" / "features").rglob("*.py"):
+54
View File
@@ -1,8 +1,10 @@
from __future__ import annotations from __future__ import annotations
import unittest import unittest
from datetime import datetime, timedelta, timezone
from backend.data.providers.tushare_client import TushareClient from backend.data.providers.tushare_client import TushareClient
from backend.data.providers.tushare_helpers import calendar_is_open
class FakeRealtimeClient(TushareClient): class FakeRealtimeClient(TushareClient):
@@ -130,6 +132,58 @@ class RealtimeDashboardTests(unittest.TestCase):
self.assertEqual(dashboard["meta"]["limit_data_source"], "derived") self.assertEqual(dashboard["meta"]["limit_data_source"], "derived")
self.assertIn("日线数据推算", dashboard["meta"]["notice"]) self.assertIn("日线数据推算", dashboard["meta"]["notice"])
def test_calendar_open_flag_accepts_string_and_bool(self):
self.assertTrue(calendar_is_open(1))
self.assertTrue(calendar_is_open("1"))
self.assertTrue(calendar_is_open(True))
self.assertFalse(calendar_is_open(0))
self.assertFalse(calendar_is_open("0"))
self.assertFalse(calendar_is_open(False))
original_query = self.client.query
def query(api_name, params=None, fields=""):
if api_name == "trade_cal":
return [
{
"cal_date": params.get("start_date"),
"is_open": "1",
"pretrade_date": "20260907",
}
]
return original_query(api_name, params, fields)
self.client.query = query
trade_date, previous = self.client.resolve_trade_context("20260908")
self.assertEqual(trade_date, "20260908")
self.assertEqual(previous, "20260907")
def test_session_clock_uses_realtime_until_official_window(self):
today = "20260908"
self.client.clock = lambda: datetime(
2026, 9, 8, 10, 5, tzinfo=timezone(timedelta(hours=8))
)
self.assertTrue(self.client.should_use_realtime(today, today))
self.client.clock = lambda: datetime(
2026, 9, 8, 16, 10, tzinfo=timezone(timedelta(hours=8))
)
self.assertFalse(self.client.should_use_realtime(today, today))
def test_realtime_dashboard_survives_missing_limit_table(self):
original_query = self.client.query
def query(api_name, params=None, fields=""):
if api_name == "stk_limit":
return []
return original_query(api_name, params, fields)
self.client.query = query
TushareClient._realtime_reference_cache.clear()
dashboard = self.client._realtime_dashboard("20260720", "20260720", "20260717")
self.assertTrue(dashboard["meta"]["realtime"])
self.assertEqual(dashboard["meta"]["quote_count"], 3)
self.assertEqual(dashboard["overview"]["limit_up_count"], 0)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+8 -5
View File
@@ -6,11 +6,12 @@
## 做什么 ## 做什么
- SQLite WAL `datahub.db`,容器名 `xiaobai-datahub`,端口 `8766` - SQLite WAL `datahub.db`,容器名 `xiaobai-datahub`,端口 `8766`
- Tushare 盘后正式数据:交易日历、股票主档、daily、daily_basic、adj_factor、index_daily、moneyflow、stk_auction - Tushare 盘后正式数据:交易日历、股票主档、daily、daily_basic、adj_factor、index_daily、moneyflow、stk_auction、limit_list_d、ths_hot/dc_hot、hm_detail、ths_daily/dc_index/sw_daily
- 盘中观察(provisional):东财/腾讯指数报价、个股最新价、分时点(`/v1/quotes/latest` `/v1/indexes/quotes` `/v1/intraday/points`);永不写入 eod_* 正式表
- 暂存 → 校验 → 整批原子发布 → 可回滚 - 暂存 → 校验 → 整批原子发布 → 可回滚
- `/v1` 稳定接口(`X-Datahub-Token` - `/v1` 稳定接口(`X-Datahub-Token`
- `/admin/` 最小管理后台(总览 / 数据源 / 调度 / 发布 / 数据集 / 审计) - `/admin/` 最小管理后台(总览 / 数据源 / 调度 / 发布 / 数据集 / 审计)
- 东财/腾讯/同花顺/选股宝/AKShare/iFinD 适配器位预留,本阶段不拉实时源 - 同花顺/选股宝/AKShare/iFinD 适配器位预留;东财/腾讯已接入盘中观察
## 单位口径(相对现站) ## 单位口径(相对现站)
@@ -91,7 +92,7 @@ python -m datahub history-backfill
- 流程:组内全部成员先在暂存表完成拉取、字段质量门、覆盖检查和跨数据集交叉校验(`cross_gates` 配置 ts_code 覆盖重叠率下限),全部达标后才在**一个 SQLite 事务**里复制正式表并翻转全部 `publications` 指针。 - 流程:组内全部成员先在暂存表完成拉取、字段质量门、覆盖检查和跨数据集交叉校验(`cross_gates` 配置 ts_code 覆盖重叠率下限),全部达标后才在**一个 SQLite 事务**里复制正式表并翻转全部 `publications` 指针。
- 任一成员失败(拉取失败、质量门拒绝、交叉校验不过、切换事务中断)→ 整批不切换,对外继续提供上一份完整正式版本,失败原因写入 `batches.error``audit_log``action=release-group`),等待晚间自动重试。 - 任一成员失败(拉取失败、质量门拒绝、交叉校验不过、切换事务中断)→ 整批不切换,对外继续提供上一份完整正式版本,失败原因写入 `batches.error``audit_log``action=release-group`),等待晚间自动重试。
- 读取侧任何时刻只会看到"旧完整版本"或"新完整版本":发布指针在单事务内统一翻转,容器重启/事务中断自动回滚,不暴露字段残缺或跨数据集混合版本。 - 读取侧任何时刻只会看到"旧完整版本"或"新完整版本":发布指针在单事务内统一翻转,容器重启/事务中断自动回滚,不暴露字段残缺或跨数据集混合版本。
- 幂等:已发布数据集自动跳过;重复执行、并发重试不会生成重复批次或覆盖正常版本(调度器另有 EOD 互斥锁)。 - 幂等:仅当一致性边界内全部成员都已发布时才整组跳过;边界内任有缺失则整组重暂存后统一切换,避免旧批次与新批次混在同一次重发中。重复执行、并发重试不会在完整边界已就绪时生成重复批次(调度器另有 EOD 互斥锁)。
## 股票主档每日刷新与发布 ## 股票主档每日刷新与发布
@@ -115,11 +116,13 @@ python -m datahub moneyflow-backfill # --trading-days 60 --end-date --for
```bash ```bash
cd xiaobai-datahub cd xiaobai-datahub
python -m datahub eod-refresh --trade-date 20260904 # 只补缺失数据集 python -m datahub eod-refresh --trade-date 20260904 # 补不完整的 A/B 边界
python -m datahub eod-refresh --trade-date 20260904 --force --dataset valuation python -m datahub eod-refresh --trade-date 20260904 --force --dataset valuation
# 强制重取重发:仍走全部质量门,生成新批次,上一批次保留可回滚 # --force 按一致性边界整组重发:valuation/daily/moneyflow/auction/stocks → A 组;
# index_daily → B 组。不可再单独切换某一个正式数据集。
``` ```
管理后台「补数」对盘后正式数据集同样走 `force_republish_boundary`,不会绕过 A/B 整批边界。
## 备份 ## 备份
+1 -1
View File
@@ -269,7 +269,7 @@ function renderRelease(data) {
async function dangerous(kind, dataset) { async function dangerous(kind, dataset) {
const date = ($("rel-date") && $("rel-date").value) || ""; const date = ($("rel-date") && $("rel-date").value) || "";
const ds = dataset || prompt("数据集(daily / valuation / moneyflow / auction / index_daily / reference", "daily"); const ds = dataset || prompt("数据集(daily/valuation/moneyflow/auction/stocks→A组整批;index_daily→B组;或 reference", "daily");
if (!ds) return; if (!ds) return;
const password = prompt("二次确认:输入管理密码"); const password = prompt("二次确认:输入管理密码");
if (!password) return; if (!password) return;
+4 -4
View File
@@ -1,13 +1,13 @@
from datahub.adapters.akshare import ADAPTER as akshare from datahub.adapters.akshare import ADAPTER as akshare
from datahub.adapters.eastmoney import ADAPTER as eastmoney from datahub.adapters.eastmoney import EastmoneyAdapter
from datahub.adapters.ifind import ADAPTER as ifind from datahub.adapters.ifind import ADAPTER as ifind
from datahub.adapters.tencent import ADAPTER as tencent from datahub.adapters.tencent import TencentAdapter
from datahub.adapters.ths import ADAPTER as ths from datahub.adapters.ths import ADAPTER as ths
from datahub.adapters.xgb import ADAPTER as xgb from datahub.adapters.xgb import ADAPTER as xgb
RESERVED = { RESERVED = {
"eastmoney": eastmoney, "eastmoney": EastmoneyAdapter(),
"tencent": tencent, "tencent": TencentAdapter(),
"ths": ths, "ths": ths,
"xgb": xgb, "xgb": xgb,
"akshare": akshare, "akshare": akshare,
+277 -2
View File
@@ -1,3 +1,278 @@
from datahub.adapters.base import ReservedAdapter from __future__ import annotations
ADAPTER = ReservedAdapter("eastmoney") import json
import time
import urllib.error
import urllib.parse
import urllib.request
from datetime import datetime
from typing import Any
from datahub.adapters.base import AdapterError, MarketAdapter
from datahub.numbers import finite_number, round4
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
EASTMONEY_CLIST_URL = "https://push2.eastmoney.com/api/qt/clist/get"
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
HIS_TRENDS_URL = "https://push2his.eastmoney.com/api/qt/stock/trends2/get"
BROWSER_UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
)
INDEX_SECIDS = {
"000001.SH": "1.000001",
"399001.SZ": "0.399001",
"399006.SZ": "0.399006",
}
class EastmoneyAdapter(MarketAdapter):
name = "eastmoney"
def __init__(self, timeout: int = 8) -> None:
self.timeout = timeout
def probe(self) -> dict[str, Any]:
started = time.perf_counter()
try:
rows = self.fetch_indices()
state = "ok" if len(rows) == 3 else "empty"
except AdapterError as exc:
return {
"provider": self.name,
"configured": True,
"state": "error",
"message": str(exc),
"latency_ms": round((time.perf_counter() - started) * 1000),
}
return {
"provider": self.name,
"configured": True,
"state": state,
"latency_ms": round((time.perf_counter() - started) * 1000),
}
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
if dataset in {"indexes_quotes", "index_quotes"}:
return self.fetch_indices()
if dataset in {"quotes", "quotes_latest"}:
codes = params.get("codes") or []
if isinstance(codes, str):
codes = [item.strip() for item in codes.split(",") if item.strip()]
return self.fetch_quotes(list(codes))
raise AdapterError(f"{self.name} unsupported dataset: {dataset}")
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
return list(rows)
def fetch_indices(self) -> list[dict[str, Any]]:
payload = self._get_json(
EASTMONEY_INDEX_URL,
{
"secids": "1.000001,0.399001,0.399006",
"fltt": "2",
"invt": "2",
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
},
referer="https://quote.eastmoney.com/",
)
rows = list((payload.get("data") or {}).get("diff") or [])
result = []
for row in rows:
code = str(row.get("f12") or "")
if code not in {"000001", "399001", "399006"}:
continue
epoch = int(finite_number(row.get("f124")) or 0)
ts_code = f"{code}.SH" if code.startswith("0") and code == "000001" else f"{code}.SZ"
if code == "000001":
ts_code = "000001.SH"
result.append(
{
"ts_code": ts_code,
"code": code,
"name": row.get("f14") or code,
"price": round4(finite_number(row.get("f2"))),
"pct_chg": round4(finite_number(row.get("f3"))),
"change_amount": round4(finite_number(row.get("f4"))),
"open": round4(finite_number(row.get("f17"))),
"high": round4(finite_number(row.get("f15"))),
"low": round4(finite_number(row.get("f16"))),
"previous_close": round4(finite_number(row.get("f18"))),
"amount": round4(finite_number(row.get("f6"))),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch
else ""
),
"source": "eastmoney_push2",
}
)
if len(result) != 3:
raise AdapterError(f"Eastmoney returned {len(result)}/3 indices")
return result
def fetch_quotes(self, codes: list[str]) -> list[dict[str, Any]]:
# Eastmoney clist does not accept arbitrary code lists well; use ulist.np for batches.
secids = []
for code in codes:
ts = str(code or "").upper()
symbol = ts.split(".")[0]
if ts.endswith(".SH") or symbol.startswith(("5", "6", "9")):
secids.append(f"1.{symbol}")
else:
secids.append(f"0.{symbol}")
if not secids:
return []
payload = self._get_json(
EASTMONEY_INDEX_URL,
{
"secids": ",".join(secids[:60]),
"fltt": "2",
"invt": "2",
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f5,f6,f8,f124",
},
referer="https://quote.eastmoney.com/",
)
rows = list((payload.get("data") or {}).get("diff") or [])
result = []
for row in rows:
symbol = str(row.get("f12") or "")
if not symbol:
continue
ts_code = f"{symbol}.SH" if symbol.startswith(("5", "6", "9")) else f"{symbol}.SZ"
epoch = int(finite_number(row.get("f124")) or 0)
result.append(
{
"ts_code": ts_code,
"name": row.get("f14") or symbol,
"price": round4(finite_number(row.get("f2"))),
"pct_chg": round4(finite_number(row.get("f3"))),
"change_amount": round4(finite_number(row.get("f4"))),
"open": round4(finite_number(row.get("f17"))),
"high": round4(finite_number(row.get("f15"))),
"low": round4(finite_number(row.get("f16"))),
"previous_close": round4(finite_number(row.get("f18"))),
"volume": round4(finite_number(row.get("f5"))),
"amount": round4(finite_number(row.get("f6"))),
"turnover_rate": round4(finite_number(row.get("f8"))),
"quote_time_epoch": epoch,
"quote_time": (
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
if epoch
else ""
),
"source": "eastmoney_push2",
}
)
return result
def fetch_intraday(self, ts_code: str, date: str = "") -> dict[str, Any]:
code = str(ts_code or "").upper()
if code in INDEX_SECIDS:
secid = INDEX_SECIDS[code]
entity = "index"
identifier = code
else:
symbol = code.split(".")[0]
market = "1" if symbol.startswith(("5", "6", "9")) else "0"
secid = f"{market}.{symbol}"
entity = "stock"
identifier = symbol
params = {
"secid": secid,
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
"iscr": "0",
}
data: dict[str, Any] = {}
points: list[dict[str, Any]] = []
last_error: Exception | None = None
for url, ndays in ((TRENDS_URL, "1"), (TRENDS_URL, "5"), (HIS_TRENDS_URL, "5")):
try:
payload = self._get_json(
url,
{**params, "ndays": ndays},
referer="https://quote.eastmoney.com/",
)
except AdapterError as exc:
last_error = exc
continue
data = payload.get("data") or {}
parsed = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
points = _preferred_session(parsed, date)
if points:
break
if not points:
raise AdapterError("No intraday chart data returned") from last_error
return {
"entity_type": entity,
"identifier": identifier,
"ts_code": code if "." in code else f"{identifier}.{'SH' if identifier.startswith(('5','6','9')) else 'SZ'}",
"name": str(data.get("name") or ""),
"code": str(data.get("code") or identifier),
"trade_date": points[-1]["date"],
"previous_close": round4(finite_number(data.get("preClose"))),
"points": points,
"source": "eastmoney_trends2",
}
def _get_json(self, url: str, params: dict[str, str], referer: str) -> dict[str, Any]:
request_url = f"{url}?{urllib.parse.urlencode(params)}"
request = urllib.request.Request(
request_url,
headers={
"Accept": "application/json,text/plain,*/*",
"User-Agent": BROWSER_UA,
"Referer": referer,
},
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
return json.loads(response.read().decode("utf-8"))
except Exception as exc:
raise AdapterError(f"eastmoney request failed: {exc}") from exc
def _preferred_session(points: list[dict[str, Any]], preferred_date: str = "") -> list[dict[str, Any]]:
if not points:
return []
want = ""
digits = str(preferred_date or "").replace("-", "")[:8]
if len(digits) == 8 and digits.isdigit():
want = f"{digits[:4]}-{digits[4:6]}-{digits[6:8]}"
if want:
matched = [point for point in points if str(point.get("date") or "") == want]
if matched:
return matched
latest = max(str(point.get("date") or "") for point in points)
if not latest:
return points
return [point for point in points if str(point.get("date") or "") == latest]
def _parse_trend(raw: Any) -> dict[str, Any] | None:
text = str(raw or "")
parts = text.split(",")
if len(parts) < 8:
return None
stamp = parts[0]
try:
when = datetime.strptime(stamp, "%Y-%m-%d %H:%M")
except ValueError:
return None
close = round4(finite_number(parts[2]))
if close <= 0:
return None
return {
"time": when.strftime("%H:%M"),
"date": when.strftime("%Y-%m-%d"),
"open": round4(finite_number(parts[1])),
"close": close,
"high": round4(finite_number(parts[3])),
"low": round4(finite_number(parts[4])),
"avg_price": round4(finite_number(parts[7] if len(parts) > 7 else parts[2])),
"volume": round4(finite_number(parts[5])),
"amount": round4(finite_number(parts[6])),
}
+98 -2
View File
@@ -1,3 +1,99 @@
from datahub.adapters.base import ReservedAdapter from __future__ import annotations
ADAPTER = ReservedAdapter("tencent") import time
import urllib.error
import urllib.request
from datetime import datetime
from typing import Any
from datahub.adapters.base import AdapterError, MarketAdapter
from datahub.numbers import finite_number, round4
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
BROWSER_UA = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
)
class TencentAdapter(MarketAdapter):
name = "tencent"
def __init__(self, timeout: int = 8) -> None:
self.timeout = timeout
def probe(self) -> dict[str, Any]:
started = time.perf_counter()
try:
rows = self.fetch_indices()
state = "ok" if len(rows) == 3 else "empty"
except AdapterError as exc:
return {
"provider": self.name,
"configured": True,
"state": "error",
"message": str(exc),
"latency_ms": round((time.perf_counter() - started) * 1000),
}
return {
"provider": self.name,
"configured": True,
"state": state,
"latency_ms": round((time.perf_counter() - started) * 1000),
}
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
if dataset in {"indexes_quotes", "index_quotes"}:
return self.fetch_indices()
raise AdapterError(f"{self.name} unsupported dataset: {dataset}")
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
return list(rows)
def fetch_indices(self) -> list[dict[str, Any]]:
request = urllib.request.Request(
TENCENT_INDEX_URL,
headers={"User-Agent": BROWSER_UA, "Referer": "https://gu.qq.com/"},
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=self.timeout) as response:
raw = response.read().decode("gb18030", errors="ignore")
except Exception as exc:
raise AdapterError(f"tencent request failed: {exc}") from exc
result = []
for line in raw.splitlines():
if '="' not in line:
continue
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
if len(fields) < 38:
continue
code = fields[2]
if code not in {"000001", "399001", "399006"}:
continue
try:
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
except ValueError as exc:
raise AdapterError(f"Tencent invalid quote time for {code}") from exc
ts_code = "000001.SH" if code == "000001" else f"{code}.SZ"
result.append(
{
"ts_code": ts_code,
"code": code,
"name": fields[1] or code,
"price": round4(finite_number(fields[3])),
"pct_chg": round4(finite_number(fields[32])),
"change_amount": round4(finite_number(fields[31])),
"open": round4(finite_number(fields[5])),
"high": round4(finite_number(fields[33])),
"low": round4(finite_number(fields[34])),
"previous_close": round4(finite_number(fields[4])),
"amount": round4(finite_number(fields[37]) * 10000),
"quote_time_epoch": int(quote_time.timestamp()),
"quote_time": quote_time.isoformat(timespec="seconds"),
"source": "tencent_qt",
}
)
if len(result) != 3:
raise AdapterError(f"Tencent returned {len(result)}/3 indices")
return result
+108 -11
View File
@@ -11,8 +11,12 @@ from datahub.normalize import (
normalize_auction, normalize_auction,
normalize_calendar, normalize_calendar,
normalize_daily, normalize_daily,
normalize_dragon_tiger,
normalize_index_daily, normalize_index_daily,
normalize_limit_event,
normalize_moneyflow, normalize_moneyflow,
normalize_popularity,
normalize_sector_daily,
normalize_stock, normalize_stock,
normalize_valuation, normalize_valuation,
) )
@@ -31,6 +35,21 @@ TUSHARE_FIELDS = {
"buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount" "buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount"
), ),
"stk_auction": "ts_code,trade_date,vol,price,amount,pre_close,turnover_rate,volume_ratio,float_share", "stk_auction": "ts_code,trade_date,vol,price,amount,pre_close,turnover_rate,volume_ratio,float_share",
"limit_list_d": (
"trade_date,ts_code,industry,name,close,pct_chg,amount,limit_amount,"
"float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time,"
"open_times,up_stat,limit_times,limit_type"
),
"ths_hot": "ts_code,ts_name,hot,rank,pct_change,current_price,concept,data_type,trade_date",
"dc_hot": "ts_code,ts_name,rank,pct_change,current_price,hot,concept,data_type,trade_date",
"hm_detail": "trade_date,ts_code,ts_name,buy_amount,sell_amount,net_amount,hm_name,hm_orgs,tag",
"hm_list": "name,desc,orgs",
"top_list": "trade_date,ts_code,name,pct_change,reason",
"top_inst": "trade_date,ts_code,exalter,buy,buy_rate,sell,sell_rate,net_buy,side,reason",
"ths_index": "ts_code,name,count,exchange,list_date,type",
"ths_daily": "ts_code,trade_date,open,high,low,close,pre_close,pct_change,vol,turnover_rate",
"dc_index": "ts_code,trade_date,name,open,high,low,close,pre_close,pct_change,vol,amount,turnover_rate",
"sw_daily": "ts_code,trade_date,name,open,high,low,close,pct_change,vol,amount",
} }
DATASET_API = { DATASET_API = {
@@ -42,12 +61,15 @@ DATASET_API = {
"index_daily": "index_daily", "index_daily": "index_daily",
"moneyflow": "moneyflow", "moneyflow": "moneyflow",
"auction": "stk_auction", "auction": "stk_auction",
"limit_events": "limit_list_d",
"popularity": "ths_hot",
"dragon_tiger": "hm_detail",
"sector_daily": "ths_daily",
} }
# Website actual index usage: market cards / 90-day charts (SH/SZ/CYB) plus
# screener 沪深300 benchmark (lookback up to 260 trading days).
WEBSITE_INDEX_CODES = ("000001.SH", "399001.SZ", "399006.SZ", "000300.SH") WEBSITE_INDEX_CODES = ("000001.SH", "399001.SZ", "399006.SZ", "000300.SH")
DEFAULT_INDEX_CODES = WEBSITE_INDEX_CODES DEFAULT_INDEX_CODES = WEBSITE_INDEX_CODES
LIMIT_TYPES = ("U", "D", "Z")
class TushareAdapter(MarketAdapter): class TushareAdapter(MarketAdapter):
@@ -85,6 +107,14 @@ class TushareAdapter(MarketAdapter):
} }
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]: def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
if dataset == "limit_events":
return self.fetch_limit_events(str(params.get("trade_date") or ""))
if dataset == "popularity":
return self.fetch_popularity(str(params.get("trade_date") or ""))
if dataset == "dragon_tiger":
return self.fetch_dragon_tiger(str(params.get("trade_date") or ""))
if dataset == "sector_daily":
return self.fetch_sector_daily(str(params.get("trade_date") or ""))
api_name = DATASET_API.get(dataset, dataset) api_name = DATASET_API.get(dataset, dataset)
fields = TUSHARE_FIELDS.get(api_name, "") fields = TUSHARE_FIELDS.get(api_name, "")
query_params = dict(params) query_params = dict(params)
@@ -93,10 +123,67 @@ class TushareAdapter(MarketAdapter):
if api_name == "trade_cal" and "exchange" not in query_params: if api_name == "trade_cal" and "exchange" not in query_params:
query_params["exchange"] = "SSE" query_params["exchange"] = "SSE"
if api_name == "index_daily" and "ts_code" not in query_params: if api_name == "index_daily" and "ts_code" not in query_params:
# Caller typically loops codes; a missing code would pull nothing useful.
query_params.setdefault("ts_code", DEFAULT_INDEX_CODES[0]) query_params.setdefault("ts_code", DEFAULT_INDEX_CODES[0])
return self._query(api_name, query_params, fields) return self._query(api_name, query_params, fields)
def fetch_limit_events(self, trade_date: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for limit_type in LIMIT_TYPES:
part = self._query(
"limit_list_d",
{"trade_date": trade_date, "limit_type": limit_type},
TUSHARE_FIELDS["limit_list_d"],
)
for row in part:
row = dict(row)
row.setdefault("limit_type", limit_type)
rows.append(row)
return rows
def fetch_popularity(self, trade_date: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for api_name, source in (("ths_hot", "ths"), ("dc_hot", "dc")):
for row in self._query(api_name, {"trade_date": trade_date}, TUSHARE_FIELDS[api_name]):
item = dict(row)
item["source"] = source
item.setdefault("trade_date", trade_date)
rows.append(item)
return rows
def fetch_dragon_tiger(self, trade_date: str) -> list[dict[str, Any]]:
details = self._query("hm_detail", {"trade_date": trade_date}, TUSHARE_FIELDS["hm_detail"])
top_rows = self._query("top_list", {"trade_date": trade_date}, TUSHARE_FIELDS["top_list"])
context = {
str(row.get("ts_code") or ""): row
for row in top_rows
if str(row.get("ts_code") or "")
}
rows: list[dict[str, Any]] = []
for row in details:
item = dict(row)
stock = context.get(str(item.get("ts_code") or ""), {})
if item.get("pct_change") is None and stock.get("pct_change") is not None:
item["pct_change"] = stock.get("pct_change")
if not item.get("reason") and stock.get("reason"):
item["reason"] = stock.get("reason")
if not item.get("ts_name") and stock.get("name"):
item["ts_name"] = stock.get("name")
rows.append(item)
return rows
def fetch_sector_daily(self, trade_date: str) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for api_name, family in (("ths_daily", "ths"), ("dc_index", "dc"), ("sw_daily", "sw")):
try:
part = self._query(api_name, {"trade_date": trade_date}, TUSHARE_FIELDS[api_name])
except AdapterError:
part = []
for row in part:
item = dict(row)
item["family"] = family
rows.append(item)
return rows
def fetch_index_daily(self, trade_date: str, codes: tuple[str, ...] = DEFAULT_INDEX_CODES) -> list[dict[str, Any]]: def fetch_index_daily(self, trade_date: str, codes: tuple[str, ...] = DEFAULT_INDEX_CODES) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = [] rows: list[dict[str, Any]] = []
for ts_code in codes: for ts_code in codes:
@@ -104,6 +191,17 @@ class TushareAdapter(MarketAdapter):
return rows return rows
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
if dataset in {"limit_events", "limit_list_d"}:
return [normalize_limit_event(row) for row in rows]
if dataset == "popularity":
return [normalize_popularity(row, source=str(row.get("source") or "")) for row in rows]
if dataset == "dragon_tiger":
return [normalize_dragon_tiger(row) for row in rows]
if dataset == "sector_daily":
return [
normalize_sector_daily(row, family=str(row.get("family") or "ths"))
for row in rows
]
mapping = { mapping = {
"calendar": normalize_calendar, "calendar": normalize_calendar,
"trade_cal": normalize_calendar, "trade_cal": normalize_calendar,
@@ -148,12 +246,11 @@ class TushareAdapter(MarketAdapter):
try: try:
with urllib.request.urlopen(request, timeout=self.timeout) as response: with urllib.request.urlopen(request, timeout=self.timeout) as response:
result = json.loads(response.read().decode("utf-8")) result = json.loads(response.read().decode("utf-8"))
except json.JSONDecodeError: except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
raise AdapterError("Tushare returned invalid json") from None raise AdapterError(f"Tushare 请求失败: {exc}") from exc
except (urllib.error.URLError, TimeoutError) as exc: if result.get("code") not in (0, "0", None):
raise AdapterError(f"Tushare request failed: {exc}") from exc raise AdapterError(str(result.get("msg") or f"Tushare error {result.get('code')}"))
if result.get("code") != 0:
raise AdapterError(result.get("msg") or "Tushare returned an unknown error")
data = result.get("data") or {} data = result.get("data") or {}
columns = data.get("fields") or [] items = data.get("items") or []
return [dict(zip(columns, item)) for item in data.get("items") or []] fields_list = data.get("fields") or (fields.split(",") if fields else [])
return [dict(zip(fields_list, item)) for item in items]
+16 -2
View File
@@ -6,7 +6,7 @@ from typing import Any
from datahub.adapters import RESERVED from datahub.adapters import RESERVED
from datahub.auth import AuthService from datahub.auth import AuthService
from datahub.db import HubDB from datahub.db import HubDB
from datahub.pipeline import Pipeline from datahub.pipeline import OFFICIAL_DATASETS, STOCKS_DATASET, Pipeline
from datahub.scheduler import Scheduler from datahub.scheduler import Scheduler
from datahub.serving import ApiError from datahub.serving import ApiError
from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd
@@ -143,8 +143,22 @@ class AdminAPI:
self._dangerous(password, confirm, f"{dataset}:{day}") self._dangerous(password, confirm, f"{dataset}:{day}")
if dataset == "reference": if dataset == "reference":
result = self.pipeline.ingest_reference(day) result = self.pipeline.ingest_reference(day)
elif dataset in OFFICIAL_DATASETS or dataset == STOCKS_DATASET:
# Manual same-day republish must rebuild the full A/B boundary.
# Gate failures and mid-switch exceptions both surface as
# FAILED_PRECONDITION so the admin API never leaks raw
# transaction errors to the client.
try:
result = self.pipeline.force_republish_boundary(dataset, day)
failures = self.pipeline.eod_failures(result)
if failures:
raise ApiError("FAILED_PRECONDITION", "; ".join(failures))
except ApiError:
raise
except Exception as exc:
raise ApiError("FAILED_PRECONDITION", str(exc)) from exc
else: else:
result = self.pipeline.run_dataset(dataset, day) raise ApiError("INVALID_ARGUMENT", f"unsupported backfill dataset: {dataset}")
self.pipeline.audit(actor, "backfill", f"{dataset}:{day}", json.dumps({"ok": True})) self.pipeline.audit(actor, "backfill", f"{dataset}:{day}", json.dumps({"ok": True}))
return result return result
+15 -14
View File
@@ -7,7 +7,7 @@ import json
import sys import sys
from datahub.hub import build_hub from datahub.hub import build_hub
from datahub.pipeline import OFFICIAL_DATASETS from datahub.pipeline import EOD_A_DATASETS, OFFICIAL_DATASETS, STOCKS_DATASET
from datahub.settings import load_settings from datahub.settings import load_settings
from datahub.timeutil import yyyymmdd from datahub.timeutil import yyyymmdd
@@ -19,15 +19,15 @@ def main(argv: list[str] | None = None) -> int:
history.add_argument("--calendar-start", default=None, help="日历起点,默认配置 calendar_start") history.add_argument("--calendar-start", default=None, help="日历起点,默认配置 calendar_start")
history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量,默认 260") history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量,默认 260")
history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期") history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期")
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据(跳过已发布数据集,仍走质量门禁)") refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据(跳过已完整发布的一致性边界,仍走质量门禁)")
refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天") refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天")
refresh.add_argument( refresh.add_argument(
"--force", action="store_true", "--force", action="store_true",
help=" --dataset 指定的数据集强制重取重发(生成新批次保留上一批次可回滚", help="强制重发 --dataset 所属的完整一致性边界(A 组或 B 组),生成新批次保留上一批次可回滚",
) )
refresh.add_argument( refresh.add_argument(
"--dataset", default=None, "--dataset", default=None,
help="配合 --force 使用:只强制重发该数据集(如 valuation", help="配合 --force:指定边界内任一成员(如 valuation→整组 Aindex_daily→整组 B",
) )
stocks_refresh = sub.add_parser("stocks-refresh", help="刷新股票主档并发布正式快照(幂等:无变化则跳过)") stocks_refresh = sub.add_parser("stocks-refresh", help="刷新股票主档并发布正式快照(幂等:无变化则跳过)")
stocks_refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天") stocks_refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD,默认今天")
@@ -54,26 +54,27 @@ def main(argv: list[str] | None = None) -> int:
if args.command == "eod-refresh": if args.command == "eod-refresh":
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd() day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
if args.force: if args.force:
datasets = tuple(sorted({args.dataset} & OFFICIAL_DATASETS)) if args.dataset else () allowed = set(OFFICIAL_DATASETS) | {STOCKS_DATASET}
if args.dataset and not datasets: if not args.dataset:
parser.error(f"unknown dataset: {args.dataset}")
if not datasets:
parser.error("--force requires --dataset (e.g. --dataset valuation)") parser.error("--force requires --dataset (e.g. --dataset valuation)")
result = {} if args.dataset not in allowed:
for dataset in datasets: parser.error(f"unknown dataset: {args.dataset}")
result[dataset] = hub.pipeline.run_dataset(dataset, day) result = hub.pipeline.force_republish_boundary(args.dataset, day)
boundary = "A" if args.dataset in EOD_A_DATASETS or args.dataset == STOCKS_DATASET else "B"
else: else:
result = hub.pipeline.run_eod_missing(day) result = hub.pipeline.run_eod_missing(day)
boundary = None
hub.pipeline.audit("cli", "eod-refresh", f"eod:{day}", json.dumps( hub.pipeline.audit("cli", "eod-refresh", f"eod:{day}", json.dumps(
{"force": bool(args.force), "dataset": args.dataset, {"force": bool(args.force), "dataset": args.dataset, "boundary": boundary,
**{name: item.get("state") for name, item in result.items() if isinstance(item, dict)}}, **{name: item.get("state") for name, item in result.items() if isinstance(item, dict)}},
ensure_ascii=False, ensure_ascii=False,
)) ))
if args.force: if args.force:
payload = {"trade_date": day, "datasets": result} failures = hub.pipeline.eod_failures(result)
payload = {"trade_date": day, "boundary": boundary, "datasets": result}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str) json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n") sys.stdout.write("\n")
return 0 return 0 if not failures else 1
missing = hub.pipeline.missing_official_datasets(day) missing = hub.pipeline.missing_official_datasets(day)
payload = {"trade_date": day, "datasets": result, "missing_after": missing} payload = {"trade_date": day, "datasets": result, "missing_after": missing}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str) json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
+180
View File
@@ -0,0 +1,180 @@
"""Extended EOD datasets beyond the first-batch A/B release groups.
These publish independently (soft): a failure here must not block daily/valuation
release. Scheduler runs them after the core EOD window.
"""
from __future__ import annotations
from typing import Any
# Independent soft datasets (not part of A/B atomic groups).
EXTENDED_SOFT_DATASETS = {
"limit_events",
"popularity",
"dragon_tiger",
"sector_daily",
}
EXTENDED_SCHEMA = """
CREATE TABLE IF NOT EXISTS eod_limit_events (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, limit_type TEXT NOT NULL,
name TEXT, industry TEXT, close REAL, pct_chg REAL, amount REAL,
limit_amount REAL, float_mv REAL, total_mv REAL, turnover_ratio REAL,
fd_amount REAL, first_time TEXT, last_time TEXT,
open_times INTEGER, up_stat TEXT, limit_times INTEGER,
batch_id TEXT NOT NULL,
PRIMARY KEY (ts_code, trade_date, limit_type, batch_id)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS staging_limit_events (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, limit_type TEXT NOT NULL, batch_id TEXT NOT NULL,
name TEXT, industry TEXT, close REAL, pct_chg REAL, amount REAL,
limit_amount REAL, float_mv REAL, total_mv REAL, turnover_ratio REAL,
fd_amount REAL, first_time TEXT, last_time TEXT,
open_times INTEGER, up_stat TEXT, limit_times INTEGER,
PRIMARY KEY (batch_id, ts_code, trade_date, limit_type)
);
CREATE TABLE IF NOT EXISTS eod_popularity (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, source TEXT NOT NULL,
ts_name TEXT, rank INTEGER, pct_change REAL, current_price REAL,
hot REAL, concept TEXT, data_type TEXT,
batch_id TEXT NOT NULL,
PRIMARY KEY (ts_code, trade_date, source, batch_id)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS staging_popularity (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, source TEXT NOT NULL, batch_id TEXT NOT NULL,
ts_name TEXT, rank INTEGER, pct_change REAL, current_price REAL,
hot REAL, concept TEXT, data_type TEXT,
PRIMARY KEY (batch_id, ts_code, trade_date, source)
);
CREATE TABLE IF NOT EXISTS eod_dragon_tiger (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, hm_name TEXT NOT NULL,
ts_name TEXT, buy_amount REAL, sell_amount REAL, net_amount REAL,
hm_orgs TEXT, tag TEXT, pct_change REAL, reason TEXT,
batch_id TEXT NOT NULL,
PRIMARY KEY (ts_code, trade_date, hm_name, batch_id)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS staging_dragon_tiger (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, hm_name TEXT NOT NULL, batch_id TEXT NOT NULL,
ts_name TEXT, buy_amount REAL, sell_amount REAL, net_amount REAL,
hm_orgs TEXT, tag TEXT, pct_change REAL, reason TEXT,
PRIMARY KEY (batch_id, ts_code, trade_date, hm_name)
);
CREATE TABLE IF NOT EXISTS eod_sector_daily (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, family TEXT NOT NULL,
name TEXT, open REAL, high REAL, low REAL, close REAL, pre_close REAL,
pct_change REAL, vol REAL, turnover_rate REAL, amount REAL,
batch_id TEXT NOT NULL,
PRIMARY KEY (ts_code, trade_date, family, batch_id)
) WITHOUT ROWID;
CREATE TABLE IF NOT EXISTS staging_sector_daily (
ts_code TEXT NOT NULL, trade_date TEXT NOT NULL, family TEXT NOT NULL, batch_id TEXT NOT NULL,
name TEXT, open REAL, high REAL, low REAL, close REAL, pre_close REAL,
pct_change REAL, vol REAL, turnover_rate REAL, amount REAL,
PRIMARY KEY (batch_id, ts_code, trade_date, family)
);
CREATE TABLE IF NOT EXISTS sector_master (
ts_code TEXT PRIMARY KEY,
name TEXT,
family TEXT NOT NULL,
exchange TEXT,
list_date TEXT,
member_count INTEGER,
type TEXT,
updated_at TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_eod_limit_date ON eod_limit_events(trade_date, batch_id);
CREATE INDEX IF NOT EXISTS idx_eod_pop_date ON eod_popularity(trade_date, batch_id);
CREATE INDEX IF NOT EXISTS idx_eod_lhb_date ON eod_dragon_tiger(trade_date, batch_id);
CREATE INDEX IF NOT EXISTS idx_eod_sector_date ON eod_sector_daily(trade_date, family, batch_id);
"""
EXTENDED_DATASET_TABLES = {
"limit_events": ("eod_limit_events", "staging_limit_events"),
"popularity": ("eod_popularity", "staging_popularity"),
"dragon_tiger": ("eod_dragon_tiger", "staging_dragon_tiger"),
"sector_daily": ("eod_sector_daily", "staging_sector_daily"),
}
EXTENDED_STAGING_INSERT: dict[str, tuple[str, Any]] = {
"limit_events": (
"INSERT INTO staging_limit_events("
"ts_code,trade_date,limit_type,batch_id,name,industry,close,pct_chg,amount,"
"limit_amount,float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time,"
"open_times,up_stat,limit_times) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
lambda r, b: (
r["ts_code"], r["trade_date"], r["limit_type"], b,
r.get("name"), r.get("industry"), r.get("close"), r.get("pct_chg"), r.get("amount"),
r.get("limit_amount"), r.get("float_mv"), r.get("total_mv"), r.get("turnover_ratio"),
r.get("fd_amount"), r.get("first_time"), r.get("last_time"),
r.get("open_times"), r.get("up_stat"), r.get("limit_times"),
),
),
"popularity": (
"INSERT INTO staging_popularity("
"ts_code,trade_date,source,batch_id,ts_name,rank,pct_change,current_price,hot,concept,data_type) "
"VALUES (?,?,?,?,?,?,?,?,?,?,?)",
lambda r, b: (
r["ts_code"], r["trade_date"], r["source"], b,
r.get("ts_name"), r.get("rank"), r.get("pct_change"), r.get("current_price"),
r.get("hot"), r.get("concept"), r.get("data_type"),
),
),
"dragon_tiger": (
"INSERT INTO staging_dragon_tiger("
"ts_code,trade_date,hm_name,batch_id,ts_name,buy_amount,sell_amount,net_amount,"
"hm_orgs,tag,pct_change,reason) VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
lambda r, b: (
r["ts_code"], r["trade_date"], r["hm_name"], b,
r.get("ts_name"), r.get("buy_amount"), r.get("sell_amount"), r.get("net_amount"),
r.get("hm_orgs"), r.get("tag"), r.get("pct_change"), r.get("reason"),
),
),
"sector_daily": (
"INSERT INTO staging_sector_daily("
"ts_code,trade_date,family,batch_id,name,open,high,low,close,pre_close,"
"pct_change,vol,turnover_rate,amount) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
lambda r, b: (
r["ts_code"], r["trade_date"], r["family"], b,
r.get("name"), r.get("open"), r.get("high"), r.get("low"), r.get("close"),
r.get("pre_close"), r.get("pct_change"), r.get("vol"), r.get("turnover_rate"),
r.get("amount"),
),
),
}
EXTENDED_EOD_COPY = {
"limit_events": (
"INSERT OR REPLACE INTO eod_limit_events "
"SELECT ts_code,trade_date,limit_type,name,industry,close,pct_chg,amount,"
"limit_amount,float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time,"
"open_times,up_stat,limit_times,batch_id "
"FROM staging_limit_events WHERE batch_id = ?"
),
"popularity": (
"INSERT OR REPLACE INTO eod_popularity "
"SELECT ts_code,trade_date,source,ts_name,rank,pct_change,current_price,hot,concept,data_type,batch_id "
"FROM staging_popularity WHERE batch_id = ?"
),
"dragon_tiger": (
"INSERT OR REPLACE INTO eod_dragon_tiger "
"SELECT ts_code,trade_date,hm_name,ts_name,buy_amount,sell_amount,net_amount,"
"hm_orgs,tag,pct_change,reason,batch_id "
"FROM staging_dragon_tiger WHERE batch_id = ?"
),
"sector_daily": (
"INSERT OR REPLACE INTO eod_sector_daily "
"SELECT ts_code,trade_date,family,name,open,high,low,close,pre_close,"
"pct_change,vol,turnover_rate,amount,batch_id "
"FROM staging_sector_daily WHERE batch_id = ?"
),
}
+5 -1
View File
@@ -7,9 +7,10 @@ from contextlib import contextmanager
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from datahub.datasets_ext import EXTENDED_DATASET_TABLES, EXTENDED_SCHEMA
from datahub.timeutil import isoformat from datahub.timeutil import isoformat
SCHEMA = """ _BASE_SCHEMA = """
CREATE TABLE IF NOT EXISTS schema_migrations ( CREATE TABLE IF NOT EXISTS schema_migrations (
version INTEGER PRIMARY KEY, version INTEGER PRIMARY KEY,
applied_at TEXT NOT NULL applied_at TEXT NOT NULL
@@ -282,6 +283,8 @@ CREATE INDEX IF NOT EXISTS idx_eod_bars_date ON eod_bars(trade_date, batch_id);
CREATE INDEX IF NOT EXISTS idx_calendar_open ON trade_calendar(is_open, cal_date); CREATE INDEX IF NOT EXISTS idx_calendar_open ON trade_calendar(is_open, cal_date);
""" """
SCHEMA = _BASE_SCHEMA + EXTENDED_SCHEMA
DATASET_TABLES = { DATASET_TABLES = {
"daily": ("eod_bars", "staging_bars"), "daily": ("eod_bars", "staging_bars"),
"valuation": ("eod_valuation", "staging_valuation"), "valuation": ("eod_valuation", "staging_valuation"),
@@ -289,6 +292,7 @@ DATASET_TABLES = {
"auction": ("eod_auction", "staging_auction"), "auction": ("eod_auction", "staging_auction"),
"index_daily": ("eod_index_bars", "staging_index_bars"), "index_daily": ("eod_index_bars", "staging_index_bars"),
"stocks": ("eod_stocks", "staging_stocks"), "stocks": ("eod_stocks", "staging_stocks"),
**EXTENDED_DATASET_TABLES,
} }
+94
View File
@@ -156,6 +156,95 @@ def normalize_stock(row: dict[str, Any]) -> dict[str, Any]:
} }
def normalize_limit_event(row: dict[str, Any]) -> dict[str, Any]:
"""limit_list_d. float_mv/total_mv/limit_amount are 万元 → yuan; amount/fd_amount already yuan."""
return {
"ts_code": _code(row.get("ts_code")),
"trade_date": _date(row.get("trade_date")),
"limit_type": str(row.get("limit_type") or "").strip().upper() or "U",
"name": str(row.get("name") or "").strip() or None,
"industry": str(row.get("industry") or "").strip() or None,
"close": round4(finite_number(row.get("close"))),
"pct_chg": round4(finite_number(row.get("pct_chg"))),
"amount": round4(finite_number(row.get("amount"))),
"limit_amount": round4(_scale(row.get("limit_amount"), AMOUNT_WAN_YUAN)),
"float_mv": round4(_scale(row.get("float_mv"), AMOUNT_WAN_YUAN)),
"total_mv": round4(_scale(row.get("total_mv"), AMOUNT_WAN_YUAN)),
"turnover_ratio": round4(finite_number(row.get("turnover_ratio"))),
"fd_amount": round4(finite_number(row.get("fd_amount"))),
"first_time": str(row.get("first_time") or "").strip() or None,
"last_time": str(row.get("last_time") or "").strip() or None,
"open_times": _optional_int(row.get("open_times")),
"up_stat": str(row.get("up_stat") or "").strip() or None,
"limit_times": _optional_int(row.get("limit_times")),
}
def normalize_popularity(row: dict[str, Any], source: str = "") -> dict[str, Any]:
src = str(source or row.get("source") or "").strip().lower() or "ths"
return {
"ts_code": _code(row.get("ts_code")),
"trade_date": _date(row.get("trade_date")),
"source": src,
"ts_name": str(row.get("ts_name") or row.get("name") or "").strip() or None,
"rank": _optional_int(row.get("rank")),
"pct_change": round4(
finite_number(row.get("pct_change") if row.get("pct_change") is not None else row.get("pct_chg"))
),
"current_price": round4(finite_number(row.get("current_price") or row.get("price"))),
"hot": round4(finite_number(row.get("hot"))),
"concept": str(row.get("concept") or "").strip() or None,
"data_type": str(row.get("data_type") or "").strip() or None,
}
def normalize_dragon_tiger(row: dict[str, Any]) -> dict[str, Any]:
"""hm_detail amounts are 万元 → yuan."""
return {
"ts_code": _code(row.get("ts_code")),
"trade_date": _date(row.get("trade_date")),
"hm_name": str(row.get("hm_name") or "未命名游资").strip() or "未命名游资",
"ts_name": str(row.get("ts_name") or row.get("name") or "").strip() or None,
"buy_amount": round4(_scale(row.get("buy_amount"), AMOUNT_WAN_YUAN)),
"sell_amount": round4(_scale(row.get("sell_amount"), AMOUNT_WAN_YUAN)),
"net_amount": round4(_scale(row.get("net_amount"), AMOUNT_WAN_YUAN)),
"hm_orgs": str(row.get("hm_orgs") or "").strip() or None,
"tag": str(row.get("tag") or "").strip() or None,
"pct_change": round4(finite_number(row.get("pct_change"))),
"reason": str(row.get("reason") or "").strip() or None,
}
def normalize_sector_daily(row: dict[str, Any], family: str = "ths") -> dict[str, Any]:
fam = str(family or row.get("family") or "ths").strip().lower()
return {
"ts_code": _code(row.get("ts_code")),
"trade_date": _date(row.get("trade_date")),
"family": fam,
"name": str(row.get("name") or "").strip() or None,
"open": round4(finite_number(row.get("open"))),
"high": round4(finite_number(row.get("high"))),
"low": round4(finite_number(row.get("low"))),
"close": round4(finite_number(row.get("close"))),
"pre_close": round4(finite_number(row.get("pre_close"))),
"pct_change": round4(
finite_number(row.get("pct_change") if row.get("pct_change") is not None else row.get("pct_chg"))
),
"vol": round4(finite_number(row.get("vol"))),
"turnover_rate": round4(finite_number(row.get("turnover_rate"))),
"amount": round4(finite_number(row.get("amount"))),
}
def _optional_int(value: Any) -> int | None:
if value in (None, ""):
return None
try:
return int(float(value))
except (TypeError, ValueError):
return None
def apply_qfq(price: float | None, factor: float | None, latest_factor: float | None) -> float | None: def apply_qfq(price: float | None, factor: float | None, latest_factor: float | None) -> float | None:
if price is None: if price is None:
return None return None
@@ -184,6 +273,11 @@ NORMALIZERS = {
"calendar": normalize_calendar, "calendar": normalize_calendar,
"stock_basic": normalize_stock, "stock_basic": normalize_stock,
"stocks": normalize_stock, "stocks": normalize_stock,
"limit_events": normalize_limit_event,
"limit_list_d": normalize_limit_event,
"popularity": normalize_popularity,
"dragon_tiger": normalize_dragon_tiger,
"sector_daily": normalize_sector_daily,
} }
+217 -54
View File
@@ -9,6 +9,11 @@ from typing import Any
from datahub.adapters.base import AdapterError from datahub.adapters.base import AdapterError
from datahub.adapters.tushare import DEFAULT_INDEX_CODES, WEBSITE_INDEX_CODES, TushareAdapter from datahub.adapters.tushare import DEFAULT_INDEX_CODES, WEBSITE_INDEX_CODES, TushareAdapter
from datahub.datasets_ext import (
EXTENDED_EOD_COPY,
EXTENDED_SOFT_DATASETS,
EXTENDED_STAGING_INSERT,
)
from datahub.db import DATASET_TABLES, HubDB from datahub.db import DATASET_TABLES, HubDB
from datahub.governance.circuit import CircuitBreaker from datahub.governance.circuit import CircuitBreaker
from datahub.governance.ratelimit import TokenBucket from datahub.governance.ratelimit import TokenBucket
@@ -21,12 +26,16 @@ from datahub.timeutil import add_days, isoformat, now_shanghai, yyyymmdd
LOGGER = get_logger() LOGGER = get_logger()
HARD_DATASETS = {"daily", "valuation", "index_daily"} HARD_DATASETS = {"daily", "valuation", "index_daily"}
SOFT_DATASETS = {"moneyflow", "auction"} SOFT_DATASETS = {"moneyflow", "auction"} | EXTENDED_SOFT_DATASETS
OFFICIAL_DATASETS = HARD_DATASETS | SOFT_DATASETS OFFICIAL_DATASETS = HARD_DATASETS | {"moneyflow", "auction"} # A/B retry scope unchanged
STOCKS_DATASET = "stocks" STOCKS_DATASET = "stocks"
STOCK_SNAPSHOT_FIELDS = ("ts_code", "symbol", "name", "area", "industry", "market", "list_status", "list_date") STOCK_SNAPSHOT_FIELDS = ("ts_code", "symbol", "name", "area", "industry", "market", "list_status", "list_date")
EOD_A_DATASETS = ("daily", "valuation", "moneyflow", "auction") EOD_A_DATASETS = ("daily", "valuation", "moneyflow", "auction")
EOD_B_DATASETS = ("index_daily",) EOD_B_DATASETS = ("index_daily",)
EOD_C_DATASETS = ("limit_events",)
EOD_D_DATASETS = ("dragon_tiger",)
EOD_E_DATASETS = ("sector_daily",)
EOD_F_DATASETS = ("popularity",)
EMPTY_BATCH_ERROR = "empty official batch: 0 valid rows" EMPTY_BATCH_ERROR = "empty official batch: 0 valid rows"
STAGING_INSERT = { STAGING_INSERT = {
@@ -80,6 +89,7 @@ STAGING_INSERT = {
r.get("close"), r.get("pct_chg"), r.get("volume"), r.get("amount"), r.get("close"), r.get("pct_chg"), r.get("volume"), r.get("amount"),
), ),
), ),
**EXTENDED_STAGING_INSERT,
} }
EOD_COPY = { EOD_COPY = {
@@ -114,6 +124,7 @@ EOD_COPY = {
"SELECT ts_code,trade_date,open,high,low,close,pct_chg,volume,amount,batch_id " "SELECT ts_code,trade_date,open,high,low,close,pct_chg,volume,amount,batch_id "
"FROM staging_index_bars WHERE batch_id = ?" "FROM staging_index_bars WHERE batch_id = ?"
), ),
**EXTENDED_EOD_COPY,
} }
@@ -348,11 +359,20 @@ class Pipeline:
is published; ``force`` re-publishes unconditionally. New listings, is published; ``force`` re-publishes unconditionally. New listings,
renames (incl. N/C prefix removal) and status changes all flow into renames (incl. N/C prefix removal) and status changes all flow into
the snapshot, which carries batch_id/published_at metadata. the snapshot, which carries batch_id/published_at metadata.
The ``stock_master`` UPSERT happens inside the same publish
transaction as the snapshot switch — fetch / quality-gate / switch
failures leave the master on the previous complete values.
""" """
day = yyyymmdd(trade_date or self.clock()) day = yyyymmdd(trade_date or self.clock())
rows = self._fetch_dataset(STOCKS_DATASET, day) try:
with self.db.write() as connection: rows = self._fetch_dataset(STOCKS_DATASET, day)
self._upsert_stock_master(connection, rows, isoformat(self.clock())) except Exception as exc:
self.audit(
"pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}",
json.dumps({"state": "failed", "error": str(exc)}, ensure_ascii=False),
)
raise
if not force: if not force:
active, snapshot = self.published_stock_snapshot(day) active, snapshot = self.published_stock_snapshot(day)
if active is not None: if active is not None:
@@ -369,7 +389,14 @@ class Pipeline:
"batch_id": active, "batch_id": active,
"rows": len(snapshot), "rows": len(snapshot),
} }
result = self.run_dataset(STOCKS_DATASET, day, prepared_rows=rows) try:
result = self.run_dataset(STOCKS_DATASET, day, prepared_rows=rows)
except Exception as exc:
self.audit(
"pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}",
json.dumps({"state": "failed", "error": str(exc)}, ensure_ascii=False),
)
raise
self.audit( self.audit(
"pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}", "pipeline", "stocks-refresh", f"{STOCKS_DATASET}:{day}",
json.dumps({"batch_id": result["batch_id"], "rows": result["rows"]}, ensure_ascii=False), json.dumps({"batch_id": result["batch_id"], "rows": result["rows"]}, ensure_ascii=False),
@@ -637,53 +664,140 @@ class Pipeline:
return [dataset for dataset in sorted(OFFICIAL_DATASETS) if dataset not in published] return [dataset for dataset in sorted(OFFICIAL_DATASETS) if dataset not in published]
def run_eod_missing(self, trade_date: str) -> dict[str, Any]: def run_eod_missing(self, trade_date: str) -> dict[str, Any]:
"""Fetch/publish every official dataset still missing for the date. """Republish every incomplete EOD consistency group for the date.
Idempotent: datasets with an existing publication are skipped, so A-group (daily/valuation/moneyflow/auction + stocks) and B-group
repeats never overwrite the current official batch. All missing (index_daily) are separate boundaries. Within a group, either the
members stage first and switch in one atomic release group; a single whole boundary is already published (idempotent skip) or every
member failure keeps the previous complete official version serving. member is re-staged and switched together — never fill only the
missing members on top of older batches from an earlier partial run.
""" """
return self.run_release_group(tuple(sorted(OFFICIAL_DATASETS)), trade_date, include_stocks=True)
def run_eod_batch_a(self, trade_date: str) -> dict[str, Any]:
return self.run_release_group(EOD_A_DATASETS, trade_date, include_stocks=True)
def run_eod_batch_b(self, trade_date: str) -> dict[str, Any]:
return self.run_release_group(EOD_B_DATASETS, trade_date)
def run_release_group(
self,
datasets: tuple[str, ...],
trade_date: str,
include_stocks: bool = False,
) -> dict[str, Any]:
"""One post-market publish/republish becomes one atomic visibility flip.
Consistency boundary: every member (official datasets, plus the daily
stocks snapshot when ``include_stocks`` and not yet published) is
fetched, staged, field-gated and cross-validated BEFORE any reader can
see it. Only when the whole group passes does a single SQLite
transaction copy all staging batches to the official tables and flip
every ``publications`` row at once. Any member failure aborts the
group: the previous complete official version keeps serving and the
reason is recorded on the batches and in the audit log.
"""
day = yyyymmdd(trade_date)
results: dict[str, Any] = {} results: dict[str, Any] = {}
staged: dict[str, dict[str, Any]] = {} results.update(self.run_eod_batch_a(trade_date))
failure: str | None = None results.update(self.run_eod_batch_b(trade_date))
pending: list[str] = [] return results
def run_eod_batch_a(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_release_group(EOD_A_DATASETS, trade_date, include_stocks=True, force=force)
def run_eod_batch_b(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_release_group(EOD_B_DATASETS, trade_date, force=force)
def run_extended_soft(self, datasets: tuple[str, ...], trade_date: str, force: bool = False) -> dict[str, Any]:
"""Publish extended soft datasets independently (not A/B atomic)."""
results: dict[str, Any] = {}
day = yyyymmdd(trade_date)
for dataset in datasets: for dataset in datasets:
if self.active_batch(dataset, day) is not None: if not force and self.active_batch(dataset, day):
results[dataset] = { results[dataset] = {
"dataset": dataset, "dataset": dataset,
"trade_date": day, "trade_date": day,
"state": "skipped", "state": "skipped",
"reason": "already_published", "reason": "already_published",
} }
else: continue
pending.append(dataset) try:
rows = self._fetch_dataset(dataset, day)
if not rows and dataset in {"popularity", "dragon_tiger"}:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "skipped",
"reason": "upstream_empty",
"rows": 0,
}
continue
results[dataset] = self.run_dataset(dataset, day, prepared_rows=rows)
except Exception as exc:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "failed",
"error": str(exc),
}
LOGGER.exception("extended soft publish failed dataset=%s date=%s", dataset, day)
return results
def run_eod_batch_c(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_extended_soft(EOD_C_DATASETS, trade_date, force=force)
def run_eod_batch_d(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_extended_soft(EOD_D_DATASETS, trade_date, force=force)
def run_eod_batch_e(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_extended_soft(EOD_E_DATASETS, trade_date, force=force)
def run_eod_batch_f(self, trade_date: str, force: bool = False) -> dict[str, Any]:
return self.run_extended_soft(EOD_F_DATASETS, trade_date, force=force)
def force_republish_boundary(self, dataset: str, trade_date: str) -> dict[str, Any]:
"""Force-republish the full A/B consistency boundary that owns ``dataset``.
CLI ``eod-refresh --force`` and admin manual backfill must not publish a
single official member alone — that would mix old and new batches inside
the same trade date. Naming any A-group member (or stocks) rebuilds the
whole A group; naming ``index_daily`` rebuilds B. Extended soft datasets
republish independently.
"""
name = str(dataset or "").strip()
if name in EOD_A_DATASETS or name == STOCKS_DATASET:
return self.run_eod_batch_a(trade_date, force=True)
if name in EOD_B_DATASETS:
return self.run_eod_batch_b(trade_date, force=True)
if name in EXTENDED_SOFT_DATASETS:
return self.run_extended_soft((name,), trade_date, force=True)
raise ValueError(f"dataset is not part of an EOD release boundary: {dataset}")
def run_release_group(
self,
datasets: tuple[str, ...],
trade_date: str,
include_stocks: bool = False,
force: bool = False,
) -> dict[str, Any]:
"""One post-market publish/republish becomes one atomic visibility flip.
Consistency boundary: every member (official datasets, plus the daily
stocks snapshot when ``include_stocks``) is fetched, staged,
field-gated and cross-validated BEFORE any reader can see it. Only
when the whole group passes does a single SQLite transaction copy
all staging batches to the official tables and flip every
``publications`` row at once. Any member failure aborts the group:
the previous complete official version keeps serving and the reason
is recorded on the batches and in the audit log.
Skip is all-or-nothing for the boundary unless ``force``: if every
official member (and stocks when required) is already published, the
group is skipped. If any official member is still missing — or
``force`` is set — every official member is re-staged, so a retry or
manual republish never mixes old and new batches in one release.
"""
day = yyyymmdd(trade_date)
results: dict[str, Any] = {}
staged: dict[str, dict[str, Any]] = {}
failure: str | None = None
missing_official = [dataset for dataset in datasets if self.active_batch(dataset, day) is None]
stocks_missing = include_stocks and self.active_batch(STOCKS_DATASET, day) is None
if not force and not missing_official and not stocks_missing:
for dataset in datasets:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "skipped",
"reason": "already_published",
}
if include_stocks:
results[STOCKS_DATASET] = {
"dataset": STOCKS_DATASET,
"trade_date": day,
"state": "skipped",
"reason": "already_published",
}
return results
# Incomplete or forced boundary → restage every official member together.
pending = list(datasets)
for dataset in pending: for dataset in pending:
if failure is not None: if failure is not None:
@@ -705,9 +819,13 @@ class Pipeline:
"error": str(exc), "error": str(exc),
} }
if include_stocks and failure is None and self.active_batch(STOCKS_DATASET, day) is None: # Stocks join the same switch when the official boundary is being
# rebuilt (missing or forced), or when only the stocks snapshot is
# still missing.
rebuild_official = bool(force or missing_official)
if include_stocks and failure is None and (rebuild_official or stocks_missing):
try: try:
stocks_plan = self._stage_stocks_snapshot(day) stocks_plan = self._stage_stocks_snapshot(day, force=rebuild_official)
except Exception as exc: except Exception as exc:
failure = f"{STOCKS_DATASET}: {exc}" failure = f"{STOCKS_DATASET}: {exc}"
results[STOCKS_DATASET] = { results[STOCKS_DATASET] = {
@@ -756,8 +874,32 @@ class Pipeline:
try: try:
self._switch_release_group(day, staged) self._switch_release_group(day, staged)
except Exception as exc: except Exception as exc:
reason = f"release group switch failed: {exc}"
for item in staged.values(): for item in staged.values():
self._abandon_batch(item["batch_id"], f"release group switch failed: {exc}") self._abandon_batch(item["batch_id"], reason)
LOGGER.warning(
"release group switch failed, previous official version keeps serving",
extra={
"hub": {
"trade_date": day,
"datasets": sorted(staged),
"reason": reason,
"event": "release_group_switch_failed",
}
},
)
self.audit(
"pipeline", "release-group", f"eod:{day}",
json.dumps(
{
"state": "failed",
"reason": reason,
"switched": [],
"force": bool(force),
},
ensure_ascii=False,
),
)
raise raise
for dataset, item in staged.items(): for dataset, item in staged.items():
results[dataset] = { results[dataset] = {
@@ -770,7 +912,8 @@ class Pipeline:
self.audit( self.audit(
"pipeline", "release-group", f"eod:{day}", "pipeline", "release-group", f"eod:{day}",
json.dumps( json.dumps(
{"state": "ok", "switched": sorted(staged)}, ensure_ascii=False {"state": "ok", "switched": sorted(staged), "force": bool(force)},
ensure_ascii=False,
), ),
) )
return results return results
@@ -820,18 +963,18 @@ class Pipeline:
"state": "degraded" if report["soft_fail"] else "published", "state": "degraded" if report["soft_fail"] else "published",
} }
def _stage_stocks_snapshot(self, trade_date: str) -> dict[str, Any] | None: def _stage_stocks_snapshot(self, trade_date: str, force: bool = False) -> dict[str, Any] | None:
"""Stage the daily stocks snapshot for a release group switch. """Stage the daily stocks snapshot for a release group switch.
Returns None when the published snapshot is already identical to Returns None when the published snapshot is already identical to
upstream (idempotent skip). The stock_master upsert is deferred into upstream (idempotent skip) unless ``force`` is set. The stock_master
the group switch transaction so the master never runs ahead of the upsert is deferred into the group switch / publish transaction so the
published snapshot. master never runs ahead of the published snapshot.
""" """
day = yyyymmdd(trade_date) day = yyyymmdd(trade_date)
active, snapshot = self.published_stock_snapshot(day) active, snapshot = self.published_stock_snapshot(day)
rows = self._fetch_dataset(STOCKS_DATASET, day) rows = self._fetch_dataset(STOCKS_DATASET, day)
if active is not None: if active is not None and not force:
upstream = sorted( upstream = sorted(
tuple(str(row.get(field)) for field in STOCK_SNAPSHOT_FIELDS) for row in rows tuple(str(row.get(field)) for field in STOCK_SNAPSHOT_FIELDS) for row in rows
) )
@@ -940,7 +1083,16 @@ class Pipeline:
) )
listed_n = int((listed or {}).get("n") or 0) listed_n = int((listed or {}).get("n") or 0)
row_n = len(rows) row_n = len(rows)
keys = [(row.get("ts_code"), row.get("trade_date")) for row in rows] if dataset == "limit_events":
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("limit_type")) for row in rows]
elif dataset == "popularity":
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("source")) for row in rows]
elif dataset == "dragon_tiger":
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("hm_name")) for row in rows]
elif dataset == "sector_daily":
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("family")) for row in rows]
else:
keys = [(row.get("ts_code"), row.get("trade_date")) for row in rows]
dup = row_n - len(set(keys)) dup = row_n - len(set(keys))
if dup: if dup:
errors.append(f"duplicate keys: {dup}") errors.append(f"duplicate keys: {dup}")
@@ -961,7 +1113,8 @@ class Pipeline:
errors.append(EMPTY_BATCH_ERROR) errors.append(EMPTY_BATCH_ERROR)
field_report = self._field_gate(dataset, trade_date, rows, errors) field_report = self._field_gate(dataset, trade_date, rows, errors)
if dataset in SOFT_DATASETS: if dataset in SOFT_DATASETS:
hard_fail = bool(dup or bad_date or empty) allow_empty = dataset in {"popularity", "dragon_tiger", "moneyflow", "auction"}
hard_fail = bool(dup or bad_date or (empty and not allow_empty))
else: else:
hard_fail = bool(errors) and (dataset in HARD_DATASETS or dataset == STOCKS_DATASET) hard_fail = bool(errors) and (dataset in HARD_DATASETS or dataset == STOCKS_DATASET)
report = { report = {
@@ -1077,6 +1230,16 @@ class Pipeline:
with self.db.write() as connection: with self.db.write() as connection:
_staging_count_or_raise(connection, dataset, trade_date, batch_id) _staging_count_or_raise(connection, dataset, trade_date, batch_id)
connection.execute(EOD_COPY[dataset], (batch_id,)) connection.execute(EOD_COPY[dataset], (batch_id,))
if dataset == STOCKS_DATASET:
staging = DATASET_TABLES[STOCKS_DATASET][1]
stock_rows = [
dict(row)
for row in connection.execute(
f"SELECT * FROM {staging} WHERE batch_id = ?",
(batch_id,),
).fetchall()
]
self._upsert_stock_master(connection, stock_rows, published_at)
if self.before_commit: if self.before_commit:
self.before_commit() self.before_commit()
_upsert_publication(connection, dataset, trade_date, batch_id, state, published_at) _upsert_publication(connection, dataset, trade_date, batch_id, state, published_at)
+233
View File
@@ -0,0 +1,233 @@
"""Provisional (盘中观察) serving: quotes, index quotes, intraday points.
Free sources only. Never writes official eod_* tables. Uses rt_cache + LKG.
"""
from __future__ import annotations
import json
import time
from datetime import datetime
from typing import Any
from datahub.adapters.eastmoney import EastmoneyAdapter
from datahub.adapters.tencent import TencentAdapter
from datahub.codes import resolve_code
from datahub.db import HubDB
from datahub.governance.lkg import LastKnownGood
from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
QUOTE_TTL = 60
INDEX_TTL = 60
INTRADAY_TTL = 20
class RealtimeApiError(RuntimeError):
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
self.message = message
def _envelope(data: Any, meta: dict[str, Any]) -> dict[str, Any]:
from datahub import SCHEMA_VERSION
return {"schema_version": SCHEMA_VERSION, "data": data, "meta": meta}
def fetch_index_quotes(db: HubDB) -> dict[str, Any]:
cache_key = "indexes:quotes"
cached = _read_cache(db, cache_key)
if cached is not None:
return cached
eastmoney = EastmoneyAdapter()
try:
rows = eastmoney.fetch_indices()
source = "eastmoney:ulist"
except Exception:
rows = TencentAdapter().fetch_indices()
source = "tencent:qt"
if len(rows) < 3:
raise RealtimeApiError("SOURCE_UNAVAILABLE", "index quotes incomplete")
payload = _envelope(
rows,
{
"tier": "provisional",
"trade_date": yyyymmdd(now_shanghai()),
"source": source,
"stale": False,
"staleness_seconds": 0,
"published_at": isoformat(now_shanghai()),
},
)
_write_cache(db, cache_key, payload, INDEX_TTL, source)
return payload
def fetch_quotes(db: HubDB, codes: list[str]) -> dict[str, Any]:
if not codes:
raise RealtimeApiError("INVALID_ARGUMENT", "codes is required")
resolved: list[str] = []
for code in codes[:60]:
item = resolve_code(db, code) or _guess_ts_code(code)
if item:
resolved.append(item)
if not resolved:
raise RealtimeApiError("INVALID_ARGUMENT", "no resolvable codes")
cache_key = "quotes:" + ",".join(sorted(resolved))
cached = _read_cache(db, cache_key)
if cached is not None:
return cached
adapter = EastmoneyAdapter()
try:
rows = adapter.fetch_quotes(resolved)
source = "eastmoney:clist"
except Exception as exc:
raise RealtimeApiError("SOURCE_UNAVAILABLE", f"quotes unavailable: {exc}") from exc
payload = _envelope(
rows,
{
"tier": "provisional",
"trade_date": yyyymmdd(now_shanghai()),
"source": source,
"stale": False,
"staleness_seconds": 0,
"published_at": isoformat(now_shanghai()),
},
)
_write_cache(db, cache_key, payload, QUOTE_TTL, source)
return payload
def fetch_intraday(db: HubDB, code: str, date: str = "") -> dict[str, Any]:
ts_code = resolve_code(db, code) or _guess_ts_code(code)
if not ts_code:
raise RealtimeApiError("INVALID_ARGUMENT", f"ambiguous code: {code}")
cache_key = f"intraday:{ts_code}:{date or 'today'}"
cached = _read_cache(db, cache_key)
if cached is not None:
return cached
adapter = EastmoneyAdapter()
try:
payload_data = adapter.fetch_intraday(ts_code, date)
source = "eastmoney:trends2"
except Exception as exc:
recovered = _load_intraday_lkg(db, ts_code, date)
if recovered is None:
raise RealtimeApiError("SOURCE_UNAVAILABLE", f"intraday unavailable: {exc}") from exc
return recovered
payload = _envelope(
payload_data,
{
"tier": "provisional",
"trade_date": yyyymmdd(payload_data.get("trade_date") or date or now_shanghai()),
"source": source,
"stale": False,
"staleness_seconds": 0,
"published_at": isoformat(now_shanghai()),
},
)
_write_cache(db, cache_key, payload, INTRADAY_TTL, source)
return payload
def _load_intraday_lkg(db: HubDB, ts_code: str, date: str = "") -> dict[str, Any] | None:
store = LastKnownGood(db)
keys = [f"intraday:{ts_code}:{date or 'today'}"]
if date:
keys.append(f"intraday:{ts_code}:today")
for key in keys:
item = store.load(key)
payload = _lkg_payload(item)
if payload is not None:
return payload
row = db.fetchone(
"SELECT * FROM last_known_good WHERE cache_key LIKE ? ORDER BY stored_at DESC LIMIT 1",
(f"intraday:{ts_code}:%",),
)
if not row:
return None
try:
raw = json.loads(row["payload"])
except json.JSONDecodeError:
return None
return _mark_stale(raw) if isinstance(raw, dict) else None
def _lkg_payload(item: dict[str, Any] | None) -> dict[str, Any] | None:
if not item:
return None
payload = item.get("payload")
return _mark_stale(payload) if isinstance(payload, dict) else None
def _mark_stale(payload: dict[str, Any]) -> dict[str, Any] | None:
data = payload.get("data")
if not isinstance(data, dict) or not data.get("points"):
return None
stamped = dict(payload)
meta = dict(stamped.get("meta") or {})
meta["stale"] = True
stamped["meta"] = meta
return stamped
def _guess_ts_code(code: str) -> str | None:
raw = str(code or "").strip().upper()
if "." in raw:
return raw
if len(raw) == 6 and raw.isdigit():
if raw.startswith(("5", "6", "9")):
return f"{raw}.SH"
return f"{raw}.SZ"
return None
def _read_cache(db: HubDB, cache_key: str) -> dict[str, Any] | None:
row = db.fetchone("SELECT * FROM rt_cache WHERE cache_key = ?", (cache_key,))
if not row:
return None
expires = str(row.get("expires_at") or "")
now = isoformat(now_shanghai())
if expires and expires < now:
return None
try:
payload = json.loads(row["payload"])
except json.JSONDecodeError:
return None
if isinstance(payload, dict) and isinstance(payload.get("meta"), dict):
stored = str(row.get("stored_at") or "")
try:
age = max(0, int(time.time() - datetime.fromisoformat(stored).timestamp()))
except Exception:
age = 0
payload["meta"]["staleness_seconds"] = age
payload["meta"]["stale"] = age > QUOTE_TTL
return payload
def _write_cache(db: HubDB, cache_key: str, payload: dict[str, Any], ttl: int, source: str) -> None:
from datetime import timedelta
now = now_shanghai()
stored = isoformat(now)
expires = isoformat(now + timedelta(seconds=ttl))
db.execute(
"""
INSERT INTO rt_cache(cache_key, payload, source, stored_at, expires_at)
VALUES (?,?,?,?,?)
ON CONFLICT(cache_key) DO UPDATE SET
payload=excluded.payload, source=excluded.source,
stored_at=excluded.stored_at, expires_at=excluded.expires_at
""",
(cache_key, json.dumps(payload, ensure_ascii=False), source, stored, expires),
)
db.execute(
"""
INSERT INTO last_known_good(cache_key, payload, source, stored_at)
VALUES (?,?,?,?)
ON CONFLICT(cache_key) DO UPDATE SET
payload=excluded.payload, source=excluded.source, stored_at=excluded.stored_at
""",
(cache_key, json.dumps(payload, ensure_ascii=False), source, stored),
)
+22 -2
View File
@@ -48,6 +48,10 @@ class Scheduler:
"precheck": self._precheck, "precheck": self._precheck,
"eod_a": self._eod_a, "eod_a": self._eod_a,
"eod_b": self._eod_b, "eod_b": self._eod_b,
"eod_c": self._eod_c,
"eod_d": self._eod_d,
"eod_e": self._eod_e,
"eod_f": self._eod_f,
"eod_retry": self._eod_retry, "eod_retry": self._eod_retry,
"stocks_refresh": self._stocks_refresh, "stocks_refresh": self._stocks_refresh,
"cleanup": self._cleanup, "cleanup": self._cleanup,
@@ -87,6 +91,10 @@ class Scheduler:
("precheck", time(8, 45)), ("precheck", time(8, 45)),
("eod_a", time(15, 5)), ("eod_a", time(15, 5)),
("eod_b", time(15, 10)), ("eod_b", time(15, 10)),
("eod_c", time(16, 40)),
("eod_d", time(16, 45)),
("eod_e", time(18, 5)),
("eod_f", time(22, 40)),
("cleanup", time(0, 30)), ("cleanup", time(0, 30)),
("backup", time(0, 40)), ("backup", time(0, 40)),
] ]
@@ -99,7 +107,7 @@ class Scheduler:
key = (job_id, day, at.strftime("%H%M")) key = (job_id, day, at.strftime("%H%M"))
if key in self._fired: if key in self._fired:
continue continue
if job_id in {"eod_a", "eod_b", "stocks_refresh"} and not open_day: if job_id in {"eod_a", "eod_b", "eod_c", "eod_d", "eod_e", "eod_f", "stocks_refresh"} and not open_day:
self._fired.add(key) self._fired.add(key)
continue continue
self._fired.add(key) self._fired.add(key)
@@ -110,7 +118,7 @@ class Scheduler:
try: try:
self.run_job(job_id, day) self.run_job(job_id, day)
except Exception: except Exception:
if job_id not in {"eod_a", "eod_b", "stocks_refresh"}: if job_id not in {"eod_a", "eod_b", "eod_c", "eod_d", "eod_e", "eod_f", "stocks_refresh"}:
raise raise
# Keep the tick alive; evening retries take over. # Keep the tick alive; evening retries take over.
LOGGER.exception("scheduled job %s failed for %s", job_id, day) LOGGER.exception("scheduled job %s failed for %s", job_id, day)
@@ -317,6 +325,18 @@ class Scheduler:
def _eod_b(self, trade_date: str) -> dict[str, Any]: def _eod_b(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_b(trade_date) return self.pipeline.run_eod_batch_b(trade_date)
def _eod_c(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_c(trade_date)
def _eod_d(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_d(trade_date)
def _eod_e(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_e(trade_date)
def _eod_f(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_f(trade_date)
def _eod_retry(self, trade_date: str) -> dict[str, Any]: def _eod_retry(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_missing(trade_date) return self.pipeline.run_eod_missing(trade_date)
+94 -1
View File
@@ -73,6 +73,20 @@ class V1API:
return self.moneyflow(q) return self.moneyflow(q)
if path == "/v1/auction": if path == "/v1/auction":
return self.auction(q) return self.auction(q)
if path == "/v1/limit-events":
return self.limit_events(q)
if path == "/v1/popularity":
return self.popularity(q)
if path == "/v1/dragon-tiger":
return self.dragon_tiger(q)
if path == "/v1/sectors":
return self.sectors(q)
if path == "/v1/quotes/latest":
return self.quotes_latest(q)
if path == "/v1/indexes/quotes":
return self.index_quotes(q)
if path == "/v1/intraday/points":
return self.intraday_points(q)
if path == "/v1/datasets/status": if path == "/v1/datasets/status":
return self.dataset_status(q.get("date") or "") return self.dataset_status(q.get("date") or "")
if path == "/v1/batches": if path == "/v1/batches":
@@ -197,9 +211,80 @@ class V1API:
def auction(self, q: dict[str, str]) -> dict[str, Any]: def auction(self, q: dict[str, str]) -> dict[str, Any]:
return self._published_rows(dataset="auction", table="eod_auction", q=q, source="tushare:stk_auction") return self._published_rows(dataset="auction", table="eod_auction", q=q, source="tushare:stk_auction")
def limit_events(self, q: dict[str, str]) -> dict[str, Any]:
return self._published_rows(
dataset="limit_events",
table="eod_limit_events",
q=q,
source="tushare:limit_list_d",
extra_filters={"limit_type": q.get("limit_type") or ""},
)
def popularity(self, q: dict[str, str]) -> dict[str, Any]:
return self._published_rows(
dataset="popularity",
table="eod_popularity",
q=q,
source="tushare:ths_hot+dc_hot",
extra_filters={"source": q.get("source") or ""},
)
def dragon_tiger(self, q: dict[str, str]) -> dict[str, Any]:
return self._published_rows(
dataset="dragon_tiger",
table="eod_dragon_tiger",
q=q,
source="tushare:hm_detail",
)
def sectors(self, q: dict[str, str]) -> dict[str, Any]:
return self._published_rows(
dataset="sector_daily",
table="eod_sector_daily",
q=q,
source="tushare:ths_daily+dc_index+sw_daily",
extra_filters={"family": q.get("family") or ""},
)
def quotes_latest(self, q: dict[str, str]) -> dict[str, Any]:
from datahub.realtime_serve import RealtimeApiError, fetch_quotes
codes = [item.strip() for item in str(q.get("codes") or "").split(",") if item.strip()]
try:
return fetch_quotes(self.db, codes)
except RealtimeApiError as exc:
raise ApiError(exc.code, exc.message) from exc
def index_quotes(self, q: dict[str, str]) -> dict[str, Any]:
from datahub.realtime_serve import RealtimeApiError, fetch_index_quotes
try:
return fetch_index_quotes(self.db)
except RealtimeApiError as exc:
raise ApiError(exc.code, exc.message) from exc
def intraday_points(self, q: dict[str, str]) -> dict[str, Any]:
from datahub.realtime_serve import RealtimeApiError, fetch_intraday
code = str(q.get("code") or "").strip()
if not code:
raise ApiError("INVALID_ARGUMENT", "code is required")
raw_date = str(q.get("date") or "").strip()
try:
trade_date = yyyymmdd(raw_date or now_shanghai())
except ValueError as exc:
raise ApiError("INVALID_ARGUMENT", str(exc)) from exc
try:
return fetch_intraday(self.db, code, trade_date)
except RealtimeApiError as exc:
raise ApiError(exc.code, exc.message) from exc
def dataset_status(self, date: str) -> dict[str, Any]: def dataset_status(self, date: str) -> dict[str, Any]:
trade_date = yyyymmdd(date or now_shanghai()) trade_date = yyyymmdd(date or now_shanghai())
datasets = ("daily", "valuation", "moneyflow", "auction", "index_daily", "stocks") datasets = (
"daily", "valuation", "moneyflow", "auction", "index_daily", "stocks",
"limit_events", "popularity", "dragon_tiger", "sector_daily",
)
items = [] items = []
for dataset in datasets: for dataset in datasets:
pub = self.db.fetchone( pub = self.db.fetchone(
@@ -244,6 +329,7 @@ class V1API:
source: str, source: str,
adjust: str = "none", adjust: str = "none",
default_code: str = "", default_code: str = "",
extra_filters: dict[str, str] | None = None,
) -> dict[str, Any]: ) -> dict[str, Any]:
trade_date = q.get("date") or q.get("trade_date") or "" trade_date = q.get("date") or q.get("trade_date") or ""
code = q.get("code") or default_code code = q.get("code") or default_code
@@ -264,6 +350,7 @@ class V1API:
if resolved is None: if resolved is None:
raise ApiError("INVALID_ARGUMENT", f"ambiguous code: {code}") raise ApiError("INVALID_ARGUMENT", f"ambiguous code: {code}")
ts_code = resolved ts_code = resolved
filters = {key: value for key, value in (extra_filters or {}).items() if value}
# For a range, use per-date published batch. Single-date is the common path. # For a range, use per-date published batch. Single-date is the common path.
if start == end: if start == end:
pub = self.db.fetchone( pub = self.db.fetchone(
@@ -282,6 +369,9 @@ class V1API:
if ts_code: if ts_code:
sql += " AND ts_code = ?" sql += " AND ts_code = ?"
params.append(ts_code) params.append(ts_code)
for key, value in filters.items():
sql += f" AND {key} = ?"
params.append(value)
sql += " ORDER BY ts_code LIMIT ? OFFSET ?" sql += " ORDER BY ts_code LIMIT ? OFFSET ?"
params.extend([limit, offset]) params.extend([limit, offset])
rows = [dict(row) for row in self.db.fetchall(sql, tuple(params))] rows = [dict(row) for row in self.db.fetchall(sql, tuple(params))]
@@ -317,6 +407,9 @@ class V1API:
if ts_code: if ts_code:
sql += " AND ts_code = ?" sql += " AND ts_code = ?"
params.append(ts_code) params.append(ts_code)
for key, value in filters.items():
sql += f" AND {key} = ?"
params.append(value)
sql += " ORDER BY ts_code" sql += " ORDER BY ts_code"
rows.extend(self.db.fetchall(sql, tuple(params))) rows.extend(self.db.fetchall(sql, tuple(params)))
sliced = rows[offset: offset + limit] sliced = rows[offset: offset + limit]
+30 -1
View File
@@ -45,6 +45,30 @@ RAW = {
{"ts_code": "600000.SH", "trade_date": "20240902", "vol": 100, "price": 10.15, "amount": 1500000, "pre_close": 10.00, "turnover_rate": 0.1, "volume_ratio": 1.2, "float_share": 2000}, {"ts_code": "600000.SH", "trade_date": "20240902", "vol": 100, "price": 10.15, "amount": 1500000, "pre_close": 10.00, "turnover_rate": 0.1, "volume_ratio": 1.2, "float_share": 2000},
{"ts_code": "000001.SZ", "trade_date": "20240902", "vol": 80, "price": 11.05, "amount": 1200000, "pre_close": 11.10, "turnover_rate": 0.2, "volume_ratio": 0.9, "float_share": 1800}, {"ts_code": "000001.SZ", "trade_date": "20240902", "vol": 80, "price": 11.05, "amount": 1200000, "pre_close": 11.10, "turnover_rate": 0.2, "volume_ratio": 0.9, "float_share": 1800},
], ],
"limit_list_d": [
{"trade_date": "20240902", "ts_code": "600000.SH", "industry": "银行", "name": "浦发银行", "close": 10.2, "pct_chg": 9.95, "amount": 1e8, "limit_amount": 5000, "float_mv": 800, "total_mv": 1000, "turnover_ratio": 5.0, "fd_amount": 2e7, "first_time": "09:30:01", "last_time": "14:55:00", "open_times": 0, "up_stat": "1/1", "limit_times": 1, "limit_type": "U"},
],
"ths_hot": [
{"ts_code": "600000.SH", "ts_name": "浦发银行", "hot": 90.0, "rank": 1, "pct_change": 1.2, "current_price": 10.2, "concept": "银行", "data_type": "热股", "trade_date": "20240902"},
],
"dc_hot": [
{"ts_code": "600000.SH", "ts_name": "浦发银行", "rank": 2, "pct_change": 1.2, "current_price": 10.2, "hot": 80.0, "concept": "银行", "data_type": "A股市场", "trade_date": "20240902"},
],
"hm_detail": [
{"trade_date": "20240902", "ts_code": "600000.SH", "ts_name": "浦发银行", "buy_amount": 1000, "sell_amount": 200, "net_amount": 800, "hm_name": "测试游资", "hm_orgs": "某某营业部", "tag": "超买"},
],
"top_list": [
{"trade_date": "20240902", "ts_code": "600000.SH", "name": "浦发银行", "pct_change": 9.95, "reason": "涨幅偏离值达7%"},
],
"ths_daily": [
{"ts_code": "885811.TI", "trade_date": "20240902", "open": 1000, "high": 1010, "low": 990, "close": 1005, "pre_close": 995, "pct_change": 1.0, "vol": 100, "turnover_rate": 1.2},
],
"dc_index": [
{"ts_code": "BK0475", "trade_date": "20240902", "name": "银行", "open": 100, "high": 101, "low": 99, "close": 100.5, "pre_close": 99.5, "pct_change": 1.0, "vol": 10, "amount": 1e8, "turnover_rate": 0.5},
],
"sw_daily": [
{"ts_code": "801780.SI", "trade_date": "20240902", "name": "银行", "open": 2000, "high": 2010, "low": 1990, "close": 2005, "pct_change": 0.8, "vol": 50, "amount": 2e8},
],
} }
@@ -66,4 +90,9 @@ def fake_transport(api_name: str, params: dict, fields: str):
start = str(params.get("start_date") or "") start = str(params.get("start_date") or "")
end = str(params.get("end_date") or "99999999") end = str(params.get("end_date") or "99999999")
return [row for row in RAW["trade_cal"] if start <= row["cal_date"] <= end] return [row for row in RAW["trade_cal"] if start <= row["cal_date"] <= end]
return list(RAW.get(api_name) or []) rows = list(RAW.get(api_name) or [])
if api_name == "limit_list_d":
limit_type = str(params.get("limit_type") or "")
if limit_type:
rows = [row for row in rows if str(row.get("limit_type") or "") == limit_type]
return rows
+181 -8
View File
@@ -21,14 +21,14 @@ class GroupTransport:
def __init__(self) -> None: def __init__(self) -> None:
self.empty: set[str] = set() self.empty: set[str] = set()
self.keep_rows: dict[str, int] = {} self.keep_rows: dict[str, int] = {}
self.stocks: list[dict] = [] self.stocks: list[dict] | None = None
self.calls: list[str] = [] self.calls: list[str] = []
def __call__(self, api_name: str, params: dict, fields: str): def __call__(self, api_name: str, params: dict, fields: str):
self.calls.append(api_name) self.calls.append(api_name)
if api_name in self.empty: if api_name in self.empty:
return [] return []
if api_name == "stock_basic" and self.stocks: if api_name == "stock_basic" and self.stocks is not None:
return [dict(row) for row in self.stocks] return [dict(row) for row in self.stocks]
rows = fake_transport(api_name, params, fields) rows = fake_transport(api_name, params, fields)
keep = self.keep_rows.get(api_name) keep = self.keep_rows.get(api_name)
@@ -113,19 +113,48 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
first = self.pipe.run_dataset("daily", TRADE_DATE) first = self.pipe.run_dataset("daily", TRADE_DATE)
self.transport.empty = {"daily_basic"} self.transport.empty = {"daily_basic"}
results = self.pipe.run_eod_missing(TRADE_DATE) results = self.pipe.run_eod_missing(TRADE_DATE)
self.assertEqual(results["daily"]["state"], "skipped") # incomplete A-group restages daily with the others; valuation fails → no A switch
self.assertEqual(results["daily"]["state"], "failed")
self.assertEqual(results["valuation"]["state"], "failed") self.assertEqual(results["valuation"]["state"], "failed")
# the already-published complete batch is untouched and keeps serving # the already-published daily batch is untouched and keeps serving
self.assertEqual(self.pipe.active_batch("daily", TRADE_DATE), first["batch_id"]) self.assertEqual(self.pipe.active_batch("daily", TRADE_DATE), first["batch_id"])
self.assertEqual( pubs = publications_map(self.db, TRADE_DATE)
publications_map(self.db, TRADE_DATE), self.assertEqual(pubs["daily"], first["batch_id"])
{"daily": first["batch_id"]}, self.assertNotIn("valuation", pubs)
) self.assertNotIn("moneyflow", pubs)
self.assertNotIn("auction", pubs)
# B-group is an independent boundary and may still publish
self.assertEqual(results["index_daily"]["state"], "published")
payload = V1API(self.db, self.pipe, self.pipe.settings).handle( payload = V1API(self.db, self.pipe, self.pipe.settings).handle(
"/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]} "/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]}
) )
self.assertEqual(payload["meta"]["batch_id"], first["batch_id"]) self.assertEqual(payload["meta"]["batch_id"], first["batch_id"])
def test_partial_group_retry_does_not_mix_batches(self) -> None:
"""Already-published A members must be restaged with missing ones."""
first_daily = self.pipe.run_dataset("daily", TRADE_DATE)
first_moneyflow = self.pipe.run_dataset("moneyflow", TRADE_DATE)
results = self.pipe.run_eod_missing(TRADE_DATE)
# A-group switched as one boundary; B-group (index) also published
for name in (*GROUP_A, "stocks"):
self.assertEqual(results[name]["state"], "published", name)
self.assertEqual(results["index_daily"]["state"], "published")
pubs = self.db.fetchall(
"SELECT dataset, active_batch, published_at FROM publications WHERE trade_date = ?",
(TRADE_DATE,),
)
by_ds = {str(row["dataset"]): row for row in pubs}
# old partial batches replaced — no cross-batch mix of the first wave
self.assertNotEqual(by_ds["daily"]["active_batch"], first_daily["batch_id"])
self.assertNotEqual(by_ds["moneyflow"]["active_batch"], first_moneyflow["batch_id"])
a_times = {by_ds[name]["published_at"] for name in (*GROUP_A, "stocks")}
self.assertEqual(len(a_times), 1)
# serving resolves the new complete A-group batches
api = V1API(self.db, self.pipe, self.pipe.settings)
daily = api.handle("/v1/bars/daily", {"date": [TRADE_DATE], "code": ["600000.SH"]})
self.assertEqual(daily["meta"]["batch_id"], results["daily"]["batch_id"])
self.assertEqual(daily["meta"]["batch_id"], by_ds["daily"]["active_batch"])
def test_reads_during_switch_see_old_state_until_commit(self) -> None: def test_reads_during_switch_see_old_state_until_commit(self) -> None:
snapshots: list[dict] = [] snapshots: list[dict] = []
@@ -155,6 +184,13 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
for table in ("eod_bars", "eod_valuation", "eod_moneyflow", "eod_auction", "eod_stocks"): for table in ("eod_bars", "eod_valuation", "eod_moneyflow", "eod_auction", "eod_stocks"):
rows = self.db.fetchall(f"SELECT * FROM {table} WHERE trade_date = ?", (TRADE_DATE,)) rows = self.db.fetchall(f"SELECT * FROM {table} WHERE trade_date = ?", (TRADE_DATE,))
self.assertEqual(rows, [], table) self.assertEqual(rows, [], table)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'release-group' ORDER BY id DESC"
)
self.assertIsNotNone(audit)
detail = str(audit["detail"])
self.assertIn("killed mid-switch", detail)
self.assertIn("failed", detail)
def test_duplicate_runs_are_idempotent(self) -> None: def test_duplicate_runs_are_idempotent(self) -> None:
self.pipe.run_eod_batch_a(TRADE_DATE) self.pipe.run_eod_batch_a(TRADE_DATE)
@@ -231,5 +267,142 @@ class ReleaseGroupSwitchTests(unittest.TestCase):
self.assertIsNone(stocks_pub) self.assertIsNone(stocks_pub)
class StocksRefreshAtomicTests(unittest.TestCase):
def setUp(self) -> None:
self.transport = GroupTransport()
self.pipe, self.db = make_pipe(self.transport)
self.pipe.ingest_reference(TRADE_DATE)
self.transport.stocks = [
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
{"ts_code": "920071.BJ", "symbol": "920071", "name": "N金钛", "area": "辽宁",
"industry": "小金属", "market": "北交所", "list_status": "L", "list_date": "20240901"},
]
first = self.pipe.refresh_stocks(TRADE_DATE)
self.assertEqual(first["state"], "published")
self.first_batch = first["batch_id"]
def test_refresh_keeps_master_when_snapshot_publish_fails(self) -> None:
self.transport.stocks = [
{"ts_code": "600000.SH", "symbol": "600000", "name": "浦发银行", "area": "上海",
"industry": "银行", "market": "主板", "list_status": "L", "list_date": "19991110"},
{"ts_code": "920071.BJ", "symbol": "920071", "name": "金钛股份", "area": "辽宁",
"industry": "小金属", "market": "北交所", "list_status": "L", "list_date": "20240901"},
]
def explode() -> None:
raise RuntimeError("snapshot switch killed")
self.pipe.before_commit = explode
with self.assertRaises(RuntimeError):
self.pipe.refresh_stocks(TRADE_DATE)
master = self.db.fetchone("SELECT name FROM stock_master WHERE ts_code = '920071.BJ'")
self.assertEqual(master["name"], "N金钛") # rename not applied
self.assertEqual(self.pipe.active_batch("stocks", TRADE_DATE), self.first_batch)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'stocks-refresh' ORDER BY id DESC"
)
self.assertIn("failed", str(audit["detail"]))
self.assertIn("snapshot switch killed", str(audit["detail"]))
def test_refresh_keeps_master_when_quality_gate_rejects(self) -> None:
self.transport.stocks = [] # empty → hard fail before publish
with self.assertRaises(Exception):
self.pipe.refresh_stocks(TRADE_DATE)
master = self.db.fetchone("SELECT name FROM stock_master WHERE ts_code = '920071.BJ'")
self.assertEqual(master["name"], "N金钛")
self.assertEqual(self.pipe.active_batch("stocks", TRADE_DATE), self.first_batch)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'stocks-refresh' ORDER BY id DESC"
)
self.assertIn("failed", str(audit["detail"]))
class ForceBoundaryEntryTests(unittest.TestCase):
"""CLI force / admin backfill must rebuild the full A/B boundary."""
def setUp(self) -> None:
self.transport = GroupTransport()
self.pipe, self.db = make_pipe(self.transport)
self.pipe.ingest_reference(TRADE_DATE)
self.first = self.pipe.run_eod_batch_a(TRADE_DATE)
self.pipe.run_eod_batch_b(TRADE_DATE)
def test_force_republish_valuation_rebuilds_whole_a_group(self) -> None:
before = publications_map(self.db, TRADE_DATE)
results = self.pipe.force_republish_boundary("valuation", TRADE_DATE)
self.assertEqual({item["state"] for item in results.values()}, {"published"})
after = publications_map(self.db, TRADE_DATE)
for name in (*GROUP_A, "stocks"):
self.assertNotEqual(after[name], before[name], name)
self.assertEqual(after[name], results[name]["batch_id"], name)
# B-group left alone
self.assertEqual(after["index_daily"], before["index_daily"])
pubs = self.db.fetchall(
"SELECT dataset, published_at FROM publications WHERE trade_date = ?",
(TRADE_DATE,),
)
a_times = {row["published_at"] for row in pubs if row["dataset"] in {*GROUP_A, "stocks"}}
self.assertEqual(len(a_times), 1)
def test_force_republish_index_rebuilds_only_b_group(self) -> None:
before = publications_map(self.db, TRADE_DATE)
results = self.pipe.force_republish_boundary("index_daily", TRADE_DATE)
self.assertEqual(results["index_daily"]["state"], "published")
after = publications_map(self.db, TRADE_DATE)
self.assertNotEqual(after["index_daily"], before["index_daily"])
for name in GROUP_A:
self.assertEqual(after[name], before[name], name)
def test_admin_backfill_official_dataset_uses_boundary(self) -> None:
from datahub.admin_api import AdminAPI
from datahub.auth import AuthService
from datahub.crypto import SecretVault
from datahub.scheduler import Scheduler
from datahub.serving import ApiError
vault = SecretVault(self.pipe.settings.encryption_key)
auth = AuthService(self.db, vault, self.pipe.settings.api_token, "StartPass1")
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth)
before = publications_map(self.db, TRADE_DATE)
result = admin.backfill("moneyflow", TRADE_DATE, "StartPass1", f"moneyflow:{TRADE_DATE}", "tester")
self.assertEqual(result["moneyflow"]["state"], "published")
after = publications_map(self.db, TRADE_DATE)
for name in (*GROUP_A, "stocks"):
self.assertNotEqual(after[name], before[name], name)
# bad password / wrong confirm still rejected
with self.assertRaises(ApiError):
admin.backfill("daily", TRADE_DATE, "wrong", f"daily:{TRADE_DATE}", "tester")
def test_admin_backfill_switch_crash_is_failed_precondition(self) -> None:
from datahub.admin_api import AdminAPI
from datahub.auth import AuthService
from datahub.crypto import SecretVault
from datahub.scheduler import Scheduler
from datahub.serving import ApiError
vault = SecretVault(self.pipe.settings.encryption_key)
auth = AuthService(self.db, vault, self.pipe.settings.api_token, "StartPass1")
admin = AdminAPI(self.db, self.pipe, Scheduler(self.db, self.pipe), auth)
before = publications_map(self.db, TRADE_DATE)
def explode() -> None:
raise RuntimeError("killed mid-switch")
self.pipe.before_commit = explode
with self.assertRaises(ApiError) as ctx:
admin.backfill("valuation", TRADE_DATE, "StartPass1", f"valuation:{TRADE_DATE}", "tester")
self.assertEqual(ctx.exception.code, "FAILED_PRECONDITION")
self.assertIn("killed mid-switch", ctx.exception.message)
# previous complete A/B versions keep serving
self.assertEqual(publications_map(self.db, TRADE_DATE), before)
audit = self.db.fetchone(
"SELECT * FROM audit_log WHERE action = 'release-group' ORDER BY id DESC"
)
self.assertIsNotNone(audit)
self.assertIn("failed", str(audit["detail"]))
self.assertIn("killed mid-switch", str(audit["detail"]))
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
@@ -0,0 +1,65 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from datahub.adapters.tushare import TushareAdapter
from datahub.crypto import SecretVault
from datahub.hub import Hub
from datahub.settings import Settings
from tests.fixtures import TRADE_DATE, fake_transport
class ExtendedEodTests(unittest.TestCase):
def setUp(self) -> None:
self.tmp = tempfile.TemporaryDirectory()
key = SecretVault.generate_key()
settings = Settings(
host="127.0.0.1",
port=0,
encryption_key=key,
api_token="k" * 32,
admin_password="StartPass1",
tushare_token="tushare-secret",
db_path=Path(self.tmp.name) / "hub.db",
backup_dir=Path(self.tmp.name) / "backups",
scheduler_enabled=False,
quality={"daily_row_ratio": 0.5, "null_rate_max": 0.5, "list_limit_default": 5000, "list_limit_max": 5000},
)
adapter = TushareAdapter("tushare-secret", transport=fake_transport)
self.hub = Hub(settings, adapter=adapter)
self.hub.pipeline.ingest_reference(TRADE_DATE)
for dataset in ("daily", "valuation", "moneyflow", "auction", "index_daily"):
self.hub.pipeline.run_dataset(dataset, TRADE_DATE)
def tearDown(self) -> None:
self.hub.stop()
self.tmp.cleanup()
def test_extended_soft_datasets_publish_and_serve(self) -> None:
results = self.hub.pipeline.run_extended_soft(
("limit_events", "popularity", "dragon_tiger", "sector_daily"),
TRADE_DATE,
)
for name in ("limit_events", "popularity", "dragon_tiger", "sector_daily"):
self.assertEqual(results[name]["state"], "published", results[name])
api = self.hub.api
limits = api.handle("/v1/limit-events", {"date": [TRADE_DATE]})
self.assertGreaterEqual(len(limits["data"]), 1)
self.assertEqual(limits["meta"]["tier"], "official")
pop = api.handle("/v1/popularity", {"date": [TRADE_DATE], "source": ["ths"]})
self.assertEqual(pop["data"][0]["source"], "ths")
lhb = api.handle("/v1/dragon-tiger", {"date": [TRADE_DATE]})
self.assertEqual(lhb["data"][0]["hm_name"], "测试游资")
# hub stores 万元→元
self.assertEqual(lhb["data"][0]["buy_amount"], 10_000_000.0)
sectors = api.handle("/v1/sectors", {"date": [TRADE_DATE], "family": ["ths"]})
self.assertEqual(sectors["data"][0]["family"], "ths")
status = api.handle("/v1/datasets/status", {"date": [TRADE_DATE]})
names = {item["dataset"] for item in status["data"]}
self.assertTrue({"limit_events", "popularity", "dragon_tiger", "sector_daily"} <= names)
if __name__ == "__main__":
unittest.main()
+7 -1
View File
@@ -19,11 +19,17 @@ class LayoutTests(unittest.TestCase):
def test_reserved_adapters_present(self) -> None: def test_reserved_adapters_present(self) -> None:
from datahub.adapters import RESERVED from datahub.adapters import RESERVED
for name in ("eastmoney", "tencent", "ths", "xgb", "akshare", "ifind"): for name in ("ths", "xgb", "akshare", "ifind"):
self.assertIn(name, RESERVED) self.assertIn(name, RESERVED)
probe = RESERVED[name].probe() probe = RESERVED[name].probe()
self.assertEqual(probe["state"], "reserved") self.assertEqual(probe["state"], "reserved")
self.assertFalse(probe["configured"]) self.assertFalse(probe["configured"])
for name in ("eastmoney", "tencent"):
self.assertIn(name, RESERVED)
probe = RESERVED[name].probe()
# Live free adapters: probe may be ok/error/empty depending on network.
self.assertIn(probe["state"], {"ok", "empty", "error"})
self.assertTrue(probe["configured"])
if __name__ == "__main__": if __name__ == "__main__":
+20 -11
View File
@@ -228,25 +228,34 @@ class GateRetryInterplayTests(unittest.TestCase):
class ForceRepublishTests(unittest.TestCase): class ForceRepublishTests(unittest.TestCase):
def test_run_dataset_over_published_keeps_prev_for_rollback(self) -> None: def test_force_boundary_republish_keeps_prev_for_rollback(self) -> None:
transport = ValuationTransport() transport = ValuationTransport()
pipe, db = make_pipe(transport) pipe, db = make_pipe(transport)
pipe.ingest_reference(TRADE_DATE) pipe.ingest_reference(TRADE_DATE)
first = pipe.run_dataset("valuation", TRADE_DATE) first = pipe.run_eod_batch_a(TRADE_DATE)
first_val = first["valuation"]["batch_id"]
first_daily = first["daily"]["batch_id"]
transport.mode = "vr_all_null" transport.mode = "vr_all_null"
with self.assertRaises(QualityError): blocked = pipe.force_republish_boundary("valuation", TRADE_DATE)
pipe.run_dataset("valuation", TRADE_DATE) # gate holds: bad re-publish refused self.assertEqual(blocked["valuation"]["state"], "failed")
self.assertEqual(pipe.active_batch("valuation", TRADE_DATE), first_val)
self.assertEqual(pipe.active_batch("daily", TRADE_DATE), first_daily)
transport.mode = "ok" transport.mode = "ok"
second = pipe.run_dataset("valuation", TRADE_DATE) # CLI --force path second = pipe.force_republish_boundary("valuation", TRADE_DATE)
self.assertNotEqual(first["batch_id"], second["batch_id"]) self.assertEqual(second["valuation"]["state"], "published")
pub = db.fetchone( self.assertNotEqual(second["valuation"]["batch_id"], first_val)
"SELECT * FROM publications WHERE dataset='valuation' AND trade_date=?", self.assertNotEqual(second["daily"]["batch_id"], first_daily)
pubs = db.fetchall(
"SELECT dataset, active_batch, prev_batch, published_at FROM publications WHERE trade_date=?",
(TRADE_DATE,), (TRADE_DATE,),
) )
self.assertEqual(pub["active_batch"], second["batch_id"]) by_ds = {str(row["dataset"]): row for row in pubs}
self.assertEqual(pub["prev_batch"], first["batch_id"]) a_times = {by_ds[name]["published_at"] for name in ("daily", "valuation", "moneyflow", "auction", "stocks")}
self.assertEqual(len(a_times), 1)
self.assertEqual(by_ds["valuation"]["active_batch"], second["valuation"]["batch_id"])
self.assertEqual(by_ds["valuation"]["prev_batch"], first_val)
rolled = pipe.rollback("valuation", TRADE_DATE, actor="cli") rolled = pipe.rollback("valuation", TRADE_DATE, actor="cli")
self.assertEqual(rolled["active_batch"], first["batch_id"]) self.assertEqual(rolled["active_batch"], first_val)
if __name__ == "__main__": if __name__ == "__main__":
@@ -0,0 +1,176 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from datahub.adapters.base import AdapterError
from datahub.adapters.eastmoney import HIS_TRENDS_URL, TRENDS_URL, EastmoneyAdapter
from datahub.db import HubDB
from datahub.realtime_serve import fetch_intraday
from datahub.serving import ApiError, V1API
from datahub.timeutil import now_shanghai, yyyymmdd
class FakeEastmoney(EastmoneyAdapter):
def __init__(self) -> None:
super().__init__(timeout=2)
self.urls: list[str] = []
def _get_json(self, url, params, referer):
self.urls.append(f"{url}|{params.get('ndays')}")
if url == TRENDS_URL:
return {"data": {"name": "中国平安", "code": "601318", "preClose": 56.36, "trends": []}}
if url == HIS_TRENDS_URL:
return {
"data": {
"name": "中国平安",
"code": "601318",
"preClose": 55.8,
"trends": [
"2026-09-07 09:30,55.80,55.90,56.00,55.70,100,5580.00,55.900",
"2026-09-07 15:00,56.10,56.20,56.30,56.00,200,11240.00,56.150",
"2026-09-08 09:30,0,0,0,0,0,0.00,0",
],
}
}
raise AdapterError(f"unexpected url {url}")
class EastmoneyIntradayLookbackTests(unittest.TestCase):
def test_empty_today_uses_latest_available_session(self):
adapter = FakeEastmoney()
payload = adapter.fetch_intraday("601318.SH")
self.assertEqual(adapter.urls, [f"{TRENDS_URL}|1", f"{TRENDS_URL}|5", f"{HIS_TRENDS_URL}|5"])
self.assertEqual(payload["trade_date"], "2026-09-07")
self.assertEqual([point["time"] for point in payload["points"]], ["09:30", "15:00"])
self.assertEqual(payload["points"][0]["close"], 55.9)
def test_preferred_date_keeps_that_session(self):
adapter = FakeEastmoney()
payload = adapter.fetch_intraday("601318.SH", "20260907")
self.assertEqual(payload["trade_date"], "2026-09-07")
self.assertEqual(len(payload["points"]), 2)
class IntradayLkgTests(unittest.TestCase):
def setUp(self) -> None:
self.tmp = tempfile.TemporaryDirectory()
self.db = HubDB(Path(self.tmp.name) / "hub.db")
def tearDown(self) -> None:
self.tmp.cleanup()
def test_source_failure_returns_last_known_good(self):
from datahub.realtime_serve import _envelope, _write_cache
payload = _envelope(
{
"entity_type": "stock",
"ts_code": "601318.SH",
"trade_date": "2026-09-07",
"previous_close": 55.8,
"points": [{"date": "2026-09-07", "time": "09:30", "close": 55.9}],
},
{
"tier": "provisional",
"trade_date": "20260907",
"source": "eastmoney:trends2",
"stale": False,
},
)
_write_cache(self.db, "intraday:601318.SH:today", payload, 20, "eastmoney:trends2")
self.db.execute(
"UPDATE rt_cache SET expires_at = ? WHERE cache_key = ?",
("2000-01-01T00:00:00+08:00", "intraday:601318.SH:today"),
)
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
mocked.return_value.fetch_intraday.side_effect = AdapterError("down")
recovered = fetch_intraday(self.db, "601318.SH")
self.assertTrue(recovered["meta"]["stale"])
self.assertEqual(recovered["data"]["points"][0]["close"], 55.9)
def test_source_failure_without_lkg_raises(self):
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
mocked.return_value.fetch_intraday.side_effect = AdapterError("down")
with self.assertRaises(Exception) as ctx:
fetch_intraday(self.db, "000001.SZ")
self.assertIn("intraday unavailable", str(ctx.exception))
class ServingIntradayDateTests(unittest.TestCase):
def setUp(self) -> None:
self.tmp = tempfile.TemporaryDirectory()
self.db = HubDB(Path(self.tmp.name) / "hub.db")
self.api = V1API(self.db, pipeline=None, settings=None)
def tearDown(self) -> None:
self.tmp.cleanup()
def _assert_usable_intraday(self, payload: dict) -> None:
data = payload["data"]
points = [point for point in data.get("points") or [] if float(point.get("close") or 0) > 0]
self.assertGreaterEqual(len(points), 1)
self.assertTrue(str(data.get("trade_date") or ""))
self.assertFalse((payload.get("meta") or {}).get("stale"))
def test_serving_omitted_or_empty_date_uses_today_and_returns_points(self) -> None:
today = yyyymmdd(now_shanghai())
omitted = self.api.handle("/v1/intraday/points", {"code": ["601318"]})
empty = self.api.handle("/v1/intraday/points", {"code": ["601318"], "date": [""]})
explicit = self.api.handle("/v1/intraday/points", {"code": ["601318"], "date": [today]})
self._assert_usable_intraday(omitted)
self._assert_usable_intraday(empty)
self._assert_usable_intraday(explicit)
self.assertEqual(omitted["data"]["trade_date"], empty["data"]["trade_date"])
self.assertEqual(explicit["data"]["trade_date"], omitted["data"]["trade_date"])
def test_serving_normalizes_empty_date_to_today_and_keeps_history(self) -> None:
today = yyyymmdd(now_shanghai())
captured: list[str] = []
def fake_fetch(db, code, date=""):
captured.append(date)
return {
"schema_version": 1,
"data": {
"trade_date": f"{date[:4]}-{date[4:6]}-{date[6:8]}",
"points": [{"date": f"{date[:4]}-{date[4:6]}-{date[6:8]}", "time": "09:30", "close": 55.9}],
},
"meta": {"stale": False, "trade_date": date},
}
with patch("datahub.realtime_serve.fetch_intraday", side_effect=fake_fetch):
omitted = self.api.handle("/v1/intraday/points", {"code": ["601318"]})
empty = self.api.handle("/v1/intraday/points", {"code": ["601318"], "date": [" "]})
history = self.api.handle("/v1/intraday/points", {"code": ["601318"], "date": ["20260907"]})
self.assertEqual(captured, [today, today, "20260907"])
self.assertEqual(omitted["data"]["trade_date"], f"{today[:4]}-{today[4:6]}-{today[6:8]}")
self.assertEqual(empty["data"]["trade_date"], omitted["data"]["trade_date"])
self.assertEqual(history["data"]["trade_date"], "2026-09-07")
def test_serving_invalid_date_is_invalid_argument(self) -> None:
with self.assertRaises(ApiError) as ctx:
self.api.handle("/v1/intraday/points", {"code": ["601318"], "date": ["not-a-date"]})
self.assertEqual(ctx.exception.code, "INVALID_ARGUMENT")
self.assertIn("invalid trade_date", ctx.exception.message)
def test_serving_missing_code_is_invalid_argument(self) -> None:
with self.assertRaises(ApiError) as ctx:
self.api.handle("/v1/intraday/points", {"date": [yyyymmdd(now_shanghai())]})
self.assertEqual(ctx.exception.code, "INVALID_ARGUMENT")
self.assertIn("code is required", ctx.exception.message)
def test_serving_no_data_keeps_source_unavailable(self) -> None:
with patch("datahub.realtime_serve.EastmoneyAdapter") as mocked:
mocked.return_value.fetch_intraday.side_effect = AdapterError("No intraday chart data returned")
with self.assertRaises(ApiError) as ctx:
self.api.handle("/v1/intraday/points", {"code": ["000001"]})
self.assertEqual(ctx.exception.code, "SOURCE_UNAVAILABLE")
self.assertIn("intraday unavailable", ctx.exception.message)
if __name__ == "__main__":
unittest.main()