feat(HEL-463): 接入剩余行情数据到 datahub
扩展盘后正式集（涨跌停/人气/龙虎榜/板块日线）与盘中观察 API（报价/指数/分时），网站 bridge 按开关接入并回退旧链路；问天改为按数据依赖跟随开关，不再整栈强制旧路径。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
16ba83ec01
commit
605f97e5df
@@ -9,6 +9,11 @@ from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.adapters.tushare import DEFAULT_INDEX_CODES, WEBSITE_INDEX_CODES, TushareAdapter
|
||||
from datahub.datasets_ext import (
|
||||
EXTENDED_EOD_COPY,
|
||||
EXTENDED_SOFT_DATASETS,
|
||||
EXTENDED_STAGING_INSERT,
|
||||
)
|
||||
from datahub.db import DATASET_TABLES, HubDB
|
||||
from datahub.governance.circuit import CircuitBreaker
|
||||
from datahub.governance.ratelimit import TokenBucket
|
||||
@@ -21,12 +26,16 @@ from datahub.timeutil import add_days, isoformat, now_shanghai, yyyymmdd
|
||||
LOGGER = get_logger()
|
||||
|
||||
HARD_DATASETS = {"daily", "valuation", "index_daily"}
|
||||
SOFT_DATASETS = {"moneyflow", "auction"}
|
||||
OFFICIAL_DATASETS = HARD_DATASETS | SOFT_DATASETS
|
||||
SOFT_DATASETS = {"moneyflow", "auction"} | EXTENDED_SOFT_DATASETS
|
||||
OFFICIAL_DATASETS = HARD_DATASETS | {"moneyflow", "auction"} # A/B retry scope unchanged
|
||||
STOCKS_DATASET = "stocks"
|
||||
STOCK_SNAPSHOT_FIELDS = ("ts_code", "symbol", "name", "area", "industry", "market", "list_status", "list_date")
|
||||
EOD_A_DATASETS = ("daily", "valuation", "moneyflow", "auction")
|
||||
EOD_B_DATASETS = ("index_daily",)
|
||||
EOD_C_DATASETS = ("limit_events",)
|
||||
EOD_D_DATASETS = ("dragon_tiger",)
|
||||
EOD_E_DATASETS = ("sector_daily",)
|
||||
EOD_F_DATASETS = ("popularity",)
|
||||
EMPTY_BATCH_ERROR = "empty official batch: 0 valid rows"
|
||||
|
||||
STAGING_INSERT = {
|
||||
@@ -80,6 +89,7 @@ STAGING_INSERT = {
|
||||
r.get("close"), r.get("pct_chg"), r.get("volume"), r.get("amount"),
|
||||
),
|
||||
),
|
||||
**EXTENDED_STAGING_INSERT,
|
||||
}
|
||||
|
||||
EOD_COPY = {
|
||||
@@ -114,6 +124,7 @@ EOD_COPY = {
|
||||
"SELECT ts_code,trade_date,open,high,low,close,pct_chg,volume,amount,batch_id "
|
||||
"FROM staging_index_bars WHERE batch_id = ?"
|
||||
),
|
||||
**EXTENDED_EOD_COPY,
|
||||
}
|
||||
|
||||
|
||||
@@ -672,19 +683,69 @@ class Pipeline:
|
||||
def run_eod_batch_b(self, trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
return self.run_release_group(EOD_B_DATASETS, trade_date, force=force)
|
||||
|
||||
def run_extended_soft(self, datasets: tuple[str, ...], trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
"""Publish extended soft datasets independently (not A/B atomic)."""
|
||||
results: dict[str, Any] = {}
|
||||
day = yyyymmdd(trade_date)
|
||||
for dataset in datasets:
|
||||
if not force and self.active_batch(dataset, day):
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "skipped",
|
||||
"reason": "already_published",
|
||||
}
|
||||
continue
|
||||
try:
|
||||
rows = self._fetch_dataset(dataset, day)
|
||||
if not rows and dataset in {"popularity", "dragon_tiger"}:
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "skipped",
|
||||
"reason": "upstream_empty",
|
||||
"rows": 0,
|
||||
}
|
||||
continue
|
||||
results[dataset] = self.run_dataset(dataset, day, prepared_rows=rows)
|
||||
except Exception as exc:
|
||||
results[dataset] = {
|
||||
"dataset": dataset,
|
||||
"trade_date": day,
|
||||
"state": "failed",
|
||||
"error": str(exc),
|
||||
}
|
||||
LOGGER.exception("extended soft publish failed dataset=%s date=%s", dataset, day)
|
||||
return results
|
||||
|
||||
def run_eod_batch_c(self, trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
return self.run_extended_soft(EOD_C_DATASETS, trade_date, force=force)
|
||||
|
||||
def run_eod_batch_d(self, trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
return self.run_extended_soft(EOD_D_DATASETS, trade_date, force=force)
|
||||
|
||||
def run_eod_batch_e(self, trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
return self.run_extended_soft(EOD_E_DATASETS, trade_date, force=force)
|
||||
|
||||
def run_eod_batch_f(self, trade_date: str, force: bool = False) -> dict[str, Any]:
|
||||
return self.run_extended_soft(EOD_F_DATASETS, trade_date, force=force)
|
||||
|
||||
def force_republish_boundary(self, dataset: str, trade_date: str) -> dict[str, Any]:
|
||||
"""Force-republish the full A/B consistency boundary that owns ``dataset``.
|
||||
|
||||
CLI ``eod-refresh --force`` and admin manual backfill must not publish a
|
||||
single official member alone — that would mix old and new batches inside
|
||||
the same trade date. Naming any A-group member (or stocks) rebuilds the
|
||||
whole A group; naming ``index_daily`` rebuilds B.
|
||||
whole A group; naming ``index_daily`` rebuilds B. Extended soft datasets
|
||||
republish independently.
|
||||
"""
|
||||
name = str(dataset or "").strip()
|
||||
if name in EOD_A_DATASETS or name == STOCKS_DATASET:
|
||||
return self.run_eod_batch_a(trade_date, force=True)
|
||||
if name in EOD_B_DATASETS:
|
||||
return self.run_eod_batch_b(trade_date, force=True)
|
||||
if name in EXTENDED_SOFT_DATASETS:
|
||||
return self.run_extended_soft((name,), trade_date, force=True)
|
||||
raise ValueError(f"dataset is not part of an EOD release boundary: {dataset}")
|
||||
|
||||
def run_release_group(
|
||||
@@ -1022,7 +1083,16 @@ class Pipeline:
|
||||
)
|
||||
listed_n = int((listed or {}).get("n") or 0)
|
||||
row_n = len(rows)
|
||||
keys = [(row.get("ts_code"), row.get("trade_date")) for row in rows]
|
||||
if dataset == "limit_events":
|
||||
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("limit_type")) for row in rows]
|
||||
elif dataset == "popularity":
|
||||
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("source")) for row in rows]
|
||||
elif dataset == "dragon_tiger":
|
||||
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("hm_name")) for row in rows]
|
||||
elif dataset == "sector_daily":
|
||||
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("family")) for row in rows]
|
||||
else:
|
||||
keys = [(row.get("ts_code"), row.get("trade_date")) for row in rows]
|
||||
dup = row_n - len(set(keys))
|
||||
if dup:
|
||||
errors.append(f"duplicate keys: {dup}")
|
||||
@@ -1043,7 +1113,8 @@ class Pipeline:
|
||||
errors.append(EMPTY_BATCH_ERROR)
|
||||
field_report = self._field_gate(dataset, trade_date, rows, errors)
|
||||
if dataset in SOFT_DATASETS:
|
||||
hard_fail = bool(dup or bad_date or empty)
|
||||
allow_empty = dataset in {"popularity", "dragon_tiger", "moneyflow", "auction"}
|
||||
hard_fail = bool(dup or bad_date or (empty and not allow_empty))
|
||||
else:
|
||||
hard_fail = bool(errors) and (dataset in HARD_DATASETS or dataset == STOCKS_DATASET)
|
||||
report = {
|
||||
|
||||
Reference in New Issue
Block a user