leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(HEL-494): 修复个股缺失指标、问天遮罩、四爻外显并回补250日K

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
2026-09-08 16:26:13 +08:00
co-authored by Cursor multica-agent
parent c8a9376adb
commit 3e828b346c
24 changed files with 1011 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/config/hub-quality.config.json
+1
View File
@@ -14,6 +14,7 @@
"list_limit_max": 5000,
"calendar_start": "20160101",
"index_history_trading_days": 260,
"daily_history_trading_days": 250,
"eod_retry_start": "15:15",
"eod_retry_interval_minutes": 30,
"eod_retry_cutoff": "23:30",
xiaobai-datahub/datahub/admin_api.py
+1 -1
View File
@@ -94,7 +94,7 @@ class AdminAPI:
{"id": "eod_retry", "at": "15:15-23:30", "title": "盘后未出数自动重试（每 30 分钟，成功即停）"},
{"id": "eod_revise", "at": "20:00-23:20", "title": "估值发布后复核（轻量比对，有修订才整组原子追补）"},
{"id": "stocks_refresh", "at": stocks_times, "title": "股票主档刷新与正式发布（新上市/更名，无变化跳过）"},
{"id": "history_backfill", "at": "manual", "title": "回补历史日历与指数日 K"},
{"id": "history_backfill", "at": "manual", "title": "回补历史日历、个股日 K 与指数日 K"},
{"id": "cleanup", "at": "00:30", "title": "清理 staging / 日志"},
{"id": "backup", "at": "00:40", "title": "SQLite 备份"},
],
xiaobai-datahub/datahub/cli.py
+4 -2
View File
@@ -15,10 +15,11 @@ from datahub.timeutil import yyyymmdd
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="xiaobai-datahub CLI")
sub = parser.add_subparsers(dest="command", required=True)
history = sub.add_parser("history-backfill", help="回补 2016 年起交易日历和网站所用指数日 K")
history = sub.add_parser("history-backfill", help="回补交易日历、个股日 K（默认 250 日）和网站所用指数日 K")
history.add_argument("--calendar-start", default=None, help="日历起点，默认配置 calendar_start")
history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量，默认 260")
history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期")
history.add_argument("--daily-days", type=int, default=None, help="个股日 K 回补交易日数量，默认 250")
history.add_argument("--force", action="store_true", help="覆盖已发布的个股日 K / 指数日期")
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据（跳过已完整发布的一致性边界，仍走质量门禁）")
refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD，默认今天")
refresh.add_argument(
@@ -46,6 +47,7 @@ def main(argv: list[str] | None = None) -> int:
result = hub.pipeline.backfill_history(
calendar_start=args.calendar_start,
index_days=args.index_days,
daily_days=args.daily_days,
force=args.force,
)
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
xiaobai-datahub/datahub/pipeline.py
+98 -10
View File
@@ -491,24 +491,96 @@ class Pipeline:
)
return sorted(str(row["cal_date"]) for row in rows)
def backfill_daily_history(
self,
end_date: str | None = None,
trading_days: int | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Incrementally publish official daily bars for the website K-line window.
Same-day EOD still uses the atomic A-group. History backfill publishes
daily (with adj_factor) first so K-line coverage cannot be blocked by
the looser moneyflow universe, then valuation and moneyflow/auction
best-effort. Complete daily dates are skipped unless ``force``.
"""
end = yyyymmdd(end_date or self.clock())
limit = int(trading_days or self.settings.daily_history_trading_days)
open_dates = self.open_trade_dates(end, limit)
if not open_dates:
return {
"start": None,
"end": end,
"requested_days": 0,
"published": [],
"skipped": [],
"failed": [{"error": "calendar has no open dates on or before end"}],
"ok": False,
}
start = open_dates[0]
published: list[dict[str, Any]] = []
skipped: list[str] = []
failed: list[dict[str, Any]] = []
for day in open_dates:
if not force and self.active_batch("daily", day):
skipped.append(day)
continue
try:
daily = self.run_dataset("daily", day)
datasets = {"daily": daily.get("state")}
try:
valuation = self.run_dataset("valuation", day)
datasets["valuation"] = valuation.get("state")
except Exception as exc:
datasets["valuation"] = f"failed:{exc}"[:180]
for name in ("moneyflow", "auction"):
try:
extra = self.run_dataset(name, day)
datasets[name] = extra.get("state")
except Exception as exc:
datasets[name] = f"failed:{exc}"[:180]
published.append({"trade_date": day, "datasets": datasets})
except Exception as exc:
failed.append({"trade_date": day, "error": str(exc)})
return {
"start": start,
"end": end,
"requested_days": len(open_dates),
"published": published,
"skipped": skipped,
"failed": failed,
"ok": not failed,
}
def backfill_history(
self,
trade_date: str | None = None,
calendar_start: str | None = None,
index_days: int | None = None,
daily_days: int | None = None,
codes: tuple[str, ...] | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Idempotent calendar + website-index history backfill."""
"""Idempotent calendar + stock daily + website-index history backfill."""
day = yyyymmdd(trade_date or self.clock())
calendar = self.ingest_reference(day, start=calendar_start)
daily = self.backfill_daily_history(
end_date=day,
trading_days=daily_days,
force=force,
)
index = self.backfill_index_history(
end_date=day,
trading_days=index_days,
codes=codes,
force=force,
)
return {"calendar": calendar, "index_daily": index, "ok": bool(index.get("ok"))}
return {
"calendar": calendar,
"daily": daily,
"index_daily": index,
"ok": bool(daily.get("ok")) and bool(index.get("ok")),
}
def backfill_index_history(
self,
@@ -803,10 +875,7 @@ class Pipeline:
"published_rows": len(published),
"upstream_rows": 0,
}
listed = self.db.fetchone(
"SELECT COUNT(*) AS n FROM stock_master WHERE list_status = 'L'",
)
listed_n = int((listed or {}).get("n") or 0)
listed_n = self._listed_count(day)
floor = float(self.settings.quality.get("daily_row_ratio") or 0.98)
if listed_n and len(upstream) / listed_n < floor:
return {
@@ -1314,14 +1383,33 @@ class Pipeline:
if isinstance(item, dict) and item.get("state") == "failed"
]
def _listed_count(self, trade_date: str = "") -> int:
"""Count listed names that already existed on ``trade_date``.
Historical daily bars must not be judged against later IPOs, or a
correct past session fails the 0.98 row-ratio gate.
"""
day = yyyymmdd(trade_date) if trade_date else ""
if day:
listed = self.db.fetchone(
"""
SELECT COUNT(*) AS n FROM stock_master
WHERE list_status = 'L'
AND (list_date IS NULL OR TRIM(list_date) = '' OR list_date <= ?)
""",
(day,),
)
else:
listed = self.db.fetchone(
"SELECT COUNT(*) AS n FROM stock_master WHERE list_status = 'L'"
)
return int((listed or {}).get("n") or 0)
def validate(self, dataset: str, batch_id: str, trade_date: str, rows: list[dict[str, Any]]) -> dict[str, Any]:
quality = self.settings.quality
errors: list[str] = []
warnings: list[str] = []
listed = self.db.fetchone(
"SELECT COUNT(*) AS n FROM stock_master WHERE list_status = 'L'",
)
listed_n = int((listed or {}).get("n") or 0)
listed_n = self._listed_count(trade_date)
row_n = len(rows)
if dataset == "limit_events":
keys = [(row.get("ts_code"), row.get("trade_date"), row.get("limit_type")) for row in rows]
xiaobai-datahub/datahub/settings.py
+4
View File
@@ -56,6 +56,10 @@ class Settings:
def index_history_trading_days(self) -> int:
return int(self.quality.get("index_history_trading_days") or 260)
@property
def daily_history_trading_days(self) -> int:
return int(self.quality.get("daily_history_trading_days") or 250)
@property
def moneyflow_history_trading_days(self) -> int:
return int(self.quality.get("moneyflow_history_trading_days") or 60)
xiaobai-datahub/tests/test_history_backfill.py
+56 -1
View File
@@ -5,7 +5,7 @@ from datetime import date, timedelta
from datahub.coverage import calendar_coverage, point_coverage, published_range_coverage
from datahub.serving import V1API
from tests.fixtures import TRADE_DATE, fake_transport
from tests.fixtures import RAW, TRADE_DATE, fake_transport
from tests.test_pipeline import make_pipeline
@@ -48,6 +48,8 @@ def history_transport(open_dates: list[str], extra_closed: list[str] | None = No
}
)
dated_apis = ("daily", "daily_basic", "adj_factor", "moneyflow", "stk_auction")
def transport(api_name, params, fields):
if api_name == "trade_cal":
start = str(params.get("start_date") or "")
@@ -66,6 +68,11 @@ def history_transport(open_dates: list[str], extra_closed: list[str] | None = No
if end:
rows = [row for row in rows if row["trade_date"] <= end]
return rows
if api_name in dated_apis:
day = str(params.get("trade_date") or "")
if day not in open_set:
return []
return [{**row, "trade_date": day} for row in RAW.get(api_name) or []]
return fake_transport(api_name, params, fields)
return transport
@@ -221,6 +228,54 @@ class HistoryBackfillTests(unittest.TestCase):
self.assertEqual(result["rows"], 1)
self.assertEqual(calls["n"], before)
def test_daily_history_is_idempotent_and_covers_requested_days(self) -> None:
open_dates = consecutive_open_days(TRADE_DATE, 5)
pipe, db = make_pipeline(
quality={
"index_history_trading_days": 5,
"daily_history_trading_days": 5,
"calendar_start": open_dates[0],
}
)
pipe.adapter._transport = history_transport(open_dates)
first = pipe.backfill_history(TRADE_DATE, index_days=5, daily_days=5)
self.assertTrue(first["ok"])
self.assertEqual(first["daily"]["requested_days"], 5)
self.assertEqual(len(first["daily"]["published"]), 5)
pubs = db.fetchall("SELECT trade_date FROM publications WHERE dataset='daily'")
self.assertEqual(sorted(row["trade_date"] for row in pubs), open_dates)
for day in open_dates:
rows = db.fetchall(
"""
SELECT COUNT(*) AS n FROM eod_bars
WHERE trade_date = ? AND batch_id = (
SELECT active_batch FROM publications WHERE dataset='daily' AND trade_date = ?
)
""",
(day, day),
)
self.assertEqual(rows[0]["n"], 2)
second = pipe.backfill_daily_history(end_date=TRADE_DATE, trading_days=5)
self.assertTrue(second["ok"])
self.assertEqual(second["published"], [])
self.assertEqual(second["skipped"], open_dates)
def test_daily_row_ratio_ignores_later_ipos(self) -> None:
pipe, db = make_pipeline()
pipe.adapter._transport = history_transport([TRADE_DATE])
pipe.ingest_reference(TRADE_DATE, start=TRADE_DATE)
with db.write() as connection:
connection.execute(
"INSERT INTO stock_master(ts_code, symbol, name, list_status, list_date, updated_at) "
"VALUES (?,?,?,?,?,?)",
("688001.SH", "688001", "未来上市", "L", "20250101", "2024-09-02T00:00:00+08:00"),
)
self.assertEqual(pipe._listed_count(TRADE_DATE), 2)
result = pipe.run_eod_batch_a(TRADE_DATE)
self.assertEqual(pipe.eod_failures(result), [])
self.assertEqual(result["daily"]["state"], "published")
def test_coverage_helpers_point_and_calendar(self) -> None:
pipe, db = make_pipeline()
pipe.ingest_reference(TRADE_DATE)