leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(HEL-421): 回补历史日历和指数并标记区间不完整

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
2026-09-02 22:16:05 +08:00
co-authored by Cursor multica-agent
parent 25ff6bbe06
commit a836cda1b2
18 changed files with 715 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/datahub/__main__.py
+4
View File
@@ -0,0 +1,4 @@
from datahub.cli import main
if __name__ == "__main__":
raise SystemExit(main())
xiaobai-datahub/datahub/adapters/tushare.py
+4 -1
View File
@@ -44,7 +44,10 @@ DATASET_API = {
"auction": "stk_auction",
}
DEFAULT_INDEX_CODES = ("000001.SH", "399001.SZ", "399006.SZ", "000300.SH")
# Website actual index usage: market cards / 90-day charts (SH/SZ/CYB) plus
# screener 沪深300 benchmark (lookback up to 260 trading days).
WEBSITE_INDEX_CODES = ("000001.SH", "399001.SZ", "399006.SZ", "000300.SH")
DEFAULT_INDEX_CODES = WEBSITE_INDEX_CODES
class TushareAdapter(MarketAdapter):
xiaobai-datahub/datahub/admin_api.py
+11 -5
View File
@@ -88,6 +88,7 @@ class AdminAPI:
{"id": "precheck", "at": "08:45", "title": "盘前预检"},
{"id": "eod_a", "at": "15:05", "title": "盘后批 A daily/valuation/moneyflow/auction"},
{"id": "eod_b", "at": "15:10", "title": "盘后批 B index_daily"},
{"id": "history_backfill", "at": "manual", "title": "回补历史日历与指数日 K"},
{"id": "cleanup", "at": "00:30", "title": "清理 staging / 日志"},
{"id": "backup", "at": "00:40", "title": "SQLite 备份"},
],
@@ -130,12 +131,17 @@ class AdminAPI:
return result
def backfill(self, dataset: str, trade_date: str, password: str, confirm: str, actor: str) -> dict[str, Any]:
self._dangerous(password, confirm, f"{dataset}:{trade_date}")
if dataset == "reference":
result = self.pipeline.ingest_reference(trade_date)
day = yyyymmdd(trade_date or now_shanghai())
if dataset == "history":
self._dangerous(password, confirm, "history:full")
result = self.pipeline.backfill_history(day)
else:
result = self.pipeline.run_dataset(dataset, trade_date)
self.pipeline.audit(actor, "backfill", f"{dataset}:{trade_date}", json.dumps({"ok": True}))
self._dangerous(password, confirm, f"{dataset}:{day}")
if dataset == "reference":
result = self.pipeline.ingest_reference(day)
else:
result = self.pipeline.run_dataset(dataset, day)
self.pipeline.audit(actor, "backfill", f"{dataset}:{day}", json.dumps({"ok": True}))
return result
def _dangerous(self, password: str, confirm: str, expected: str) -> None:
xiaobai-datahub/datahub/cli.py
+38
View File
@@ -0,0 +1,38 @@
"""Command-line entry for one-shot datahub operations."""
from __future__ import annotations
import argparse
import json
import sys
from datahub.hub import build_hub
from datahub.settings import load_settings
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="xiaobai-datahub CLI")
sub = parser.add_subparsers(dest="command", required=True)
history = sub.add_parser("history-backfill", help="回补 2016 年起交易日历和网站所用指数日 K")
history.add_argument("--calendar-start", default=None, help="日历起点，默认配置 calendar_start")
history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量，默认 260")
history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期")
args = parser.parse_args(argv)
settings = load_settings()
hub = build_hub(settings)
if args.command == "history-backfill":
result = hub.pipeline.backfill_history(
calendar_start=args.calendar_start,
index_days=args.index_days,
force=args.force,
)
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0 if result.get("ok") else 1
parser.error(f"unknown command: {args.command}")
return 2
if __name__ == "__main__":
raise SystemExit(main())
xiaobai-datahub/datahub/coverage.py
+130
View File
@@ -0,0 +1,130 @@
from __future__ import annotations
from typing import Any, Iterable
from datahub.db import HubDB
from datahub.timeutil import iter_yyyymmdd, yyyymmdd
MISSING_SAMPLE_LIMIT = 10
def coverage_payload(
*,
kind: str,
start: str,
end: str,
expected: Iterable[str],
available: Iterable[str],
extra: dict[str, Any] | None = None,
) -> dict[str, Any]:
start = yyyymmdd(start)
end = yyyymmdd(end)
expected_list = sorted({yyyymmdd(item) for item in expected if item})
available_set = {yyyymmdd(item) for item in available if item}
missing = [item for item in expected_list if item not in available_set]
payload: dict[str, Any] = {
"kind": kind,
"complete": not missing,
"requested_from": start,
"requested_to": end,
"available_from": min(available_set) if available_set else None,
"available_to": max(available_set) if available_set else None,
"expected_count": len(expected_list),
"available_count": len(available_set),
"missing_count": len(missing),
"missing_sample": missing[:MISSING_SAMPLE_LIMIT],
}
if extra:
payload.update(extra)
return payload
def calendar_coverage(db: HubDB, start: str, end: str, exchange: str = "SSE") -> dict[str, Any]:
start = yyyymmdd(start)
end = yyyymmdd(end)
expected = list(iter_yyyymmdd(start, end))
rows = db.fetchall(
"SELECT cal_date FROM trade_calendar WHERE exchange = ? AND cal_date >= ? AND cal_date <= ?",
(exchange, start, end),
)
return coverage_payload(
kind="calendar",
start=start,
end=end,
expected=expected,
available=(row["cal_date"] for row in rows),
extra={"exchange": exchange},
)
def published_range_coverage(
db: HubDB,
dataset: str,
start: str,
end: str,
ts_code: str = "",
table: str = "",
) -> dict[str, Any]:
start = yyyymmdd(start)
end = yyyymmdd(end)
calendar = calendar_coverage(db, start, end)
open_rows = db.fetchall(
"""
SELECT cal_date FROM trade_calendar
WHERE exchange = 'SSE' AND is_open = 1 AND cal_date >= ? AND cal_date <= ?
ORDER BY cal_date
""",
(start, end),
)
expected_open = [row["cal_date"] for row in open_rows]
pubs = db.fetchall(
"""
SELECT trade_date, active_batch FROM publications
WHERE dataset = ? AND trade_date >= ? AND trade_date <= ?
ORDER BY trade_date
""",
(dataset, start, end),
)
published_dates = [row["trade_date"] for row in pubs]
available = list(published_dates)
extra: dict[str, Any] = {
"dataset": dataset,
"calendar_complete": calendar["complete"],
"calendar_missing_count": calendar["missing_count"],
}
if ts_code and table and pubs:
present_code: list[str] = []
for pub in pubs:
hit = db.fetchone(
f"SELECT 1 AS ok FROM {table} WHERE trade_date = ? AND batch_id = ? AND ts_code = ? LIMIT 1",
(pub["trade_date"], pub["active_batch"], ts_code),
)
if hit:
present_code.append(pub["trade_date"])
available = present_code
extra["code"] = ts_code
payload = coverage_payload(
kind="published_range",
start=start,
end=end,
expected=expected_open,
available=available,
extra=extra,
)
if not calendar["complete"]:
payload["complete"] = False
payload["calendar_missing_sample"] = calendar["missing_sample"]
return payload
def point_coverage(trade_date: str, dataset: str = "") -> dict[str, Any]:
day = yyyymmdd(trade_date)
payload = coverage_payload(
kind="point",
start=day,
end=day,
expected=[day],
available=[day],
extra={"dataset": dataset} if dataset else None,
)
return payload
xiaobai-datahub/datahub/pipeline.py
+168 -13
View File
@@ -7,7 +7,7 @@ from datetime import timedelta
from typing import Any
from datahub.adapters.base import AdapterError
from datahub.adapters.tushare import DEFAULT_INDEX_CODES, TushareAdapter
from datahub.adapters.tushare import DEFAULT_INDEX_CODES, WEBSITE_INDEX_CODES, TushareAdapter
from datahub.db import DATASET_TABLES, HubDB
from datahub.governance.circuit import CircuitBreaker
from datahub.governance.ratelimit import TokenBucket
@@ -141,11 +141,22 @@ class Pipeline:
seq = int((row or {}).get("n") or 0) + 1
return f"{trade_date}-{dataset}-{seq:03d}"
def ingest_reference(self, trade_date: str | None = None) -> dict[str, Any]:
"""Refresh trade calendar (window) and stock master. Not versioned by batch."""
def ingest_reference(
self,
trade_date: str | None = None,
start: str | None = None,
end: str | None = None,
) -> dict[str, Any]:
"""Refresh trade calendar and stock master. Not versioned by batch.
Calendar defaults to 2016-01-01 through today+30 so a 5-year website
query is not silently truncated. UPSERT makes repeats safe.
"""
day = yyyymmdd(trade_date or self.clock())
start = add_days(day, -400)
end = add_days(day, 30)
start = yyyymmdd(start or self.settings.calendar_start)
end = yyyymmdd(end or add_days(day, 30))
if start > end:
start, end = end, start
calendar = self.adapter.normalize(
"calendar",
self._guarded_fetch("calendar", {"exchange": "SSE", "start_date": start, "end_date": end}),
@@ -180,9 +191,150 @@ class Pipeline:
row.get("list_date"), fetched_at,
),
)
return {"calendar": len(calendar), "stocks": len(stocks), "trade_date": day}
return {
"calendar": len(calendar),
"stocks": len(stocks),
"trade_date": day,
"calendar_from": start,
"calendar_to": end,
}
def run_dataset(self, dataset: str, trade_date: str, attempts: int | None = None) -> dict[str, Any]:
def open_trade_dates(self, end: str, limit: int) -> list[str]:
end = yyyymmdd(end)
rows = self.db.fetchall(
"""
SELECT cal_date FROM trade_calendar
WHERE exchange = 'SSE' AND is_open = 1 AND cal_date <= ?
ORDER BY cal_date DESC
LIMIT ?
""",
(end, max(1, int(limit))),
)
return sorted(str(row["cal_date"]) for row in rows)
def backfill_history(
self,
trade_date: str | None = None,
calendar_start: str | None = None,
index_days: int | None = None,
codes: tuple[str, ...] | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Idempotent calendar + website-index history backfill."""
day = yyyymmdd(trade_date or self.clock())
calendar = self.ingest_reference(day, start=calendar_start)
index = self.backfill_index_history(
end_date=day,
trading_days=index_days,
codes=codes,
force=force,
)
return {"calendar": calendar, "index_daily": index, "ok": bool(index.get("ok"))}
def backfill_index_history(
self,
end_date: str | None = None,
trading_days: int | None = None,
codes: tuple[str, ...] | None = None,
force: bool = False,
) -> dict[str, Any]:
"""Incrementally publish official index bars for website index codes.
One range fetch per code, then per-day publish. Already published dates
are skipped unless ``force``. Failures are recorded and do not roll back
successful days.
"""
end = yyyymmdd(end_date or self.clock())
limit = int(trading_days or self.settings.index_history_trading_days)
codes = tuple(codes or WEBSITE_INDEX_CODES)
open_dates = self.open_trade_dates(end, limit)
if not open_dates:
return {
"start": None,
"end": end,
"codes": list(codes),
"requested_days": 0,
"published": [],
"skipped": [],
"failed": [{"error": "calendar has no open dates on or before end"}],
"ok": False,
}
start = open_dates[0]
complete_dates = set() if force else self._index_dates_with_all_codes(start, end, codes)
targets = [day for day in open_dates if day not in complete_dates]
skipped = [day for day in open_dates if day in complete_dates]
by_date: dict[str, list[dict[str, Any]]] = {day: [] for day in targets}
failed: list[dict[str, Any]] = []
for ts_code in codes:
try:
raw = retry_call(
lambda code=ts_code: self._guarded_fetch(
"index_daily",
{"ts_code": code, "start_date": start, "end_date": end},
),
attempts=self.settings.max_publish_attempts,
base_delay=0.05,
sleeper=lambda _d: time.sleep(_d),
)
for row in self.adapter.normalize("index_daily", raw):
day = str(row.get("trade_date") or "")
if day in by_date:
by_date[day].append(row)
except Exception as exc:
failed.append({"ts_code": ts_code, "error": str(exc)})
published: list[dict[str, Any]] = []
for day in targets:
rows = by_date.get(day) or []
try:
result = self.run_dataset("index_daily", day, prepared_rows=rows)
published.append(
{
"trade_date": day,
"batch_id": result["batch_id"],
"rows": result["rows"],
"state": result["state"],
}
)
except Exception as exc:
failed.append({"trade_date": day, "error": str(exc), "rows": len(rows)})
return {
"start": start,
"end": end,
"codes": list(codes),
"requested_days": len(open_dates),
"published": published,
"skipped": skipped,
"failed": failed,
"ok": not failed,
}
def _index_dates_with_all_codes(self, start: str, end: str, codes: tuple[str, ...]) -> set[str]:
pubs = self.db.fetchall(
"""
SELECT trade_date, active_batch FROM publications
WHERE dataset = 'index_daily' AND trade_date >= ? AND trade_date <= ?
""",
(start, end),
)
needed = set(codes)
complete: set[str] = set()
for pub in pubs:
rows = self.db.fetchall(
"SELECT DISTINCT ts_code FROM eod_index_bars WHERE trade_date = ? AND batch_id = ?",
(pub["trade_date"], pub["active_batch"]),
)
have = {str(row["ts_code"]) for row in rows}
if needed <= have:
complete.add(str(pub["trade_date"]))
return complete
def run_dataset(
self,
dataset: str,
trade_date: str,
attempts: int | None = None,
prepared_rows: list[dict[str, Any]] | None = None,
) -> dict[str, Any]:
trade_date = yyyymmdd(trade_date)
batch_id = self.next_batch_id(dataset, trade_date)
max_attempts = attempts or self.settings.max_publish_attempts
@@ -190,12 +342,15 @@ class Pipeline:
rows: list[dict[str, Any]] = []
try:
self._set_batch(batch_id, dataset, trade_date, "fetching", 1)
rows = retry_call(
lambda: self._fetch_dataset(dataset, trade_date),
attempts=max_attempts,
base_delay=0.05,
sleeper=lambda _d: None if attempts == 1 else time.sleep(_d),
)
if prepared_rows is None:
rows = retry_call(
lambda: self._fetch_dataset(dataset, trade_date),
attempts=max_attempts,
base_delay=0.05,
sleeper=lambda _d: None if attempts == 1 else time.sleep(_d),
)
else:
rows = list(prepared_rows)
self._stage(dataset, batch_id, rows)
self._set_batch(batch_id, dataset, trade_date, "staged", 1, rows_in=len(rows), rows_out=len(rows))
self._set_batch(batch_id, dataset, trade_date, "validating", 1)
xiaobai-datahub/datahub/scheduler.py
+4
View File
@@ -37,6 +37,7 @@ class Scheduler:
"eod_b": self._eod_b,
"cleanup": self._cleanup,
"backup": self._backup,
"history_backfill": self._history_backfill,
}
self._stop = threading.Event()
self._thread: threading.Thread | None = None
@@ -125,6 +126,9 @@ class Scheduler:
def _eod_b(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_b(trade_date)
def _history_backfill(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.backfill_history(trade_date)
def _cleanup(self, trade_date: str) -> dict[str, Any]:
result = self.pipeline.cleanup()
if now_shanghai().weekday() == 6:
xiaobai-datahub/datahub/serving.py
+31 -11
View File
@@ -6,6 +6,7 @@ from urllib.parse import parse_qs
from datahub import SCHEMA_VERSION
from datahub.codes import resolve_code
from datahub.coverage import calendar_coverage, point_coverage, published_range_coverage
from datahub.db import HubDB
from datahub.normalize import qfq_bar
from datahub.numbers import finite_number
@@ -129,7 +130,8 @@ class V1API:
}
for row in rows
]
return envelope(items, self._official_meta("calendar", end if items else start, source="tushare:trade_cal"))
meta = self._official_meta("calendar", end if items else start, source="tushare:trade_cal")
return envelope(items, attach_coverage(meta, calendar_coverage(self.db, start, end)))
def stocks(self, updated_since: str, q: dict[str, str]) -> dict[str, Any]:
limit, offset = self._page(q)
@@ -272,7 +274,7 @@ class V1API:
"staleness_seconds": 0,
"state": pub["state"],
}
return envelope(rows, meta)
return envelope(rows, attach_coverage(meta, point_coverage(start, dataset)))
# multi-day: walk published dates
pubs = self.db.fetchall(
"SELECT * FROM publications WHERE dataset = ? AND trade_date >= ? AND trade_date <= ? ORDER BY trade_date",
@@ -294,17 +296,28 @@ class V1API:
if adjust == "qfq" and dataset == "daily":
sliced = self._apply_qfq(sliced)
last = pubs[-1]
coverage = published_range_coverage(
self.db,
dataset,
start,
end,
ts_code=ts_code,
table=table,
)
return envelope(
sliced,
{
"tier": "official",
"trade_date": last["trade_date"],
"published_at": last["published_at"],
"source": source,
"batch_id": last["active_batch"],
"stale": False,
"staleness_seconds": 0,
},
attach_coverage(
{
"tier": "official",
"trade_date": last["trade_date"],
"published_at": last["published_at"],
"source": source,
"batch_id": last["active_batch"],
"stale": False,
"staleness_seconds": 0,
},
coverage,
),
)
def _apply_qfq(self, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -359,6 +372,13 @@ def add_default(days: int) -> str:
return (now_shanghai() + timedelta(days=days)).strftime("%Y%m%d")
def attach_coverage(meta: dict[str, Any], coverage: dict[str, Any]) -> dict[str, Any]:
merged = dict(meta)
merged["coverage"] = coverage
merged["incomplete"] = not bool(coverage.get("complete"))
return merged
def parse_query(raw: str) -> dict[str, list[str]]:
return parse_qs(raw, keep_blank_values=True)
xiaobai-datahub/datahub/settings.py
+8
View File
@@ -48,6 +48,14 @@ class Settings:
def list_limit_max(self) -> int:
return int(self.quality.get("list_limit_max") or 5000)
@property
def calendar_start(self) -> str:
return str(self.quality.get("calendar_start") or "20160101")
@property
def index_history_trading_days(self) -> int:
return int(self.quality.get("index_history_trading_days") or 260)
def load_settings(
env: dict[str, str] | None = None,
xiaobai-datahub/datahub/timeutil.py
+10
View File
@@ -58,6 +58,16 @@ def add_days(trade_date: str, days: int) -> str:
return (parse_trade_date(trade_date) + timedelta(days=days)).strftime("%Y%m%d")
def iter_yyyymmdd(start: str, end: str):
cursor = parse_trade_date(start)
last = parse_trade_date(end)
if cursor > last:
return
while cursor <= last:
yield cursor.strftime("%Y%m%d")
cursor += timedelta(days=1)
def utc_timestamp(value: Any) -> str:
if isinstance(value, datetime):
return isoformat(value)