feat(HEL-421): 回补历史日历和指数并标记区间不完整
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
co-authored by
Cursor
multica-agent
Cursor
multica-agent
parent
25ff6bbe06
commit
a836cda1b2
@@ -7,7 +7,7 @@ from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
from datahub.adapters.base import AdapterError
|
||||
from datahub.adapters.tushare import DEFAULT_INDEX_CODES, TushareAdapter
|
||||
from datahub.adapters.tushare import DEFAULT_INDEX_CODES, WEBSITE_INDEX_CODES, TushareAdapter
|
||||
from datahub.db import DATASET_TABLES, HubDB
|
||||
from datahub.governance.circuit import CircuitBreaker
|
||||
from datahub.governance.ratelimit import TokenBucket
|
||||
@@ -141,11 +141,22 @@ class Pipeline:
|
||||
seq = int((row or {}).get("n") or 0) + 1
|
||||
return f"{trade_date}-{dataset}-{seq:03d}"
|
||||
|
||||
def ingest_reference(self, trade_date: str | None = None) -> dict[str, Any]:
|
||||
"""Refresh trade calendar (window) and stock master. Not versioned by batch."""
|
||||
def ingest_reference(
|
||||
self,
|
||||
trade_date: str | None = None,
|
||||
start: str | None = None,
|
||||
end: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Refresh trade calendar and stock master. Not versioned by batch.
|
||||
|
||||
Calendar defaults to 2016-01-01 through today+30 so a 5-year website
|
||||
query is not silently truncated. UPSERT makes repeats safe.
|
||||
"""
|
||||
day = yyyymmdd(trade_date or self.clock())
|
||||
start = add_days(day, -400)
|
||||
end = add_days(day, 30)
|
||||
start = yyyymmdd(start or self.settings.calendar_start)
|
||||
end = yyyymmdd(end or add_days(day, 30))
|
||||
if start > end:
|
||||
start, end = end, start
|
||||
calendar = self.adapter.normalize(
|
||||
"calendar",
|
||||
self._guarded_fetch("calendar", {"exchange": "SSE", "start_date": start, "end_date": end}),
|
||||
@@ -180,9 +191,150 @@ class Pipeline:
|
||||
row.get("list_date"), fetched_at,
|
||||
),
|
||||
)
|
||||
return {"calendar": len(calendar), "stocks": len(stocks), "trade_date": day}
|
||||
return {
|
||||
"calendar": len(calendar),
|
||||
"stocks": len(stocks),
|
||||
"trade_date": day,
|
||||
"calendar_from": start,
|
||||
"calendar_to": end,
|
||||
}
|
||||
|
||||
def run_dataset(self, dataset: str, trade_date: str, attempts: int | None = None) -> dict[str, Any]:
|
||||
def open_trade_dates(self, end: str, limit: int) -> list[str]:
|
||||
end = yyyymmdd(end)
|
||||
rows = self.db.fetchall(
|
||||
"""
|
||||
SELECT cal_date FROM trade_calendar
|
||||
WHERE exchange = 'SSE' AND is_open = 1 AND cal_date <= ?
|
||||
ORDER BY cal_date DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
(end, max(1, int(limit))),
|
||||
)
|
||||
return sorted(str(row["cal_date"]) for row in rows)
|
||||
|
||||
def backfill_history(
|
||||
self,
|
||||
trade_date: str | None = None,
|
||||
calendar_start: str | None = None,
|
||||
index_days: int | None = None,
|
||||
codes: tuple[str, ...] | None = None,
|
||||
force: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Idempotent calendar + website-index history backfill."""
|
||||
day = yyyymmdd(trade_date or self.clock())
|
||||
calendar = self.ingest_reference(day, start=calendar_start)
|
||||
index = self.backfill_index_history(
|
||||
end_date=day,
|
||||
trading_days=index_days,
|
||||
codes=codes,
|
||||
force=force,
|
||||
)
|
||||
return {"calendar": calendar, "index_daily": index, "ok": bool(index.get("ok"))}
|
||||
|
||||
def backfill_index_history(
|
||||
self,
|
||||
end_date: str | None = None,
|
||||
trading_days: int | None = None,
|
||||
codes: tuple[str, ...] | None = None,
|
||||
force: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
"""Incrementally publish official index bars for website index codes.
|
||||
|
||||
One range fetch per code, then per-day publish. Already published dates
|
||||
are skipped unless ``force``. Failures are recorded and do not roll back
|
||||
successful days.
|
||||
"""
|
||||
end = yyyymmdd(end_date or self.clock())
|
||||
limit = int(trading_days or self.settings.index_history_trading_days)
|
||||
codes = tuple(codes or WEBSITE_INDEX_CODES)
|
||||
open_dates = self.open_trade_dates(end, limit)
|
||||
if not open_dates:
|
||||
return {
|
||||
"start": None,
|
||||
"end": end,
|
||||
"codes": list(codes),
|
||||
"requested_days": 0,
|
||||
"published": [],
|
||||
"skipped": [],
|
||||
"failed": [{"error": "calendar has no open dates on or before end"}],
|
||||
"ok": False,
|
||||
}
|
||||
start = open_dates[0]
|
||||
complete_dates = set() if force else self._index_dates_with_all_codes(start, end, codes)
|
||||
targets = [day for day in open_dates if day not in complete_dates]
|
||||
skipped = [day for day in open_dates if day in complete_dates]
|
||||
by_date: dict[str, list[dict[str, Any]]] = {day: [] for day in targets}
|
||||
failed: list[dict[str, Any]] = []
|
||||
for ts_code in codes:
|
||||
try:
|
||||
raw = retry_call(
|
||||
lambda code=ts_code: self._guarded_fetch(
|
||||
"index_daily",
|
||||
{"ts_code": code, "start_date": start, "end_date": end},
|
||||
),
|
||||
attempts=self.settings.max_publish_attempts,
|
||||
base_delay=0.05,
|
||||
sleeper=lambda _d: time.sleep(_d),
|
||||
)
|
||||
for row in self.adapter.normalize("index_daily", raw):
|
||||
day = str(row.get("trade_date") or "")
|
||||
if day in by_date:
|
||||
by_date[day].append(row)
|
||||
except Exception as exc:
|
||||
failed.append({"ts_code": ts_code, "error": str(exc)})
|
||||
published: list[dict[str, Any]] = []
|
||||
for day in targets:
|
||||
rows = by_date.get(day) or []
|
||||
try:
|
||||
result = self.run_dataset("index_daily", day, prepared_rows=rows)
|
||||
published.append(
|
||||
{
|
||||
"trade_date": day,
|
||||
"batch_id": result["batch_id"],
|
||||
"rows": result["rows"],
|
||||
"state": result["state"],
|
||||
}
|
||||
)
|
||||
except Exception as exc:
|
||||
failed.append({"trade_date": day, "error": str(exc), "rows": len(rows)})
|
||||
return {
|
||||
"start": start,
|
||||
"end": end,
|
||||
"codes": list(codes),
|
||||
"requested_days": len(open_dates),
|
||||
"published": published,
|
||||
"skipped": skipped,
|
||||
"failed": failed,
|
||||
"ok": not failed,
|
||||
}
|
||||
|
||||
def _index_dates_with_all_codes(self, start: str, end: str, codes: tuple[str, ...]) -> set[str]:
|
||||
pubs = self.db.fetchall(
|
||||
"""
|
||||
SELECT trade_date, active_batch FROM publications
|
||||
WHERE dataset = 'index_daily' AND trade_date >= ? AND trade_date <= ?
|
||||
""",
|
||||
(start, end),
|
||||
)
|
||||
needed = set(codes)
|
||||
complete: set[str] = set()
|
||||
for pub in pubs:
|
||||
rows = self.db.fetchall(
|
||||
"SELECT DISTINCT ts_code FROM eod_index_bars WHERE trade_date = ? AND batch_id = ?",
|
||||
(pub["trade_date"], pub["active_batch"]),
|
||||
)
|
||||
have = {str(row["ts_code"]) for row in rows}
|
||||
if needed <= have:
|
||||
complete.add(str(pub["trade_date"]))
|
||||
return complete
|
||||
|
||||
def run_dataset(
|
||||
self,
|
||||
dataset: str,
|
||||
trade_date: str,
|
||||
attempts: int | None = None,
|
||||
prepared_rows: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
trade_date = yyyymmdd(trade_date)
|
||||
batch_id = self.next_batch_id(dataset, trade_date)
|
||||
max_attempts = attempts or self.settings.max_publish_attempts
|
||||
@@ -190,12 +342,15 @@ class Pipeline:
|
||||
rows: list[dict[str, Any]] = []
|
||||
try:
|
||||
self._set_batch(batch_id, dataset, trade_date, "fetching", 1)
|
||||
rows = retry_call(
|
||||
lambda: self._fetch_dataset(dataset, trade_date),
|
||||
attempts=max_attempts,
|
||||
base_delay=0.05,
|
||||
sleeper=lambda _d: None if attempts == 1 else time.sleep(_d),
|
||||
)
|
||||
if prepared_rows is None:
|
||||
rows = retry_call(
|
||||
lambda: self._fetch_dataset(dataset, trade_date),
|
||||
attempts=max_attempts,
|
||||
base_delay=0.05,
|
||||
sleeper=lambda _d: None if attempts == 1 else time.sleep(_d),
|
||||
)
|
||||
else:
|
||||
rows = list(prepared_rows)
|
||||
self._stage(dataset, batch_id, rows)
|
||||
self._set_batch(batch_id, dataset, trade_date, "staged", 1, rows_in=len(rows), rows_out=len(rows))
|
||||
self._set_batch(batch_id, dataset, trade_date, "validating", 1)
|
||||
|
||||
Reference in New Issue
Block a user