leefer/xiaobai-review
1
1
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(HEL-435): 盘后未出数时晚间自动重试并提供安全补跑

Browse Source 
Co-authored-by: multica-agent <github@multica.ai>
This commit is contained in:
总工
2026-09-03 22:30:15 +08:00
co-authored by multica-agent
parent a836cda1b2
commit c9892050c3
9 changed files with 532 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
xiaobai-datahub/datahub/admin_api.py
+2
View File
@@ -38,6 +38,7 @@ class AdminAPI:
"trade_date": today,
"session_phase": session_phase(now_shanghai(), is_open),
"is_open_day": is_open,
"eod_status": self.scheduler.eod_status(today),
"publications": pubs,
"anomalies": failed,
"recent_calls": _public_calls(calls),
@@ -88,6 +89,7 @@ class AdminAPI:
{"id": "precheck", "at": "08:45", "title": "盘前预检"},
{"id": "eod_a", "at": "15:05", "title": "盘后批 A daily/valuation/moneyflow/auction"},
{"id": "eod_b", "at": "15:10", "title": "盘后批 B index_daily"},
{"id": "eod_retry", "at": "15:15-23:30", "title": "盘后未出数自动重试（每 30 分钟，成功即停）"},
{"id": "history_backfill", "at": "manual", "title": "回补历史日历与指数日 K"},
{"id": "cleanup", "at": "00:30", "title": "清理 staging / 日志"},
{"id": "backup", "at": "00:40", "title": "SQLite 备份"},
xiaobai-datahub/datahub/cli.py
+15
View File
@@ -8,6 +8,7 @@ import sys
from datahub.hub import build_hub
from datahub.settings import load_settings
from datahub.timeutil import yyyymmdd
def main(argv: list[str] | None = None) -> int:
@@ -17,6 +18,8 @@ def main(argv: list[str] | None = None) -> int:
history.add_argument("--calendar-start", default=None, help="日历起点，默认配置 calendar_start")
history.add_argument("--index-days", type=int, default=None, help="指数回补交易日数量，默认 260")
history.add_argument("--force", action="store_true", help="覆盖已发布的指数日期")
refresh = sub.add_parser("eod-refresh", help="对指定交易日补跑盘后正式数据（跳过已发布数据集，仍走质量门禁）")
refresh.add_argument("--trade-date", default=None, help="交易日 YYYYMMDD，默认今天")
args = parser.parse_args(argv)
settings = load_settings()
@@ -30,6 +33,18 @@ def main(argv: list[str] | None = None) -> int:
json.dump(result, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0 if result.get("ok") else 1
if args.command == "eod-refresh":
day = yyyymmdd(args.trade_date) if args.trade_date else yyyymmdd()
result = hub.pipeline.run_eod_missing(day)
hub.pipeline.audit("cli", "eod-refresh", f"eod:{day}", json.dumps(
{name: item.get("state") for name, item in result.items() if isinstance(item, dict)},
ensure_ascii=False,
))
missing = hub.pipeline.missing_official_datasets(day)
payload = {"trade_date": day, "datasets": result, "missing_after": missing}
json.dump(payload, sys.stdout, ensure_ascii=False, indent=2, default=str)
sys.stdout.write("\n")
return 0 if not missing else 1
parser.error(f"unknown command: {args.command}")
return 2
xiaobai-datahub/datahub/db.py
+11
View File
@@ -212,6 +212,17 @@ CREATE TABLE IF NOT EXISTS job_runs (
detail TEXT
);
CREATE TABLE IF NOT EXISTS eod_progress (
trade_date TEXT PRIMARY KEY,
state TEXT NOT NULL,
attempts INTEGER NOT NULL DEFAULT 0,
last_attempt_at TEXT,
next_retry_at TEXT,
finished_at TEXT,
detail TEXT,
updated_at TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS audit_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
actor TEXT NOT NULL,
xiaobai-datahub/datahub/pipeline.py
+55 -5
View File
@@ -22,6 +22,8 @@ LOGGER = get_logger()
HARD_DATASETS = {"daily", "valuation", "index_daily"}
SOFT_DATASETS = {"moneyflow", "auction"}
OFFICIAL_DATASETS = HARD_DATASETS | SOFT_DATASETS
EOD_A_DATASETS = ("daily", "valuation", "moneyflow", "auction")
EOD_B_DATASETS = ("index_daily",)
EMPTY_BATCH_ERROR = "empty official batch: 0 valid rows"
STAGING_INSERT = {
@@ -379,14 +381,62 @@ class Pipeline:
self._set_batch(batch_id, dataset, trade_date, "failed", 1, error=str(exc), finished=True)
raise
def missing_official_datasets(self, trade_date: str) -> list[str]:
"""Official datasets without an active publication for the date."""
day = yyyymmdd(trade_date)
placeholders = ",".join("?" for _ in OFFICIAL_DATASETS)
rows = self.db.fetchall(
f"SELECT dataset FROM publications WHERE trade_date = ? AND dataset IN ({placeholders})",
(day, *sorted(OFFICIAL_DATASETS)),
)
published = {str(row["dataset"]) for row in rows}
return [dataset for dataset in sorted(OFFICIAL_DATASETS) if dataset not in published]
def run_eod_missing(self, trade_date: str) -> dict[str, Any]:
"""Fetch/publish every official dataset still missing for the date.
Idempotent: datasets with an existing publication are skipped, so
repeats never overwrite the current official batch. Per-dataset
failures are collected instead of aborting the remaining datasets.
"""
return self._run_eod_datasets(tuple(sorted(OFFICIAL_DATASETS)), trade_date)
def run_eod_batch_a(self, trade_date: str) -> dict[str, Any]:
results = {}
for dataset in ("daily", "valuation", "moneyflow", "auction"):
results[dataset] = self.run_dataset(dataset, trade_date)
return results
return self._run_eod_datasets(EOD_A_DATASETS, trade_date)
def run_eod_batch_b(self, trade_date: str) -> dict[str, Any]:
return {"index_daily": self.run_dataset("index_daily", trade_date)}
return self._run_eod_datasets(EOD_B_DATASETS, trade_date)
def _run_eod_datasets(self, datasets: tuple[str, ...], trade_date: str) -> dict[str, Any]:
day = yyyymmdd(trade_date)
results: dict[str, Any] = {}
for dataset in datasets:
if self.active_batch(dataset, day) is not None:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "skipped",
"reason": "already_published",
}
continue
try:
results[dataset] = self.run_dataset(dataset, day)
except Exception as exc:
results[dataset] = {
"dataset": dataset,
"trade_date": day,
"state": "failed",
"error": str(exc),
}
return results
@staticmethod
def eod_failures(results: dict[str, Any]) -> list[str]:
return [
f"{name}: {item.get('error')}"
for name, item in results.items()
if isinstance(item, dict) and item.get("state") == "failed"
]
def validate(self, dataset: str, batch_id: str, trade_date: str, rows: list[dict[str, Any]]) -> dict[str, Any]:
quality = self.settings.quality
xiaobai-datahub/datahub/scheduler.py
+196 -5
View File
@@ -2,7 +2,7 @@ from __future__ import annotations
import threading
from collections.abc import Callable
from datetime import datetime, time
from datetime import datetime, time, timedelta
from typing import Any
from datahub.db import HubDB
@@ -14,6 +14,8 @@ LOGGER = get_logger()
JobFn = Callable[[str], Any]
EOD_JOB_IDS = {"eod_a", "eod_b", "eod_retry"}
def is_open_day(db: HubDB, day: str) -> bool:
row = db.fetchone(
@@ -25,8 +27,19 @@ def is_open_day(db: HubDB, day: str) -> bool:
return int(row["is_open"]) == 1
def _hhmm(value: str) -> time:
return datetime.strptime(value, "%H:%M").time()
class Scheduler:
"""Calendar-driven in-process scheduler. Non-trading days skip EOD fetches."""
"""Calendar-driven in-process scheduler. Non-trading days skip EOD fetches.
EOD datasets that failed to publish (e.g. upstream not ready at 15:05)
are retried automatically every ``eod_retry_interval_minutes`` between
``eod_retry_start`` and ``eod_retry_cutoff``. Progress is persisted in
``eod_progress`` so a container restart catches up instead of waiting
for the next day, and completed days are never re-fetched.
"""
def __init__(self, db: HubDB, pipeline: Pipeline, jobs: dict[str, JobFn] | None = None) -> None:
self.db = db
@@ -35,6 +48,7 @@ class Scheduler:
"precheck": self._precheck,
"eod_a": self._eod_a,
"eod_b": self._eod_b,
"eod_retry": self._eod_retry,
"cleanup": self._cleanup,
"backup": self._backup,
"history_backfill": self._history_backfill,
@@ -42,6 +56,7 @@ class Scheduler:
self._stop = threading.Event()
self._thread: threading.Thread | None = None
self._fired: set[tuple[str, str, str]] = set()
self._eod_lock = threading.Lock()
def start(self, interval_seconds: float = 30.0) -> None:
if self._thread and self._thread.is_alive():
@@ -63,9 +78,9 @@ class Scheduler:
self._thread.join(timeout)
def tick(self, clock: datetime | None = None) -> list[str]:
now = clock or now_shanghai()
now = now_shanghai(clock)
day = yyyymmdd(now)
current = now.timetz() if False else now.time()
current = now.time()
ran: list[str] = []
plan = [
("precheck", time(8, 45)),
@@ -85,14 +100,183 @@ class Scheduler:
self._fired.add(key)
continue
self._fired.add(key)
self.run_job(job_id, day)
if job_id in {"eod_a", "eod_b"}:
# Record the attempt before running: even a crash must not
# hide that today's first EOD try already happened.
self._record_eod_attempt(day, now)
try:
self.run_job(job_id, day)
except Exception:
if job_id not in {"eod_a", "eod_b"}:
raise
# Keep the tick alive; evening retries take over.
LOGGER.exception("scheduled job %s failed for %s", job_id, day)
ran.append(job_id)
if job_id in {"eod_a", "eod_b"}:
self._settle_eod(day)
ran.extend(self._eod_retry_tick(now, day, open_day))
return ran
# ------------------------------------------------------------------
# EOD retry window
# ------------------------------------------------------------------
def _eod_retry_tick(self, now: datetime, day: str, open_day: bool) -> list[str]:
if not open_day:
return []
settings = self.pipeline.settings
current = now.time()
start = _hhmm(settings.eod_retry_start)
cutoff = _hhmm(settings.eod_retry_cutoff)
interval = timedelta(minutes=settings.eod_retry_interval_minutes)
missing = self.pipeline.missing_official_datasets(day)
row = self.eod_progress(day)
if not missing:
if row is None or row["state"] != "done":
self._save_eod_progress(day, state="done", finished_at=isoformat(now))
return []
if current < start:
return []
if row and row["state"] == "cutoff_failed":
return []
if current >= cutoff:
detail = "截止时间已到，缺失数据集: " + ",".join(missing)
self._save_eod_progress(day, state="cutoff_failed", finished_at=isoformat(now), detail=detail)
with self.db.write() as connection:
connection.execute(
"INSERT INTO job_runs(job_id, state, started_at, finished_at, error, attempt, detail)"
" VALUES ('eod_retry','failed',?,?,?,?,?)",
(isoformat(now), isoformat(now), detail, int((row or {}).get("attempts") or 0), "eod cutoff reached"),
)
LOGGER.warning(
"eod retry window closed without data",
extra={"hub": {"trade_date": day, "missing": missing, "reason": "eod_cutoff"}},
)
return []
last = None
if row and row["last_attempt_at"]:
try:
last = datetime.fromisoformat(str(row["last_attempt_at"]))
except ValueError:
last = None
if last is not None and now_shanghai(last).replace(tzinfo=None) + interval > now.replace(tzinfo=None):
return []
if "eod_retry" not in self.jobs:
return []
self._record_eod_attempt(day, now)
ran = []
try:
self.run_job("eod_retry", day)
except Exception:
# job_runs already carries the failure; the window keeps retrying.
LOGGER.warning("eod retry failed for %s", day, exc_info=True)
ran.append("eod_retry")
self._settle_eod(day)
return ran
def _settle_eod(self, day: str) -> None:
"""Flip the day to done as soon as every official dataset is published."""
if not self.pipeline.missing_official_datasets(day):
row = self.eod_progress(day)
if row is None or row["state"] != "done":
self._save_eod_progress(day, state="done", finished_at=isoformat())
def eod_progress(self, day: str) -> dict[str, Any] | None:
return self.db.fetchone("SELECT * FROM eod_progress WHERE trade_date = ?", (day,))
def eod_status(self, trade_date: str | None = None, clock: datetime | None = None) -> dict[str, Any]:
"""Human/admin facing view: 等待上游 / 下次重试 / 已成功 / 已截止失败."""
day = yyyymmdd(trade_date or now_shanghai(clock))
now = now_shanghai(clock)
row = self.eod_progress(day)
open_day = is_open_day(self.db, day)
missing = self.pipeline.missing_official_datasets(day)
if row and row["state"] == "done":
state = "done"
elif not open_day:
state = "closed_day"
elif not missing:
state = "done"
elif row and row["state"] == "cutoff_failed":
state = "cutoff_failed"
elif now.time() < _hhmm("15:05"):
state = "pending_first_attempt"
else:
state = "waiting_upstream"
return {
"trade_date": day,
"is_open_day": open_day,
"state": state,
"missing_datasets": missing,
"attempts": int((row or {}).get("attempts") or 0),
"last_attempt_at": (row or {}).get("last_attempt_at"),
"next_retry_at": (row or {}).get("next_retry_at") if state == "waiting_upstream" else None,
"finished_at": (row or {}).get("finished_at"),
"detail": (row or {}).get("detail"),
}
def _record_eod_attempt(self, day: str, now: datetime) -> None:
row = self.eod_progress(day)
attempts = int((row or {}).get("attempts") or 0) + 1
interval = self.pipeline.settings.eod_retry_interval_minutes
self._save_eod_progress(
day,
state="waiting_upstream",
attempts=attempts,
last_attempt_at=isoformat(now),
next_retry_at=isoformat(now + timedelta(minutes=interval)),
)
def _save_eod_progress(self, day: str, **fields: Any) -> None:
columns = [
"trade_date", "state", "attempts", "last_attempt_at",
"next_retry_at", "finished_at", "detail", "updated_at",
]
with self.db.write() as connection:
existing = connection.execute(
"SELECT trade_date FROM eod_progress WHERE trade_date = ?",
(day,),
).fetchone()
if existing is None:
payload = {name: None for name in columns}
payload.update({"trade_date": day, "state": "waiting_upstream", "attempts": 0})
payload.update(fields)
payload["updated_at"] = isoformat()
placeholders = ",".join("?" for _ in columns)
connection.execute(
f"INSERT INTO eod_progress({','.join(columns)}) VALUES ({placeholders})",
tuple(payload[name] for name in columns),
)
else:
assignments = ", ".join(f"{name} = ?" for name in fields)
connection.execute(
f"UPDATE eod_progress SET {assignments}, updated_at = ? WHERE trade_date = ?",
(*fields.values(), isoformat(), day),
)
# ------------------------------------------------------------------
# Job execution
# ------------------------------------------------------------------
def run_job(self, job_id: str, trade_date: str) -> dict[str, Any]:
fn = self.jobs.get(job_id)
if fn is None:
raise KeyError(job_id)
if job_id in EOD_JOB_IDS:
if not self._eod_lock.acquire(blocking=False):
return {
"job_id": job_id,
"state": "skipped",
"detail": "another EOD job is already running",
}
try:
return self._run_job(fn, job_id, trade_date)
finally:
self._eod_lock.release()
return self._run_job(fn, job_id, trade_date)
def _run_job(self, fn: JobFn, job_id: str, trade_date: str) -> dict[str, Any]:
started = isoformat()
run_id = None
with self.db.write() as connection:
@@ -103,6 +287,10 @@ class Scheduler:
run_id = cur.lastrowid
try:
result = fn(trade_date) or {}
if isinstance(result, dict):
failures = self.pipeline.eod_failures(result) if job_id in EOD_JOB_IDS else []
if failures:
raise RuntimeError("; ".join(failures))
with self.db.write() as connection:
connection.execute(
"UPDATE job_runs SET state=?, finished_at=?, rows_out=?, detail=? WHERE id=?",
@@ -126,6 +314,9 @@ class Scheduler:
def _eod_b(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_batch_b(trade_date)
def _eod_retry(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.run_eod_missing(trade_date)
def _history_backfill(self, trade_date: str) -> dict[str, Any]:
return self.pipeline.backfill_history(trade_date)
xiaobai-datahub/datahub/settings.py
+12
View File
@@ -56,6 +56,18 @@ class Settings:
def index_history_trading_days(self) -> int:
return int(self.quality.get("index_history_trading_days") or 260)
@property
def eod_retry_start(self) -> str:
return str(self.quality.get("eod_retry_start") or "15:15")
@property
def eod_retry_interval_minutes(self) -> int:
return int(self.quality.get("eod_retry_interval_minutes") or 30)
@property
def eod_retry_cutoff(self) -> str:
return str(self.quality.get("eod_retry_cutoff") or "23:30")
def load_settings(
env: dict[str, str] | None = None,