feat(HEL-478): 估值发布后晚间复核并原子追补上游修订

盘后成功发布后继续轻量比对 daily_basic 网站字段，发现修订才走质量门与整组原子切换，避免 17:10 快照落后于晚间上游改写。

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: multica-agent <github@multica.ai>

xiaobai-datahub/datahub/scheduler.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import threading
 from collections.abc import Callable
 from datetime import datetime, time, timedelta
@@ -8,13 +9,14 @@ from typing import Any
 from datahub.db import HubDB
 from datahub.logutil import get_logger
 from datahub.pipeline import Pipeline
+from datahub.revision import revision_datasets
 from datahub.timeutil import isoformat, now_shanghai, yyyymmdd
 
 LOGGER = get_logger()
 
 JobFn = Callable[[str], Any]
 
-EOD_JOB_IDS = {"eod_a", "eod_b", "eod_retry"}
+EOD_JOB_IDS = {"eod_a", "eod_b", "eod_retry", "eod_revise"}
 
 
 def is_open_day(db: HubDB, day: str) -> bool:
@@ -27,6 +29,20 @@ def is_open_day(db: HubDB, day: str) -> bool:
     return int(row["is_open"]) == 1
 
 
+def previous_open_day(db: HubDB, day: str) -> str | None:
+    row = db.fetchone(
+        """
+        SELECT cal_date FROM trade_calendar
+        WHERE exchange = 'SSE' AND is_open = 1 AND cal_date < ?
+        ORDER BY cal_date DESC LIMIT 1
+        """,
+        (day,),
+    )
+    if row is None:
+        return None
+    return str(row["cal_date"])
+
+
 def _hhmm(value: str) -> time:
     return datetime.strptime(value, "%H:%M").time()
 
@@ -49,6 +65,7 @@ class Scheduler:
             "eod_a": self._eod_a,
             "eod_b": self._eod_b,
             "eod_retry": self._eod_retry,
+            "eod_revise": self._eod_revise,
             "stocks_refresh": self._stocks_refresh,
             "cleanup": self._cleanup,
             "backup": self._backup,
@@ -118,6 +135,8 @@ class Scheduler:
             if job_id in {"eod_a", "eod_b"}:
                 self._settle_eod(day)
         ran.extend(self._eod_retry_tick(now, day, open_day))
+        ran.extend(self._revision_review_tick(now, day, open_day))
+        ran.extend(self._revision_catchup_tick(now, day))
         return ran
 
     # ------------------------------------------------------------------
@@ -219,6 +238,201 @@ class Scheduler:
             "detail": (row or {}).get("detail"),
         }
 
+    def revision_progress(self, day: str) -> dict[str, Any] | None:
+        return self.db.fetchone("SELECT * FROM revision_progress WHERE trade_date = ?", (day,))
+
+    def revision_status(self, trade_date: str | None = None, clock: datetime | None = None) -> dict[str, Any]:
+        """等待复核 / 复核失败 / 已追平 / 已截止."""
+        day = yyyymmdd(trade_date or now_shanghai(clock))
+        row = self.revision_progress(day)
+        open_day = is_open_day(self.db, day)
+        published = self._revision_ready(day)
+        if row and row["state"] in {"aligned", "review_failed", "cutoff", "waiting_review"}:
+            state = str(row["state"])
+        elif not open_day:
+            state = "closed_day"
+        elif not published:
+            state = "pending_publish"
+        else:
+            state = "waiting_review"
+        return {
+            "trade_date": day,
+            "is_open_day": open_day,
+            "state": state,
+            "datasets": list(revision_datasets(self.pipeline.settings.quality)),
+            "attempts": int((row or {}).get("attempts") or 0),
+            "last_attempt_at": (row or {}).get("last_attempt_at"),
+            "next_retry_at": (row or {}).get("next_retry_at") if state in {"waiting_review", "review_failed"} else None,
+            "finished_at": (row or {}).get("finished_at"),
+            "catchup_done": bool(int((row or {}).get("catchup_done") or 0)),
+            "detail": (row or {}).get("detail"),
+            "window": f"{self.pipeline.settings.revision_review_start}-{self.pipeline.settings.revision_review_cutoff}",
+        }
+
+    def _revision_ready(self, day: str) -> bool:
+        return all(
+            self.pipeline.active_batch(dataset, day)
+            for dataset in revision_datasets(self.pipeline.settings.quality)
+        )
+
+    def _revision_due(self, now: datetime, row: dict[str, Any] | None) -> bool:
+        if row is None or not row.get("last_attempt_at"):
+            return True
+        try:
+            last = datetime.fromisoformat(str(row["last_attempt_at"]))
+        except ValueError:
+            return True
+        interval = timedelta(minutes=self.pipeline.settings.revision_review_interval_minutes)
+        return now_shanghai(last).replace(tzinfo=None) + interval <= now.replace(tzinfo=None)
+
+    def _revision_review_tick(self, now: datetime, day: str, open_day: bool) -> list[str]:
+        if not open_day or not self._revision_ready(day):
+            return []
+        settings = self.pipeline.settings
+        current = now.time()
+        start = _hhmm(settings.revision_review_start)
+        cutoff = _hhmm(settings.revision_review_cutoff)
+        row = self.revision_progress(day)
+        if current < start:
+            if row is None:
+                self._save_revision_progress(day, state="waiting_review")
+            return []
+        if current >= cutoff:
+            if row is None or row["state"] not in {"aligned", "cutoff"}:
+                detail = "复核窗口已截止"
+                self._save_revision_progress(
+                    day, state="cutoff", finished_at=isoformat(now), detail=detail,
+                )
+                with self.db.write() as connection:
+                    connection.execute(
+                        "INSERT INTO job_runs(job_id, state, started_at, finished_at, error, attempt, detail)"
+                        " VALUES ('eod_revise','failed',?,?,?,?,?)",
+                        (
+                            isoformat(now), isoformat(now), detail,
+                            int((row or {}).get("attempts") or 0), "revision cutoff reached",
+                        ),
+                    )
+            elif row["state"] == "aligned" and not row.get("finished_at"):
+                self._save_revision_progress(day, finished_at=isoformat(now))
+            return []
+        if not self._revision_due(now, row):
+            return []
+        if "eod_revise" not in self.jobs:
+            return []
+        return self._run_revision_job(day, now, catchup=False)
+
+    def _revision_catchup_tick(self, now: datetime, day: str) -> list[str]:
+        prev = previous_open_day(self.db, day)
+        if prev is None or prev >= day:
+            return []
+        if not self._revision_ready(prev):
+            return []
+        row = self.revision_progress(prev)
+        if row and int(row.get("catchup_done") or 0):
+            return []
+        if not self._revision_due(now, row):
+            return []
+        if "eod_revise" not in self.jobs:
+            return []
+        return self._run_revision_job(prev, now, catchup=True)
+
+    def _run_revision_job(self, day: str, now: datetime, catchup: bool) -> list[str]:
+        attempts = int((self.revision_progress(day) or {}).get("attempts") or 0) + 1
+        interval = self.pipeline.settings.revision_review_interval_minutes
+        self._save_revision_progress(
+            day,
+            state="waiting_review",
+            attempts=attempts,
+            last_attempt_at=isoformat(now),
+            next_retry_at=isoformat(now + timedelta(minutes=interval)),
+        )
+        ran: list[str] = []
+        try:
+            out = self.run_job("eod_revise", day)
+        except Exception as exc:
+            LOGGER.warning("revision review failed for %s: %s", day, exc)
+            self._save_revision_progress(
+                day,
+                state="review_failed",
+                detail="复核失败，保留上一完整版本",
+            )
+            ran.append("eod_revise")
+            return ran
+        ran.append("eod_revise")
+        if out.get("state") == "skipped":
+            return ran
+        result = out.get("result") if isinstance(out.get("result"), dict) else {}
+        failed = [
+            name for name, item in result.items()
+            if isinstance(item, dict) and item.get("state") == "failed"
+        ]
+        review = result.get("review") if isinstance(result.get("review"), dict) else None
+        watched = [
+            result[name]
+            for name in revision_datasets(self.pipeline.settings.quality)
+            if isinstance(result.get(name), dict)
+        ]
+        diff_blob = None
+        if review and review.get("diffs"):
+            diff_blob = json.dumps(review.get("diffs"), ensure_ascii=False)
+        else:
+            for item in watched:
+                if item.get("diffs"):
+                    diff_blob = json.dumps(item.get("diffs"), ensure_ascii=False)
+                    break
+        revised = bool(review and review.get("reason") == "revised")
+        matched = any(item.get("reason") == "unchanged" or item.get("state") == "aligned" for item in watched)
+        if failed:
+            self._save_revision_progress(
+                day,
+                state="review_failed",
+                detail="复核失败，保留上一完整版本",
+                last_diff=diff_blob,
+            )
+        elif revised or matched:
+            fields: dict[str, Any] = {
+                "state": "aligned",
+                "finished_at": isoformat(now),
+                "detail": "已追平" if revised else "已追平（无变化）",
+                "last_diff": diff_blob,
+            }
+            if catchup:
+                fields["catchup_done"] = 1
+            self._save_revision_progress(day, **fields)
+        return ran
+
+    def _save_revision_progress(self, day: str, **fields: Any) -> None:
+        columns = [
+            "trade_date", "state", "attempts", "last_attempt_at",
+            "next_retry_at", "finished_at", "catchup_done", "last_diff", "detail", "updated_at",
+        ]
+        with self.db.write() as connection:
+            existing = connection.execute(
+                "SELECT trade_date FROM revision_progress WHERE trade_date = ?",
+                (day,),
+            ).fetchone()
+            if existing is None:
+                payload = {name: None for name in columns}
+                payload.update({
+                    "trade_date": day,
+                    "state": "waiting_review",
+                    "attempts": 0,
+                    "catchup_done": 0,
+                })
+                payload.update(fields)
+                payload["updated_at"] = isoformat()
+                placeholders = ",".join("?" for _ in columns)
+                connection.execute(
+                    f"INSERT INTO revision_progress({','.join(columns)}) VALUES ({placeholders})",
+                    tuple(payload[name] for name in columns),
+                )
+            else:
+                assignments = ", ".join(f"{name} = ?" for name in fields)
+                connection.execute(
+                    f"UPDATE revision_progress SET {assignments}, updated_at = ? WHERE trade_date = ?",
+                    (*fields.values(), isoformat(), day),
+                )
+
     def _record_eod_attempt(self, day: str, now: datetime) -> None:
         row = self.eod_progress(day)
         attempts = int((row or {}).get("attempts") or 0) + 1
@@ -320,6 +534,9 @@ class Scheduler:
     def _eod_retry(self, trade_date: str) -> dict[str, Any]:
         return self.pipeline.run_eod_missing(trade_date)
 
+    def _eod_revise(self, trade_date: str) -> dict[str, Any]:
+        return self.pipeline.review_published_revisions(trade_date)
+
     def _stocks_refresh(self, trade_date: str) -> dict[str, Any]:
         return self.pipeline.refresh_stocks(trade_date)