feat(HEL-382): 搭建 datahub 底座和盘后正式数据链路

新增独立 xiaobai-datahub 服务（SQLite WAL、Tushare 盘后发布、/v1 契约和管理后台），不改现站页面与数据链路。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
2026-09-02 12:05:26 +08:00
parent c2ebc0ab91
commit 3498dd7a4b
52 changed files with 4259 additions and 0 deletions
@@ -0,0 +1,13 @@
+from datahub.governance.circuit import CircuitBreaker, CircuitState
+from datahub.governance.lkg import LastKnownGood
+from datahub.governance.ratelimit import TokenBucket
+from datahub.governance.retry import RetryError, retry_call
+
+__all__ = [
+    "CircuitBreaker",
+    "CircuitState",
+    "LastKnownGood",
+    "RetryError",
+    "TokenBucket",
+    "retry_call",
+]
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass
+
+
+@dataclass
+class CircuitState:
+    state: str = "closed"  # closed | open | half_open
+    consec_failures: int = 0
+    opened_at: float | None = None
+    cooldown_until: float = 0.0
+    last_error: str = ""
+    last_ok_at: float | None = None
+
+
+class CircuitBreaker:
+    """Sliding-window breaker: 5 consecutive failures or >50% of 60s window → open."""
+
+    def __init__(
+        self,
+        failure_threshold: int = 5,
+        window_seconds: float = 60.0,
+        open_seconds: float = 120.0,
+        max_open_seconds: float = 600.0,
+        clock=time.monotonic,
+    ) -> None:
+        self.failure_threshold = failure_threshold
+        self.window_seconds = window_seconds
+        self.open_seconds = open_seconds
+        self.max_open_seconds = max_open_seconds
+        self._clock = clock
+        self._lock = threading.Lock()
+        self._events: deque[tuple[float, bool]] = deque()
+        self.status = CircuitState()
+        self._open_stretch = open_seconds
+
+    def allow(self) -> bool:
+        with self._lock:
+            self._refresh_locked()
+            if self.status.state == "open":
+                return False
+            if self.status.state == "half_open":
+                # single probe in flight: caller must record success/failure
+                return True
+            return True
+
+    def record_success(self) -> CircuitState:
+        with self._lock:
+            now = self._clock()
+            self._events.append((now, True))
+            self.status.last_ok_at = now
+            self.status.consec_failures = 0
+            self.status.last_error = ""
+            self._open_stretch = self.open_seconds
+            self.status.state = "closed"
+            self.status.opened_at = None
+            self.status.cooldown_until = 0.0
+            return self._copy()
+
+    def record_failure(self, error: str = "") -> CircuitState:
+        with self._lock:
+            now = self._clock()
+            self._events.append((now, False))
+            self.status.consec_failures += 1
+            self.status.last_error = error
+            self._prune_locked(now)
+            failures = sum(1 for _, ok in self._events if not ok)
+            total = len(self._events)
+            rate = (failures / total) if total else 0.0
+            trip = self.status.consec_failures >= self.failure_threshold or (
+                total >= self.failure_threshold and rate > 0.5
+            )
+            if trip:
+                self.status.state = "open"
+                self.status.opened_at = now
+                self.status.cooldown_until = now + self._open_stretch
+                self._open_stretch = min(self.max_open_seconds, self._open_stretch * 2)
+            return self._copy()
+
+    def snapshot(self) -> CircuitState:
+        with self._lock:
+            self._refresh_locked()
+            return self._copy()
+
+    def _refresh_locked(self) -> None:
+        now = self._clock()
+        self._prune_locked(now)
+        if self.status.state == "open" and now >= self.status.cooldown_until:
+            self.status.state = "half_open"
+
+    def _prune_locked(self, now: float) -> None:
+        cutoff = now - self.window_seconds
+        while self._events and self._events[0][0] < cutoff:
+            self._events.popleft()
+
+    def _copy(self) -> CircuitState:
+        return CircuitState(
+            state=self.status.state,
+            consec_failures=self.status.consec_failures,
+            opened_at=self.status.opened_at,
+            cooldown_until=self.status.cooldown_until,
+            last_error=self.status.last_error,
+            last_ok_at=self.status.last_ok_at,
+        )
@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from datahub.db import HubDB
+from datahub.timeutil import isoformat, now_shanghai
+
+
+class LastKnownGood:
+    def __init__(self, db: HubDB) -> None:
+        self.db = db
+
+    def store(self, cache_key: str, payload: Any, source: str) -> None:
+        self.db.execute(
+            """
+            INSERT INTO last_known_good(cache_key, payload, source, stored_at)
+            VALUES (?, ?, ?, ?)
+            ON CONFLICT(cache_key) DO UPDATE SET
+              payload=excluded.payload, source=excluded.source, stored_at=excluded.stored_at
+            """,
+            (cache_key, json.dumps(payload, ensure_ascii=False), source, isoformat()),
+        )
+
+    def load(self, cache_key: str) -> dict[str, Any] | None:
+        row = self.db.fetchone("SELECT * FROM last_known_good WHERE cache_key = ?", (cache_key,))
+        if not row:
+            return None
+        return {
+            "payload": json.loads(row["payload"]),
+            "source": row["source"],
+            "stored_at": row["stored_at"],
+        }
+
+    def put_rt(self, cache_key: str, payload: Any, source: str, ttl_seconds: int) -> None:
+        now = now_shanghai()
+        expires = isoformat(now.replace(microsecond=0))
+        # expires_at stored as iso; compute by adding ttl via timestamp
+        from datetime import timedelta
+
+        self.db.execute(
+            """
+            INSERT INTO rt_cache(cache_key, payload, source, stored_at, expires_at)
+            VALUES (?, ?, ?, ?, ?)
+            ON CONFLICT(cache_key) DO UPDATE SET
+              payload=excluded.payload, source=excluded.source,
+              stored_at=excluded.stored_at, expires_at=excluded.expires_at
+            """,
+            (
+                cache_key,
+                json.dumps(payload, ensure_ascii=False),
+                source,
+                isoformat(now),
+                isoformat(now + timedelta(seconds=ttl_seconds)),
+            ),
+        )
+        self.store(cache_key, payload, source)
+
+    def get_rt(self, cache_key: str, max_stale_seconds: int | None = None) -> dict[str, Any] | None:
+        row = self.db.fetchone("SELECT * FROM rt_cache WHERE cache_key = ?", (cache_key,))
+        if not row:
+            lkg = self.load(cache_key)
+            if not lkg:
+                return None
+            return {**lkg, "stale": True}
+        stored_at = row["stored_at"]
+        expired = row["expires_at"] < isoformat()
+        result = {
+            "payload": json.loads(row["payload"]),
+            "source": row["source"],
+            "stored_at": stored_at,
+            "stale": expired,
+        }
+        if expired and max_stale_seconds is not None:
+            from datetime import datetime
+
+            try:
+                stored = datetime.fromisoformat(stored_at)
+                age = (now_shanghai() - stored).total_seconds()
+            except ValueError:
+                age = max_stale_seconds + 1
+            if age > max_stale_seconds:
+                return None
+        return result
@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import threading
+import time
+
+
+class TokenBucket:
+    def __init__(self, rate_per_minute: float, capacity: float | None = None, clock=time.monotonic) -> None:
+        self.rate_per_second = max(0.001, rate_per_minute / 60.0)
+        self.capacity = float(capacity if capacity is not None else rate_per_minute)
+        self._tokens = self.capacity
+        self._updated = clock()
+        self._clock = clock
+        self._lock = threading.Lock()
+
+    def acquire(self, tokens: float = 1.0, block: bool = True) -> bool:
+        while True:
+            with self._lock:
+                now = self._clock()
+                elapsed = max(0.0, now - self._updated)
+                self._tokens = min(self.capacity, self._tokens + elapsed * self.rate_per_second)
+                self._updated = now
+                if self._tokens >= tokens:
+                    self._tokens -= tokens
+                    return True
+                wait = (tokens - self._tokens) / self.rate_per_second
+            if not block:
+                return False
+            time.sleep(min(wait, 0.05))
+
+    @property
+    def remaining(self) -> float:
+        with self._lock:
+            now = self._clock()
+            elapsed = max(0.0, now - self._updated)
+            return min(self.capacity, self._tokens + elapsed * self.rate_per_second)
@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+import time
+from collections.abc import Callable
+from typing import TypeVar
+
+T = TypeVar("T")
+
+
+class RetryError(RuntimeError):
+    def __init__(self, message: str, attempts: int, last_error: BaseException | None = None) -> None:
+        super().__init__(message)
+        self.attempts = attempts
+        self.last_error = last_error
+
+
+def retry_call(
+    fn: Callable[[], T],
+    attempts: int = 5,
+    base_delay: float = 0.2,
+    max_delay: float = 8.0,
+    sleeper: Callable[[float], None] = time.sleep,
+    retry_on: tuple[type[BaseException], ...] = (Exception,),
+) -> T:
+    last: BaseException | None = None
+    for attempt in range(1, max(1, attempts) + 1):
+        try:
+            return fn()
+        except retry_on as exc:
+            last = exc
+            if attempt >= attempts:
+                break
+            delay = min(max_delay, base_delay * (2 ** (attempt - 1)))
+            sleeper(delay)
+    raise RetryError(f"retry exhausted after {attempts} attempts: {last}", attempts, last)