from __future__ import annotations from http import HTTPStatus from typing import Any from urllib.parse import parse_qs from datahub import SCHEMA_VERSION from datahub.codes import resolve_code from datahub.coverage import calendar_coverage, point_coverage, published_range_coverage from datahub.db import HubDB from datahub.normalize import qfq_bar from datahub.numbers import finite_number from datahub.pipeline import Pipeline from datahub.settings import Settings from datahub.timeutil import isoformat, now_shanghai, session_phase, yyyymmdd ERROR_STATUS = { "UNAUTHORIZED": HTTPStatus.UNAUTHORIZED, "INVALID_ARGUMENT": HTTPStatus.BAD_REQUEST, "RATE_LIMITED": HTTPStatus.TOO_MANY_REQUESTS, "SOURCE_UNAVAILABLE": HTTPStatus.SERVICE_UNAVAILABLE, "DATASET_NOT_PUBLISHED": HTTPStatus.NOT_FOUND, "STALE_DATA": HTTPStatus.OK, "INTERNAL": HTTPStatus.INTERNAL_SERVER_ERROR, } class ApiError(Exception): def __init__(self, code: str, message: str, retry_after: int | None = None, extra: dict[str, Any] | None = None) -> None: super().__init__(message) self.code = code self.message = message self.retry_after = retry_after self.extra = extra or {} def payload(self) -> dict[str, Any]: body: dict[str, Any] = {"code": self.code, "message": self.message} if self.retry_after is not None: body["retry_after"] = self.retry_after body.update(self.extra) return {"error": body} @property def status(self) -> HTTPStatus: return ERROR_STATUS.get(self.code, HTTPStatus.INTERNAL_SERVER_ERROR) def envelope(data: Any, meta: dict[str, Any]) -> dict[str, Any]: return {"schema_version": SCHEMA_VERSION, "data": data, "meta": meta} class V1API: def __init__(self, db: HubDB, pipeline: Pipeline, settings: Settings) -> None: self.db = db self.pipeline = pipeline self.settings = settings def handle(self, path: str, query: dict[str, list[str]]) -> dict[str, Any]: q = {key: values[-1] if values else "" for key, values in query.items()} if path == "/v1/health": return self.health() if path == "/v1/calendar": return self.calendar(q.get("from") or "", q.get("to") or "") if path == "/v1/stocks": return self.stocks(q.get("updated_since") or "", q) if path == "/v1/bars/daily": return self.daily_bars(q) if path == "/v1/indexes/bars": return self.index_bars(q) if path == "/v1/valuation": return self.valuation(q) if path == "/v1/moneyflow": return self.moneyflow(q) if path == "/v1/auction": return self.auction(q) if path == "/v1/datasets/status": return self.dataset_status(q.get("date") or "") if path == "/v1/batches": return self.batches(q.get("date") or "", q.get("dataset") or "") raise ApiError("INVALID_ARGUMENT", f"unknown endpoint: {path}") def health(self) -> dict[str, Any]: today = yyyymmdd(now_shanghai()) cal = self.db.fetchone( "SELECT is_open FROM trade_calendar WHERE exchange = 'SSE' AND cal_date = ?", (today,), ) is_open = bool(cal and cal["is_open"] == 1) sources = self.db.fetchall("SELECT * FROM src_health") return envelope( { "status": "ok", "session_phase": session_phase(now_shanghai(), is_open), "trade_date": today, "is_open_day": is_open, "sources": [ { "provider": row["provider"], "endpoint_class": row["endpoint_class"], "state": row["state"], "last_ok_at": row["last_ok_at"], "consec_failures": row["consec_failures"], } for row in sources ], }, {"tier": "official", "trade_date": today, "source": "datahub", "stale": False, "staleness_seconds": 0}, ) def calendar(self, start: str, end: str) -> dict[str, Any]: start = yyyymmdd(start or add_default(-30)) end = yyyymmdd(end or add_default(5)) rows = self.db.fetchall( """ SELECT cal_date, is_open, pretrade_date, (SELECT MAX(cal_date) FROM trade_calendar t2 WHERE t2.exchange = 'SSE' AND t2.is_open = 1 AND t2.cal_date < t1.cal_date) AS prev_open FROM trade_calendar t1 WHERE exchange = 'SSE' AND cal_date >= ? AND cal_date <= ? ORDER BY cal_date """, (start, end), ) items = [ { "cal_date": row["cal_date"], "is_open": bool(row["is_open"]), "pretrade_date": row["pretrade_date"], "prev_open": row["prev_open"], } for row in rows ] meta = self._official_meta("calendar", end if items else start, source="tushare:trade_cal") return envelope(items, attach_coverage(meta, calendar_coverage(self.db, start, end))) def stocks(self, updated_since: str, q: dict[str, str]) -> dict[str, Any]: limit, offset = self._page(q) if updated_since: rows = self.db.fetchall( "SELECT * FROM stock_master WHERE updated_at >= ? ORDER BY ts_code LIMIT ? OFFSET ?", (updated_since, limit, offset), ) else: rows = self.db.fetchall( "SELECT * FROM stock_master ORDER BY ts_code LIMIT ? OFFSET ?", (limit, offset), ) return envelope(rows, self._official_meta("stocks", yyyymmdd(), source="tushare:stock_basic")) def daily_bars(self, q: dict[str, str]) -> dict[str, Any]: return self._published_rows( dataset="daily", table="eod_bars", q=q, source="tushare:daily", adjust=q.get("adjust") or "none", ) def index_bars(self, q: dict[str, str]) -> dict[str, Any]: return self._published_rows( dataset="index_daily", table="eod_index_bars", q=q, source="tushare:index_daily", default_code="000001.SH", ) def valuation(self, q: dict[str, str]) -> dict[str, Any]: return self._published_rows(dataset="valuation", table="eod_valuation", q=q, source="tushare:daily_basic") def moneyflow(self, q: dict[str, str]) -> dict[str, Any]: return self._published_rows(dataset="moneyflow", table="eod_moneyflow", q=q, source="tushare:moneyflow") def auction(self, q: dict[str, str]) -> dict[str, Any]: return self._published_rows(dataset="auction", table="eod_auction", q=q, source="tushare:stk_auction") def dataset_status(self, date: str) -> dict[str, Any]: trade_date = yyyymmdd(date or now_shanghai()) datasets = ("daily", "valuation", "moneyflow", "auction", "index_daily") items = [] for dataset in datasets: pub = self.db.fetchone( "SELECT * FROM publications WHERE dataset = ? AND trade_date = ?", (dataset, trade_date), ) batch = None if pub: batch = self.db.fetchone("SELECT * FROM batches WHERE batch_id = ?", (pub["active_batch"],)) items.append( { "dataset": dataset, "trade_date": trade_date, "state": (pub or {}).get("state") or "unpublished", "batch_id": (pub or {}).get("active_batch"), "published_at": (pub or {}).get("published_at"), "rows_out": (batch or {}).get("rows_out"), "quality": _parse_json((batch or {}).get("quality_json")), } ) return envelope(items, self._official_meta("status", trade_date, source="datahub")) def batches(self, date: str, dataset: str) -> dict[str, Any]: trade_date = yyyymmdd(date or now_shanghai()) if dataset: rows = self.db.fetchall( "SELECT * FROM batches WHERE trade_date = ? AND dataset = ? ORDER BY started_at", (trade_date, dataset), ) else: rows = self.db.fetchall( "SELECT * FROM batches WHERE trade_date = ? ORDER BY started_at", (trade_date,), ) return envelope(rows, self._official_meta("batches", trade_date, source="datahub")) def _published_rows( self, dataset: str, table: str, q: dict[str, str], source: str, adjust: str = "none", default_code: str = "", ) -> dict[str, Any]: trade_date = q.get("date") or q.get("trade_date") or "" code = q.get("code") or default_code start = q.get("from") or "" end = q.get("to") or "" if trade_date: trade_date = yyyymmdd(trade_date) start = end = trade_date if not start or not end: if not trade_date: raise ApiError("INVALID_ARGUMENT", "date or from/to is required") else: start = yyyymmdd(start) end = yyyymmdd(end) ts_code = "" if code: resolved = resolve_code(self.db, code) if resolved is None: raise ApiError("INVALID_ARGUMENT", f"ambiguous code: {code}") ts_code = resolved # For a range, use per-date published batch. Single-date is the common path. if start == end: pub = self.db.fetchone( "SELECT * FROM publications WHERE dataset = ? AND trade_date = ?", (dataset, start), ) if not pub: raise ApiError( "DATASET_NOT_PUBLISHED", f"{dataset} {start} 尚未发布", extra={"expected_at": "15:05+08:00"}, ) limit, offset = self._page(q) sql = f"SELECT * FROM {table} WHERE trade_date = ? AND batch_id = ?" params: list[Any] = [start, pub["active_batch"]] if ts_code: sql += " AND ts_code = ?" params.append(ts_code) sql += " ORDER BY ts_code LIMIT ? OFFSET ?" params.extend([limit, offset]) rows = [dict(row) for row in self.db.fetchall(sql, tuple(params))] if adjust == "qfq" and dataset == "daily": rows = self._apply_qfq(rows) meta = { "tier": "official", "trade_date": start, "published_at": pub["published_at"], "source": source, "batch_id": pub["active_batch"], "stale": False, "staleness_seconds": 0, "state": pub["state"], } return envelope(rows, attach_coverage(meta, point_coverage(start, dataset))) # multi-day: walk published dates pubs = self.db.fetchall( "SELECT * FROM publications WHERE dataset = ? AND trade_date >= ? AND trade_date <= ? ORDER BY trade_date", (dataset, start, end), ) if not pubs: raise ApiError("DATASET_NOT_PUBLISHED", f"{dataset} {start}-{end} 尚未发布") rows: list[dict[str, Any]] = [] limit, offset = self._page(q) for pub in pubs: sql = f"SELECT * FROM {table} WHERE trade_date = ? AND batch_id = ?" params = [pub["trade_date"], pub["active_batch"]] if ts_code: sql += " AND ts_code = ?" params.append(ts_code) sql += " ORDER BY ts_code" rows.extend(self.db.fetchall(sql, tuple(params))) sliced = rows[offset: offset + limit] if adjust == "qfq" and dataset == "daily": sliced = self._apply_qfq(sliced) last = pubs[-1] coverage = published_range_coverage( self.db, dataset, start, end, ts_code=ts_code, table=table, ) return envelope( sliced, attach_coverage( { "tier": "official", "trade_date": last["trade_date"], "published_at": last["published_at"], "source": source, "batch_id": last["active_batch"], "stale": False, "staleness_seconds": 0, }, coverage, ), ) def _apply_qfq(self, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: by_code: dict[str, list[dict[str, Any]]] = {} for row in rows: by_code.setdefault(str(row["ts_code"]), []).append(row) out: list[dict[str, Any]] = [] for code, group in by_code.items(): latest = None factors = [finite_number(item.get("adj_factor")) for item in group] factors = [item for item in factors if item] if factors: latest = max(factors) else: extra = self.db.fetchone( "SELECT MAX(adj_factor) AS f FROM eod_bars WHERE ts_code = ?", (code,), ) latest = finite_number((extra or {}).get("f"), 1.0) out.extend(qfq_bar(item, latest) for item in group) return out def _page(self, q: dict[str, str]) -> tuple[int, int]: try: limit = int(q.get("limit") or self.settings.list_limit_default) offset = int(q.get("offset") or 0) except ValueError as exc: raise ApiError("INVALID_ARGUMENT", "limit/offset must be integers") from exc limit = max(1, min(limit, self.settings.list_limit_max)) offset = max(0, offset) return limit, offset def _official_meta(self, dataset: str, trade_date: str, source: str) -> dict[str, Any]: pub = self.db.fetchone( "SELECT * FROM publications WHERE dataset = ? AND trade_date = ?", (dataset, trade_date), ) return { "tier": "official", "trade_date": trade_date, "published_at": (pub or {}).get("published_at"), "source": source, "batch_id": (pub or {}).get("active_batch"), "stale": False, "staleness_seconds": 0, } def add_default(days: int) -> str: from datetime import timedelta return (now_shanghai() + timedelta(days=days)).strftime("%Y%m%d") def attach_coverage(meta: dict[str, Any], coverage: dict[str, Any]) -> dict[str, Any]: merged = dict(meta) merged["coverage"] = coverage merged["incomplete"] = not bool(coverage.get("complete")) return merged def parse_query(raw: str) -> dict[str, list[str]]: return parse_qs(raw, keep_blank_values=True) def _parse_json(raw: Any) -> Any: if not raw: return None if isinstance(raw, dict): return raw import json try: return json.loads(str(raw)) except json.JSONDecodeError: return None