restore main tree to deployed 3bafd30 (HEL-529 校准)

Revert the full revert 6f99ee9 so main's tree content is byte-identical to 3bafd30, which is the commit currently deployed and healthy on the 8766 xiaobai-datahub container (image hel531-3bafd30). No history rewrite, no force push; this is a normal forward commit on top of main. Co-authored-by: multica-agent <github@multica.ai>
2026-09-15 15:03:14 +08:00
parent 6f99ee9d9e
commit 78931f9c42
8 changed files with 826 additions and 219 deletions
@@ -31,10 +31,18 @@ class AdminAPI:
        )
        is_open = bool(cal and int(cal["is_open"]) == 1)
        pubs = self.db.fetchall("SELECT * FROM publications WHERE trade_date = ?", (today,))
-        failed = self.db.fetchall(
-            "SELECT * FROM batches WHERE trade_date = ? AND state IN ('failed','staged')",
-            (today,),
-        )
+        # HEL-529 fix 2: "待处理"只保留当前仍未恢复的最新异常。一条失败/停滞
+        # 批次若已被同数据集更晚的成功批次或当日发布解决，就不再是当前故障，
+        # 历史记录仍完整保留在 batches/audit 明细里，不在此重复展示。
+        latest_by_dataset: dict[str, dict[str, Any]] = {}
+        for row in self.db.fetchall("SELECT * FROM batches WHERE trade_date = ? ORDER BY started_at, batch_id", (today,)):
+            latest_by_dataset[row["dataset"]] = row
+        published_datasets = {row["dataset"] for row in pubs if row["state"] == "published"}
+        anomalies = [
+            row
+            for dataset, row in latest_by_dataset.items()
+            if dataset not in published_datasets and row["state"] in ("failed", "staged")
+        ]
        calls = self.db.fetchall(
            "SELECT * FROM src_calls ORDER BY id DESC LIMIT 20",
        )
@@ -45,7 +53,7 @@ class AdminAPI:
            "eod_status": self.scheduler.eod_status(today),
            "revision_status": self.scheduler.revision_status(today),
            "publications": pubs,
-            "anomalies": failed,
+            "anomalies": anomalies,
            "recent_calls": _public_calls(calls),
            "source_count": len(self.db.fetchall("SELECT provider FROM src_health")),
        }
@@ -38,6 +38,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "calendar",
        "tier": "official",
+        "update_freq": "每日 08:45 预检",
        "v1_endpoint": "/v1/calendar",
        "primary_source": "tushare:trade_cal",
        "backup_source": None,
@@ -46,6 +47,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "stocks",
        "tier": "official",
+        "update_freq": "每日 20:00 / 23:10",
        "v1_endpoint": "/v1/stocks",
        "primary_source": "tushare:stock_basic",
        "backup_source": None,
@@ -54,6 +56,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "daily",
        "tier": "official",
+        "update_freq": "盘后 15:05 · 重试至 23:30",
        "v1_endpoint": "/v1/bars/daily",
        "primary_source": "tushare:daily",
        "backup_source": None,
@@ -62,6 +65,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "valuation",
        "tier": "official",
+        "update_freq": "盘后 15:05 · 复核 20:00",
        "v1_endpoint": "/v1/valuation",
        "primary_source": "tushare:daily_basic",
        "backup_source": None,
@@ -70,6 +74,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "moneyflow",
        "tier": "official",
+        "update_freq": "盘后 15:05 · 重试至 23:30",
        "v1_endpoint": "/v1/moneyflow",
        "primary_source": "tushare:moneyflow",
        "backup_source": None,
@@ -78,6 +83,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "auction",
        "tier": "official",
+        "update_freq": "盘后 15:05",
        "v1_endpoint": "/v1/auction",
        "primary_source": "tushare:stk_auction",
        "backup_source": None,
@@ -86,6 +92,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "index_daily",
        "tier": "official",
+        "update_freq": "盘后 15:10 · 重试至 23:30",
        "v1_endpoint": "/v1/indexes/bars",
        "primary_source": "tushare:index_daily",
        "backup_source": None,
@@ -94,6 +101,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "limit_events",
        "tier": "official",
+        "update_freq": "盘后 16:40 · 重试至 23:30",
        "v1_endpoint": "/v1/limit-events",
        "primary_source": "tushare:limit_list_d",
        "backup_source": None,
@@ -102,6 +110,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "popularity",
        "tier": "official",
+        "update_freq": "盘后 22:40",
        "v1_endpoint": "/v1/popularity",
        "primary_source": "tushare:ths_hot+dc_hot",
        "backup_source": None,
@@ -110,6 +119,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "dragon_tiger",
        "tier": "official",
+        "update_freq": "盘后 16:45 · 重试至 23:30",
        "v1_endpoint": "/v1/dragon-tiger",
        "primary_source": "tushare:hm_detail",
        "backup_source": None,
@@ -118,6 +128,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "sector_daily",
        "tier": "official",
+        "update_freq": "盘后 15:20 · 重试至 23:30",
        "v1_endpoint": "/v1/sectors",
        "primary_source": "tushare:ths_daily+dc_index+sw_daily",
        "backup_source": None,
@@ -126,6 +137,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "quotes_latest",
        "tier": "provisional",
+        "update_freq": "盘中 · 缓存 60s",
        "v1_endpoint": "/v1/quotes/latest",
        "primary_source": "eastmoney:ulist/clist",
        "backup_source": "tencent:qt",
@@ -134,6 +146,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "index_quotes",
        "tier": "provisional",
+        "update_freq": "盘中 · 缓存 60s",
        "v1_endpoint": "/v1/indexes/quotes",
        "primary_source": "eastmoney:ulist",
        "backup_source": "tencent:qt",
@@ -142,6 +155,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "sectors_quote",
        "tier": "provisional",
+        "update_freq": "盘中 · 缓存 60s",
        "v1_endpoint": "/v1/sectors/quote",
        "primary_source": "eastmoney:sw",
        "backup_source": None,
@@ -150,6 +164,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "limit_pool",
        "tier": "provisional",
+        "update_freq": "盘中 · 缓存 60s",
        "v1_endpoint": "/v1/limit-pool",
        "primary_source": "eastmoney:zt_pool",
        "backup_source": None,
@@ -158,6 +173,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "intraday_points",
        "tier": "provisional",
+        "update_freq": "盘中 · 缓存 20s",
        "v1_endpoint": "/v1/intraday/points",
        "primary_source": "eastmoney:trends2",
        "backup_source": None,
@@ -166,6 +182,7 @@ DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "ifind_wencai",
        "tier": "licensed",
+        "update_freq": "按需调用",
        "v1_endpoint": "/v1/query (api_name=ifind_wencai)",
        "primary_source": "ifind:smart_stock_picking",
        "backup_source": None,
@@ -99,6 +99,58 @@ STAGING_INSERT = {
    **EXTENDED_STAGING_INSERT,
 }

+# Staging-table business keys (PRIMARY KEY minus the constant batch_id),
+# mirroring the PRIMARY KEY clauses declared in datahub/db.py and
+# datahub/datasets_ext.py. Used only to collapse within-batch duplicates so
+# a single upstream response cannot fail the whole batch on a UNIQUE
+# constraint (HEL-529: 2026-09-14 dc_hot returned 4 duplicate ts_codes and
+# hm_detail 43 duplicate (ts_code, hm_name) keys in one response, which has
+# blocked popularity/dragon_tiger publishing every day since 09-07).
+STAGING_KEY_FIELDS = {
+    "stocks": ("ts_code", "trade_date"),
+    "daily": ("ts_code", "trade_date"),
+    "valuation": ("ts_code", "trade_date"),
+    "moneyflow": ("ts_code", "trade_date"),
+    "auction": ("ts_code", "trade_date"),
+    "index_daily": ("ts_code", "trade_date"),
+    "limit_events": ("ts_code", "trade_date", "limit_type"),
+    "popularity": ("ts_code", "trade_date", "source"),
+    "dragon_tiger": ("ts_code", "trade_date", "hm_name"),
+    "sector_daily": ("ts_code", "trade_date", "family"),
+}
+
+
+def _dedupe_staging_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Collapse within-batch duplicates on the staging table's business key.
+
+    Deterministic: keeps the LAST occurrence of each key (the same row the
+    eod copy's INSERT OR REPLACE would keep), preserves first-seen order, and
+    never touches rows across batches. Datasets without a declared business
+    key are returned unchanged.
+    """
+    fields = STAGING_KEY_FIELDS.get(dataset)
+    if not fields:
+        return rows
+    seen: dict[tuple, int] = {}
+    out: list[dict[str, Any]] = []
+    dropped = 0
+    for row in rows:
+        key = tuple(row.get(field) for field in fields)
+        if key in seen:
+            out[seen[key]] = row
+            dropped += 1
+        else:
+            seen[key] = len(out)
+            out.append(row)
+    if dropped:
+        LOGGER.info(
+            "staging dedupe: %s collapsed %d duplicate rows within batch (kept last)",
+            dataset,
+            dropped,
+            extra={"hub": {"dataset": dataset, "deduped": dropped}},
+        )
+    return out
+
 EOD_COPY = {
    "stocks": (
        "INSERT OR REPLACE INTO eod_stocks "
@@ -1720,6 +1772,7 @@ class Pipeline:

    def _stage(self, dataset: str, batch_id: str, rows: list[dict[str, Any]]) -> None:
        sql, mapper = STAGING_INSERT[dataset]
+        rows = _dedupe_staging_rows(dataset, rows)
        with self.db.write() as connection:
            connection.execute(
                f"DELETE FROM {DATASET_TABLES[dataset][1]} WHERE batch_id = ?",
@@ -31,21 +31,21 @@ CATALOG: list[dict[str, Any]] = [
        "credential_key": "tushare_token",
        "status_source": "src_health (legacy, kept) + provider_health (unified, HEL-543)",
        "interfaces": [
-            {"interface": "trade_cal", "capability": "交易日历", "datasets": ["calendar"]},
-            {"interface": "stock_basic", "capability": "股票主档", "datasets": ["stocks"]},
-            {"interface": "daily", "capability": "个股日K", "datasets": ["daily"]},
-            {"interface": "adj_factor", "capability": "复权因子", "datasets": ["daily"]},
-            {"interface": "daily_basic", "capability": "估值", "datasets": ["valuation"]},
-            {"interface": "index_daily", "capability": "指数日K", "datasets": ["index_daily"]},
-            {"interface": "moneyflow", "capability": "资金流", "datasets": ["moneyflow"]},
-            {"interface": "stk_auction", "capability": "集合竞价", "datasets": ["auction"]},
-            {"interface": "limit_list_d", "capability": "涨跌停池", "datasets": ["limit_events"]},
-            {"interface": "ths_hot", "capability": "同花顺人气榜", "datasets": ["popularity"]},
-            {"interface": "dc_hot", "capability": "东方财富人气榜", "datasets": ["popularity"]},
-            {"interface": "hm_detail", "capability": "龙虎榜游资明细", "datasets": ["dragon_tiger"]},
-            {"interface": "ths_daily", "capability": "同花顺概念行情", "datasets": ["sector_daily"]},
-            {"interface": "dc_index", "capability": "东方财富板块行情", "datasets": ["sector_daily"]},
-            {"interface": "sw_daily", "capability": "申万行业行情", "datasets": ["sector_daily"]},
+            {"interface": "trade_cal", "capability": "交易日历", "datasets": ["calendar"], "group": "日历 / 主档"},
+            {"interface": "stock_basic", "capability": "股票主档", "datasets": ["stocks"], "group": "日历 / 主档"},
+            {"interface": "daily", "capability": "个股日K", "datasets": ["daily"], "group": "盘后 A 批"},
+            {"interface": "adj_factor", "capability": "复权因子", "datasets": ["daily"], "group": "盘后 A 批"},
+            {"interface": "daily_basic", "capability": "估值", "datasets": ["valuation"], "group": "盘后 A 批"},
+            {"interface": "index_daily", "capability": "指数日K", "datasets": ["index_daily"], "group": "指数 B 批"},
+            {"interface": "moneyflow", "capability": "资金流", "datasets": ["moneyflow"], "group": "盘后 A 批"},
+            {"interface": "stk_auction", "capability": "集合竞价", "datasets": ["auction"], "group": "盘后 A 批"},
+            {"interface": "limit_list_d", "capability": "涨跌停池", "datasets": ["limit_events"], "group": "扩展软批"},
+            {"interface": "ths_hot", "capability": "同花顺人气榜", "datasets": ["popularity"], "group": "扩展软批"},
+            {"interface": "dc_hot", "capability": "东方财富人气榜", "datasets": ["popularity"], "group": "扩展软批"},
+            {"interface": "hm_detail", "capability": "龙虎榜游资明细", "datasets": ["dragon_tiger"], "group": "扩展软批"},
+            {"interface": "ths_daily", "capability": "同花顺概念行情", "datasets": ["sector_daily"], "group": "扩展软批"},
+            {"interface": "dc_index", "capability": "东方财富板块行情", "datasets": ["sector_daily"], "group": "扩展软批"},
+            {"interface": "sw_daily", "capability": "申万行业行情", "datasets": ["sector_daily"], "group": "扩展软批"},
        ],
    },
    {
@@ -55,13 +55,13 @@ CATALOG: list[dict[str, Any]] = [
        "credential_key": None,
        "status_source": "provider_health (unified, HEL-543)",
        "interfaces": [
-            {"interface": "indices", "capability": "指数实时报价", "datasets": ["index_quotes"]},
-            {"interface": "market_quotes", "capability": "全市场实时快照", "datasets": ["quotes_latest"]},
-            {"interface": "named_quotes", "capability": "指定个股实时报价", "datasets": ["quotes_latest"]},
-            {"interface": "sector_quote", "capability": "申万板块实时报价(单个)", "datasets": ["sectors_quote"]},
-            {"interface": "sector_quotes_batch", "capability": "申万板块批量报价(预热)", "datasets": ["sectors_quote"]},
-            {"interface": "limit_pool", "capability": "涨停/炸板池(盘中)", "datasets": ["limit_pool"]},
-            {"interface": "intraday", "capability": "分时走势", "datasets": ["intraday_points"]},
+            {"interface": "indices", "capability": "指数实时报价", "datasets": ["index_quotes"], "group": "实时快照"},
+            {"interface": "market_quotes", "capability": "全市场实时快照", "datasets": ["quotes_latest"], "group": "实时快照"},
+            {"interface": "named_quotes", "capability": "指定个股实时报价", "datasets": ["quotes_latest"], "group": "实时快照"},
+            {"interface": "sector_quote", "capability": "申万板块实时报价(单个)", "datasets": ["sectors_quote"], "group": "实时快照"},
+            {"interface": "sector_quotes_batch", "capability": "申万板块批量报价(预热)", "datasets": ["sectors_quote"], "group": "实时快照"},
+            {"interface": "limit_pool", "capability": "涨停/炸板池(盘中)", "datasets": ["limit_pool"], "group": "实时快照"},
+            {"interface": "intraday", "capability": "分时走势", "datasets": ["intraday_points"], "group": "实时快照"},
        ],
    },
    {
@@ -71,13 +71,14 @@ CATALOG: list[dict[str, Any]] = [
        "credential_key": None,
        "status_source": "provider_health (unified, HEL-543)",
        "interfaces": [
-            {"interface": "indices", "capability": "指数实时报价(东财失败时备用)", "datasets": ["index_quotes"]},
+            {"interface": "indices", "capability": "指数实时报价(东财失败时备用)", "datasets": ["index_quotes"], "group": "实时备援"},
            {
                "interface": "market_quotes_fallback",
                "capability": "全市场快照(备用；按本地股票主档逐只请求拼接)",
                "datasets": ["quotes_latest"],
+                "group": "实时备援",
            },
-            {"interface": "named_quotes", "capability": "指定个股实时报价(东财失败时备用)", "datasets": ["quotes_latest"]},
+            {"interface": "named_quotes", "capability": "指定个股实时报价(东财失败时备用)", "datasets": ["quotes_latest"], "group": "实时备援"},
        ],
    },
    {
@@ -87,11 +88,11 @@ CATALOG: list[dict[str, Any]] = [
        "credential_key": "ifind_refresh_token",
        "status_source": "provider_health (unified, HEL-543) + adapter.status()",
        "interfaces": [
-            {"interface": "wencai", "capability": "问财自然语言选股", "datasets": ["ifind_wencai"]},
-            {"interface": "snapshots", "capability": "快照", "datasets": ["ifind_snapshots"]},
-            {"interface": "history", "capability": "历史行情", "datasets": ["ifind_history"]},
-            {"interface": "realtime", "capability": "实时行情", "datasets": ["ifind_realtime"]},
-            {"interface": "intraday", "capability": "分时(高频)", "datasets": ["ifind_intraday"]},
+            {"interface": "wencai", "capability": "问财自然语言选股", "datasets": ["ifind_wencai"], "group": "预留接口"},
+            {"interface": "snapshots", "capability": "快照", "datasets": ["ifind_snapshots"], "group": "预留接口"},
+            {"interface": "history", "capability": "历史行情", "datasets": ["ifind_history"], "group": "预留接口"},
+            {"interface": "realtime", "capability": "实时行情", "datasets": ["ifind_realtime"], "group": "预留接口"},
+            {"interface": "intraday", "capability": "分时(高频)", "datasets": ["ifind_intraday"], "group": "预留接口"},
        ],
    },
    {
@@ -161,5 +162,35 @@ def snapshot(db: Any, auth: Any = None) -> list[dict[str, Any]]:
        except Exception:
            health_rows = []
        item["live_interfaces"] = health_rows
+        # HEL-529 fix 3: 已登记 ≠ 已观测 ≠ 健康。观测记录有两种真实写法：
+        # eastmoney/tencent 的 observe() 直接写接口名；tushare 官方管线
+        # _log_call() 写的是数据集名（如 daily_basic 接口对应的数据集
+        # valuation）。这里按「接口名 或 该接口声明的 datasets 之一」双向
+        # 匹配，并为每个接口标注观测依据，禁止把"暂无观测"显示成"未配置"。
+        by_interface = {str(row["interface"]): row for row in health_rows}
+        for iface in item["interfaces"]:
+            row = by_interface.get(str(iface["interface"]))
+            basis = "interface" if row is not None else ""
+            if row is None:
+                for dataset in iface.get("datasets", []):
+                    candidate = by_interface.get(str(dataset))
+                    if candidate is not None:
+                        row = candidate
+                        basis = "dataset"
+                        break
+            if row is None:
+                iface["observed"] = False
+                iface["observed_basis"] = ""
+                iface["observed_state"] = ""
+                iface["observed_at"] = ""
+                iface["observed_latency_ms"] = None
+                iface["observed_note"] = ""
+            else:
+                iface["observed"] = True
+                iface["observed_basis"] = basis
+                iface["observed_state"] = str(row["state"] or "")
+                iface["observed_at"] = str(row["last_ok_at"] or row["updated_at"] or "")
+                iface["observed_latency_ms"] = row["last_latency_ms"]
+                iface["observed_note"] = str(row["last_error"] or row["last_fallback_reason"] or "")
        result.append(item)
    return result