xiaobai-review/xiaobai-datahub/datahub/lineage.py

"""Read-only lineage/impact inventory (HEL-543).

Answers, without changing any routing decision: for a given main-site data
item, which datahub dataset backs it, which provider/interface currently
serves it (primary and backup), and — when a provider/interface is
unhealthy — which datasets and, best-effort, which main-site consumers are
affected.

Every row cites where it was verified so a reviewer does not have to trust
a paraphrase:

- ``v1_endpoint``/``primary_source``/``backup_source`` are taken verbatim
  from ``datahub/serving.py`` (the ``source=`` string literal passed to
  ``_published_rows``/``_official_meta``) or from the provider/interface
  pairs wired into ``datahub/realtime_serve.py`` for HEL-543.
- ``known_consumers`` lists only call sites this round actually found via
  code search in the ``xiaobai-review`` website tree (cited as
  ``file:line`` in the comment above each dataset). Anything not backed by
  a citation is left out rather than guessed; a fuller page-by-page map is
  tracked separately (HEL-549) and can extend this table later without
  touching its shape.

This module never talks to a provider and never mutates anything; it only
reads ``provider_health``/``provider_call_log`` (HEL-543) and the existing
``publications``/``batches`` tables to attach live status to each row.
"""

from __future__ import annotations

from typing import Any

# Verified against backend/features/screener/data_sync.py (calendar,
# stock_basic, daily, daily_basic, index_daily-as-benchmark, stk_auction,
# moneyflow, ths_hot, dc_hot all called via `self.client.query(...)`) and
# backend/features/heaven/market_context.py (stock_basic, index_daily via
# `self._tushare_client().query(...)` / `client.query(...)`).
DATASETS: list[dict[str, Any]] = [
    {
        "dataset": "calendar",
        "tier": "official",
        "v1_endpoint": "/v1/calendar",
        "primary_source": "tushare:trade_cal",
        "backup_source": None,
        "known_consumers": ["智能选股(data_sync.py 交易日历解析)", "问天(交易日推算)"],
    },
    {
        "dataset": "stocks",
        "tier": "official",
        "v1_endpoint": "/v1/stocks",
        "primary_source": "tushare:stock_basic",
        "backup_source": None,
        "known_consumers": ["智能选股(股票主档)", "问天(market_context.py 股票代码/名称解析)"],
    },
    {
        "dataset": "daily",
        "tier": "official",
        "v1_endpoint": "/v1/bars/daily",
        "primary_source": "tushare:daily",
        "backup_source": None,
        "known_consumers": ["智能选股(data_sync.py 日K因子)", "交易复盘/个股详情日K图表"],
    },
    {
        "dataset": "valuation",
        "tier": "official",
        "v1_endpoint": "/v1/valuation",
        "primary_source": "tushare:daily_basic",
        "backup_source": None,
        "known_consumers": ["智能选股(data_sync.py 估值因子)"],
    },
    {
        "dataset": "moneyflow",
        "tier": "official",
        "v1_endpoint": "/v1/moneyflow",
        "primary_source": "tushare:moneyflow",
        "backup_source": None,
        "known_consumers": ["智能选股(data_sync.py 资金流因子)", "个股详情资金流"],
    },
    {
        "dataset": "auction",
        "tier": "official",
        "v1_endpoint": "/v1/auction",
        "primary_source": "tushare:stk_auction",
        "backup_source": None,
        "known_consumers": ["智能选股(data_sync.py 竞价快照)", "竞价板块"],
    },
    {
        "dataset": "index_daily",
        "tier": "official",
        "v1_endpoint": "/v1/indexes/bars",
        "primary_source": "tushare:index_daily",
        "backup_source": None,
        "known_consumers": ["问天(market_context.py 指数近20日走势)", "智能选股(基准回看)"],
    },
    {
        "dataset": "limit_events",
        "tier": "official",
        "v1_endpoint": "/v1/limit-events",
        "primary_source": "tushare:limit_list_d",
        "backup_source": None,
        "known_consumers": ["涨停梯队(历史/盘后视图)"],
    },
    {
        "dataset": "popularity",
        "tier": "official",
        "v1_endpoint": "/v1/popularity",
        "primary_source": "tushare:ths_hot+dc_hot",
        "backup_source": None,
        "known_consumers": ["人气榜", "智能选股(data_sync.py 人气因子)"],
    },
    {
        "dataset": "dragon_tiger",
        "tier": "official",
        "v1_endpoint": "/v1/dragon-tiger",
        "primary_source": "tushare:hm_detail",
        "backup_source": None,
        "known_consumers": ["龙虎榜"],
    },
    {
        "dataset": "sector_daily",
        "tier": "official",
        "v1_endpoint": "/v1/sectors",
        "primary_source": "tushare:ths_daily+dc_index+sw_daily",
        "backup_source": None,
        "known_consumers": ["主题轮动", "板块梯队"],
    },
    {
        "dataset": "quotes_latest",
        "tier": "provisional",
        "v1_endpoint": "/v1/quotes/latest",
        "primary_source": "eastmoney:ulist/clist",
        "backup_source": "tencent:qt",
        "known_consumers": ["竞价/股票池盘中价格", "情绪周期盘中快照"],
    },
    {
        "dataset": "index_quotes",
        "tier": "provisional",
        "v1_endpoint": "/v1/indexes/quotes",
        "primary_source": "eastmoney:ulist",
        "backup_source": "tencent:qt",
        "known_consumers": ["首页大盘指数条"],
    },
    {
        "dataset": "sectors_quote",
        "tier": "provisional",
        "v1_endpoint": "/v1/sectors/quote",
        "primary_source": "eastmoney:sw",
        "backup_source": None,
        "known_consumers": ["主题轮动盘中板块报价"],
    },
    {
        "dataset": "limit_pool",
        "tier": "provisional",
        "v1_endpoint": "/v1/limit-pool",
        "primary_source": "eastmoney:zt_pool",
        "backup_source": None,
        "known_consumers": ["涨停梯队盘中视图"],
    },
    {
        "dataset": "intraday_points",
        "tier": "provisional",
        "v1_endpoint": "/v1/intraday/points",
        "primary_source": "eastmoney:trends2",
        "backup_source": None,
        "known_consumers": ["个股详情分时图"],
    },
    {
        "dataset": "ifind_wencai",
        "tier": "licensed",
        "v1_endpoint": "/v1/query (api_name=ifind_wencai)",
        "primary_source": "ifind:smart_stock_picking",
        "backup_source": None,
        "known_consumers": ["问师(自然语言选股，需 iFinD 凭证)"],
    },
]

_KNOWN_PROVIDERS_BY_SOURCE_PREFIX = ("tushare", "eastmoney", "tencent", "ifind")


def _providers_for(primary_source: str, backup_source: str | None) -> list[str]:
    providers: list[str] = []
    for source in (primary_source, backup_source or ""):
        for provider in _KNOWN_PROVIDERS_BY_SOURCE_PREFIX:
            if source.startswith(provider) and provider not in providers:
                providers.append(provider)
    return providers


def snapshot(db: Any, trade_date: str = "") -> list[dict[str, Any]]:
    """Attach live status to the static lineage table. Read-only; never
    raises (a per-row status lookup failure just leaves that row's status
    empty rather than failing the whole snapshot)."""
    result: list[dict[str, Any]] = []
    for entry in DATASETS:
        row = dict(entry)
        providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
        row["providers"] = providers
        live: list[dict[str, Any]] = []
        try:
            if db is not None and providers:
                placeholders = ",".join("?" for _ in providers)
                live = db.fetchall(
                    f"SELECT provider, interface, state, last_error, last_fallback_reason, "
                    f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
                    tuple(providers),
                )
        except Exception:
            live = []
        row["live_provider_health"] = live
        if entry["tier"] == "official":
            pub = None
            try:
                if db is not None and trade_date:
                    pub = db.fetchone(
                        "SELECT dataset, trade_date, state, published_at FROM publications "
                        "WHERE dataset = ? AND trade_date = ?",
                        (entry["dataset"], trade_date),
                    )
            except Exception:
                pub = None
            row["publication"] = pub
        result.append(row)
    return result


def affected(db: Any, provider: str = "", interface: str = "") -> list[dict[str, Any]]:
    """Read-only: which datasets/pages are impacted by a given provider (and,
    optionally, a specific interface) right now. Does not change routing."""
    provider = str(provider or "").strip()
    interface = str(interface or "").strip()
    result: list[dict[str, Any]] = []
    for entry in DATASETS:
        providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
        if provider and provider not in providers:
            continue
        row = dict(entry)
        row["providers"] = providers
        health: list[dict[str, Any]] = []
        try:
            if db is not None:
                if interface:
                    health = db.fetchall(
                        "SELECT provider, interface, state, last_error, last_fallback_reason, "
                        "consec_failures, updated_at FROM provider_health "
                        "WHERE provider = ? AND interface = ?",
                        (provider, interface),
                    )
                elif providers:
                    placeholders = ",".join("?" for _ in providers)
                    health = db.fetchall(
                        f"SELECT provider, interface, state, last_error, last_fallback_reason, "
                        f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
                        tuple(providers),
                    )
        except Exception:
            health = []
        row["live_provider_health"] = health
        result.append(row)
    return result