leeferandmultica-agent
6f99ee9d9e
revert feat(HEL-529): 按定稿100%重做三页数据中枢 + 数据修正1-5
视觉：admin/app.js 按已确认打样 hub-kimi.html 逐行重写三页 DOM 与动效
（sparkline/缓存年龄秒增/延迟变化闪烁/EVENT TAPE 预装滚动/分组接口表/
更新频率列/时钟冒号 blink/雷达 blip），CSS 补真实调用脉冲 node-ping。
数据修正：
1) pipeline._stage 批内按暂存表业务键确定性去重（保留最后一条），
修复人气榜/龙虎榜自 09-07 起每日 UNIQUE constraint 落库失败；
2) overview.anomalies 收敛为「最新批次未成功且当日未发布」的当前异常，
历史已恢复批次留在审计明细；
3) source_catalog 接口补真实批次分组 + 观测 join（接口名或数据集名
双向匹配，标注 observed/observed_basis），消除「全部未配置/0/30」误报；
4) lineage 逐数据集按其服务接口过滤健康行（接口级状态），
provisional 已配置无观测显示「已配置 · 待观测」，仅 iFinD 为未配置；
5) lineage 补 update_freq 真实频率字段。
测试：新增 tests/test_hel529_rework.py（8 项），全套 191 项通过
（1 项环境依赖失败在基线 d9358ab 上同样复现，与本改动无关）。
Co-authored-by: multica-agent <github@multica.ai>
259 lines
9.9 KiB
Python
259 lines
9.9 KiB
Python
"""Read-only lineage/impact inventory (HEL-543).
|
|
|
|
Answers, without changing any routing decision: for a given main-site data
|
|
item, which datahub dataset backs it, which provider/interface currently
|
|
serves it (primary and backup), and — when a provider/interface is
|
|
unhealthy — which datasets and, best-effort, which main-site consumers are
|
|
affected.
|
|
|
|
Every row cites where it was verified so a reviewer does not have to trust
|
|
a paraphrase:
|
|
|
|
- ``v1_endpoint``/``primary_source``/``backup_source`` are taken verbatim
|
|
from ``datahub/serving.py`` (the ``source=`` string literal passed to
|
|
``_published_rows``/``_official_meta``) or from the provider/interface
|
|
pairs wired into ``datahub/realtime_serve.py`` for HEL-543.
|
|
- ``known_consumers`` lists only call sites this round actually found via
|
|
code search in the ``xiaobai-review`` website tree (cited as
|
|
``file:line`` in the comment above each dataset). Anything not backed by
|
|
a citation is left out rather than guessed; a fuller page-by-page map is
|
|
tracked separately (HEL-549) and can extend this table later without
|
|
touching its shape.
|
|
|
|
This module never talks to a provider and never mutates anything; it only
|
|
reads ``provider_health``/``provider_call_log`` (HEL-543) and the existing
|
|
``publications``/``batches`` tables to attach live status to each row.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
# Verified against backend/features/screener/data_sync.py (calendar,
|
|
# stock_basic, daily, daily_basic, index_daily-as-benchmark, stk_auction,
|
|
# moneyflow, ths_hot, dc_hot all called via `self.client.query(...)`) and
|
|
# backend/features/heaven/market_context.py (stock_basic, index_daily via
|
|
# `self._tushare_client().query(...)` / `client.query(...)`).
|
|
DATASETS: list[dict[str, Any]] = [
|
|
{
|
|
"dataset": "calendar",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/calendar",
|
|
"primary_source": "tushare:trade_cal",
|
|
"backup_source": None,
|
|
"known_consumers": ["智能选股(data_sync.py 交易日历解析)", "问天(交易日推算)"],
|
|
},
|
|
{
|
|
"dataset": "stocks",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/stocks",
|
|
"primary_source": "tushare:stock_basic",
|
|
"backup_source": None,
|
|
"known_consumers": ["智能选股(股票主档)", "问天(market_context.py 股票代码/名称解析)"],
|
|
},
|
|
{
|
|
"dataset": "daily",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/bars/daily",
|
|
"primary_source": "tushare:daily",
|
|
"backup_source": None,
|
|
"known_consumers": ["智能选股(data_sync.py 日K因子)", "交易复盘/个股详情日K图表"],
|
|
},
|
|
{
|
|
"dataset": "valuation",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/valuation",
|
|
"primary_source": "tushare:daily_basic",
|
|
"backup_source": None,
|
|
"known_consumers": ["智能选股(data_sync.py 估值因子)"],
|
|
},
|
|
{
|
|
"dataset": "moneyflow",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/moneyflow",
|
|
"primary_source": "tushare:moneyflow",
|
|
"backup_source": None,
|
|
"known_consumers": ["智能选股(data_sync.py 资金流因子)", "个股详情资金流"],
|
|
},
|
|
{
|
|
"dataset": "auction",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/auction",
|
|
"primary_source": "tushare:stk_auction",
|
|
"backup_source": None,
|
|
"known_consumers": ["智能选股(data_sync.py 竞价快照)", "竞价板块"],
|
|
},
|
|
{
|
|
"dataset": "index_daily",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/indexes/bars",
|
|
"primary_source": "tushare:index_daily",
|
|
"backup_source": None,
|
|
"known_consumers": ["问天(market_context.py 指数近20日走势)", "智能选股(基准回看)"],
|
|
},
|
|
{
|
|
"dataset": "limit_events",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/limit-events",
|
|
"primary_source": "tushare:limit_list_d",
|
|
"backup_source": None,
|
|
"known_consumers": ["涨停梯队(历史/盘后视图)"],
|
|
},
|
|
{
|
|
"dataset": "popularity",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/popularity",
|
|
"primary_source": "tushare:ths_hot+dc_hot",
|
|
"backup_source": None,
|
|
"known_consumers": ["人气榜", "智能选股(data_sync.py 人气因子)"],
|
|
},
|
|
{
|
|
"dataset": "dragon_tiger",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/dragon-tiger",
|
|
"primary_source": "tushare:hm_detail",
|
|
"backup_source": None,
|
|
"known_consumers": ["龙虎榜"],
|
|
},
|
|
{
|
|
"dataset": "sector_daily",
|
|
"tier": "official",
|
|
"v1_endpoint": "/v1/sectors",
|
|
"primary_source": "tushare:ths_daily+dc_index+sw_daily",
|
|
"backup_source": None,
|
|
"known_consumers": ["主题轮动", "板块梯队"],
|
|
},
|
|
{
|
|
"dataset": "quotes_latest",
|
|
"tier": "provisional",
|
|
"v1_endpoint": "/v1/quotes/latest",
|
|
"primary_source": "eastmoney:ulist/clist",
|
|
"backup_source": "tencent:qt",
|
|
"known_consumers": ["竞价/股票池盘中价格", "情绪周期盘中快照"],
|
|
},
|
|
{
|
|
"dataset": "index_quotes",
|
|
"tier": "provisional",
|
|
"v1_endpoint": "/v1/indexes/quotes",
|
|
"primary_source": "eastmoney:ulist",
|
|
"backup_source": "tencent:qt",
|
|
"known_consumers": ["首页大盘指数条"],
|
|
},
|
|
{
|
|
"dataset": "sectors_quote",
|
|
"tier": "provisional",
|
|
"v1_endpoint": "/v1/sectors/quote",
|
|
"primary_source": "eastmoney:sw",
|
|
"backup_source": None,
|
|
"known_consumers": ["主题轮动盘中板块报价"],
|
|
},
|
|
{
|
|
"dataset": "limit_pool",
|
|
"tier": "provisional",
|
|
"v1_endpoint": "/v1/limit-pool",
|
|
"primary_source": "eastmoney:zt_pool",
|
|
"backup_source": None,
|
|
"known_consumers": ["涨停梯队盘中视图"],
|
|
},
|
|
{
|
|
"dataset": "intraday_points",
|
|
"tier": "provisional",
|
|
"v1_endpoint": "/v1/intraday/points",
|
|
"primary_source": "eastmoney:trends2",
|
|
"backup_source": None,
|
|
"known_consumers": ["个股详情分时图"],
|
|
},
|
|
{
|
|
"dataset": "ifind_wencai",
|
|
"tier": "licensed",
|
|
"v1_endpoint": "/v1/query (api_name=ifind_wencai)",
|
|
"primary_source": "ifind:smart_stock_picking",
|
|
"backup_source": None,
|
|
"known_consumers": ["问师(自然语言选股，需 iFinD 凭证)"],
|
|
},
|
|
]
|
|
|
|
_KNOWN_PROVIDERS_BY_SOURCE_PREFIX = ("tushare", "eastmoney", "tencent", "ifind")
|
|
|
|
|
|
def _providers_for(primary_source: str, backup_source: str | None) -> list[str]:
|
|
providers: list[str] = []
|
|
for source in (primary_source, backup_source or ""):
|
|
for provider in _KNOWN_PROVIDERS_BY_SOURCE_PREFIX:
|
|
if source.startswith(provider) and provider not in providers:
|
|
providers.append(provider)
|
|
return providers
|
|
|
|
|
|
def snapshot(db: Any, trade_date: str = "") -> list[dict[str, Any]]:
|
|
"""Attach live status to the static lineage table. Read-only; never
|
|
raises (a per-row status lookup failure just leaves that row's status
|
|
empty rather than failing the whole snapshot)."""
|
|
result: list[dict[str, Any]] = []
|
|
for entry in DATASETS:
|
|
row = dict(entry)
|
|
providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
|
|
row["providers"] = providers
|
|
live: list[dict[str, Any]] = []
|
|
try:
|
|
if db is not None and providers:
|
|
placeholders = ",".join("?" for _ in providers)
|
|
live = db.fetchall(
|
|
f"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
|
f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
|
|
tuple(providers),
|
|
)
|
|
except Exception:
|
|
live = []
|
|
row["live_provider_health"] = live
|
|
if entry["tier"] == "official":
|
|
pub = None
|
|
try:
|
|
if db is not None and trade_date:
|
|
pub = db.fetchone(
|
|
"SELECT dataset, trade_date, state, published_at FROM publications "
|
|
"WHERE dataset = ? AND trade_date = ?",
|
|
(entry["dataset"], trade_date),
|
|
)
|
|
except Exception:
|
|
pub = None
|
|
row["publication"] = pub
|
|
result.append(row)
|
|
return result
|
|
|
|
|
|
def affected(db: Any, provider: str = "", interface: str = "") -> list[dict[str, Any]]:
|
|
"""Read-only: which datasets/pages are impacted by a given provider (and,
|
|
optionally, a specific interface) right now. Does not change routing."""
|
|
provider = str(provider or "").strip()
|
|
interface = str(interface or "").strip()
|
|
result: list[dict[str, Any]] = []
|
|
for entry in DATASETS:
|
|
providers = _providers_for(entry["primary_source"], entry.get("backup_source"))
|
|
if provider and provider not in providers:
|
|
continue
|
|
row = dict(entry)
|
|
row["providers"] = providers
|
|
health: list[dict[str, Any]] = []
|
|
try:
|
|
if db is not None:
|
|
if interface:
|
|
health = db.fetchall(
|
|
"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
|
"consec_failures, updated_at FROM provider_health "
|
|
"WHERE provider = ? AND interface = ?",
|
|
(provider, interface),
|
|
)
|
|
elif providers:
|
|
placeholders = ",".join("?" for _ in providers)
|
|
health = db.fetchall(
|
|
f"SELECT provider, interface, state, last_error, last_fallback_reason, "
|
|
f"consec_failures, updated_at FROM provider_health WHERE provider IN ({placeholders})",
|
|
tuple(providers),
|
|
)
|
|
except Exception:
|
|
health = []
|
|
row["live_provider_health"] = health
|
|
result.append(row)
|
|
return result
|