605f97e5df
扩展盘后正式集（涨跌停/人气/龙虎榜/板块日线）与盘中观察 API（报价/指数/分时），网站 bridge 按开关接入并回退旧链路；问天改为按数据依赖跟随开关，不再整栈强制旧路径。 Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
296 lines
12 KiB
Python
296 lines
12 KiB
Python
"""Canonical field normalization for Tushare-native rows.
|
||
|
||
Units (architecture §7.1):
|
||
- price: 4 decimal REAL
|
||
- pct_chg: percent, 4 decimal REAL
|
||
- volume: shares (Tushare daily/index vol is 手 → ×100)
|
||
- amount: yuan (Tushare daily/index amount is 千元 → ×1000)
|
||
- moneyflow amounts: yuan (Tushare is 万元 → ×1e4)
|
||
- daily_basic total_mv / circ_mv: yuan (Tushare is 万元 → ×1e4)
|
||
- stk_auction.amount is already yuan in Tushare; volume 手 → ×100
|
||
|
||
Existing xiaobai-review stores Tushare native units and converts at display time.
|
||
Hub converts once at ingest. Golden tests compare hub output against applying
|
||
these same factors to review-native rows.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
from datahub.numbers import finite_number, round4
|
||
|
||
AMOUNT_THOUSAND_YUAN = 1000.0
|
||
AMOUNT_WAN_YUAN = 10000.0
|
||
VOLUME_LOT = 100.0
|
||
|
||
DAILY_FIELDS = ("ts_code", "trade_date", "open", "high", "low", "close", "pct_chg", "vol", "amount")
|
||
VALUATION_FIELDS = (
|
||
"ts_code", "trade_date", "turnover_rate", "volume_ratio",
|
||
"total_mv", "circ_mv", "pe_ttm", "pb", "ps_ttm", "dv_ttm",
|
||
)
|
||
MONEYFLOW_FIELDS = (
|
||
"ts_code", "trade_date",
|
||
"buy_sm_amount", "sell_sm_amount", "buy_md_amount", "sell_md_amount",
|
||
"buy_lg_amount", "sell_lg_amount", "buy_elg_amount", "sell_elg_amount",
|
||
"net_mf_amount",
|
||
)
|
||
AUCTION_FIELDS = (
|
||
"ts_code", "trade_date", "vol", "price", "amount", "pre_close",
|
||
"turnover_rate", "volume_ratio", "float_share",
|
||
)
|
||
INDEX_FIELDS = ("ts_code", "trade_date", "open", "high", "low", "close", "pct_chg", "vol", "amount")
|
||
CALENDAR_FIELDS = ("exchange", "cal_date", "is_open", "pretrade_date")
|
||
STOCK_FIELDS = ("ts_code", "symbol", "name", "area", "industry", "market", "list_status", "list_date")
|
||
|
||
|
||
def _code(value: Any) -> str:
|
||
return str(value or "").strip().upper()
|
||
|
||
|
||
def _date(value: Any) -> str:
|
||
return str(value or "").replace("-", "")[:8]
|
||
|
||
|
||
def review_daily_to_canonical(row: dict[str, Any]) -> dict[str, Any]:
|
||
"""Convert a review-stored daily row (Tushare native units) to hub canonical."""
|
||
return normalize_daily(row)
|
||
|
||
|
||
def normalize_daily(row: dict[str, Any], adj_factor: float | None = None) -> dict[str, Any]:
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"open": round4(finite_number(row.get("open"))),
|
||
"high": round4(finite_number(row.get("high"))),
|
||
"low": round4(finite_number(row.get("low"))),
|
||
"close": round4(finite_number(row.get("close"))),
|
||
"pct_chg": round4(finite_number(row.get("pct_chg"))),
|
||
"volume": round4(_scale(row.get("vol"), VOLUME_LOT)),
|
||
"amount": round4(_scale(row.get("amount"), AMOUNT_THOUSAND_YUAN)),
|
||
"adj_factor": round4(finite_number(adj_factor if adj_factor is not None else row.get("adj_factor"))),
|
||
}
|
||
|
||
|
||
def normalize_valuation(row: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"turnover_rate": round4(finite_number(row.get("turnover_rate"))),
|
||
"volume_ratio": round4(finite_number(row.get("volume_ratio"))),
|
||
"total_mv": round4(_scale(row.get("total_mv"), AMOUNT_WAN_YUAN)),
|
||
"circ_mv": round4(_scale(row.get("circ_mv"), AMOUNT_WAN_YUAN)),
|
||
"pe_ttm": round4(finite_number(row.get("pe_ttm"))),
|
||
"pb": round4(finite_number(row.get("pb"))),
|
||
"ps_ttm": round4(finite_number(row.get("ps_ttm"))),
|
||
"dv_ttm": round4(finite_number(row.get("dv_ttm"))),
|
||
}
|
||
|
||
|
||
def normalize_moneyflow(row: dict[str, Any]) -> dict[str, Any]:
|
||
converted = {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
}
|
||
for field in MONEYFLOW_FIELDS[2:]:
|
||
converted[field] = round4(_scale(row.get(field), AMOUNT_WAN_YUAN))
|
||
return converted
|
||
|
||
|
||
def normalize_auction(row: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"volume": round4(_scale(row.get("vol") if row.get("vol") is not None else row.get("volume"), VOLUME_LOT)),
|
||
"price": round4(finite_number(row.get("price"))),
|
||
"amount": round4(finite_number(row.get("amount"))),
|
||
"pre_close": round4(finite_number(row.get("pre_close"))),
|
||
"turnover_rate": round4(finite_number(row.get("turnover_rate"))),
|
||
"volume_ratio": round4(finite_number(row.get("volume_ratio"))),
|
||
"float_share": round4(_scale(row.get("float_share"), AMOUNT_WAN_YUAN) if row.get("float_share") is not None else None),
|
||
}
|
||
|
||
|
||
def normalize_index_daily(row: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"open": round4(finite_number(row.get("open"))),
|
||
"high": round4(finite_number(row.get("high"))),
|
||
"low": round4(finite_number(row.get("low"))),
|
||
"close": round4(finite_number(row.get("close"))),
|
||
"pct_chg": round4(finite_number(row.get("pct_chg"))),
|
||
"volume": round4(_scale(row.get("vol"), VOLUME_LOT)),
|
||
"amount": round4(_scale(row.get("amount"), AMOUNT_THOUSAND_YUAN)),
|
||
}
|
||
|
||
|
||
def normalize_calendar(row: dict[str, Any]) -> dict[str, Any]:
|
||
is_open = row.get("is_open")
|
||
if is_open in (True, "1", 1, "Y", "y"):
|
||
open_flag = 1
|
||
elif is_open in (False, "0", 0, "N", "n", None, ""):
|
||
open_flag = 0
|
||
else:
|
||
open_flag = int(is_open)
|
||
return {
|
||
"exchange": str(row.get("exchange") or "SSE"),
|
||
"cal_date": _date(row.get("cal_date") or row.get("calDate")),
|
||
"is_open": open_flag,
|
||
"pretrade_date": _date(row.get("pretrade_date")) or None,
|
||
}
|
||
|
||
|
||
def normalize_stock(row: dict[str, Any]) -> dict[str, Any]:
|
||
ts_code = _code(row.get("ts_code"))
|
||
symbol = str(row.get("symbol") or "").strip() or (ts_code.split(".")[0] if ts_code else "")
|
||
return {
|
||
"ts_code": ts_code,
|
||
"symbol": symbol,
|
||
"name": str(row.get("name") or "").strip(),
|
||
"area": str(row.get("area") or "").strip() or None,
|
||
"industry": str(row.get("industry") or "").strip() or None,
|
||
"market": str(row.get("market") or "").strip() or None,
|
||
"list_status": str(row.get("list_status") or "L").strip() or "L",
|
||
"list_date": _date(row.get("list_date")) or None,
|
||
}
|
||
|
||
|
||
def normalize_limit_event(row: dict[str, Any]) -> dict[str, Any]:
|
||
"""limit_list_d. float_mv/total_mv/limit_amount are 万元 → yuan; amount/fd_amount already yuan."""
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"limit_type": str(row.get("limit_type") or "").strip().upper() or "U",
|
||
"name": str(row.get("name") or "").strip() or None,
|
||
"industry": str(row.get("industry") or "").strip() or None,
|
||
"close": round4(finite_number(row.get("close"))),
|
||
"pct_chg": round4(finite_number(row.get("pct_chg"))),
|
||
"amount": round4(finite_number(row.get("amount"))),
|
||
"limit_amount": round4(_scale(row.get("limit_amount"), AMOUNT_WAN_YUAN)),
|
||
"float_mv": round4(_scale(row.get("float_mv"), AMOUNT_WAN_YUAN)),
|
||
"total_mv": round4(_scale(row.get("total_mv"), AMOUNT_WAN_YUAN)),
|
||
"turnover_ratio": round4(finite_number(row.get("turnover_ratio"))),
|
||
"fd_amount": round4(finite_number(row.get("fd_amount"))),
|
||
"first_time": str(row.get("first_time") or "").strip() or None,
|
||
"last_time": str(row.get("last_time") or "").strip() or None,
|
||
"open_times": _optional_int(row.get("open_times")),
|
||
"up_stat": str(row.get("up_stat") or "").strip() or None,
|
||
"limit_times": _optional_int(row.get("limit_times")),
|
||
}
|
||
|
||
|
||
def normalize_popularity(row: dict[str, Any], source: str = "") -> dict[str, Any]:
|
||
src = str(source or row.get("source") or "").strip().lower() or "ths"
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"source": src,
|
||
"ts_name": str(row.get("ts_name") or row.get("name") or "").strip() or None,
|
||
"rank": _optional_int(row.get("rank")),
|
||
"pct_change": round4(
|
||
finite_number(row.get("pct_change") if row.get("pct_change") is not None else row.get("pct_chg"))
|
||
),
|
||
"current_price": round4(finite_number(row.get("current_price") or row.get("price"))),
|
||
"hot": round4(finite_number(row.get("hot"))),
|
||
"concept": str(row.get("concept") or "").strip() or None,
|
||
"data_type": str(row.get("data_type") or "").strip() or None,
|
||
}
|
||
|
||
|
||
def normalize_dragon_tiger(row: dict[str, Any]) -> dict[str, Any]:
|
||
"""hm_detail amounts are 万元 → yuan."""
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"hm_name": str(row.get("hm_name") or "未命名游资").strip() or "未命名游资",
|
||
"ts_name": str(row.get("ts_name") or row.get("name") or "").strip() or None,
|
||
"buy_amount": round4(_scale(row.get("buy_amount"), AMOUNT_WAN_YUAN)),
|
||
"sell_amount": round4(_scale(row.get("sell_amount"), AMOUNT_WAN_YUAN)),
|
||
"net_amount": round4(_scale(row.get("net_amount"), AMOUNT_WAN_YUAN)),
|
||
"hm_orgs": str(row.get("hm_orgs") or "").strip() or None,
|
||
"tag": str(row.get("tag") or "").strip() or None,
|
||
"pct_change": round4(finite_number(row.get("pct_change"))),
|
||
"reason": str(row.get("reason") or "").strip() or None,
|
||
}
|
||
|
||
|
||
def normalize_sector_daily(row: dict[str, Any], family: str = "ths") -> dict[str, Any]:
|
||
fam = str(family or row.get("family") or "ths").strip().lower()
|
||
return {
|
||
"ts_code": _code(row.get("ts_code")),
|
||
"trade_date": _date(row.get("trade_date")),
|
||
"family": fam,
|
||
"name": str(row.get("name") or "").strip() or None,
|
||
"open": round4(finite_number(row.get("open"))),
|
||
"high": round4(finite_number(row.get("high"))),
|
||
"low": round4(finite_number(row.get("low"))),
|
||
"close": round4(finite_number(row.get("close"))),
|
||
"pre_close": round4(finite_number(row.get("pre_close"))),
|
||
"pct_change": round4(
|
||
finite_number(row.get("pct_change") if row.get("pct_change") is not None else row.get("pct_chg"))
|
||
),
|
||
"vol": round4(finite_number(row.get("vol"))),
|
||
"turnover_rate": round4(finite_number(row.get("turnover_rate"))),
|
||
"amount": round4(finite_number(row.get("amount"))),
|
||
}
|
||
|
||
|
||
def _optional_int(value: Any) -> int | None:
|
||
if value in (None, ""):
|
||
return None
|
||
try:
|
||
return int(float(value))
|
||
except (TypeError, ValueError):
|
||
return None
|
||
|
||
|
||
def apply_qfq(price: float | None, factor: float | None, latest_factor: float | None) -> float | None:
|
||
if price is None:
|
||
return None
|
||
current = factor if factor not in (None, 0) else 1.0
|
||
latest = latest_factor if latest_factor not in (None, 0) else current
|
||
return round4(price * current / latest)
|
||
|
||
|
||
def qfq_bar(row: dict[str, Any], latest_factor: float | None) -> dict[str, Any]:
|
||
factor = finite_number(row.get("adj_factor"), 1.0) or 1.0
|
||
out = dict(row)
|
||
for field in ("open", "high", "low", "close"):
|
||
out[field] = apply_qfq(finite_number(row.get(field)), factor, latest_factor)
|
||
return out
|
||
|
||
|
||
NORMALIZERS = {
|
||
"daily": normalize_daily,
|
||
"valuation": normalize_valuation,
|
||
"daily_basic": normalize_valuation,
|
||
"moneyflow": normalize_moneyflow,
|
||
"auction": normalize_auction,
|
||
"stk_auction": normalize_auction,
|
||
"index_daily": normalize_index_daily,
|
||
"trade_cal": normalize_calendar,
|
||
"calendar": normalize_calendar,
|
||
"stock_basic": normalize_stock,
|
||
"stocks": normalize_stock,
|
||
"limit_events": normalize_limit_event,
|
||
"limit_list_d": normalize_limit_event,
|
||
"popularity": normalize_popularity,
|
||
"dragon_tiger": normalize_dragon_tiger,
|
||
"sector_daily": normalize_sector_daily,
|
||
}
|
||
|
||
|
||
def normalize_rows(dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
fn = NORMALIZERS.get(dataset)
|
||
if fn is None:
|
||
raise ValueError(f"unknown dataset: {dataset}")
|
||
return [fn(row) for row in rows]
|
||
|
||
|
||
def _scale(value: Any, factor: float) -> float | None:
|
||
number = finite_number(value)
|
||
if number is None:
|
||
return None
|
||
return number * factor
|