diff --git a/backend/application.py b/backend/application.py index 4295902..b38b31a 100644 --- a/backend/application.py +++ b/backend/application.py @@ -18,7 +18,7 @@ from backend.bootstrap.container import build_application_container from backend.bootstrap.settings import load_runtime_settings from backend.http import HttpTransportMixin from backend.llm import LLMGateway, LLMGatewayError -from chart_data_provider import ChartDataError +from backend.features.market import ChartDataError, MarketServiceMixin from backend.bootstrap.config import ( DATA_DIR, MENTOR_SKILLS_DIR, @@ -39,7 +39,7 @@ from heaven_engine import ( build_personal_field, hexagram_from_lines, ) -from ifind_client import IfindError +from backend.data.providers.ifind_client import IfindError from llm_strategy import LLMCompilerError, compile_strategy_with_llm, test_llm_connection from mentor_agent import MentorAgentError, stream_with_mentor from market_insights import MarketInsightsService @@ -61,7 +61,7 @@ from sentiment_engine import ( build_sentiment_history, latest_contiguous_history, ) -from tushare_client import TushareClient, TushareError, _sector_coverage_issue +from backend.data.providers.tushare_client import TushareClient, TushareError, _sector_coverage_issue SCREENER_LIBRARY_VERSION = 8 @@ -103,23 +103,6 @@ LEGACY_SECRET_KEYS = { "LLM_FALLBACK_MODEL", } -SEARCH_INDEXES = ( - {"id": "000001.SH", "code": "000001.SH", "name": "上证指数", "type": "index", "subtitle": "沪市综合指数"}, - {"id": "399001.SZ", "code": "399001.SZ", "name": "深证成指", "type": "index", "subtitle": "深市成份指数"}, - {"id": "399006.SZ", "code": "399006.SZ", "name": "创业板指", "type": "index", "subtitle": "创业板核心指数"}, -) -SEARCH_TYPE_LABELS = { - "stock": "股票", - "sector": "板块", - "theme": "题材", - "index": "指数", -} -THS_SEARCH_TYPES = { - "I": ("sector", "行业板块"), - "R": ("sector", "地域板块"), - "N": ("theme", "概念题材"), -} - MENTOR_DATA_PROFILES = { "emotion": { "kobe92-perspective", "niepanchongsheng-perspective", @@ -158,7 +141,7 @@ MENTOR_ETF_UNIVERSE = ( ) -class DashboardService: +class DashboardService(MarketServiceMixin): def __init__(self) -> None: runtime = load_runtime_settings() self.vault = SecretVault(runtime.encryption_key) @@ -213,12 +196,6 @@ class DashboardService: initial_delay_seconds=3, ) - def _tushare_client(self) -> TushareClient: - gateway = getattr(self, "data_gateway", None) - if gateway is not None: - return gateway.tushare() - # Compatibility for isolated legacy unit-test service stubs. - return TushareClient(self.token) def _load_system_credentials(self, environment: dict[str, str]) -> dict[str, Any]: encrypted = self.database.get_system_setting("credentials") @@ -756,181 +733,6 @@ class DashboardService: def _public_personal_profile(personal: dict[str, Any]) -> dict[str, Any]: return AccountService.public_personal_profile(personal) - def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]: - normalized_date = normalize_date(trade_date) - now = datetime.now().astimezone() - if ( - normalized_date == now.strftime("%Y%m%d") - and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time() - ): - previous = self.database.get_latest_real_snapshot(normalized_date, strictly_before=True) - if previous: - carried = self._carry_dashboard(previous, normalized_date, "盘前沿用最近交易日收盘行情") - return self._apply_reason_overrides(self._with_storage(carried, cached=True)) - if not force: - snapshot = self.database.get_snapshot(normalized_date) - if snapshot and str((snapshot.get("meta") or {}).get("source") or "") != "demo": - snapshot = copy.deepcopy(snapshot) - if normalized_date != now.strftime("%Y%m%d"): - snapshot.setdefault("meta", {}).update( - {"realtime": False, "market_status": "closed"} - ) - if not self._dashboard_sentiment_ready(snapshot): - snapshot = self._enrich_dashboard_sentiment(snapshot, normalized_date) - self.database.save_snapshot( - normalized_date, - str((snapshot.get("meta") or {}).get("source") or "tushare"), - snapshot, - ) - snapshot.setdefault("meta", {})["requested_date"] = self._display_compact_date(normalized_date) - return self._apply_reason_overrides(self._with_storage(snapshot, cached=True)) - resolved = self.database.get_data_snapshot( - "dashboard_request_v1", normalized_date - ) - if resolved and str((resolved.get("meta") or {}).get("source") or "") != "demo": - resolved = copy.deepcopy(resolved) - resolved.setdefault("meta", {})["requested_date"] = self._display_compact_date( - normalized_date - ) - return self._apply_reason_overrides( - self._with_storage(resolved, cached=True) - ) - if datetime.strptime(normalized_date, "%Y%m%d").weekday() >= 5: - previous = self.database.get_latest_real_snapshot(normalized_date) - if previous: - carried = self._carry_dashboard( - previous, - normalized_date, - "非交易日沿用最近交易日收盘行情", - ) - self.database.save_data_snapshot( - "dashboard_request_v1", normalized_date, "sqlite", carried - ) - return self._apply_reason_overrides( - self._with_storage(carried, cached=True) - ) - return self.sync_dashboard(normalized_date) - - @staticmethod - def _dashboard_sentiment_ready(dashboard: dict[str, Any]) -> bool: - overview = dashboard.get("overview") or {} - return int(overview.get("sentiment_engine_version") or 0) == SENTIMENT_ENGINE_VERSION and all( - key in overview - for key in ( - "sentiment_score", - "sentiment_label", - "sentiment_phase", - "sentiment_direction", - "sentiment_components", - ) - ) - - @staticmethod - def _display_compact_date(compact: str) -> str: - return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" - - def _carry_dashboard( - self, snapshot: dict[str, Any], requested_date: str, reason: str - ) -> dict[str, Any]: - carried = copy.deepcopy(snapshot) - meta = carried.setdefault("meta", {}) - meta.update( - { - "requested_date": self._display_compact_date(requested_date), - "carried_forward": True, - "realtime": False, - "market_status": "closed", - "notice": reason, - } - ) - return carried - - def _realtime_snapshot_due( - self, - normalized_date: str, - snapshot: dict[str, Any], - ) -> bool: - if not self.configured or normalized_date != date.today().strftime("%Y%m%d"): - return False - now = datetime.now().astimezone() - local_time = now.time().replace(tzinfo=None) - realtime_start = datetime.strptime("09:15", "%H:%M").time() - morning_end = datetime.strptime("11:35", "%H:%M").time() - afternoon_start = datetime.strptime("12:55", "%H:%M").time() - realtime_end = datetime.strptime("15:05", "%H:%M").time() - in_session = ( - realtime_start <= local_time < morning_end - or afternoon_start <= local_time < realtime_end - ) - if not in_session: - return False - meta = snapshot.get("meta") or {} - snapshot_trade_date = str(meta.get("trade_date") or "").replace("-", "") - if snapshot_trade_date and snapshot_trade_date != normalized_date: - return False - if not meta.get("realtime"): - return True - try: - updated_at = datetime.fromisoformat(str(meta.get("updated_at") or "")) - if updated_at.tzinfo is None: - updated_at = updated_at.replace(tzinfo=now.tzinfo) - except ValueError: - return True - age_seconds = (now - updated_at.astimezone(now.tzinfo)).total_seconds() - return age_seconds >= 8 - - def sync_dashboard(self, trade_date: str) -> dict[str, Any]: - normalized_date = normalize_date(trade_date) - source = "tushare" - with self.sync_lock: - sync_id = self.database.start_sync(normalized_date, source) - try: - if not self.configured: - raise TushareError("公共行情尚未配置") - dashboard = self._tushare_client().dashboard(normalized_date) - - dashboard["meta"]["source"] = source - dashboard["meta"]["requested_date"] = self._display_compact_date(normalized_date) - dashboard = self._enrich_dashboard_sentiment(dashboard, normalized_date) - record_count = self._record_count(dashboard) - actual_date = normalize_date( - str(dashboard.get("meta", {}).get("trade_date") or normalized_date) - ) - self.database.save_snapshot(actual_date, source, dashboard) - if actual_date != normalized_date: - dashboard.setdefault("meta", {}).update( - { - "carried_forward": True, - "realtime": False, - "market_status": "closed", - } - ) - self.database.save_data_snapshot( - "dashboard_request_v1", normalized_date, source, dashboard - ) - self.database.finish_sync( - sync_id, - "success", - record_count, - dashboard.get("meta", {}).get("notice", ""), - source, - ) - return self._apply_reason_overrides(self._with_storage(dashboard, cached=False)) - except TushareError as exc: - fallback = self.database.get_latest_real_snapshot(normalized_date) - if fallback: - carried = self._carry_dashboard( - fallback, normalized_date, f"最新行情暂不可用，沿用最近收盘快照：{exc}" - ) - self.database.finish_sync( - sync_id, "fallback", self._record_count(carried), str(exc), "tushare" - ) - return self._apply_reason_overrides(self._with_storage(carried, cached=True)) - self.database.finish_sync(sync_id, "failed", message=str(exc)) - raise ValueError("暂无可用的真实行情快照，请等待后台完成首次同步。") from exc - except Exception as exc: - self.database.finish_sync(sync_id, "failed", message=str(exc)) - raise def _enrich_dashboard_sentiment( self, @@ -1126,9 +928,6 @@ class DashboardService: **self.database.status(), } - def realtime_aggregate_health(self, sector: str = "") -> dict[str, Any]: - sector = validate_text(sector, "板块名称", 50) - return self.realtime_aggregator.health_snapshot(sector) def _market_insights(self) -> MarketInsightsService: if not self.configured: @@ -3580,663 +3379,6 @@ class DashboardService: "rows": [], } - def _search_market_directory(self) -> list[dict[str, Any]]: - cached = self.database.get_data_snapshot("search_directory", "ths") or {} - cached_items = list(cached.get("items") or []) - if cached_items and int(cached.get("schema_version") or 0) >= 2: - return cached_items - if not self.configured: - return cached_items - - try: - rows = self._tushare_client().query( - "ths_index", - {}, - "ts_code,name,count,exchange,list_date,type", - ) - except TushareError: - return cached_items - - items = [] - for row in rows: - mapping = THS_SEARCH_TYPES.get(str(row.get("type") or "").upper()) - code = str(row.get("ts_code") or "").strip().upper() - name = str(row.get("name") or "").strip() - if not mapping or not code or not name or str(row.get("exchange") or "").upper() != "A": - continue - entity_type, subtitle = mapping - items.append( - { - "id": code, - "code": code, - "name": name, - "type": entity_type, - "subtitle": subtitle, - "member_count": int(float(row.get("count") or 0)), - } - ) - if items: - self.database.save_data_snapshot( - "search_directory", "ths", "tushare", {"schema_version": 2, "items": items} - ) - return items - - @staticmethod - def _search_match_score(item: dict[str, Any], query: str) -> tuple[int, int, str]: - name = str(item.get("name") or "").casefold() - code = str(item.get("code") or item.get("id") or "").casefold() - needle = query.casefold() - if code == needle: - rank = 0 - elif name == needle: - rank = 1 - elif code.startswith(needle): - rank = 2 - elif name.startswith(needle): - rank = 3 - else: - rank = 4 - return rank, len(name), code - - def search_entities(self, query: str, trade_date: str) -> dict[str, Any]: - needle = str(query or "").strip() - normalized_date = normalize_date(trade_date) - groups: dict[str, list[dict[str, Any]]] = { - "stocks": [], - "sectors": [], - "themes": [], - "indices": [], - } - if not needle: - return {"query": "", "trade_date": normalized_date, "groups": groups} - - stocks = [] - for row in self.database.search_stock_master(needle, 12): - stocks.append( - { - "id": str(row.get("code") or ""), - "code": str(row.get("code") or ""), - "name": str(row.get("name") or "--"), - "type": "stock", - "type_label": SEARCH_TYPE_LABELS["stock"], - "industry": str(row.get("industry") or "其他"), - "market": str(row.get("market") or ""), - "subtitle": " · ".join( - part for part in (str(row.get("industry") or ""), str(row.get("market") or "")) if part - ) or "A股", - } - ) - groups["stocks"] = stocks[:8] - - market_items = list(self._search_market_directory()) + [dict(item) for item in SEARCH_INDEXES] - matched = [ - item for item in market_items - if needle.casefold() in str(item.get("name") or "").casefold() - or needle.casefold() in str(item.get("code") or "").casefold() - ] - matched.sort(key=lambda item: self._search_match_score(item, needle)) - group_keys = {"sector": "sectors", "theme": "themes", "index": "indices"} - for item in matched: - group_key = group_keys.get(str(item.get("type") or "")) - if not group_key or len(groups[group_key]) >= 8: - continue - groups[group_key].append( - { - **item, - "type_label": SEARCH_TYPE_LABELS[str(item["type"])], - } - ) - return {"query": needle, "trade_date": normalized_date, "groups": groups} - - def get_search_detail( - self, entity_type: str, identifier: str, trade_date: str - ) -> dict[str, Any]: - entity_type = str(entity_type or "").strip().lower() - identifier = str(identifier or "").strip().upper() - normalized_date = normalize_date(trade_date) - if entity_type not in {"sector", "theme", "index"}: - raise ValueError("搜索详情类型不支持。") - if not re.fullmatch(r"[A-Z0-9.]{3,24}", identifier): - raise ValueError("搜索详情标识无效。") - if not self.configured: - raise ValueError("行情数据源尚未配置。") - - if entity_type == "index": - index_basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None) - if not index_basic: - raise ValueError("暂不支持该指数详情。") - return self._index_search_detail(index_basic, normalized_date) - - directory = self._search_market_directory() - basic = next( - ( - item for item in directory - if item.get("id") == identifier and item.get("type") == entity_type - ), - None, - ) - if not basic: - raise ValueError("未找到对应的板块或题材。") - return self._ths_search_detail(basic, normalized_date) - - def get_intraday_chart( - self, entity_type: str, identifier: str - ) -> dict[str, Any]: - entity_type = str(entity_type or "").strip().lower() - identifier = str(identifier or "").strip().upper() - if entity_type == "stock": - code = validate_stock_code(identifier) - chart = self.chart_data.stock_intraday(code) - type_label = SEARCH_TYPE_LABELS["stock"] - elif entity_type == "index": - basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None) - if not basic: - raise ValueError("暂不支持该指数分时行情。") - chart = self.chart_data.index_intraday(identifier) - type_label = SEARCH_TYPE_LABELS["index"] - elif entity_type in {"sector", "theme"}: - basic = next( - ( - item for item in self._search_market_directory() - if item.get("id") == identifier and item.get("type") == entity_type - ), - None, - ) - if not basic: - raise ValueError("未找到对应的板块或题材。") - chart = self.chart_data.board_intraday(identifier, str(basic.get("name") or "")) - type_label = SEARCH_TYPE_LABELS[entity_type] - else: - raise ValueError("分时行情类型不支持。") - - return { - "meta": { - "trade_date": str(chart.get("trade_date") or ""), - "previous_close": float(chart.get("previous_close") or 0), - }, - "entity": { - "id": identifier, - "code": str(chart.get("code") or identifier), - "name": str(chart.get("name") or ""), - "type": entity_type, - "type_label": type_label, - }, - "points": list(chart.get("points") or []), - } - - def _ths_search_detail( - self, basic: dict[str, Any], trade_date: str - ) -> dict[str, Any]: - client = self._tushare_client() - resolved_date, _ = client.resolve_trade_context(trade_date) - end = datetime.strptime(resolved_date, "%Y%m%d") - start_date = (end - timedelta(days=190)).strftime("%Y%m%d") - identifier = str(basic["id"]) - snapshot = client.sector_snapshot(identifier, resolved_date) - rows = client.query( - "ths_daily", - {"ts_code": identifier, "start_date": start_date, "end_date": resolved_date}, - "ts_code,trade_date,open,high,low,close,pct_change,vol,turnover_rate,total_mv,float_mv", - ) - rows.sort(key=lambda item: str(item.get("trade_date") or "")) - series = [ - { - "trade_date": self._display_compact_date(str(row.get("trade_date") or "")), - "open": float(row.get("open") or 0), - "high": float(row.get("high") or 0), - "low": float(row.get("low") or 0), - "close": float(row.get("close") or 0), - "change": float(row.get("pct_change") or 0), - "volume": float(row.get("vol") or 0), - "turnover_rate": float(row.get("turnover_rate") or 0), - } - for row in rows[-90:] - ] - try: - chart_series = self.chart_data.board_daily(identifier, resolved_date, 90) - if chart_series: - series = chart_series - except (AttributeError, ChartDataError): - pass - latest = series[-1] if series else {} - snapshot_is_current = str(snapshot.get("trade_date") or "").replace("-", "") == resolved_date - change = float( - snapshot.get("change") - if snapshot_is_current and snapshot.get("change") is not None - else latest.get("change") or 0 - ) - if latest.get("realtime"): - change = float(latest.get("change") or 0) - turnover_rate = float( - snapshot.get("turnover_rate") - if snapshot_is_current and snapshot.get("turnover_rate") is not None - else latest.get("turnover_rate") or 0 - ) - metrics = [ - {"label": "涨跌幅", "value": round(change, 2), "unit": "%", "tone": "change"}, - {"label": "换手率", "value": round(turnover_rate, 2), "unit": "%"}, - {"label": "成份数量", "value": int(float(basic.get("member_count") or 0)), "unit": "只"}, - ] - up_count = int(float(snapshot.get("up_count") or 0)) - down_count = int(float(snapshot.get("down_count") or 0)) - if up_count or down_count: - metrics.extend( - [ - {"label": "上涨家数", "value": up_count, "unit": "家"}, - {"label": "下跌家数", "value": down_count, "unit": "家"}, - ] - ) - leader = str(snapshot.get("leader") or "").strip() - if leader and leader != "--": - metrics.extend( - [ - {"label": "领涨标的", "value": leader, "unit": ""}, - {"label": "领涨幅", "value": round(float(snapshot.get("leading_pct") or 0), 2), "unit": "%", "tone": "change"}, - ] - ) - return { - "meta": { - "trade_date": self._display_compact_date(resolved_date), - "realtime": bool(snapshot.get("realtime")), - }, - "entity": { - "id": identifier, - "code": identifier, - "name": str(snapshot.get("name") or basic.get("name") or "--"), - "type": str(basic.get("type") or "sector"), - "type_label": SEARCH_TYPE_LABELS[str(basic.get("type") or "sector")], - "subtitle": str(basic.get("subtitle") or ""), - "value": float(latest.get("close") or 0), - "change": change, - }, - "series": series, - "metrics": metrics, - } - - def _index_search_detail( - self, basic: dict[str, Any], trade_date: str - ) -> dict[str, Any]: - client = self._tushare_client() - resolved_date, _ = client.resolve_trade_context(trade_date) - payload = ( - client.realtime_market_indices(resolved_date) - if client.should_use_realtime(trade_date, resolved_date) - else client.market_indices(resolved_date, 90) - ) - current = next( - (item for item in payload.get("indices") or [] if item.get("ts_code") == basic["id"]), - None, - ) - if not current: - raise ValueError("该指数暂无可用行情。") - end = datetime.strptime(resolved_date, "%Y%m%d") - rows = client.query( - "index_daily", - { - "ts_code": basic["id"], - "start_date": (end - timedelta(days=190)).strftime("%Y%m%d"), - "end_date": resolved_date, - }, - "ts_code,trade_date,open,high,low,close,pct_chg,vol,amount", - ) - rows.sort(key=lambda item: str(item.get("trade_date") or "")) - series = [ - { - "trade_date": self._display_compact_date(str(row.get("trade_date") or "")), - "open": float(row.get("open") or 0), - "high": float(row.get("high") or 0), - "low": float(row.get("low") or 0), - "close": float(row.get("close") or 0), - "change": float(row.get("pct_chg") or 0), - "volume": float(row.get("vol") or 0), - } - for row in rows[-90:] - ] - try: - chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, 90) - if chart_series: - series = chart_series - except (AttributeError, ChartDataError): - pass - latest = series[-1] if series else {} - latest_close = float(latest.get("close") or current.get("close") or 0) - latest_change = float(latest.get("change") or current.get("pct_chg") or 0) - - def series_return(days: int) -> float: - if len(series) <= days: - return 0.0 - previous = float(series[-days - 1].get("close") or 0) - return (latest_close / previous - 1) * 100 if previous > 0 else 0.0 - return { - "meta": { - "trade_date": self._display_compact_date(str(current.get("trade_date") or resolved_date)), - "realtime": bool(payload.get("realtime")), - }, - "entity": { - **basic, - "type_label": SEARCH_TYPE_LABELS["index"], - "value": latest_close, - "change": latest_change, - }, - "series": series, - "metrics": [ - {"label": "涨跌幅", "value": round(latest_change, 2), "unit": "%", "tone": "change"}, - {"label": "近5日", "value": round(series_return(5), 2), "unit": "%", "tone": "change"}, - {"label": "近20日", "value": round(series_return(20), 2), "unit": "%", "tone": "change"}, - {"label": "成交额", "value": round(float(current.get("amount_billion") or 0), 2), "unit": "亿"}, - ], - } - - def get_stock_detail( - self, code: str, trade_date: str, force: bool = False - ) -> dict[str, Any]: - code = validate_stock_code(code) - normalized_date = normalize_date(trade_date) - cache_key = f"{code}:{normalized_date}" - if not force: - cached = self.database.get_data_snapshot("stock_detail", cache_key) - if cached and str((cached.get("meta") or {}).get("source") or "") != "demo": - if not self._stock_detail_cache_needs_refresh(cached, normalized_date): - cached["meta"] = {**cached.get("meta", {}), "cached": True} - return self._prepare_stock_detail(cached, code, normalized_date) - - name, sector = self._stock_identity(code, normalized_date) - source = "tushare" - if self.configured: - try: - payload = self._tushare_client().stock_detail( - tushare_code(code), normalized_date - ) - if not payload.get("prices"): - raise TushareError("No price history returned") - except TushareError as exc: - payload = self.database.get_latest_data_snapshot( - "stock_detail", f"{code}:", cache_key, exclude_source="demo" - ) - if not payload: - raise ValueError(f"暂无 {code} 的真实行情数据：{exc}") from exc - payload = copy.deepcopy(payload) - payload["meta"] = { - **payload.get("meta", {}), - "cached": True, - "notice": "最新行情暂不可用，已沿用最近真实收盘数据。", - } - return self._prepare_stock_detail(payload, code, normalized_date) - else: - payload = self.database.get_latest_data_snapshot( - "stock_detail", f"{code}:", cache_key, exclude_source="demo" - ) - if not payload: - raise ValueError(f"暂无 {code} 的真实行情数据，请等待后台完成首次同步。") - payload = copy.deepcopy(payload) - payload["meta"] = { - **payload.get("meta", {}), - "cached": True, - "notice": "公共行情尚未配置，已沿用最近真实收盘数据。", - } - return self._prepare_stock_detail(payload, code, normalized_date) - payload["meta"]["source"] = source - payload["meta"]["cached"] = False - self.database.save_data_snapshot("stock_detail", cache_key, source, payload) - return self._prepare_stock_detail(payload, code, normalized_date) - - @staticmethod - def _stock_detail_bar_date(payload: dict[str, Any]) -> str: - prices = list(payload.get("prices") or []) - return str((prices[-1] if prices else {}).get("trade_date") or "").replace("-", "") - - def _stock_detail_cache_needs_refresh( - self, payload: dict[str, Any], requested_date: str - ) -> bool: - now = datetime.now().astimezone() - return ( - requested_date == now.strftime("%Y%m%d") - and now.time().replace(tzinfo=None) >= dt_time(15, 0) - and self._stock_detail_bar_date(payload) < requested_date - ) - - def _prepare_stock_detail( - self, payload: dict[str, Any], code: str, requested_date: str - ) -> dict[str, Any]: - result = copy.deepcopy(payload) - now = datetime.now().astimezone() - try: - result["prices"] = self.chart_data.stock_daily(code, requested_date, 90) - result["meta"] = {**(result.get("meta") or {}), "chart_source": "market_chart"} - except (AttributeError, ChartDataError): - pass - result = self._sanitize_stock_detail_prices(result, now) - actual_date = self._stock_detail_bar_date(result) - if actual_date: - result["meta"] = { - **(result.get("meta") or {}), - "trade_date": f"{actual_date[:4]}-{actual_date[4:6]}-{actual_date[6:]}", - } - today = now.strftime("%Y%m%d") - should_merge = ( - requested_date == today - and actual_date <= today - and now.weekday() < 5 - and now.time().replace(tzinfo=None) >= dt_time(9, 30) - ) - if should_merge: - quote = self._ifind_realtime_stock_quote(code) - if quote and self._valid_realtime_stock_quote(quote, today): - self._merge_realtime_stock_detail(result, quote, requested_date) - elif self.configured and actual_date < today: - client = self._tushare_client() - try: - resolved_date, _ = client.resolve_trade_context(requested_date) - if resolved_date == today: - quote = client.realtime_stock_quote(tushare_code(code), requested_date) - if self._valid_realtime_stock_quote(quote, today): - self._merge_realtime_stock_detail(result, quote, requested_date) - except TushareError: - pass - return self._enrich_stock_detail(result) - - @staticmethod - def _sanitize_stock_detail_prices( - payload: dict[str, Any], market_now: datetime - ) -> dict[str, Any]: - result = copy.deepcopy(payload) - raw_prices = list(result.get("prices") or []) - raw_latest_date = str( - (raw_prices[-1] if raw_prices else {}).get("trade_date") or "" - ).replace("-", "") - prices = [] - for bar in raw_prices: - open_price = float(bar.get("open") or 0) - high = float(bar.get("high") or 0) - low = float(bar.get("low") or 0) - close = float(bar.get("close") or 0) - if ( - open_price > 0 - and high >= max(open_price, close) - and 0 < low <= min(open_price, close) - and close > 0 - ): - prices.append(bar) - - today = market_now.strftime("%Y%m%d") - market_open = ( - market_now.weekday() < 5 - and market_now.time().replace(tzinfo=None) >= dt_time(9, 30) - ) - if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == today: - current = prices[-1] - has_market_activity = ( - float(current.get("volume") or 0) > 0 - or float(current.get("amount_billion") or 0) > 0 - ) - if not market_open or not has_market_activity: - prices.pop() - - if raw_latest_date == today and ( - not prices - or str(prices[-1].get("trade_date") or "").replace("-", "") != today - ): - result["meta"] = {**(result.get("meta") or {}), "realtime": False} - - result["prices"] = prices - if prices: - latest = prices[-1] - stock = dict(result.get("stock") or {}) - stock.update( - { - "price": float(latest.get("close") or 0), - "change": float(latest.get("change") or 0), - "amount_billion": float(latest.get("amount_billion") or 0), - } - ) - result["stock"] = stock - return result - - @staticmethod - def _valid_realtime_stock_quote(quote: dict[str, Any], trade_date: str) -> bool: - price = float(quote.get("price") or 0) - open_price = float(quote.get("open") or 0) - high = float(quote.get("high") or 0) - low = float(quote.get("low") or 0) - volume = float(quote.get("volume") or 0) - amount = float(quote.get("amount_billion") or 0) - quote_date = str(quote.get("quote_time") or "")[:10].replace("-", "") - return ( - price > 0 - and open_price > 0 - and high >= max(open_price, price) - and 0 < low <= min(open_price, price) - and (volume > 0 or amount > 0) - and (not quote_date or quote_date == trade_date) - ) - - def _ifind_realtime_stock_quote(self, code: str) -> dict[str, Any] | None: - ifind = getattr(self, "ifind", None) - if not ifind or not ifind.configured: - return None - try: - rows = ifind.real_time( - tushare_code(code), - [ - "open", "high", "low", "latest", "preClose", - "volume", "amount", "turnoverRatio", - ], - cache_ttl=10, - ) - except IfindError: - return None - row = rows[0] if rows else {} - price = float(row.get("latest") or 0) - previous_close = float(row.get("preClose") or 0) - if price <= 0: - return None - change = (price / previous_close - 1) * 100 if previous_close > 0 else 0.0 - stock = self._stock_identity(code, date.today().strftime("%Y%m%d")) - return { - "name": stock[0], - "sector": stock[1], - "price": price, - "open": float(row.get("open") or price), - "high": float(row.get("high") or price), - "low": float(row.get("low") or price), - "change": round(change, 4), - "volume": float(row.get("volume") or 0), - "volume_unit": "lots", - "amount_billion": float(row.get("amount") or 0) / 100_000_000, - "turnover_rate": float(row.get("turnoverRatio") or 0), - "quote_time": str(row.get("time") or ""), - } - - @staticmethod - def _merge_realtime_stock_detail( - payload: dict[str, Any], quote: dict[str, Any], trade_date: str - ) -> None: - display_date = f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:]}" - realtime_bar = { - "trade_date": display_date, - "open": quote["open"], - "high": quote["high"], - "low": quote["low"], - "close": quote["price"], - "change": quote["change"], - "volume": quote["volume"] if quote.get("volume_unit") == "lots" else quote["volume"] / 100, - "amount_billion": quote["amount_billion"], - "realtime": True, - } - prices = list(payload.get("prices") or []) - if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == trade_date: - prices[-1] = realtime_bar - else: - prices.append(realtime_bar) - payload["prices"] = prices[-90:] - stock = dict(payload.get("stock") or {}) - stock.update( - { - "name": quote["name"], - "industry": quote["sector"], - "price": quote["price"], - "change": quote["change"], - "amount_billion": quote["amount_billion"], - "turnover_rate": quote["turnover_rate"], - } - ) - payload["stock"] = stock - payload["meta"] = { - **(payload.get("meta") or {}), - "trade_date": display_date, - "realtime": True, - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - } - - def get_stock_preview( - self, code: str, trade_date: str, force: bool = False - ) -> dict[str, Any]: - code = validate_stock_code(code) - # Hover previews deliberately follow the latest market day, independent - # from the review date selected by the page. - detail = self.get_stock_detail(code, date.today().strftime("%Y%m%d"), force) - detail_meta = detail.get("meta") or {} - resolved_date = str(detail_meta.get("trade_date") or trade_date) - intraday_points: list[dict[str, Any]] = [] - intraday_status = "unavailable" - intraday_notice = "分时行情暂不可用。" - - intraday_trade_date = "" - intraday_previous_close = 0.0 - try: - intraday = self.chart_data.stock_intraday(code) - intraday_points = list(intraday.get("points") or []) - intraday_trade_date = str(intraday.get("trade_date") or "") - intraday_previous_close = float(intraday.get("previous_close") or 0) - if intraday_points: - intraday_status = "available" - intraday_notice = "" - else: - intraday_status = "empty" - intraday_notice = "最近交易日暂无分时数据。" - except ChartDataError: - intraday_status = "unavailable" - intraday_notice = "分时行情暂不可用，请稍后重试。" - - prices = list(detail.get("prices") or [])[-60:] - stock = dict(detail.get("stock") or {"code": code}) - realtime = bool(detail_meta.get("realtime")) - return { - "meta": { - "trade_date": resolved_date, - "source": detail_meta.get("source") or "unavailable", - "notice": detail_meta.get("notice") or "", - "intraday_status": intraday_status, - "intraday_notice": intraday_notice, - "intraday_trade_date": intraday_trade_date, - "intraday_previous_close": intraday_previous_close, - "realtime": realtime, - "refresh_interval_seconds": 10 if realtime else 0, - }, - "stock": stock, - "prices": prices, - "intraday": intraday_points, - } def save_reason(self, trade_date: str, code: str, reason: str) -> None: normalized_date = normalize_date(trade_date) @@ -4246,55 +3388,6 @@ class DashboardService: raise ValueError("涨停原因应为 1 至 200 个字符。") self.database.save_reason_override(normalized_date, code, reason) - def backfill(self, start_date: str, end_date: str) -> list[dict[str, Any]]: - start = datetime.strptime(normalize_date(start_date), "%Y%m%d").date() - end = datetime.strptime(normalize_date(end_date), "%Y%m%d").date() - if start > end: - raise ValueError("开始日期不能晚于结束日期。") - weekdays = [] - current = start - while current <= end: - if current.weekday() < 5: - weekdays.append(current) - current += timedelta(days=1) - if len(weekdays) > 15: - raise ValueError("单次最多回补 15 个工作日。") - results = [] - for day in weekdays: - dashboard = self.sync_dashboard(day.strftime("%Y%m%d")) - results.append( - { - "requested_date": day.isoformat(), - "trade_date": dashboard["meta"]["trade_date"], - "source": dashboard["meta"]["source"], - "records": self._record_count(dashboard), - } - ) - return results - - def _stock_identity(self, code: str, trade_date: str) -> tuple[str, str]: - snapshot = self.database.get_snapshot(trade_date) or {} - for key in ("limits", "broken", "down_limits"): - for row in snapshot.get(key) or []: - if str(row.get("code")) == code: - return row.get("name") or "--", row.get("sector") or "其他" - for item in self.database.list_watchlist(self.current_user_id): - if item["code"] == code: - return item["name"], item["sector"] or "其他" - return "--", "其他" - - def _enrich_stock_detail(self, payload: dict[str, Any]) -> dict[str, Any]: - result = dict(payload) - stock = dict(payload.get("stock") or {}) - code = str(stock.get("code") or "") - watched = { - item["code"]: item - for item in self.database.list_watchlist(self.current_user_id) - } - stock["watchlist"] = watched.get(code) - result["stock"] = stock - result["notes"] = self.database.list_notes(self.current_user_id, code=code) - return result def _apply_reason_overrides(self, dashboard: dict[str, Any]) -> dict[str, Any]: trade_date = str(dashboard.get("meta", {}).get("trade_date", "")).replace("-", "") @@ -4578,22 +3671,6 @@ class DashboardService: } return result - def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]: - result = dict(dashboard) - result["meta"] = { - **dashboard.get("meta", {}), - "storage": "sqlite", - "cached": cached, - } - return result - - @staticmethod - def _record_count(dashboard: dict[str, Any]) -> int: - return sum( - len(dashboard.get(key) or []) - for key in ("limits", "broken", "down_limits", "yesterday_limits") - ) - SERVICE = DashboardService() diff --git a/backend/bootstrap/container.py b/backend/bootstrap/container.py index 6131c40..0929a06 100644 --- a/backend/bootstrap/container.py +++ b/backend/bootstrap/container.py @@ -10,12 +10,12 @@ from backend.features.alerts import AlertService from backend.features.review import TradeJournalService from backend.features.screener import StrategyTrackingService from backend.jobs import InProcessJobRunner, JobRegistry, SQLiteJobRunRepository -from chart_data_provider import MarketChartClient from database import ReviewDatabase -from ifind_client import IfindHttpClient from mentor_agent import MentorSkillRegistry -from realtime_aggregator import WebRealtimeAggregator from screener import ScreenerEngine +from backend.data.providers.ifind_client import IfindHttpClient +from backend.data.realtime import WebRealtimeAggregator +from backend.features.market.charts import MarketChartClient @dataclass(frozen=True) diff --git a/backend/data/__init__.py b/backend/data/__init__.py index 2b06ec9..1c8c7bd 100644 --- a/backend/data/__init__.py +++ b/backend/data/__init__.py @@ -1,4 +1,3 @@ -from .gateway import DataGateway, build_data_gateway from .policy import DataPolicyError, DataSourcePolicy from .quality import DataQualityError, DataQualityGate, QualityEvidence, QualityReport @@ -12,3 +11,11 @@ __all__ = [ "QualityReport", "build_data_gateway", ] + + +def __getattr__(name: str): + if name in {"DataGateway", "build_data_gateway"}: + from .gateway import DataGateway, build_data_gateway + + return {"DataGateway": DataGateway, "build_data_gateway": build_data_gateway}[name] + raise AttributeError(name) diff --git a/backend/data/gateway.py b/backend/data/gateway.py index 933bc93..fcd436c 100644 --- a/backend/data/gateway.py +++ b/backend/data/gateway.py @@ -8,10 +8,10 @@ from backend.data.contracts import DataUsage from backend.data.policy import DataSourcePolicy from backend.data.providers import IfindProvider, TushareProvider from backend.data.quality import DataQualityGate, QualityEvidence, QualityReport -from chart_data_provider import EastmoneyChartClient, MarketChartClient -from ifind_client import IfindHttpClient -from realtime_aggregator import WebRealtimeAggregator -from tushare_client import TushareClient +from backend.data.providers.ifind_client import IfindHttpClient +from backend.data.providers.tushare_client import TushareClient +from backend.data.realtime import WebRealtimeAggregator +from backend.features.market.charts import EastmoneyChartClient, MarketChartClient @dataclass(frozen=True) diff --git a/backend/data/providers/ifind.py b/backend/data/providers/ifind.py index c64a4dd..b663416 100644 --- a/backend/data/providers/ifind.py +++ b/backend/data/providers/ifind.py @@ -1,6 +1,6 @@ from __future__ import annotations -from ifind_client import IfindHttpClient +from backend.data.providers.ifind_client import IfindHttpClient class IfindProvider: diff --git a/backend/data/providers/ifind_client.py b/backend/data/providers/ifind_client.py new file mode 100644 index 0000000..2b30fd9 --- /dev/null +++ b/backend/data/providers/ifind_client.py @@ -0,0 +1,385 @@ +from __future__ import annotations + +import copy +import json +import threading +import time +import urllib.error +import urllib.request +from datetime import datetime, timedelta +from typing import Any + + +class IfindError(RuntimeError): + pass + + +class IfindHttpClient: + BASE_URL = "https://quantapi.51ifind.com/api/v1" + AUTH_ENDPOINT = "get_access_token" + AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303} + + def __init__( + self, + refresh_token: str = "", + access_token: str = "", + timeout: int = 15, + ) -> None: + self.timeout = max(3, int(timeout)) + self._refresh_token = str(refresh_token or "").strip() + self._access_token = str(access_token or "").strip() + self._access_expires_at: datetime | None = None + self._token_lock = threading.Lock() + self._cache_lock = threading.Lock() + self._cache: dict[str, dict[str, Any]] = {} + + @property + def configured(self) -> bool: + return bool(self._refresh_token or self._access_token) + + def set_credentials(self, refresh_token: str, access_token: str = "") -> None: + refresh_token = str(refresh_token or "").strip() + access_token = str(access_token or "").strip() + with self._token_lock: + refresh_changed = refresh_token != self._refresh_token + self._refresh_token = refresh_token + if access_token or refresh_changed: + self._access_token = access_token + self._access_expires_at = None + if refresh_changed: + with self._cache_lock: + self._cache.clear() + + def status(self) -> dict[str, Any]: + return { + "configured": self.configured, + "access_ready": bool(self._access_token), + "access_expires_at": ( + self._access_expires_at.isoformat(timespec="seconds") + if self._access_expires_at + else "" + ), + } + + def test_connection(self) -> dict[str, Any]: + payload = self.real_time( + "000001.SH", + ["open", "high", "low", "latest", "preClose"], + cache_ttl=0, + ) + return { + "ok": bool(payload), + "sample_time": str(payload[0].get("time") or "") if payload else "", + } + + def real_time( + self, + codes: str | list[str], + indicators: list[str], + cache_ttl: int = 10, + ) -> list[dict[str, Any]]: + code_text = self._codes(codes) + payload = self._request( + "real_time_quotation", + {"codes": code_text, "indicators": ",".join(indicators)}, + cache_key=f"rq:{code_text}:{','.join(indicators)}", + cache_ttl=cache_ttl, + ) + return self._table_rows(payload) + + def history( + self, + codes: str | list[str], + indicators: list[str], + start_date: str, + end_date: str, + cache_ttl: int = 300, + ) -> list[dict[str, Any]]: + code_text = self._codes(codes) + payload = self._request( + "cmd_history_quotation", + { + "codes": code_text, + "indicators": ",".join(indicators), + "startdate": self._display_date(start_date), + "enddate": self._display_date(end_date), + "functionpara": {"CPS": "forward1", "Fill": "Omit"}, + }, + cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}", + cache_ttl=cache_ttl, + ) + return self._table_rows(payload) + + def intraday( + self, + code: str, + start_time: str, + end_time: str, + cache_ttl: int = 20, + ) -> list[dict[str, Any]]: + indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"] + payload = self._request( + "high_frequency", + { + "codes": self._codes(code), + "indicators": ",".join(indicators), + "starttime": start_time, + "endtime": end_time, + "functionpara": { + "CPS": "forward1", + "Fill": "Previous", + "Timeformat": "LocalTime", + "Interval": "1", + "Limitstart": "09:30:00", + "Limitend": "15:00:00", + }, + }, + cache_key=f"hf:{code}:{start_time}:{end_time}", + cache_ttl=cache_ttl, + ) + return self._table_rows(payload) + + def snapshots( + self, + codes: str | list[str], + indicators: list[str], + start_time: str, + end_time: str, + cache_ttl: int = 8, + ) -> list[dict[str, Any]]: + code_text = self._codes(codes) + payload = self._request( + "snap_shot", + { + "codes": code_text, + "indicators": ",".join(indicators), + "starttime": start_time, + "endtime": end_time, + }, + cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}", + cache_ttl=cache_ttl, + ) + return self._table_rows(payload) + + def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]: + normalized = " ".join(str(query or "").split()) + if not normalized: + raise IfindError("问财查询不能为空。") + payload = self._request( + "smart_stock_picking", + {"searchstring": normalized, "searchtype": search_type}, + cache_key=f"wc:{search_type}:{normalized}", + cache_ttl=cache_ttl, + ) + return self._table_rows(payload) + + def report_query( + self, + codes: str | list[str], + begin_date: str, + end_date: str, + cache_ttl: int = 300, + ) -> list[dict[str, Any]]: + code_text = self._codes(codes) + payload = self._request( + "report_query", + { + "codes": code_text, + "beginrDate": self._display_date(begin_date), + "endrDate": self._display_date(end_date), + "outputpara": ( + "reportDate:Y,thscode:Y,secName:Y,ctime:Y," + "reportTitle:Y,pdfURL:Y,seq:Y" + ), + }, + cache_key=f"report:{code_text}:{begin_date}:{end_date}", + cache_ttl=cache_ttl, + ) + return self._table_rows(payload) + + def _request( + self, + endpoint: str, + body: dict[str, Any], + cache_key: str = "", + cache_ttl: int = 0, + ) -> dict[str, Any]: + if not self.configured: + raise IfindError("iFinD 尚未配置。") + if cache_key and cache_ttl > 0: + cached = self._cached(cache_key, cache_ttl) + if cached is not None: + return cached + + payload = self._post(endpoint, body, self._ensure_access_token()) + if self._is_auth_error(payload) and self._refresh_token: + self._invalidate_access_token() + payload = self._post(endpoint, body, self._ensure_access_token(force=True)) + self._validate_payload(payload) + if cache_key and cache_ttl > 0: + with self._cache_lock: + self._cache[cache_key] = { + "created_at": time.time(), + "payload": copy.deepcopy(payload), + } + return payload + + def _ensure_access_token(self, force: bool = False) -> str: + with self._token_lock: + now = datetime.now().astimezone().replace(tzinfo=None) + token_valid = bool(self._access_token) and ( + self._access_expires_at is None + or self._access_expires_at > now + timedelta(minutes=2) + ) + if token_valid and not force: + return self._access_token + if not self._refresh_token: + if self._access_token: + return self._access_token + raise IfindError("iFinD Refresh Token 尚未配置。") + payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token) + self._validate_payload(payload) + data = payload.get("data") or {} + token = str(data.get("access_token") or "").strip() + if not token: + raise IfindError("iFinD 未返回 Access Token。") + expires_at = self._parse_datetime(data.get("expired_time")) + self._access_token = token + self._access_expires_at = expires_at + return token + + def _post( + self, + endpoint: str, + body: dict[str, Any], + access_token: str, + refresh_token: str = "", + ) -> dict[str, Any]: + headers = { + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "XiaobaiReviewWeb/1.0", + "ifindlang": "cn", + } + if access_token: + headers["access_token"] = access_token + if refresh_token: + headers["refresh_token"] = refresh_token + request = urllib.request.Request( + f"{self.BASE_URL}/{endpoint}", + data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), + headers=headers, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = "" + try: + detail_payload = json.loads(exc.read().decode("utf-8", errors="replace")) + detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "") + except (json.JSONDecodeError, OSError): + pass + raise IfindError(f"iFinD HTTP {exc.code}{f'：{detail[:160]}' if detail else ''}") from exc + except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc: + raise IfindError("iFinD 数据请求失败。") from exc + if not isinstance(payload, dict): + raise IfindError("iFinD 返回格式不正确。") + return payload + + def _cached(self, key: str, ttl: int) -> dict[str, Any] | None: + with self._cache_lock: + cached = self._cache.get(key) + if not cached: + return None + if time.time() - float(cached.get("created_at") or 0) > ttl: + self._cache.pop(key, None) + return None + return copy.deepcopy(cached["payload"]) + + def _invalidate_access_token(self) -> None: + with self._token_lock: + self._access_token = "" + self._access_expires_at = None + + @classmethod + def _validate_payload(cls, payload: dict[str, Any]) -> None: + try: + error_code = int(payload.get("errorcode") or 0) + except (TypeError, ValueError): + error_code = -1 + if error_code != 0: + message = str(payload.get("errmsg") or "未知错误") + raise IfindError(f"iFinD 返回错误：{message[:200]}") + + @classmethod + def _is_auth_error(cls, payload: dict[str, Any]) -> bool: + try: + error_code = int(payload.get("errorcode") or 0) + except (TypeError, ValueError): + error_code = 0 + message = str(payload.get("errmsg") or "").casefold() + return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message + + @staticmethod + def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]: + tables = payload.get("tables") or [] + if isinstance(tables, dict): + tables = [tables] + rows: list[dict[str, Any]] = [] + for block in tables if isinstance(tables, list) else []: + if not isinstance(block, dict): + continue + table = block.get("table") or {} + if not isinstance(table, dict): + continue + times = block.get("time") or [] + codes = block.get("thscode") or block.get("thscodes") or [] + if isinstance(codes, str): + codes = [codes] + lengths = [len(value) for value in table.values() if isinstance(value, list)] + row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0]) + for index in range(row_count): + row: dict[str, Any] = {} + if isinstance(times, list) and index < len(times): + row["time"] = times[index] + if codes: + row["thscode"] = codes[index] if index < len(codes) else codes[0] + for field, values in table.items(): + if isinstance(values, list): + row[field] = values[index] if index < len(values) else None + elif index == 0: + row[field] = values + rows.append(row) + return rows + + @staticmethod + def _codes(codes: str | list[str]) -> str: + if isinstance(codes, list): + values = [str(code or "").strip().upper() for code in codes] + else: + values = [part.strip().upper() for part in str(codes or "").split(",")] + values = [value for value in values if value] + if not values: + raise IfindError("iFinD 证券代码不能为空。") + if len(values) > 100: + raise IfindError("iFinD 单次证券代码过多。") + return ",".join(values) + + @staticmethod + def _display_date(value: str) -> str: + compact = str(value or "").replace("-", "") + if len(compact) != 8 or not compact.isdigit(): + raise IfindError("iFinD 日期格式不正确。") + return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}" + + @staticmethod + def _parse_datetime(value: Any) -> datetime | None: + text = str(value or "").strip() + if not text: + return None + try: + return datetime.fromisoformat(text) + except ValueError: + return None diff --git a/backend/data/providers/tushare.py b/backend/data/providers/tushare.py index a2eeda0..cfae27b 100644 --- a/backend/data/providers/tushare.py +++ b/backend/data/providers/tushare.py @@ -2,7 +2,7 @@ from __future__ import annotations from collections.abc import Callable -from tushare_client import TushareClient +from backend.data.providers.tushare_client import TushareClient class TushareProvider: diff --git a/backend/data/providers/tushare_client.py b/backend/data/providers/tushare_client.py new file mode 100644 index 0000000..a2f65b0 --- /dev/null +++ b/backend/data/providers/tushare_client.py @@ -0,0 +1,2175 @@ +from __future__ import annotations + +import json +import math +import re +import urllib.error +import urllib.request +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, time as dt_time, timedelta +from threading import Lock +from typing import Any, ClassVar + +from sentiment_engine import apply_sentiment_to_dashboard + + +TUSHARE_URL = "http://api.tushare.pro" + + +class TushareError(RuntimeError): + pass + + +@dataclass +class TushareClient: + token: str + timeout: int = 30 + _realtime_reference_cache: ClassVar[dict[str, dict[str, Any]]] = {} + _realtime_reference_lock: ClassVar[Lock] = Lock() + _capital_cache: ClassVar[dict[str, dict[str, Any]]] = {} + _latest_realtime_market: ClassVar[dict[str, dict[str, Any]]] = {} + _stock_activity_cache: ClassVar[dict[str, dict[str, Any]]] = {} + _stock_listing_cache: ClassVar[dict[str, Any]] = {} + _stock_listing_lock: ClassVar[Lock] = Lock() + _suspension_cache: ClassVar[dict[str, dict[str, str] | None]] = {} + _suspension_lock: ClassVar[Lock] = Lock() + + def query( + self, + api_name: str, + params: dict[str, Any] | None = None, + fields: str = "", + ) -> list[dict[str, Any]]: + payload = json.dumps( + { + "api_name": api_name, + "token": self.token, + "params": params or {}, + "fields": fields, + } + ).encode("utf-8") + request = urllib.request.Request( + TUSHARE_URL, + data=payload, + headers={"Content-Type": "application/json", "User-Agent": "XiaobaiReviewWeb/0.2"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + result = json.loads(response.read().decode("utf-8")) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: + raise TushareError(f"Tushare request failed: {exc}") from exc + + if result.get("code") != 0: + raise TushareError(result.get("msg") or "Tushare returned an unknown error") + + data = result.get("data") or {} + columns = data.get("fields") or [] + return [dict(zip(columns, item)) for item in data.get("items") or []] + + def dashboard(self, requested_date: str) -> dict[str, Any]: + trade_date, previous_trade_date = self.resolve_trade_context(requested_date) + if self.should_use_realtime(requested_date, trade_date): + return self._realtime_dashboard( + requested_date, + trade_date, + previous_trade_date, + ) + + daily = self._load_daily(trade_date) + if ( + not daily + and requested_date == datetime.now().astimezone().strftime("%Y%m%d") + and trade_date == requested_date + and datetime.now().astimezone().time().replace(tzinfo=None) >= dt_time(9, 15) + ): + return self._realtime_dashboard( + requested_date, + trade_date, + previous_trade_date, + ) + if not daily: + raise TushareError(f"No daily data returned for {trade_date}") + + notices: list[str] = [] + try: + limit_rows = self._load_limit_lists(trade_date) + previous_limit_rows = self._load_limit_type(previous_trade_date, "U") + if not limit_rows: + notices.append("涨跌停高级接口当日数据尚未更新，已使用日线数据推算。") + limit_rows = self._derive_limits(trade_date, daily) + except TushareError as exc: + notices.append(f"涨跌停高级接口不可用，已使用日线数据推算：{exc}") + limit_rows = self._derive_limits(trade_date, daily) + previous_daily = self._load_daily(previous_trade_date) + previous_limit_rows = [ + row for row in self._derive_limits(previous_trade_date, previous_daily) + if row.get("limit_type") == "U" + ] + + up_rows = [row for row in limit_rows if row.get("limit_type") == "U"] + down_rows = [row for row in limit_rows if row.get("limit_type") == "D"] + broken_rows = [row for row in limit_rows if row.get("limit_type") == "Z"] + limits = [self._normalize_limit(row, "涨停") for row in up_rows] + broken = [self._normalize_limit(row, "炸板") for row in broken_rows] + down_limits = [self._normalize_limit(row, "跌停") for row in down_rows] + previous_limits = [self._normalize_limit(row, "涨停") for row in previous_limit_rows] + yesterday_limits = _build_yesterday_performance( + previous_limits, + daily, + limits, + broken, + down_limits, + ) + sectors = _build_sectors(limits) + previous_sectors = _build_sectors(previous_limits) + + dashboard = { + "meta": { + "requested_date": _display_date(requested_date), + "trade_date": _display_date(trade_date), + "previous_trade_date": _display_date(previous_trade_date), + "source": "tushare", + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "notice": "；".join(notices), + }, + "overview": _build_overview(daily, up_rows, down_rows, broken_rows), + "limits": limits, + "broken": broken, + "down_limits": down_limits, + "yesterday_limits": yesterday_limits, + "limit_performance": _build_limit_performance(yesterday_limits), + "ladders": _build_ladders(limits), + "sectors": sectors, + "sector_rotation": _build_sector_rotation(sectors, previous_sectors), + } + return apply_sentiment_to_dashboard(dashboard) + + @staticmethod + def should_use_realtime(requested_date: str, trade_date: str) -> bool: + """Use rt_k for today's open market until end-of-day datasets settle.""" + now = datetime.now().astimezone() + today = now.strftime("%Y%m%d") + return ( + requested_date == today + and trade_date == today + and dt_time(9, 15) <= now.time().replace(tzinfo=None) < dt_time(16, 30) + ) + + def _realtime_dashboard( + self, + requested_date: str, + trade_date: str, + previous_trade_date: str, + ) -> dict[str, Any]: + reference = self._load_realtime_reference(trade_date, previous_trade_date) + basic_rows = list(reference["basic_rows"]) + codes = ",".join( + str(row.get("ts_code") or "") for row in basic_rows if row.get("ts_code") + ) + if not codes: + raise TushareError("No active stock codes available for rt_k") + quotes = self.query("rt_k", {"ts_code": codes}) + if not quotes: + raise TushareError(f"No realtime data returned for {trade_date}") + + basic_map = {str(row.get("ts_code") or ""): row for row in basic_rows} + daily: list[dict[str, Any]] = [] + for quote in quotes: + close = _number(quote.get("close")) + previous_close = _number(quote.get("pre_close")) + if close <= 0 or previous_close <= 0: + continue + basic = basic_map.get(str(quote.get("ts_code") or ""), {}) + daily.append( + { + **quote, + "trade_date": trade_date, + "name": str(quote.get("name") or basic.get("name") or "--").strip(), + "industry": basic.get("industry") or "其他", + "pct_chg": round((close / previous_close - 1) * 100, 4), + "amount_unit": "yuan", + } + ) + with self._realtime_reference_lock: + self._latest_realtime_market[trade_date] = { + "rows": daily, + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + } + if len(self._latest_realtime_market) > 3: + oldest = next(iter(self._latest_realtime_market)) + self._latest_realtime_market.pop(oldest, None) + + limit_rows = self._derive_limits( + trade_date, + daily, + price_limits=list(reference["price_limits"]), + basic_rows=basic_rows, + previous_limit_rows=list(reference["previous_limit_rows"]), + capital_rows=list(reference["capital_rows"]), + ) + previous_limit_rows = list(reference["previous_limit_rows"]) + up_rows = [row for row in limit_rows if row.get("limit_type") == "U"] + down_rows = [row for row in limit_rows if row.get("limit_type") == "D"] + broken_rows = [row for row in limit_rows if row.get("limit_type") == "Z"] + limits = [self._normalize_limit(row, "涨停") for row in up_rows] + broken = [self._normalize_limit(row, "炸板") for row in broken_rows] + down_limits = [self._normalize_limit(row, "跌停") for row in down_rows] + previous_limits = [self._normalize_limit(row, "涨停") for row in previous_limit_rows] + yesterday_limits = _build_yesterday_performance( + previous_limits, + daily, + limits, + broken, + down_limits, + ) + sectors = _build_sectors(limits) + previous_sectors = _build_sectors(previous_limits) + now = datetime.now().astimezone() + market_status = _realtime_market_status(now.time().replace(tzinfo=None)) + dashboard = { + "meta": { + "requested_date": _display_date(requested_date), + "trade_date": _display_date(trade_date), + "previous_trade_date": _display_date(previous_trade_date), + "source": "tushare", + "mode": "realtime", + "realtime": True, + "market_status": market_status, + "refresh_mode": "manual", + "auto_refresh": False, + "quote_count": len(daily), + "updated_at": now.isoformat(timespec="seconds"), + "notice": "盘中行情由 Tushare rt_k 实时计算；涨停原因、封板时间和开板次数以盘后榜单校正为准。", + }, + "overview": _build_overview(daily, up_rows, down_rows, broken_rows), + "limits": limits, + "broken": broken, + "down_limits": down_limits, + "yesterday_limits": yesterday_limits, + "limit_performance": _build_limit_performance(yesterday_limits), + "ladders": _build_ladders(limits), + "sectors": sectors, + "sector_rotation": _build_sector_rotation(sectors, previous_sectors), + } + return apply_sentiment_to_dashboard(dashboard) + + def _load_realtime_reference( + self, + trade_date: str, + previous_trade_date: str, + ) -> dict[str, Any]: + cache_key = f"{trade_date}:{previous_trade_date}" + with self._realtime_reference_lock: + cached = self._realtime_reference_cache.get(cache_key) + if cached: + return cached + + basic_rows = self.query( + "stock_basic", + {"exchange": "", "list_status": "L"}, + "ts_code,name,industry,market,list_date", + ) + price_limits = self.query( + "stk_limit", + {"trade_date": trade_date}, + "ts_code,trade_date,up_limit,down_limit", + ) + previous_limit_rows = self._load_limit_type(previous_trade_date, "U") + capital_rows = self.query( + "daily_basic", + {"trade_date": previous_trade_date}, + "ts_code,trade_date,total_share,float_share,free_share,total_mv,circ_mv", + ) + if not basic_rows or not price_limits: + raise TushareError(f"Realtime reference data is incomplete for {trade_date}") + result = { + "basic_rows": basic_rows, + "price_limits": price_limits, + "previous_limit_rows": previous_limit_rows, + "capital_rows": capital_rows, + } + with self._realtime_reference_lock: + self._realtime_reference_cache[cache_key] = result + if len(self._realtime_reference_cache) > 3: + oldest = next(iter(self._realtime_reference_cache)) + self._realtime_reference_cache.pop(oldest, None) + return result + + def realtime_stock_quote( + self, + ts_code: str, + reference_date: str = "", + ) -> dict[str, Any]: + rows = self.query("rt_k", {"ts_code": ts_code}) + if not rows: + raise TushareError(f"No realtime quote returned for {ts_code}") + row = rows[0] + close = _number(row.get("close")) + previous_close = _number(row.get("pre_close")) + if close <= 0 or previous_close <= 0: + raise TushareError(f"Realtime quote is unavailable for {ts_code}") + + basic: dict[str, Any] = {} + with self._realtime_reference_lock: + references = list(self._realtime_reference_cache.values()) + for reference in reversed(references): + basic = next( + ( + item for item in reference.get("basic_rows") or [] + if str(item.get("ts_code") or "") == ts_code + ), + {}, + ) + if basic: + break + if not basic: + basics = self.query( + "stock_basic", + {"ts_code": ts_code}, + "ts_code,name,industry,market,list_date", + ) + basic = basics[0] if basics else {} + capital = self._latest_capital(ts_code, reference_date) + float_share = _number(capital.get("float_share")) + # rt_k volume is shares; daily_basic float_share is reported in 10k shares. + turnover_rate = _number(row.get("vol")) / float_share / 100 if float_share else 0 + market_date = reference_date or datetime.now().astimezone().strftime("%Y%m%d") + self._ensure_realtime_market_cache(market_date) + with self._realtime_reference_lock: + market_rows = list((self._latest_realtime_market.get(market_date) or {}).get("rows") or []) + references = list(self._realtime_reference_cache.values()) + capital_map: dict[str, dict[str, Any]] = {} + for reference in reversed(references): + capital_map = { + str(item.get("ts_code") or ""): item + for item in reference.get("capital_rows") or [] + } + if capital_map: + break + market_amounts = [_number(item.get("amount")) for item in market_rows if _number(item.get("amount")) > 0] + amount_percentile = _value_percentile(_number(row.get("amount")), market_amounts) + market_turnovers = [] + for item in market_rows: + item_capital = capital_map.get(str(item.get("ts_code") or ""), {}) + item_float_share = _number(item_capital.get("float_share")) + if item_float_share: + market_turnovers.append(_number(item.get("vol")) / item_float_share / 100) + market_turnover = ( + sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0 + ) + turnover_relative = turnover_rate / market_turnover if market_turnover else 0 + activity = self._stock_activity_metrics( + ts_code, + market_date, + _number(row.get("vol")) / 100, + ) + return { + "code": ts_code.split(".")[0], + "ts_code": ts_code, + "name": str(row.get("name") or basic.get("name") or "--").strip(), + "sector": basic.get("industry") or "其他", + "price": round(close, 3), + "change": round((close / previous_close - 1) * 100, 4), + "open": round(_number(row.get("open")), 3), + "high": round(_number(row.get("high")), 3), + "low": round(_number(row.get("low")), 3), + "previous_close": round(previous_close, 3), + "amount_billion": round(_number(row.get("amount")) / 100000000, 3), + "volume": _number(row.get("vol")), + "trade_count": int(_number(row.get("num"))), + "turnover_rate": round(turnover_rate, 4), + "market_turnover_rate": round(market_turnover, 4), + "turnover_relative": round(turnover_relative, 4), + "amount_percentile": round(amount_percentile * 100, 2), + "volume_activity_ratio": activity.get("volume_activity_ratio", 0), + "activity_history_date": activity.get("history_trade_date", ""), + "activity_source": activity.get("source", "unavailable"), + "float_share_10k": float_share, + "capital_trade_date": str(capital.get("trade_date") or ""), + "turnover_source": "rt_volume/latest_float_share" if float_share else "unavailable", + "data_source": "tushare", + "realtime": True, + } + + def _stock_activity_metrics( + self, + ts_code: str, + reference_date: str, + current_volume_lots: float, + ) -> dict[str, Any]: + cache_key = f"{ts_code}:{reference_date}" + with self._realtime_reference_lock: + history = self._stock_activity_cache.get(cache_key) + if history is None: + try: + end = datetime.strptime(reference_date, "%Y%m%d") + except ValueError: + end = datetime.now().astimezone().replace(tzinfo=None) + rows = self.query( + "daily", + { + "ts_code": ts_code, + "start_date": (end - timedelta(days=30)).strftime("%Y%m%d"), + "end_date": reference_date, + }, + "ts_code,trade_date,vol,amount", + ) + completed = [ + item for item in rows + if str(item.get("trade_date") or "") < reference_date and _number(item.get("vol")) > 0 + ] + completed.sort(key=lambda item: str(item.get("trade_date") or "")) + recent = completed[-5:] + history = { + "average_volume_lots": ( + sum(_number(item.get("vol")) for item in recent) / len(recent) + if recent else 0 + ), + "history_trade_date": str(recent[-1].get("trade_date") or "") if recent else "", + } + with self._realtime_reference_lock: + self._stock_activity_cache[cache_key] = history + if len(self._stock_activity_cache) > 256: + oldest = next(iter(self._stock_activity_cache)) + self._stock_activity_cache.pop(oldest, None) + average_volume = _number(history.get("average_volume_lots")) + progress = _trading_session_progress(datetime.now().astimezone().time().replace(tzinfo=None)) + expected_volume = average_volume * progress + ratio = current_volume_lots / expected_volume if expected_volume else 0 + return { + **history, + "volume_activity_ratio": round(ratio, 4), + "session_progress": round(progress, 4), + "source": "rt_volume/5d_average_at_same_progress" if expected_volume else "unavailable", + } + + def realtime_factor_snapshot(self, requested_date: str) -> dict[str, Any]: + trade_date, previous_trade_date = self.resolve_trade_context(requested_date) + reference = self._load_realtime_reference(trade_date, previous_trade_date) + codes = [ + str(row.get("ts_code") or "") + for row in reference.get("basic_rows") or [] + if row.get("ts_code") + ] + quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") + capital_map = { + str(row.get("ts_code") or ""): row + for row in reference.get("capital_rows") or [] + } + rows = [] + for quote in quotes: + ts_code = str(quote.get("ts_code") or "") + close = _number(quote.get("close")) + previous_close = _number(quote.get("pre_close")) + if not ts_code or close <= 0 or previous_close <= 0: + continue + capital = capital_map.get(ts_code, {}) + float_share = _number(capital.get("float_share")) + rows.append( + { + "ts_code": ts_code, + "trade_date": trade_date, + "open": _number(quote.get("open")), + "high": _number(quote.get("high")), + "low": _number(quote.get("low")), + "close": close, + "pct_chg": (close / previous_close - 1) * 100, + "vol": _number(quote.get("vol")) / 100, + "amount": _number(quote.get("amount")), + "turnover_rate": ( + _number(quote.get("vol")) / float_share / 100 if float_share else 0 + ), + "capital_trade_date": str(capital.get("trade_date") or ""), + } + ) + if not rows: + raise TushareError(f"No realtime factor snapshot returned for {trade_date}") + return { + "trade_date": trade_date, + "previous_trade_date": previous_trade_date, + "source": "tushare_rt_k", + "realtime": True, + "rows": rows, + } + + def _ensure_realtime_market_cache(self, requested_date: str) -> list[dict[str, Any]]: + with self._realtime_reference_lock: + cached = list( + (self._latest_realtime_market.get(requested_date) or {}).get("rows") or [] + ) + if cached: + return cached + trade_date, previous_trade_date = self.resolve_trade_context(requested_date) + if trade_date != requested_date: + return [] + reference = self._load_realtime_reference(trade_date, previous_trade_date) + codes = [ + str(row.get("ts_code") or "") + for row in reference.get("basic_rows") or [] + if row.get("ts_code") + ] + quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") + rows = [ + row for row in quotes + if _number(row.get("close")) > 0 and _number(row.get("pre_close")) > 0 + ] + with self._realtime_reference_lock: + self._latest_realtime_market[trade_date] = { + "rows": rows, + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + } + return rows + + def _latest_capital(self, ts_code: str, reference_date: str = "") -> dict[str, Any]: + end_date = reference_date or datetime.now().astimezone().strftime("%Y%m%d") + cache_key = f"{ts_code}:{end_date}" + with self._realtime_reference_lock: + cached = self._capital_cache.get(cache_key) + if cached: + return cached + try: + end = datetime.strptime(end_date, "%Y%m%d") + except ValueError: + end = datetime.now().astimezone().replace(tzinfo=None) + end_date = end.strftime("%Y%m%d") + start_date = (end - timedelta(days=20)).strftime("%Y%m%d") + rows = self.query( + "daily_basic", + {"ts_code": ts_code, "start_date": start_date, "end_date": end_date}, + "ts_code,trade_date,turnover_rate,volume_ratio,total_share,float_share," + "free_share,total_mv,circ_mv", + ) + rows.sort(key=lambda item: str(item.get("trade_date") or "")) + result = rows[-1] if rows else {} + with self._realtime_reference_lock: + self._capital_cache[cache_key] = result + if len(self._capital_cache) > 256: + oldest = next(iter(self._capital_cache)) + self._capital_cache.pop(oldest, None) + return result + + def market_indices(self, requested_date: str, lookback_days: int = 45) -> dict[str, Any]: + trade_date, _ = self.resolve_trade_context(requested_date) + end = datetime.strptime(trade_date, "%Y%m%d") + start_date = (end - timedelta(days=max(30, lookback_days * 2))).strftime("%Y%m%d") + index_names = { + "000001.SH": "上证指数", + "399001.SZ": "深证成指", + "399006.SZ": "创业板指", + } + indices = [] + for ts_code, name in index_names.items(): + rows = self.query( + "index_daily", + {"ts_code": ts_code, "start_date": start_date, "end_date": trade_date}, + "ts_code,trade_date,close,pct_chg,vol,amount", + ) + rows.sort(key=lambda item: str(item.get("trade_date") or "")) + if not rows: + continue + latest = rows[-1] + close = _number(latest.get("close")) + close_5d = _number(rows[-6].get("close")) if len(rows) >= 6 else _number(rows[0].get("close")) + close_20d = _number(rows[-21].get("close")) if len(rows) >= 21 else _number(rows[0].get("close")) + indices.append( + { + "ts_code": ts_code, + "name": name, + "trade_date": str(latest.get("trade_date") or trade_date), + "close": close, + "pct_chg": round(_number(latest.get("pct_chg")), 3), + "return_5d": round((close / close_5d - 1) * 100, 3) if close_5d else 0, + "return_20d": round((close / close_20d - 1) * 100, 3) if close_20d else 0, + "amount_billion": round(_number(latest.get("amount")) / 100000, 2), + } + ) + if not indices: + raise TushareError(f"No index data returned for {trade_date}") + return { + "trade_date": trade_date, + "source": "tushare", + "realtime": False, + "precise": all(item["trade_date"] == trade_date for item in indices), + "indices": indices, + "aggregate": { + "average_pct_chg": round(sum(item["pct_chg"] for item in indices) / len(indices), 3), + "average_return_5d": round(sum(item["return_5d"] for item in indices) / len(indices), 3), + "average_return_20d": round(sum(item["return_20d"] for item in indices) / len(indices), 3), + }, + } + + def realtime_market_indices(self, requested_date: str) -> dict[str, Any]: + trade_date, _ = self.resolve_trade_context(requested_date) + index_names = { + "000001.SH": "上证指数", + "399001.SZ": "深证成指", + "399006.SZ": "创业板指", + } + rows = self.query("rt_idx_k", {"ts_code": ",".join(index_names)}, "") + row_map = {str(row.get("ts_code") or ""): row for row in rows} + indices = [] + for ts_code, name in index_names.items(): + row = row_map.get(ts_code) + if not row: + continue + close = _number(row.get("close")) + previous_close = _number(row.get("pre_close")) + if close <= 0 or previous_close <= 0: + continue + history = self.query( + "index_daily", + { + "ts_code": ts_code, + "start_date": (datetime.strptime(trade_date, "%Y%m%d") - timedelta(days=20)).strftime("%Y%m%d"), + "end_date": trade_date, + }, + "ts_code,trade_date,close,pct_chg", + ) + history.sort(key=lambda item: str(item.get("trade_date") or "")) + previous_closes = [ + _number(item.get("close")) for item in history + if str(item.get("trade_date") or "") < trade_date and _number(item.get("close")) > 0 + ] + close_5d = previous_closes[-5] if len(previous_closes) >= 5 else previous_closes[0] if previous_closes else previous_close + indices.append( + { + "ts_code": ts_code, + "name": str(row.get("name") or name).strip(), + "trade_date": trade_date, + "close": close, + "pct_chg": round((close / previous_close - 1) * 100, 3), + "return_5d": round((close / close_5d - 1) * 100, 3) if close_5d else 0, + "amount_billion": round(_number(row.get("amount")) / 100000000, 2), + } + ) + if len(indices) != len(index_names): + raise TushareError("Realtime index quotes are incomplete") + return { + "trade_date": trade_date, + "source": "tushare_rt_idx_k", + "realtime": True, + "precise": True, + "indices": indices, + "aggregate": { + "average_pct_chg": round(sum(item["pct_chg"] for item in indices) / len(indices), 3), + "average_return_5d": round(sum(item["return_5d"] for item in indices) / len(indices), 3), + "average_return_20d": 0, + }, + } + + def sw_stock_industry(self, ts_code: str, trade_date: str) -> dict[str, Any]: + """Return the Shenwan industry active for a stock on trade_date.""" + rows = [] + for is_new in ("Y", "N"): + rows.extend( + self.query( + "index_member_all", + {"ts_code": ts_code, "is_new": is_new}, + "l1_code,l1_name,l2_code,l2_name,l3_code,l3_name," + "ts_code,name,in_date,out_date,is_new", + ) + ) + rows = _reconcile_membership_rows(rows) + matched = [row for row in rows if _membership_active_on(row, trade_date)] + if not matched: + matched = [ + row for row in rows + if row.get("is_new") == "Y" + and str(row.get("in_date") or "") <= trade_date + ] + if not matched: + raise TushareError(f"No Shenwan industry returned for {ts_code}") + row = max( + matched, + key=lambda item: ( + str(item.get("in_date") or ""), + 1 if item.get("is_new") == "Y" else 0, + str(item.get("l3_code") or item.get("l2_code") or ""), + ), + ) + return { + "l1_code": str(row.get("l1_code") or ""), + "l1_name": str(row.get("l1_name") or ""), + "l2_code": str(row.get("l2_code") or ""), + "l2_name": str(row.get("l2_name") or ""), + "l3_code": str(row.get("l3_code") or ""), + "l3_name": str(row.get("l3_name") or ""), + "in_date": str(row.get("in_date") or ""), + "out_date": str(row.get("out_date") or ""), + "is_new": str(row.get("is_new") or ""), + } + + def sw_sector_snapshot( + self, + ts_code: str, + requested_date: str, + realtime_expected: bool = False, + allow_realtime_close: bool = False, + ) -> dict[str, Any]: + """Build the single Shenwan L2 sector context used by heaven trend.""" + trade_date, previous_trade_date = self.resolve_trade_context(requested_date) + industry = self.sw_stock_industry(ts_code, trade_date) + sector_code = str(industry.get("l2_code") or "") + if not sector_code: + raise TushareError(f"Shenwan L2 code is unavailable for {ts_code}") + members = self._sw_sector_members(sector_code, trade_date) + if not members: + raise TushareError(f"No Shenwan members returned for {sector_code}") + raw_member_count = len(members) + members, excluded_members = _filter_members_by_listing( + members, + self._stock_listing_reference(), + trade_date, + ) + if not members: + raise TushareError(f"No listed Shenwan members returned for {sector_code}") + + if realtime_expected: + snapshot = self._sw_realtime_sector_snapshot( + industry, + members, + trade_date, + previous_trade_date, + finalized=False, + ) + snapshot.update({ + "raw_member_count": raw_member_count, + "excluded_member_count": len(excluded_members), + "excluded_members": excluded_members, + }) + return snapshot + + member_set = {str(item.get("ts_code") or "") for item in members} + member_names = { + str(item.get("ts_code") or ""): str(item.get("name") or "") + for item in members + } + member_rows = [ + row for row in self._load_daily(trade_date) + if str(row.get("ts_code") or "") in member_set + ] + quoted_codes = {str(row.get("ts_code") or "") for row in member_rows} + suspended_members = self._confirmed_suspended_members( + members, quoted_codes, trade_date + ) + up_count = sum(_number(row.get("pct_chg")) > 0 for row in member_rows) + down_count = sum(_number(row.get("pct_chg")) < 0 for row in member_rows) + leader = max(member_rows, key=lambda row: _number(row.get("pct_chg")), default={}) + leader_code = str(leader.get("ts_code") or "") + equal_change = ( + sum(_number(row.get("pct_chg")) for row in member_rows) / len(member_rows) + if member_rows else 0 + ) + coverage = len(member_rows) / max(len(members), 1) * 100 + explained_count = len(member_rows) + len(suspended_members) + explained_coverage = explained_count / max(len(members), 1) * 100 + coverage_issue = _sector_coverage_issue( + len(members), + len(member_rows), + explained_coverage, + explained_count, + ) + inner_precise = not coverage_issue + inner_error = coverage_issue + amount_billion = sum(_number(row.get("amount")) for row in member_rows) / 100000 + rows = self.query( + "sw_daily", + {"ts_code": sector_code, "trade_date": trade_date}, + "ts_code,trade_date,name,close,pct_change,vol,amount,pe,pb,float_mv,total_mv", + ) + daily = rows[0] if rows else {} + actual_trade_date = str(daily.get("trade_date") or "") + outer_precise = actual_trade_date == trade_date + outer_error = "" if outer_precise else ( + f"No Shenwan daily returned for {sector_code} on {trade_date}" + ) + if not outer_precise and allow_realtime_close: + try: + return self._sw_realtime_sector_snapshot( + industry, + members, + trade_date, + previous_trade_date, + finalized=True, + ) + except TushareError as exc: + outer_error = f"{outer_error}; realtime close fallback failed: {exc}" + + official_change = _number(daily.get("pct_change")) if outer_precise else None + return { + "code": sector_code, + "name": industry.get("l2_name") or daily.get("name") or sector_code, + "leader": str(leader.get("name") or member_names.get(leader_code) or "--"), + "leader_code": leader_code, + "leading_pct": round(_number(leader.get("pct_chg")), 3), + "change": round(official_change, 3) if official_change is not None else None, + "member_equal_change": round(equal_change, 3), + "turnover_rate": 0, + "up_count": up_count, + "down_count": down_count, + "flat_count": len(member_rows) - up_count - down_count, + "member_count": len(members), + "raw_member_count": raw_member_count, + "excluded_member_count": len(excluded_members), + "excluded_members": excluded_members, + "quote_count": len(member_rows), + "coverage": round(coverage, 1), + "explained_count": explained_count, + "explained_coverage": round(explained_coverage, 1), + "suspended_count": len(suspended_members), + "suspended_members": suspended_members, + "strength": round(max(0, min(100, 50 + (official_change if official_change is not None else equal_change) * 5)), 1), + "amount_billion": round(amount_billion, 2), + "count": 0, + "max_streak": 0, + "source": "tushare_sw_daily+member_daily" if outer_precise else "tushare_member_daily", + "inner_source": "tushare_member_daily", + "outer_source": "tushare_sw_daily" if outer_precise else "unavailable", + "taxonomy": "sw_l2", + "industry": industry, + "trade_date": trade_date, + "inner_trade_date": trade_date if member_rows else "", + "outer_trade_date": actual_trade_date, + "realtime": False, + "finalized": True, + "inner_precise": inner_precise, + "outer_precise": outer_precise, + "precise": inner_precise and outer_precise, + "inner_error": inner_error, + "outer_error": outer_error, + "schema_version": 6, + "methodology": "外显使用申万二级行业官方日线；内核独立使用当日成分日线宽度与等权涨跌聚合", + } + + def _sw_sector_members( + self, + sector_code: str, + trade_date: str, + ) -> list[dict[str, Any]]: + rows = [] + for is_new in ("Y", "N"): + rows.extend( + self.query( + "index_member_all", + {"l2_code": sector_code, "is_new": is_new}, + "l2_code,l2_name,ts_code,name,in_date,out_date,is_new", + ) + ) + deduped: dict[str, dict[str, Any]] = {} + for row in _reconcile_membership_rows(rows): + code = str(row.get("ts_code") or "") + if code and _membership_active_on(row, trade_date): + current = deduped.get(code) + if current is None or str(row.get("in_date") or "") > str(current.get("in_date") or ""): + deduped[code] = row + return list(deduped.values()) + + def sw_sector_members(self, sector_code: str, trade_date: str) -> list[dict[str, Any]]: + """Return constituents active in a Shenwan L2 industry on the target date.""" + return self._sw_sector_members(sector_code, trade_date) + + def _stock_listing_reference(self) -> dict[str, dict[str, Any]]: + now = datetime.now().astimezone() + with self._stock_listing_lock: + loaded_at = self._stock_listing_cache.get("loaded_at") + cached = self._stock_listing_cache.get("rows") + if ( + isinstance(loaded_at, datetime) + and isinstance(cached, dict) + and now - loaded_at < timedelta(hours=6) + ): + return cached + + rows: list[dict[str, Any]] = [] + try: + for status in ("L", "D", "P"): + rows.extend(self.query( + "stock_basic", + {"list_status": status}, + "ts_code,name,list_status,list_date,delist_date", + )) + except TushareError: + # Unknown status must remain in the denominator so a reference-data + # failure cannot silently improve coverage. + return {} + reference = { + str(row.get("ts_code") or ""): dict(row) + for row in rows + if row.get("ts_code") + } + with self._stock_listing_lock: + type(self)._stock_listing_cache = {"loaded_at": now, "rows": reference} + return reference + + def _confirmed_suspended_members( + self, + members: list[dict[str, Any]], + quoted_codes: set[str], + trade_date: str, + ) -> list[dict[str, str]]: + suspended: list[dict[str, str]] = [] + for member in members: + code = str(member.get("ts_code") or "") + if not code or code in quoted_codes: + continue + cache_key = f"{trade_date}:{code}" + with self._suspension_lock: + cached = self._suspension_cache.get(cache_key, "missing") + if cached == "missing": + try: + rows = self.query( + "suspend_d", + {"ts_code": code}, + "ts_code,suspend_date,resume_date,ann_date,suspend_reason,reason_type", + ) + except TushareError: + rows = [] + active = [ + row for row in rows + if str(row.get("suspend_date") or "") + and str(row.get("suspend_date") or "") <= trade_date + and ( + not str(row.get("resume_date") or "") + or trade_date < str(row.get("resume_date") or "") + ) + ] + row = max( + active, + key=lambda item: str(item.get("suspend_date") or ""), + default=None, + ) + cached = ({ + "ts_code": code, + "name": str(member.get("name") or code), + "suspend_date": str(row.get("suspend_date") or ""), + "resume_date": str(row.get("resume_date") or ""), + "reason": str(row.get("suspend_reason") or row.get("reason_type") or "已确认停牌"), + } if row else None) + with self._suspension_lock: + type(self)._suspension_cache[cache_key] = cached + if isinstance(cached, dict): + suspended.append(cached) + return suspended + + def _sw_realtime_sector_snapshot( + self, + industry: dict[str, Any], + members: list[dict[str, Any]], + trade_date: str, + previous_trade_date: str, + finalized: bool = False, + ) -> dict[str, Any]: + sector_code = str(industry.get("l2_code") or "") + sw_rows = self.query( + "rt_sw_k", + {"ts_code": sector_code}, + "ts_code,name,trade_time,close,pre_close,high,open,low,vol,amount,pct_change", + ) + sw_row = sw_rows[0] if sw_rows else {} + trade_time = str(sw_row.get("trade_time") or "") + quote_date = trade_time[:10].replace("-", "") + quote_clock = trade_time[11:19] if len(trade_time) >= 19 else "" + outer_precise = bool(sw_row and quote_date == trade_date) + if finalized and (not quote_clock or quote_clock < "15:00:00"): + outer_precise = False + official_change = _number(sw_row.get("pct_change")) + if not official_change: + close = _number(sw_row.get("close")) + pre_close = _number(sw_row.get("pre_close")) + official_change = (close / pre_close - 1) * 100 if close and pre_close else 0 + if not outer_precise: + official_change = None + outer_error = "" + if not sw_row: + outer_error = f"No Shenwan realtime index returned for {sector_code}" + elif quote_date != trade_date: + outer_error = f"Shenwan realtime index date is {quote_date or 'unknown'}, expected {trade_date}" + elif finalized and (not quote_clock or quote_clock < "15:00:00"): + outer_error = f"Shenwan realtime index is not a close snapshot ({trade_time})" + + valid: list[dict[str, Any]] = [] + codes: list[str] = [] + reference: dict[str, Any] = {} + inner_error = "" + try: + reference = self._load_realtime_reference(trade_date, previous_trade_date) + active_codes = { + str(row.get("ts_code") or "") + for row in reference.get("basic_rows") or [] + if row.get("ts_code") + } + codes = [ + str(row.get("ts_code") or "") + for row in members + if str(row.get("ts_code") or "") in active_codes + ] + if codes: + quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") + for row in quotes: + close = _number(row.get("close")) + previous_close = _number(row.get("pre_close")) + if close <= 0 or previous_close <= 0: + continue + valid.append({**row, "change": (close / previous_close - 1) * 100}) + else: + inner_error = f"No active Shenwan members returned for {sector_code}" + except TushareError as exc: + inner_error = str(exc) + + coverage = len(valid) / max(len(codes), 1) * 100 + valid_codes = {str(item.get("ts_code") or "") for item in valid} + suspended_members = self._confirmed_suspended_members( + members, valid_codes, trade_date + ) + explained_count = len(valid) + len(suspended_members) + explained_coverage = explained_count / max(len(codes), 1) * 100 + coverage_issue = _sector_coverage_issue( + len(codes), len(valid), explained_coverage, explained_count + ) + inner_precise = bool(codes) and not coverage_issue + if not inner_precise and not inner_error: + inner_error = coverage_issue or "申万实时有效成分为空" + up_count = sum(item["change"] > 0 for item in valid) + down_count = sum(item["change"] < 0 for item in valid) + leader = max(valid, key=lambda item: item["change"], default={}) + leader_code = str(leader.get("ts_code") or "") + member_names = { + str(item.get("ts_code") or ""): str(item.get("name") or "") + for item in members + } + equal_change = sum(item["change"] for item in valid) / len(valid) if valid else 0 + amount_billion = sum(_number(item.get("amount")) for item in valid) / 100000000 + try: + self._ensure_realtime_market_cache(trade_date) + with self._realtime_reference_lock: + market_rows = list( + (self._latest_realtime_market.get(trade_date) or {}).get("rows") or [] + ) + except TushareError as exc: + market_rows = [] + inner_precise = False + inner_error = inner_error or str(exc) + capital_map = { + str(item.get("ts_code") or ""): item + for item in reference.get("capital_rows") or [] + } + sector_turnovers = [] + for item in valid: + capital = capital_map.get(str(item.get("ts_code") or ""), {}) + float_share = _number(capital.get("float_share")) + if float_share: + sector_turnovers.append(_number(item.get("vol")) / float_share / 100) + market_turnovers = [] + for item in market_rows: + capital = capital_map.get(str(item.get("ts_code") or ""), {}) + float_share = _number(capital.get("float_share")) + if float_share: + market_turnovers.append(_number(item.get("vol")) / float_share / 100) + average_turnover = sum(sector_turnovers) / len(sector_turnovers) if sector_turnovers else 0 + market_turnover = sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0 + relative_turnover = average_turnover / market_turnover if market_turnover else 0 + if not relative_turnover: + inner_precise = False + inner_error = inner_error or "Shenwan member relative turnover is unavailable" + return { + "code": sector_code, + "name": str(industry.get("l2_name") or sw_row.get("name") or ""), + "leader": str(leader.get("name") or member_names.get(leader_code) or "--").strip(), + "leader_code": leader_code, + "leading_pct": round(_number(leader.get("change")), 3), + "change": round(official_change, 3) if official_change is not None else None, + "member_equal_change": round(equal_change, 3), + "turnover_rate": round(average_turnover, 4), + "market_turnover_rate": round(market_turnover, 4), + "relative_turnover": round(relative_turnover, 4), + "up_count": up_count, + "down_count": down_count, + "flat_count": len(valid) - up_count - down_count, + "member_count": len(codes), + "quote_count": len(valid), + "coverage": round(coverage, 1), + "explained_count": explained_count, + "explained_coverage": round(explained_coverage, 1), + "suspended_count": len(suspended_members), + "suspended_members": suspended_members, + "strength": round(max(0, min(100, 50 + (official_change if official_change is not None else equal_change) * 5)), 1), + "amount_billion": round(amount_billion, 2), + "count": sum(item["change"] >= 9.5 for item in valid), + "max_streak": 0, + "source": "tushare_rt_sw_k+sw_members_rt_k", + "inner_source": "tushare_sw_members+rt_k", + "outer_source": "tushare_rt_sw_k", + "taxonomy": "sw_l2", + "industry": industry, + "trade_date": trade_date, + "inner_trade_date": trade_date if valid else "", + "outer_trade_date": quote_date, + "trade_time": trade_time, + "realtime": True, + "finalized": finalized, + "inner_precise": inner_precise, + "outer_precise": outer_precise, + "precise": inner_precise and outer_precise, + "inner_error": inner_error, + "outer_error": outer_error, + "schema_version": 6, + "methodology": "外显使用申万官方 rt_sw_k；内核独立使用申万成分 rt_k 宽度与相对换手聚合", + } + + def sector_snapshot( + self, + identifier: str, + requested_date: str, + realtime_expected: bool | None = None, + ) -> dict[str, Any]: + trade_date, _ = self.resolve_trade_context(requested_date) + raw_identifier = identifier.strip() + if not raw_identifier: + raise TushareError("Sector identifier is empty") + errors = [] + now = datetime.now().astimezone() + if realtime_expected is None: + realtime_expected = ( + trade_date == now.strftime("%Y%m%d") + and dt_time(9, 15) <= now.time().replace(tzinfo=None) <= dt_time(15, 5) + ) + try: + dc_params = {"trade_date": trade_date} + if re.fullmatch(r"[A-Z0-9.]+", raw_identifier.upper()) and "." in raw_identifier: + dc_params["ts_code"] = raw_identifier.upper() + else: + dc_params["name"] = raw_identifier + dc_rows = self.query( + "dc_index", + dc_params, + "ts_code,trade_date,name,leading,leading_code,pct_change,leading_pct," + "total_mv,turnover_rate,up_num,down_num", + ) + if not dc_rows and "name" in dc_params: + dc_rows = self.query( + "dc_index", + {"trade_date": trade_date}, + "ts_code,trade_date,name,leading,leading_code,pct_change,leading_pct," + "total_mv,turnover_rate,up_num,down_num", + ) + dc_row = _match_sector_row(dc_rows, raw_identifier) + if dc_row and not realtime_expected: + change = _number(dc_row.get("pct_change")) + actual_trade_date = str(dc_row.get("trade_date") or "") + return { + "code": dc_row.get("ts_code") or "", + "name": dc_row.get("name") or raw_identifier, + "leader": dc_row.get("leading") or "--", + "leader_code": dc_row.get("leading_code") or "", + "leading_pct": _number(dc_row.get("leading_pct")), + "change": change, + "turnover_rate": _number(dc_row.get("turnover_rate")), + "up_count": int(_number(dc_row.get("up_num"))), + "down_count": int(_number(dc_row.get("down_num"))), + "total_mv": _number(dc_row.get("total_mv")), + "strength": round(max(0, min(100, 50 + change * 5)), 1), + "amount_billion": 0, + "count": 0, + "max_streak": 0, + "source": "tushare_dc", + "trade_date": actual_trade_date, + "realtime": False, + "precise": actual_trade_date == trade_date, + } + except TushareError as exc: + errors.append(f"DC: {exc}") + + ts_code = raw_identifier.upper() + if re.fullmatch(r"\d{6}", ts_code): + ts_code = f"{ts_code}.TI" + try: + if re.fullmatch(r"\d{6}\.TI", ts_code): + index_rows = self.query( + "ths_index", + {"ts_code": ts_code}, + "ts_code,name,count,exchange,list_date,type", + ) + else: + index_rows = self.query( + "ths_index", + {}, + "ts_code,name,count,exchange,list_date,type", + ) + basic = _match_sector_row(index_rows, raw_identifier) + if not basic: + raise TushareError(f"No THS sector returned for {raw_identifier}") + except TushareError as exc: + errors.append(f"THS: {exc}") + raise TushareError("; ".join(errors)) from exc + actual_code = str(basic.get("ts_code") or ts_code) + if realtime_expected: + try: + realtime_sector = self._realtime_sector_snapshot(actual_code, basic, trade_date) + if realtime_sector: + return realtime_sector + except TushareError as exc: + errors.append(f"THS realtime members: {exc}") + daily_rows = self.query( + "ths_daily", + {"ts_code": actual_code, "trade_date": trade_date}, + "ts_code,trade_date,close,pct_change,vol,turnover_rate,total_mv,float_mv", + ) + daily = daily_rows[0] if daily_rows else {} + actual_trade_date = str(daily.get("trade_date") or "") + change = _number(daily.get("pct_change")) + return { + "code": actual_code, + "name": basic.get("name") or raw_identifier, + "leader": "--", + "change": change, + "leading_pct": change, + "turnover_rate": _number(daily.get("turnover_rate")), + "up_count": 0, + "down_count": 0, + "strength": round(max(0, min(100, 50 + change * 5)), 1), + "amount_billion": 0, + "count": 0, + "max_streak": 0, + "source": "tushare_ths", + "trade_date": actual_trade_date, + "realtime": False, + "precise": actual_trade_date == trade_date, + } + + def _realtime_sector_snapshot( + self, + sector_code: str, + basic: dict[str, Any], + trade_date: str, + ) -> dict[str, Any] | None: + members = self.query( + "ths_member", + {"ts_code": sector_code, "is_new": "Y"}, + "ts_code,con_code,con_name,is_new", + ) + codes = [str(row.get("con_code") or "") for row in members if row.get("con_code")] + if not codes: + return None + quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") + valid = [] + for row in quotes: + close = _number(row.get("close")) + previous_close = _number(row.get("pre_close")) + if close <= 0 or previous_close <= 0: + continue + valid.append( + { + **row, + "change": (close / previous_close - 1) * 100, + } + ) + minimum = max(1, math.ceil(len(codes) * 0.9)) + if len(valid) < minimum: + raise TushareError( + f"Realtime sector coverage is insufficient ({len(valid)}/{len(codes)})" + ) + up_count = sum(item["change"] > 0 for item in valid) + down_count = sum(item["change"] < 0 for item in valid) + flat_count = len(valid) - up_count - down_count + leader = max(valid, key=lambda item: item["change"]) + change = sum(item["change"] for item in valid) / len(valid) + amount_billion = sum(_number(item.get("amount")) for item in valid) / 100000000 + self._ensure_realtime_market_cache(trade_date) + with self._realtime_reference_lock: + references = list(self._realtime_reference_cache.values()) + market_rows = list((self._latest_realtime_market.get(trade_date) or {}).get("rows") or []) + capital_map: dict[str, dict[str, Any]] = {} + for reference in reversed(references): + capital_map = { + str(item.get("ts_code") or ""): item + for item in reference.get("capital_rows") or [] + } + if capital_map: + break + sector_turnovers = [] + for item in valid: + capital = capital_map.get(str(item.get("ts_code") or ""), {}) + float_share = _number(capital.get("float_share")) + if float_share: + sector_turnovers.append(_number(item.get("vol")) / float_share / 100) + market_turnovers = [] + for item in market_rows: + capital = capital_map.get(str(item.get("ts_code") or ""), {}) + float_share = _number(capital.get("float_share")) + if float_share: + market_turnovers.append(_number(item.get("vol")) / float_share / 100) + average_turnover = sum(sector_turnovers) / len(sector_turnovers) if sector_turnovers else 0 + market_turnover = sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0 + relative_turnover = average_turnover / market_turnover if market_turnover else 0 + return { + "code": sector_code, + "name": basic.get("name") or sector_code, + "leader": str(leader.get("name") or "--").strip(), + "leader_code": leader.get("ts_code") or "", + "leading_pct": round(leader["change"], 3), + "change": round(change, 3), + "turnover_rate": round(average_turnover, 4), + "market_turnover_rate": round(market_turnover, 4), + "relative_turnover": round(relative_turnover, 4), + "up_count": up_count, + "down_count": down_count, + "flat_count": flat_count, + "member_count": len(codes), + "quote_count": len(valid), + "coverage": round(len(valid) / len(codes) * 100, 1), + "strength": round(max(0, min(100, 50 + change * 5)), 1), + "amount_billion": round(amount_billion, 2), + "count": sum(item["change"] >= 9.5 for item in valid), + "max_streak": 0, + "source": "tushare_rt_ths_members", + "trade_date": trade_date, + "realtime": True, + "precise": True, + "methodology": "同花顺行业最新成分股的 rt_k 等权涨跌、宽度与成交额聚合", + } + + def hot_money_profiles(self) -> dict[str, Any]: + rows = self.query("hm_list", {}, "name,desc,orgs") + profiles: list[dict[str, Any]] = [] + seen_names: set[str] = set() + for row in rows: + name = str(row.get("name") or "").strip() + if not name or name in seen_names: + continue + seen_names.add(name) + description = _text(row.get("desc")) + organization_text = _text(row.get("orgs")) + parsed_organizations: Any = None + if organization_text.startswith("["): + try: + parsed_organizations = json.loads(organization_text) + except json.JSONDecodeError: + parsed_organizations = None + organization_parts = ( + parsed_organizations + if isinstance(parsed_organizations, list) + else re.split(r"[,，;；\n]+", organization_text) + ) + organizations = list(dict.fromkeys( + _text(part) + for part in organization_parts + if _text(part) + )) + profiles.append( + { + "id": f"hot-money-profile-{len(profiles) + 1}", + "name": name, + "description": description, + "organizations": organizations, + "organization_count": len(organizations), + } + ) + return { + "meta": { + "source": "tushare", + "status": "success" if profiles else "empty", + "schema_version": 1, + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "notice": "", + }, + "summary": { + "profile_count": len(profiles), + "described_count": sum(bool(item["description"]) for item in profiles), + "organization_count": sum(item["organization_count"] for item in profiles), + }, + "profiles": profiles, + } + + def dragon_tiger(self, requested_date: str) -> dict[str, Any]: + trade_date, _ = self.resolve_trade_context(requested_date) + detail_rows = self.query( + "hm_detail", + {"trade_date": trade_date}, + "trade_date,ts_code,ts_name,buy_amount,sell_amount,net_amount," + "hm_name,hm_orgs,tag", + ) + + notices: list[str] = [] + try: + directory_rows = self.query("hm_list", {}, "name,desc,orgs") + except TushareError as exc: + directory_rows = [] + notices.append(f"游资名录暂不可用：{exc}") + directory = { + str(row.get("name") or "").strip(): { + "description": _text(row.get("desc")), + "orgs": _text(row.get("orgs")), + } + for row in directory_rows + if str(row.get("name") or "").strip() + } + + # 个股龙虎榜仅用于补充涨幅和上榜原因，不参与游资身份识别。 + try: + top_rows = self.query( + "top_list", + {"trade_date": trade_date}, + "trade_date,ts_code,name,pct_change,reason", + ) + except TushareError as exc: + top_rows = [] + notices.append(f"个股龙虎榜辅助信息暂不可用：{exc}") + stock_context: dict[str, dict[str, Any]] = {} + for row in top_rows: + ts_code = str(row.get("ts_code") or "") + if ts_code and ts_code not in stock_context: + stock_context[ts_code] = row + + groups: dict[str, dict[str, Any]] = {} + for row in detail_rows: + trader_name = str(row.get("hm_name") or "未命名游资").strip() + ts_code = str(row.get("ts_code") or "").strip() + stock = stock_context.get(ts_code, {}) + directory_item = directory.get(trader_name, {}) + seat_name = _text(row.get("hm_orgs")) or directory_item.get("orgs") or "--" + buy = round(_number(row.get("buy_amount")) / 1000000, 2) + sell = round(_number(row.get("sell_amount")) / 1000000, 2) + net_buy = round(_number(row.get("net_amount")) / 1000000, 2) + group = groups.setdefault( + trader_name, + { + "name": trader_name, + "description": directory_item.get("description") or "", + "directory_orgs": directory_item.get("orgs") or "", + "identity_type": "trader", + "identity_source": "tushare_hm", + "recognized": True, + "buy_million": 0.0, + "sell_million": 0.0, + "net_buy_million": 0.0, + "seat_names": set(), + "stock_codes": set(), + "operations": [], + }, + ) + group["buy_million"] += buy + group["sell_million"] += sell + group["net_buy_million"] += net_buy + if seat_name != "--": + group["seat_names"].add(seat_name) + code = ts_code.split(".")[0] + if code: + group["stock_codes"].add(code) + group["operations"].append( + { + "code": code, + "ts_code": ts_code, + "name": row.get("ts_name") or stock.get("name") or "--", + "change": ( + _number(stock.get("pct_change")) + if stock.get("pct_change") is not None + else None + ), + "direction": "买入" if net_buy > 0 else "卖出" if net_buy < 0 else "持平", + "buy_million": buy, + "sell_million": sell, + "net_buy_million": net_buy, + "seat_name": seat_name, + "seat_alias": trader_name, + "tag": _text(row.get("tag")) or "--", + "reason": _text(stock.get("reason")) or "--", + } + ) + + traders = list(groups.values()) + traders.sort(key=lambda item: abs(item["net_buy_million"]), reverse=True) + for index, group in enumerate(traders, start=1): + group["id"] = f"hot-money-{index}" + group["buy_million"] = round(group["buy_million"], 2) + group["sell_million"] = round(group["sell_million"], 2) + group["net_buy_million"] = round(group["net_buy_million"], 2) + group["seat_count"] = len(group.pop("seat_names")) + group["stock_count"] = len(group.pop("stock_codes")) + group["operation_count"] = len(group["operations"]) + group["operations"].sort( + key=lambda item: abs(float(item.get("net_buy_million") or 0)), reverse=True + ) + + operation_count = sum(item["operation_count"] for item in traders) + active_stocks = { + operation["code"] for item in traders for operation in item["operations"] + if operation["code"] + } + net_buy_total = round(sum(item["net_buy_million"] for item in traders), 2) + status = "success" if detail_rows else "partial" if top_rows else "empty" + if not detail_rows: + notices.insert( + 0, + f"当日有 {len(stock_context)} 只股票上榜，但未返回可识别的游资每日明细。" + if top_rows + else "该交易日未返回龙虎榜或游资每日明细。", + ) + return { + "meta": { + "requested_date": _display_date(requested_date), + "trade_date": _display_date(trade_date), + "source": "tushare", + "status": status, + "schema_version": 3, + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "notice": "；".join(notices), + }, + "summary": { + "trader_count": len(traders), + "identity_count": len(traders), + "operation_count": operation_count, + "active_stock_count": len(active_stocks), + "seat_net_buy_million": net_buy_total, + "unclassified_count": 0, + "directory_count": len(directory), + "official_stock_count": len(stock_context), + }, + "traders": traders, + "unclassified_seats": [], + "rows": [], + } + + def stock_detail(self, ts_code: str, requested_date: str) -> dict[str, Any]: + trade_date, _ = self.resolve_trade_context(requested_date) + end = datetime.strptime(trade_date, "%Y%m%d") + start_date = (end - timedelta(days=190)).strftime("%Y%m%d") + daily = self.query( + "daily", + {"ts_code": ts_code, "start_date": start_date, "end_date": trade_date}, + "ts_code,trade_date,open,high,low,close,pct_chg,vol,amount", + ) + factors = self.query( + "adj_factor", + {"ts_code": ts_code, "start_date": start_date, "end_date": trade_date}, + "ts_code,trade_date,adj_factor", + ) + basics = self.query( + "stock_basic", + {"ts_code": ts_code}, + "ts_code,symbol,name,area,industry,market,list_date", + ) + daily_basics = self.query( + "daily_basic", + {"ts_code": ts_code, "trade_date": trade_date}, + "ts_code,trade_date,turnover_rate,volume_ratio,total_mv,circ_mv", + ) + moneyflow = self.query( + "moneyflow", + {"ts_code": ts_code, "trade_date": trade_date}, + "ts_code,trade_date,buy_sm_amount,sell_sm_amount,buy_md_amount,sell_md_amount," + "buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount", + ) + factor_map = {row["trade_date"]: _number(row.get("adj_factor"), 1) for row in factors} + latest_factor = max(factor_map.values(), default=1) or 1 + prices = [] + for row in sorted(daily, key=lambda item: item.get("trade_date", ""))[-90:]: + factor = factor_map.get(row.get("trade_date"), latest_factor) + ratio = factor / latest_factor + prices.append( + { + "trade_date": _display_date(str(row.get("trade_date", ""))), + "open": round(_number(row.get("open")) * ratio, 3), + "high": round(_number(row.get("high")) * ratio, 3), + "low": round(_number(row.get("low")) * ratio, 3), + "close": round(_number(row.get("close")) * ratio, 3), + "change": _number(row.get("pct_chg")), + "volume": _number(row.get("vol")), + "amount_billion": round(_number(row.get("amount")) / 100000, 2), + } + ) + flow = moneyflow[0] if moneyflow else {} + basic = basics[0] if basics else {} + daily_basic = daily_basics[0] if daily_basics else {} + latest = prices[-1] if prices else {} + actual_trade_date = max( + (str(row.get("trade_date") or "") for row in daily), + default=trade_date, + ) or trade_date + return { + "meta": { + "requested_date": _display_date(requested_date), + "trade_date": _display_date(actual_trade_date), + "source": "tushare", + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "notice": "", + }, + "stock": { + "code": ts_code.split(".")[0], + "ts_code": ts_code, + "name": basic.get("name") or "--", + "industry": basic.get("industry") or "其他", + "area": basic.get("area") or "--", + "market": basic.get("market") or "--", + "list_date": _display_date(str(basic.get("list_date") or "")), + "price": latest.get("close", 0), + "change": latest.get("change", 0), + "turnover_rate": _number(daily_basic.get("turnover_rate")), + "volume_ratio": _number(daily_basic.get("volume_ratio")), + "amount_billion": latest.get("amount_billion", 0), + }, + "prices": prices, + "moneyflow": { + "net_million": round(_number(flow.get("net_mf_amount")) / 100, 2), + "large_million": round( + (_number(flow.get("buy_lg_amount")) + _number(flow.get("buy_elg_amount")) + - _number(flow.get("sell_lg_amount")) - _number(flow.get("sell_elg_amount"))) / 100, + 2, + ), + "medium_million": round( + (_number(flow.get("buy_md_amount")) - _number(flow.get("sell_md_amount"))) / 100, + 2, + ), + "small_million": round( + (_number(flow.get("buy_sm_amount")) - _number(flow.get("sell_sm_amount"))) / 100, + 2, + ), + }, + } + + def stock_intraday(self, ts_code: str, requested_date: str) -> dict[str, Any]: + trade_date, _ = self.resolve_trade_context(requested_date) + display_date = _display_date(trade_date) + rows = self.query( + "stk_mins", + { + "ts_code": ts_code, + "freq": "1min", + "start_date": f"{display_date} 09:00:00", + "end_date": f"{display_date} 15:30:00", + }, + "ts_code,trade_time,open,close,high,low,vol,amount", + ) + points = [] + for row in sorted(rows, key=lambda item: str(item.get("trade_time") or "")): + trade_time = str(row.get("trade_time") or "") + if not trade_time: + continue + points.append( + { + "time": trade_time[-8:-3] if len(trade_time) >= 8 else trade_time, + "open": round(_number(row.get("open")), 3), + "high": round(_number(row.get("high")), 3), + "low": round(_number(row.get("low")), 3), + "close": round(_number(row.get("close")), 3), + "volume": _number(row.get("vol")), + "amount": _number(row.get("amount")), + } + ) + return {"trade_date": display_date, "points": points} + + def resolve_trade_context(self, requested: str) -> tuple[str, str]: + requested_rows = self.query( + "trade_cal", + {"exchange": "SSE", "start_date": requested, "end_date": requested}, + "cal_date,is_open,pretrade_date", + ) + if not requested_rows: + trade_date = requested + else: + row = requested_rows[0] + trade_date = row["cal_date"] if row.get("is_open") == 1 else row.get("pretrade_date", requested) + + resolved_rows = self.query( + "trade_cal", + {"exchange": "SSE", "start_date": trade_date, "end_date": trade_date}, + "cal_date,is_open,pretrade_date", + ) + previous = resolved_rows[0].get("pretrade_date") if resolved_rows else "" + return trade_date, previous or trade_date + + def _load_daily(self, trade_date: str) -> list[dict[str, Any]]: + return self.query( + "daily", + {"trade_date": trade_date}, + "ts_code,trade_date,open,high,low,close,pct_chg,amount", + ) + + def _load_limit_type(self, trade_date: str, limit_type: str) -> list[dict[str, Any]]: + fields = ( + "trade_date,ts_code,industry,name,close,pct_chg,amount,limit_amount," + "float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time," + "open_times,up_stat,limit_times" + ) + rows = self.query( + "limit_list_d", + {"trade_date": trade_date, "limit_type": limit_type}, + fields, + ) + for row in rows: + row["limit_type"] = limit_type + row["amount_unit"] = "yuan" + return rows + + def _load_limit_lists(self, trade_date: str) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for limit_type in ("U", "D", "Z"): + rows.extend(self._load_limit_type(trade_date, limit_type)) + return rows + + def _derive_limits( + self, + trade_date: str, + daily: list[dict[str, Any]], + price_limits: list[dict[str, Any]] | None = None, + basic_rows: list[dict[str, Any]] | None = None, + previous_limit_rows: list[dict[str, Any]] | None = None, + capital_rows: list[dict[str, Any]] | None = None, + ) -> list[dict[str, Any]]: + if price_limits is None: + price_limits = self.query( + "stk_limit", + {"trade_date": trade_date}, + "ts_code,trade_date,up_limit,down_limit", + ) + limit_map = {row["ts_code"]: row for row in price_limits} + if basic_rows is None: + basic_rows = self.query( + "stock_basic", + {"list_status": "L"}, + "ts_code,name,industry", + ) + basic_map = {row["ts_code"]: row for row in basic_rows} + previous_limit_map = { + str(row.get("ts_code") or ""): row for row in (previous_limit_rows or []) + } + capital_map = { + str(row.get("ts_code") or ""): row for row in (capital_rows or []) + } + + result: list[dict[str, Any]] = [] + for row in daily: + bounds = limit_map.get(row.get("ts_code")) + if not bounds or row.get("close") is None: + continue + limit_type = "" + if _prices_equal(row["close"], bounds.get("up_limit")): + limit_type = "U" + elif _prices_equal(row["close"], bounds.get("down_limit")): + limit_type = "D" + elif _prices_equal(row.get("high"), bounds.get("up_limit")): + limit_type = "Z" + if not limit_type: + continue + basic = basic_map.get(row["ts_code"], {}) + previous_limit = previous_limit_map.get(str(row.get("ts_code") or ""), {}) + streak = ( + max(1, int(_number(previous_limit.get("limit_times"), 1)) + 1) + if limit_type == "U" and previous_limit + else 1 + ) + item = { + **row, + "name": basic.get("name", "--"), + "industry": basic.get("industry") or "其他", + "limit_type": limit_type, + "limit_times": streak, + "open_times": 1 if limit_type == "Z" else 0, + "amount_unit": row.get("amount_unit") or "thousand_yuan", + } + if row.get("amount_unit") == "yuan": + capital = capital_map.get(str(row.get("ts_code") or ""), {}) + if not capital and capital_rows is None: + capital = self._latest_capital(str(row.get("ts_code") or ""), trade_date) + float_share = _number(capital.get("float_share")) + item["turnover_ratio"] = ( + _number(row.get("vol")) / float_share / 100 if float_share else 0 + ) + item["turnover_source"] = ( + "rt_volume/latest_float_share" if float_share else "unavailable" + ) + item["capital_trade_date"] = str(capital.get("trade_date") or "") + result.append(item) + return result + + @staticmethod + def _normalize_limit(row: dict[str, Any], status: str) -> dict[str, Any]: + amount = _number(row.get("amount")) + if row.get("amount_unit") == "thousand_yuan": + amount_billion = amount / 100000 + else: + amount_billion = amount / 100000000 + return { + "code": str(row.get("ts_code", "")).split(".")[0], + "ts_code": row.get("ts_code", ""), + "name": row.get("name") or "--", + "price": _number(row.get("close")), + "change": _number(row.get("pct_chg")), + "sector": row.get("industry") or "其他", + "reason": row.get("industry") or "待补充", + "first_time": _display_time(row.get("first_time")), + "last_time": _display_time(row.get("last_time")), + "open_times": int(_number(row.get("open_times"))), + "streak": max(1, int(_number(row.get("limit_times"), 1))), + "turnover_rate": _number(row.get("turnover_ratio")), + "turnover_source": row.get("turnover_source") or "provider", + "capital_trade_date": row.get("capital_trade_date") or "", + "amount_billion": round(amount_billion, 2), + "seal_amount_million": round(_number(row.get("fd_amount")) / 10000, 0), + "float_mv_billion": round(_number(row.get("float_mv")) / 100000000, 1), + "status": status, + } + + +def _number(value: Any, default: float = 0.0) -> float: + try: + number = float(value) + return number if math.isfinite(number) else default + except (TypeError, ValueError): + return default + + +def _text(value: Any) -> str: + if isinstance(value, (list, tuple, set)): + return "、".join(str(item).strip() for item in value if str(item).strip()) + return str(value or "").strip() + + +def _filter_members_by_listing( + members: list[dict[str, Any]], + listing_reference: dict[str, dict[str, Any]], + trade_date: str, +) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: + eligible: list[dict[str, Any]] = [] + excluded: list[dict[str, str]] = [] + for member in members: + code = str(member.get("ts_code") or "") + listing = listing_reference.get(code) + if not listing: + eligible.append(member) + continue + list_date = str(listing.get("list_date") or "") + delist_date = str(listing.get("delist_date") or "") + reason = "" + effective_date = "" + if delist_date and delist_date <= trade_date: + reason = "目标日期前已退市" + effective_date = delist_date + elif list_date and list_date > trade_date: + reason = "目标日期尚未上市" + effective_date = list_date + if not reason: + eligible.append(member) + continue + excluded.append({ + "ts_code": code, + "name": str(member.get("name") or listing.get("name") or code), + "reason": reason, + "effective_date": effective_date, + }) + return eligible, excluded + + +def _sector_coverage_issue( + member_count: int, + quote_count: int, + coverage: float | None = None, + explained_count: int | None = None, +) -> str: + members = max(0, int(member_count or 0)) + quotes = max(0, min(int(quote_count or 0), members)) + if members <= 0: + if coverage is not None and float(coverage) >= 90: + return "" + if coverage is not None: + return "行业成分行情覆盖率低于90%" + return "申万有效成分为空" + explained = quotes if explained_count is None else max( + quotes, min(int(explained_count or 0), members) + ) + actual_coverage = ( + float(coverage) + if coverage is not None + else explained / members * 100 + ) + missing = members - explained + if members <= 7 and missing: + return f"小型行业有效成分状态仅确认 {explained}/{members}，要求全部可解释" + if members <= 20 and (actual_coverage < 90 or missing > 1): + return f"中型行业有效成分状态仅确认 {explained}/{members}，要求覆盖率至少90%且最多缺1只" + if members > 20 and actual_coverage < 90: + return f"行业有效成分状态仅确认 {explained}/{members}，覆盖率低于90%" + return "" + + +def _membership_active_on(row: dict[str, Any], trade_date: str) -> bool: + start = str(row.get("in_date") or "") + end = str(row.get("out_date") or "") + return (not start or start <= trade_date) and (not end or end > trade_date) + + +def _reconcile_membership_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Merge duplicate Y/N membership rows before evaluating their date interval.""" + reconciled: dict[tuple[str, str, str, str, str], dict[str, Any]] = {} + for raw in rows: + row = dict(raw) + key = ( + str(row.get("ts_code") or ""), + str(row.get("l1_code") or ""), + str(row.get("l2_code") or ""), + str(row.get("l3_code") or ""), + str(row.get("in_date") or ""), + ) + current = reconciled.get(key) + if current is None: + reconciled[key] = row + continue + current_end = str(current.get("out_date") or "") + candidate_end = str(row.get("out_date") or "") + if candidate_end and not current_end: + current["out_date"] = candidate_end + current["is_new"] = row.get("is_new") or current.get("is_new") + for field, value in row.items(): + if not current.get(field) and value not in (None, ""): + current[field] = value + return list(reconciled.values()) + + +def _match_sector_row(rows: list[dict[str, Any]], identifier: str) -> dict[str, Any] | None: + if not rows: + return None + target = identifier.strip().upper() + code_match = next( + (row for row in rows if str(row.get("ts_code") or "").strip().upper() == target), + None, + ) + if code_match: + return code_match + + def normalized(value: Any) -> str: + text = str(value or "").strip().replace(" ", "") + for suffix in ("板块", "概念", "行业"): + text = text.removesuffix(suffix) + aliases = { + "元器件": "元件", + "电子元器件": "元件", + } + return aliases.get(text, text) + + target_name = normalized(identifier) + exact = [row for row in rows if normalized(row.get("name")) == target_name] + if exact: + return min(exact, key=_sector_match_priority) + fuzzy = [ + row for row in rows + if target_name and ( + target_name in normalized(row.get("name")) + or normalized(row.get("name")) in target_name + ) + ] + return min( + fuzzy, + key=lambda row: (len(normalized(row.get("name"))), *_sector_match_priority(row)), + ) if fuzzy else None + + +def _sector_match_priority(row: dict[str, Any]) -> tuple[int, int, int]: + code = str(row.get("ts_code") or "") + exchange = str(row.get("exchange") or "").upper() + return ( + 0 if exchange == "A" else 1, + 0 if code.startswith("881") else 1, + 0 if _number(row.get("count")) > 0 else 1, + ) + + +def _prices_equal(left: Any, right: Any) -> bool: + if left is None or right is None: + return False + return abs(_number(left) - _number(right)) < 0.005 + + +def _value_percentile(value: float, population: list[float]) -> float: + valid = sorted(item for item in population if item >= 0) + if not valid: + return 0.0 + below = sum(item < value for item in valid) + equal = sum(item == value for item in valid) + return (below + equal * 0.5) / len(valid) + + +def _trading_session_progress(current_time: dt_time) -> float: + morning_start = dt_time(9, 30) + morning_end = dt_time(11, 30) + afternoon_start = dt_time(13, 0) + afternoon_end = dt_time(15, 0) + if current_time <= morning_start: + return 0.05 + if current_time <= morning_end: + minutes = (current_time.hour * 60 + current_time.minute) - (9 * 60 + 30) + return max(0.05, min(0.5, minutes / 240)) + if current_time < afternoon_start: + return 0.5 + if current_time <= afternoon_end: + minutes = (current_time.hour * 60 + current_time.minute) - 13 * 60 + return max(0.5, min(1.0, 0.5 + minutes / 240)) + return 1.0 + + +def _display_time(value: Any) -> str: + raw = str(value or "").replace(":", "").zfill(6) + if not raw.strip("0"): + return "--" + return f"{raw[:2]}:{raw[2:4]}:{raw[4:6]}" + + +def _display_date(value: str) -> str: + return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value + + +def _realtime_market_status(current_time: dt_time) -> str: + if current_time < dt_time(9, 25): + return "pre_open" + if current_time < dt_time(9, 30): + return "auction" + if current_time <= dt_time(11, 30) or dt_time(13, 0) <= current_time <= dt_time(15, 0): + return "trading" + if current_time < dt_time(13, 0): + return "lunch_break" + return "closed" + + +def _build_overview( + daily: list[dict[str, Any]], + up_rows: list[dict[str, Any]], + down_rows: list[dict[str, Any]], + broken_rows: list[dict[str, Any]], +) -> dict[str, Any]: + up_count = sum(1 for row in daily if _number(row.get("pct_chg")) > 0) + down_count = sum(1 for row in daily if _number(row.get("pct_chg")) < 0) + flat_count = len(daily) - up_count - down_count + amount_billion = sum( + _number(row.get("amount")) + / (100000000 if row.get("amount_unit") == "yuan" else 100000) + for row in daily + ) + limit_count = len(up_rows) + broken_count = len(broken_rows) + seal_rate = round(limit_count / max(limit_count + broken_count, 1) * 100, 1) + return { + "up_count": up_count, + "down_count": down_count, + "flat_count": flat_count, + "limit_up_count": limit_count, + "limit_down_count": len(down_rows), + "broken_count": broken_count, + "amount_billion": round(amount_billion, 1), + "seal_rate": seal_rate, + } + + +def _build_ladders(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + groups: dict[int, list[dict[str, Any]]] = {} + for row in rows: + groups.setdefault(int(row.get("streak") or 1), []).append(row) + return [ + { + "level": level, + "label": "首板" if level == 1 else f"{level}板", + "count": len(stocks), + "stocks": sorted(stocks, key=lambda item: item.get("first_time") or "99:99:99"), + } + for level, stocks in sorted(groups.items(), reverse=True) + ] + + +def _build_sectors(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + counts = Counter(row.get("sector") or "其他" for row in rows) + result: list[dict[str, Any]] = [] + for name, count in counts.most_common(20): + stocks = [row for row in rows if (row.get("sector") or "其他") == name] + max_streak = max(item.get("streak", 1) for item in stocks) + leader = max(stocks, key=lambda item: (item.get("streak", 1), item.get("amount_billion", 0))) + result.append( + { + "name": name, + "count": count, + "strength": min(100, 44 + count * 8 + max_streak * 5), + "amount_billion": round(sum(item.get("amount_billion", 0) for item in stocks), 1), + "leader": leader.get("name", "--"), + "change": round(sum(item.get("change", 0) for item in stocks) / count, 2), + "max_streak": max_streak, + } + ) + return result + + +def _build_yesterday_performance( + previous_limits: list[dict[str, Any]], + daily: list[dict[str, Any]], + current_limits: list[dict[str, Any]], + current_broken: list[dict[str, Any]], + current_down: list[dict[str, Any]], +) -> list[dict[str, Any]]: + daily_map = {str(row.get("ts_code", "")).split(".")[0]: row for row in daily} + limit_map = {row["code"]: row for row in current_limits} + broken_codes = {row["code"] for row in current_broken} + down_codes = {row["code"] for row in current_down} + result = [] + for previous in previous_limits: + code = previous["code"] + daily_row = daily_map.get(code, {}) + current = limit_map.get(code) + if current: + outcome = "晋级" + elif code in broken_codes: + outcome = "炸板" + elif code in down_codes: + outcome = "跌停" + else: + outcome = "断板" + result.append( + { + "code": code, + "name": previous["name"], + "prior_streak": previous.get("streak", 1), + "current_streak": current.get("streak", 0) if current else 0, + "current_change": _number(daily_row.get("pct_chg")), + "current_price": _number(daily_row.get("close")), + "sector": previous.get("sector", "其他"), + "reason": previous.get("reason", "待补充"), + "outcome": outcome, + } + ) + return result + + +def _build_limit_performance(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + result = [] + for level in sorted({int(row.get("prior_streak") or 1) for row in rows}, reverse=True): + group = [row for row in rows if int(row.get("prior_streak") or 1) == level] + advanced = sum(row.get("outcome") == "晋级" for row in group) + positive = sum(_number(row.get("current_change")) > 0 for row in group) + result.append( + { + "level": level, + "label": "昨日首板" if level == 1 else f"昨日{level}板", + "count": len(group), + "advanced": advanced, + "advance_rate": round(advanced / len(group) * 100, 1), + "positive_rate": round(positive / len(group) * 100, 1), + "average_change": round(sum(_number(row.get("current_change")) for row in group) / len(group), 2), + } + ) + return result + + +def _build_sector_rotation( + current: list[dict[str, Any]], previous: list[dict[str, Any]] +) -> list[dict[str, Any]]: + previous_map = {row["name"]: row for row in previous} + result = [] + for index, sector in enumerate(current, start=1): + previous_count = int(previous_map.get(sector["name"], {}).get("count", 0)) + delta = int(sector["count"]) - previous_count + result.append( + { + **sector, + "rank": index, + "previous_count": previous_count, + "delta": delta, + "trend": "升温" if delta > 0 else "降温" if delta < 0 else "持平", + } + ) + return result diff --git a/backend/data/realtime.py b/backend/data/realtime.py new file mode 100644 index 0000000..d566df2 --- /dev/null +++ b/backend/data/realtime.py @@ -0,0 +1,426 @@ +from __future__ import annotations + +import copy +import http.client +import json +import time +import urllib.error +import urllib.parse +import urllib.request +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from datetime import datetime +from threading import Lock +from typing import Any, ClassVar + + +class RealtimeAggregateError(RuntimeError): + pass + + +EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get" +EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get" +TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006" +THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool" +XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail" +BROWSER_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/138.0.0.0 Safari/537.36" +) + + +@dataclass +class WebRealtimeAggregator: + timeout: int = 8 + retry_attempts: int = 3 + retry_delay_seconds: float = 0.2 + response_cache_ttl_seconds: int = 90 + _sector_cache: ClassVar[dict[str, Any]] = {} + _sector_cache_lock: ClassVar[Lock] = Lock() + _response_cache: ClassVar[dict[str, dict[str, Any]]] = {} + _response_cache_lock: ClassVar[Lock] = Lock() + + def health_snapshot(self, sector: str = "") -> dict[str, Any]: + started = time.perf_counter() + sources: dict[str, dict[str, Any]] = {} + indices: list[dict[str, Any]] = [] + sector_payload: dict[str, Any] | None = None + + indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices) + if sector.strip(): + sector_payload, sources["eastmoney_sector"] = self._capture( + lambda: self.eastmoney_sector(sector) + ) + ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool) + xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool) + + index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []] + now = datetime.now().astimezone() + max_skew = 120 if now.hour >= 15 else 15 + index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew + ready = ( + bool(indices) + and len(indices) == 3 + and index_consistent + and (not sector.strip() or bool(sector_payload)) + ) + return { + "ready": ready, + "isolated": True, + "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + "elapsed_ms": round((time.perf_counter() - started) * 1000), + "indices": indices or [], + "index_consistent": index_consistent, + "sector": sector_payload, + "sources": sources, + "observations": { + "ths_limit_pool": ths_observation, + "xgb_limit_pool": xgb_observation, + }, + "policy": { + "integration": "heaven_realtime_fallback", + "max_index_time_skew_seconds": max_skew, + "notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。", + }, + } + + def eastmoney_indices(self) -> list[dict[str, Any]]: + try: + payload = self._get_json( + EASTMONEY_INDEX_URL, + { + "secids": "1.000001,0.399001,0.399006", + "fltt": "2", + "invt": "2", + "fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124", + }, + referer="https://quote.eastmoney.com/", + ) + except RealtimeAggregateError: + return self.tencent_indices() + cache_meta = payload.get("_aggregate_cache") or {} + rows = list((payload.get("data") or {}).get("diff") or []) + result = [] + for row in rows: + code = str(row.get("f12") or "") + if code not in {"000001", "399001", "399006"}: + continue + epoch = int(_number(row.get("f124"))) + result.append( + { + "code": code, + "name": row.get("f14") or code, + "price": _number(row.get("f2")), + "change": _number(row.get("f3")), + "change_amount": _number(row.get("f4")), + "open": _number(row.get("f17")), + "high": _number(row.get("f15")), + "low": _number(row.get("f16")), + "previous_close": _number(row.get("f18")), + "amount_billion": round(_number(row.get("f6")) / 100000000, 2), + "quote_time_epoch": epoch, + "quote_time": ( + datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds") + if epoch else "" + ), + "source": ( + "eastmoney_push2_cache" if cache_meta else "eastmoney_push2" + ), + "cache_age_seconds": cache_meta.get("age_seconds", 0), + } + ) + if len(result) != 3: + raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices") + return result + + def tencent_indices(self) -> list[dict[str, Any]]: + raw, cache_age = self._get_text( + TENCENT_INDEX_URL, + referer="https://gu.qq.com/", + encoding="gb18030", + ) + result = [] + for line in raw.splitlines(): + if '="' not in line: + continue + fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~") + if len(fields) < 38: + continue + code = fields[2] + if code not in {"000001", "399001", "399006"}: + continue + try: + quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone() + except ValueError as exc: + raise RealtimeAggregateError( + f"Tencent returned invalid quote time for {code}" + ) from exc + result.append( + { + "code": code, + "name": fields[1] or code, + "price": _number(fields[3]), + "change": _number(fields[32]), + "change_amount": _number(fields[31]), + "open": _number(fields[5]), + "high": _number(fields[33]), + "low": _number(fields[34]), + "previous_close": _number(fields[4]), + "amount_billion": round(_number(fields[37]) / 10000, 2), + "quote_time_epoch": int(quote_time.timestamp()), + "quote_time": quote_time.isoformat(timespec="seconds"), + "source": "tencent_qt_cache" if cache_age else "tencent_qt", + "cache_age_seconds": cache_age, + } + ) + if len(result) != 3: + raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices") + return result + + def eastmoney_sector(self, query: str) -> dict[str, Any]: + target = _normalize_sector(query) + candidates = self._eastmoney_sector_catalog() + matched = _match_sector(candidates, target) + if not matched: + raise RealtimeAggregateError(f"Eastmoney sector not found: {query}") + epoch = int(_number(matched.get("f124"))) + return { + "code": matched.get("f12") or "", + "name": matched.get("f14") or query, + "price": _number(matched.get("f2")), + "change": _number(matched.get("f3")), + "change_amount": _number(matched.get("f4")), + "turnover_rate": _number(matched.get("f8")), + "up_count": int(_number(matched.get("f104"))), + "down_count": int(_number(matched.get("f105"))), + "leader": matched.get("f128") or "--", + "leader_code": matched.get("f140") or "", + "leading_pct": _number(matched.get("f136")), + "quote_time_epoch": epoch, + "quote_time": ( + datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds") + if epoch else "" + ), + "source": "eastmoney_push2", + "match_query": query, + } + + def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]: + now = time.time() + with self._sector_cache_lock: + cached = self._sector_cache.get("eastmoney") + if cached and now - float(cached.get("created_at") or 0) < 600: + return list(cached.get("rows") or []) + + def load_page(page: int) -> list[dict[str, Any]]: + payload = self._get_json( + EASTMONEY_SECTOR_URL, + { + "pn": str(page), + "pz": "100", + "po": "1", + "np": "1", + "fltt": "2", + "invt": "2", + "fid": "f3", + "fs": "m:90+t:2", + "fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124", + }, + referer="https://quote.eastmoney.com/center/boardlist.html", + ) + return list((payload.get("data") or {}).get("diff") or []) + + with ThreadPoolExecutor(max_workers=5) as executor: + pages = list(executor.map(load_page, range(1, 6))) + rows = [row for page in pages for row in page] + if not rows: + raise RealtimeAggregateError("Eastmoney sector catalog is empty") + with self._sector_cache_lock: + self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows} + return rows + + def ths_limit_pool(self) -> dict[str, Any]: + payload = self._get_json( + THS_LIMIT_URL, + {"page": "1", "limit": "3", "field": "199112"}, + referer="https://data.10jqka.com.cn/limit_up/", + ) + data = payload.get("data") or payload + return { + "available": True, + "keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [], + "source": "ths_web_dataapi", + } + + def xgb_limit_pool(self) -> dict[str, Any]: + payload = self._get_json( + XGB_POOL_URL, + {"pool_name": "limit_up"}, + referer="https://xuangubao.cn/", + ) + data = payload.get("data") or {} + rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or [] + return { + "available": True, + "count": len(rows) if isinstance(rows, list) else 0, + "source": "xuangubao_web_api", + } + + def _capture(self, operation): + started = time.perf_counter() + try: + value = operation() + return value, { + "ok": True, + "elapsed_ms": round((time.perf_counter() - started) * 1000), + "error": "", + } + except Exception as exc: + return None, { + "ok": False, + "elapsed_ms": round((time.perf_counter() - started) * 1000), + "error": str(exc)[:500], + } + + def _get_json( + self, + url: str, + params: dict[str, str], + referer: str, + ) -> dict[str, Any]: + request_url = f"{url}?{urllib.parse.urlencode(params)}" + last_error: Exception | None = None + attempts = max(1, int(self.retry_attempts)) + for attempt in range(attempts): + request = urllib.request.Request( + request_url, + headers={ + "Accept": "application/json,text/plain,*/*", + "Connection": "close", + "Referer": referer, + "User-Agent": BROWSER_USER_AGENT, + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + content_type = response.headers.get("Content-Type", "") + raw = response.read().decode("utf-8", errors="replace") + if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")): + raise RealtimeAggregateError( + f"non-JSON response: {raw[:120].strip()}" + ) + payload = json.loads(raw) + if not isinstance(payload, dict): + raise RealtimeAggregateError("unexpected response shape") + if payload.get("rc") not in (None, 0): + raise RealtimeAggregateError(f"provider rc={payload.get('rc')}") + with self._response_cache_lock: + self._response_cache[request_url] = { + "created_at": time.time(), + "payload": copy.deepcopy(payload), + } + return payload + except ( + urllib.error.URLError, + TimeoutError, + ConnectionError, + OSError, + http.client.HTTPException, + json.JSONDecodeError, + RealtimeAggregateError, + ) as exc: + last_error = exc + if attempt + 1 < attempts and self.retry_delay_seconds > 0: + time.sleep(self.retry_delay_seconds * (attempt + 1)) + + now = time.time() + with self._response_cache_lock: + cached = self._response_cache.get(request_url) + cache_age = now - float((cached or {}).get("created_at") or 0) + if cached and cache_age <= self.response_cache_ttl_seconds: + payload = copy.deepcopy(cached.get("payload") or {}) + payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)} + return payload + raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error + + def _get_text( + self, + request_url: str, + referer: str, + encoding: str = "utf-8", + ) -> tuple[str, float]: + cache_key = f"text:{request_url}" + last_error: Exception | None = None + attempts = max(1, int(self.retry_attempts)) + for attempt in range(attempts): + request = urllib.request.Request( + request_url, + headers={ + "Accept": "text/plain,*/*", + "Connection": "close", + "Referer": referer, + "User-Agent": BROWSER_USER_AGENT, + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + raw = response.read().decode(encoding, errors="replace") + if not raw.strip(): + raise RealtimeAggregateError("empty text response") + with self._response_cache_lock: + self._response_cache[cache_key] = { + "created_at": time.time(), + "payload": raw, + } + return raw, 0 + except ( + urllib.error.URLError, + TimeoutError, + ConnectionError, + OSError, + http.client.HTTPException, + RealtimeAggregateError, + ) as exc: + last_error = exc + if attempt + 1 < attempts and self.retry_delay_seconds > 0: + time.sleep(self.retry_delay_seconds * (attempt + 1)) + + now = time.time() + with self._response_cache_lock: + cached = self._response_cache.get(cache_key) + cache_age = now - float((cached or {}).get("created_at") or 0) + if cached and cache_age <= self.response_cache_ttl_seconds: + return str(cached.get("payload") or ""), round(cache_age, 1) + raise RealtimeAggregateError( + f"text request failed after {attempts} attempts: {last_error}" + ) from last_error + + +def _normalize_sector(value: Any) -> str: + text = str(value or "").strip().replace(" ", "") + for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"): + text = text.replace(suffix, "") + aliases = {"元器件": "元件", "电子元器件": "元件"} + return aliases.get(text, text) + + +def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None: + exact = [row for row in rows if _normalize_sector(row.get("f14")) == target] + if exact: + return min(exact, key=lambda row: len(str(row.get("f14") or ""))) + fuzzy = [ + row for row in rows + if target and ( + target in _normalize_sector(row.get("f14")) + or _normalize_sector(row.get("f14")) in target + ) + ] + return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None + + +def _number(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default diff --git a/backend/features/market/__init__.py b/backend/features/market/__init__.py new file mode 100644 index 0000000..0cc0632 --- /dev/null +++ b/backend/features/market/__init__.py @@ -0,0 +1,13 @@ +"""Public market data, search, detail and chart feature.""" + +from .charts import ChartDataError, EastmoneyChartClient, MarketChartClient +from .repository import MarketRepositoryMixin +from .service import MarketServiceMixin + +__all__ = [ + "ChartDataError", + "EastmoneyChartClient", + "MarketChartClient", + "MarketRepositoryMixin", + "MarketServiceMixin", +] diff --git a/backend/features/market/charts.py b/backend/features/market/charts.py new file mode 100644 index 0000000..97cd8b7 --- /dev/null +++ b/backend/features/market/charts.py @@ -0,0 +1,497 @@ +from __future__ import annotations + +import http.client +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass +from datetime import datetime, time as dt_time, timedelta +from threading import Lock +from typing import Any, ClassVar + +from backend.data.providers.ifind_client import IfindError, IfindHttpClient + + +class ChartDataError(RuntimeError): + pass + + +TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get" +BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get" +BROWSER_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/138.0.0.0 Safari/537.36" +) +INDEX_SECIDS = { + "000001.SH": "1.000001", + "399001.SZ": "0.399001", + "399006.SZ": "0.399006", +} + + +class MarketChartClient: + """Prefer iFinD for display charts and retain Eastmoney as a last resort.""" + + def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None: + self.ifind = ifind + self.fallback = fallback + + def stock_intraday(self, code: str) -> dict[str, Any]: + normalized = str(code or "").strip() + if not re.fullmatch(r"\d{6}", normalized): + raise ChartDataError("Invalid stock code") + ifind_code = _stock_market_code(normalized) + try: + return self._ifind_intraday(ifind_code, "stock", normalized) + except (IfindError, ChartDataError): + return self.fallback.stock_intraday(normalized) + + def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]: + normalized = str(code or "").strip() + if not re.fullmatch(r"\d{6}", normalized): + raise ChartDataError("Invalid stock code") + return self._ifind_daily(_stock_market_code(normalized), end_date, limit) + + def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]: + normalized = str(identifier or "").strip().upper() + if normalized not in INDEX_SECIDS: + raise ChartDataError("Unsupported index") + return self._ifind_daily(normalized, end_date, limit) + + def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]: + normalized = str(identifier or "").strip().upper() + if not normalized: + raise ChartDataError("Invalid board code") + return self._ifind_daily(normalized, end_date, limit) + + def index_intraday(self, identifier: str) -> dict[str, Any]: + normalized = str(identifier or "").strip().upper() + if normalized not in INDEX_SECIDS: + raise ChartDataError("Unsupported index") + try: + return self._ifind_intraday(normalized, "index", normalized) + except (IfindError, ChartDataError): + return self.fallback.index_intraday(normalized) + + def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]: + normalized = str(identifier or "").strip().upper() + try: + return self._ifind_intraday(normalized, "board", normalized, name) + except (IfindError, ChartDataError): + return self.fallback.board_intraday(normalized, name) + + def _ifind_intraday( + self, + ifind_code: str, + entity_type: str, + identifier: str, + name: str = "", + ) -> dict[str, Any]: + if not self.ifind.configured: + raise ChartDataError("iFinD is not configured") + now = datetime.now().astimezone() + rows: list[dict[str, Any]] = [] + for offset in range(0, 8): + candidate = now.date() - timedelta(days=offset) + if candidate.weekday() >= 5: + continue + display_date = candidate.isoformat() + rows = self.ifind.intraday( + ifind_code, + f"{display_date} 09:30:00", + f"{display_date} 15:00:00", + cache_ttl=20 if offset == 0 else 6 * 60 * 60, + ) + if rows: + break + points = [point for row in rows if (point := _ifind_point(row))] + if not points: + raise ChartDataError("No iFinD intraday chart data returned") + latest_date = points[-1]["date"] + points = [point for point in points if point["date"] == latest_date] + previous_close = self._previous_close(ifind_code, latest_date, points[0]["open"]) + return { + "entity_type": entity_type, + "identifier": identifier, + "name": name, + "code": identifier, + "trade_date": latest_date, + "previous_close": previous_close, + "points": points, + "source": "ifind", + } + + def _ifind_daily( + self, ifind_code: str, end_date: str, limit: int + ) -> list[dict[str, Any]]: + if not self.ifind.configured: + raise ChartDataError("iFinD is not configured") + compact_end = str(end_date or "").replace("-", "") + if not re.fullmatch(r"\d{8}", compact_end): + raise ChartDataError("Invalid chart end date") + end = datetime.strptime(compact_end, "%Y%m%d") + start = (end - timedelta(days=max(190, limit * 3))).strftime("%Y%m%d") + try: + rows = self.ifind.history( + ifind_code, + ["open", "high", "low", "close", "volume", "amount"], + start, + compact_end, + cache_ttl=300, + ) + except IfindError as exc: + raise ChartDataError("No iFinD daily chart data returned") from exc + normalized = [] + for row in rows: + stamp = str(row.get("time") or "").strip() + trade_date = stamp[:10] + close = _number(row.get("close")) + if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", trade_date) or close <= 0: + continue + normalized.append( + { + "trade_date": trade_date, + "open": _number(row.get("open")), + "high": _number(row.get("high")), + "low": _number(row.get("low")), + "close": close, + "volume": _number(row.get("volume")), + "amount_billion": _number(row.get("amount")) / 100_000_000, + } + ) + normalized.sort(key=lambda row: row["trade_date"]) + for index, row in enumerate(normalized): + previous = normalized[index - 1]["close"] if index > 0 else 0 + row["change"] = round((row["close"] / previous - 1) * 100, 4) if previous else 0.0 + + market_now = datetime.now().astimezone() + today = market_now.strftime("%Y%m%d") + market_open = ( + market_now.weekday() < 5 + and market_now.time().replace(tzinfo=None) >= dt_time(9, 30) + ) + today_display = market_now.date().isoformat() + if normalized and normalized[-1]["trade_date"] == today_display: + current_bar = normalized[-1] + current_bar_is_valid = ( + current_bar["open"] > 0 + and current_bar["high"] >= max(current_bar["open"], current_bar["close"]) + and 0 < current_bar["low"] <= min(current_bar["open"], current_bar["close"]) + and (current_bar["volume"] > 0 or current_bar["amount_billion"] > 0) + ) + if not market_open or not current_bar_is_valid: + normalized.pop() + if compact_end == today and market_open: + try: + quote_rows = self.ifind.real_time( + ifind_code, + ["open", "high", "low", "latest", "preClose", "volume", "amount"], + cache_ttl=10, + ) + quote = quote_rows[0] if quote_rows else {} + latest = _number(quote.get("latest")) + previous = _number(quote.get("preClose")) + open_price = _number(quote.get("open")) + high = _number(quote.get("high")) + low = _number(quote.get("low")) + volume = _number(quote.get("volume")) + amount = _number(quote.get("amount")) + quote_date = str(quote.get("time") or "")[:10].replace("-", "") + quote_is_current = not quote_date or quote_date == today + has_market_activity = volume > 0 or amount > 0 + if ( + latest > 0 + and open_price > 0 + and high >= max(open_price, latest) + and 0 < low <= min(open_price, latest) + and has_market_activity + and quote_is_current + ): + realtime = { + "trade_date": end.strftime("%Y-%m-%d"), + "open": open_price, + "high": high, + "low": low, + "close": latest, + "change": round((latest / previous - 1) * 100, 4) if previous else 0.0, + "volume": volume, + "amount_billion": amount / 100_000_000, + "realtime": True, + } + if normalized and normalized[-1]["trade_date"] == realtime["trade_date"]: + normalized[-1] = realtime + else: + normalized.append(realtime) + except IfindError: + pass + if not normalized: + raise ChartDataError("No iFinD daily chart data returned") + return normalized[-max(20, min(180, int(limit))):] + + def _previous_close(self, code: str, trade_date: str, fallback: float) -> float: + today = datetime.now().astimezone().date().isoformat() + if trade_date == today: + try: + quote = self.ifind.real_time(code, ["preClose"], cache_ttl=20) + value = _number((quote[0] if quote else {}).get("preClose")) + if value > 0: + return value + except IfindError: + pass + end = datetime.strptime(trade_date, "%Y-%m-%d") + try: + rows = self.ifind.history( + code, + ["close"], + (end - timedelta(days=12)).strftime("%Y%m%d"), + end.strftime("%Y%m%d"), + cache_ttl=6 * 60 * 60, + ) + closes = [_number(row.get("close")) for row in rows if _number(row.get("close")) > 0] + if len(closes) >= 2: + return closes[-2] + except IfindError: + pass + return fallback + + +@dataclass +class EastmoneyChartClient: + """Isolated display-only minute chart source. + + The returned data must not be used by market snapshots, scoring, screening, + or divination. Its only consumer is a chart-rendering endpoint. + """ + + timeout: int = 6 + cache_ttl_seconds: int = 20 + retry_attempts: int = 2 + _cache: ClassVar[dict[str, dict[str, Any]]] = {} + _cache_lock: ClassVar[Lock] = Lock() + _board_catalog: ClassVar[dict[str, dict[str, str]]] = {} + _board_catalog_at: ClassVar[float] = 0.0 + _board_catalog_lock: ClassVar[Lock] = Lock() + + def stock_intraday(self, code: str) -> dict[str, Any]: + normalized = str(code or "").strip() + if not re.fullmatch(r"\d{6}", normalized): + raise ChartDataError("Invalid stock code") + market = "1" if normalized.startswith(("5", "6", "9")) else "0" + return self._intraday(f"{market}.{normalized}", "stock", normalized) + + def index_intraday(self, identifier: str) -> dict[str, Any]: + normalized = str(identifier or "").strip().upper() + secid = INDEX_SECIDS.get(normalized) + if not secid: + raise ChartDataError("Unsupported index") + return self._intraday(secid, "index", normalized) + + def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]: + normalized = str(identifier or "").strip().upper() + if re.fullmatch(r"BK\d{4}", normalized): + board_code = normalized + else: + board_code = self._resolve_board_code(name or identifier) + return self._intraday(f"90.{board_code}", "board", board_code) + + def _intraday(self, secid: str, entity_type: str, identifier: str) -> dict[str, Any]: + cache_key = f"{entity_type}:{identifier}" + cached = self._get_cached(cache_key) + if cached is not None: + return cached + + payload = self._request_json( + TRENDS_URL, + { + "secid": secid, + "fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13", + "fields2": "f51,f52,f53,f54,f55,f56,f57,f58", + "iscr": "0", + "ndays": "1", + }, + "https://quote.eastmoney.com/", + ) + data = payload.get("data") or {} + points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))] + if not points: + raise ChartDataError("No intraday chart data returned") + + result = { + "entity_type": entity_type, + "identifier": identifier, + "name": str(data.get("name") or ""), + "code": str(data.get("code") or identifier), + "trade_date": points[-1]["date"], + "previous_close": _number(data.get("preClose")), + "points": points, + } + with self._cache_lock: + self._cache[cache_key] = {"created_at": time.time(), "payload": result} + return result + + def _get_cached(self, cache_key: str) -> dict[str, Any] | None: + with self._cache_lock: + cached = self._cache.get(cache_key) + if not cached: + return None + if time.time() - float(cached.get("created_at") or 0) > self.cache_ttl_seconds: + with self._cache_lock: + self._cache.pop(cache_key, None) + return None + return dict(cached["payload"]) + + def _resolve_board_code(self, name: str) -> str: + normalized = _normalize_name(name) + if not normalized: + raise ChartDataError("Board name is required") + catalog = self._load_board_catalog() + item = catalog.get(normalized) + if not item: + raise ChartDataError("No matching chart board") + return item["code"] + + def _load_board_catalog(self) -> dict[str, dict[str, str]]: + now = time.time() + with self._board_catalog_lock: + if self._board_catalog and now - self._board_catalog_at < 6 * 60 * 60: + return dict(self._board_catalog) + + rows: list[dict[str, Any]] = [] + for board_type in ("1", "2", "3"): + for page in range(1, 6): + payload = self._request_json( + BOARD_LIST_URL, + { + "pn": str(page), + "pz": "100", + "po": "1", + "np": "1", + "fltt": "2", + "invt": "2", + "fid": "f3", + "fs": f"m:90+t:{board_type}", + "fields": "f12,f14", + }, + "https://quote.eastmoney.com/center/boardlist.html", + ) + page_rows = (payload.get("data") or {}).get("diff") or [] + rows.extend(page_rows) + if len(page_rows) < 100: + break + + catalog: dict[str, dict[str, str]] = {} + for row in rows: + code = str(row.get("f12") or "").strip().upper() + board_name = str(row.get("f14") or "").strip() + if re.fullmatch(r"BK\d{4}", code) and board_name: + catalog.setdefault(_normalize_name(board_name), {"code": code, "name": board_name}) + if not catalog: + raise ChartDataError("Board chart directory is unavailable") + with self._board_catalog_lock: + type(self)._board_catalog = catalog + type(self)._board_catalog_at = now + return dict(catalog) + + def _request_json( + self, url: str, params: dict[str, str], referer: str + ) -> dict[str, Any]: + request_url = f"{url}?{urllib.parse.urlencode(params)}" + last_error: Exception | None = None + for attempt in range(max(1, int(self.retry_attempts))): + request = urllib.request.Request( + request_url, + headers={ + "Accept": "application/json,text/plain,*/*", + "Connection": "close", + "Referer": referer, + "User-Agent": BROWSER_USER_AGENT, + }, + ) + try: + with urllib.request.urlopen(request, timeout=self.timeout) as response: + payload = json.loads(response.read().decode("utf-8")) + if not isinstance(payload, dict): + raise ChartDataError("Invalid intraday chart response") + return payload + except ( + urllib.error.URLError, + TimeoutError, + ConnectionError, + OSError, + http.client.HTTPException, + json.JSONDecodeError, + ChartDataError, + ) as exc: + last_error = exc + if attempt + 1 < self.retry_attempts: + time.sleep(0.12) + raise ChartDataError("Intraday chart request failed") from last_error + + +def _parse_trend(raw: Any) -> dict[str, Any] | None: + fields = str(raw or "").split(",") + if len(fields) < 8 or " " not in fields[0]: + return None + stamp = fields[0].strip() + trade_date, trade_time = stamp.split(" ", 1) + close = _number(fields[2]) + if close <= 0: + return None + return { + "date": trade_date, + "time": trade_time[:5], + "open": _number(fields[1]), + "close": close, + "high": _number(fields[3]), + "low": _number(fields[4]), + "volume": _number(fields[5]), + "amount": _number(fields[6]), + "average": _number(fields[7]), + } + + +def _ifind_point(row: dict[str, Any]) -> dict[str, Any] | None: + stamp = str(row.get("time") or "").strip() + if " " not in stamp: + return None + trade_date, trade_time = stamp.split(" ", 1) + close = _number(row.get("close")) + if close <= 0: + return None + return { + "date": trade_date, + "time": trade_time[:5], + "open": _number(row.get("open")), + "close": close, + "high": _number(row.get("high")), + "low": _number(row.get("low")), + "volume": _number(row.get("volume")), + "amount": _number(row.get("amount")), + "average": _number(row.get("avgPrice")), + } + + +def _stock_market_code(code: str) -> str: + if code.startswith(("4", "8", "9")): + suffix = "BJ" + elif code.startswith("6"): + suffix = "SH" + else: + suffix = "SZ" + return f"{code}.{suffix}" + + +def _number(value: Any) -> float: + try: + return float(value or 0) + except (TypeError, ValueError): + return 0.0 + + +def _normalize_name(value: Any) -> str: + normalized = re.sub(r"[\s·・（）()\-_/]", "", str(value or "")).casefold() + return re.sub(r"(?:概念|行业|[ⅠⅡⅢ])$", "", normalized) diff --git a/backend/features/market/repository.py b/backend/features/market/repository.py new file mode 100644 index 0000000..b36e26d --- /dev/null +++ b/backend/features/market/repository.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any + + +class MarketRepositoryMixin: + def get_snapshot(self, trade_date: str) -> dict[str, Any] | None: + with self.connect() as connection: + row = connection.execute( + "SELECT payload FROM dashboard_snapshots WHERE trade_date = ?", + (trade_date,), + ).fetchone() + if not row: + return None + try: + return json.loads(row["payload"]) + except json.JSONDecodeError: + return None + + def get_latest_real_snapshot( + self, trade_date: str, strictly_before: bool = False + ) -> dict[str, Any] | None: + operator = "<" if strictly_before else "<=" + with self.connect() as connection: + row = connection.execute( + f""" + SELECT payload FROM dashboard_snapshots + WHERE trade_date {operator} ? AND source != 'demo' + ORDER BY trade_date DESC LIMIT 1 + """, + (trade_date,), + ).fetchone() + if not row: + return None + try: + return json.loads(row["payload"]) + except json.JSONDecodeError: + return None + + def save_snapshot(self, trade_date: str, source: str, payload: dict[str, Any]) -> None: + updated_at = datetime.now().astimezone().isoformat(timespec="seconds") + record_count = sum( + len(payload.get(key) or []) + for key in ("limits", "broken", "down_limits", "yesterday_limits") + ) + content = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + with self.connect() as connection: + connection.execute( + """ + INSERT INTO dashboard_snapshots + (trade_date, source, payload, record_count, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(trade_date) DO UPDATE SET + source = excluded.source, + payload = excluded.payload, + record_count = excluded.record_count, + updated_at = excluded.updated_at + """, + (trade_date, source, content, record_count, updated_at), + ) + + def get_data_snapshot(self, kind: str, cache_key: str) -> dict[str, Any] | None: + with self.connect() as connection: + row = connection.execute( + "SELECT payload FROM data_snapshots WHERE kind = ? AND cache_key = ?", + (kind, cache_key), + ).fetchone() + if not row: + return None + try: + return json.loads(row["payload"]) + except json.JSONDecodeError: + return None + + def get_latest_data_snapshot( + self, + kind: str, + cache_key_prefix: str, + maximum_cache_key: str, + exclude_source: str = "", + ) -> dict[str, Any] | None: + source_clause = " AND source != ?" if exclude_source else "" + parameters: list[Any] = [kind, f"{cache_key_prefix}%", maximum_cache_key] + if exclude_source: + parameters.append(exclude_source) + with self.connect() as connection: + row = connection.execute( + f""" + SELECT payload FROM data_snapshots + WHERE kind = ? AND cache_key LIKE ? AND cache_key <= ?{source_clause} + ORDER BY cache_key DESC LIMIT 1 + """, + parameters, + ).fetchone() + if not row: + return None + try: + return json.loads(row["payload"]) + except json.JSONDecodeError: + return None + + def save_data_snapshot( + self, kind: str, cache_key: str, source: str, payload: dict[str, Any] + ) -> None: + updated_at = datetime.now().astimezone().isoformat(timespec="seconds") + content = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + with self.connect() as connection: + connection.execute( + """ + INSERT INTO data_snapshots (kind, cache_key, source, payload, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(kind, cache_key) DO UPDATE SET + source = excluded.source, + payload = excluded.payload, + updated_at = excluded.updated_at + """, + (kind, cache_key, source, content, updated_at), + ) + + def search_stock_master(self, query: str, limit: int = 12) -> list[dict[str, Any]]: + text = str(query or "").strip() + if not text: + return [] + escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + with self.connect() as connection: + rows = connection.execute( + """ + SELECT ts_code, code, name, industry, market, list_date + FROM stock_master + WHERE code = ? OR name = ? OR name LIKE ? ESCAPE '\\' + ORDER BY + CASE WHEN code = ? THEN 0 WHEN name = ? THEN 1 ELSE 2 END, + list_date DESC, + code + LIMIT ? + """, + (text, text, f"%{escaped}%", text, text, max(1, min(30, int(limit)))), + ).fetchall() + return [dict(row) for row in rows] + + def list_snapshot_payloads(self, end_date: str, limit: int = 260) -> list[dict[str, Any]]: + with self.connect() as connection: + rows = connection.execute( + """ + SELECT trade_date, payload FROM dashboard_snapshots + WHERE trade_date <= ? ORDER BY trade_date DESC LIMIT ? + """, + (end_date, limit), + ).fetchall() + result: list[dict[str, Any]] = [] + for row in reversed(rows): + try: + payload = json.loads(row["payload"]) + except json.JSONDecodeError: + continue + payload["_snapshot_date"] = row["trade_date"] + result.append(payload) + return result + + def start_sync(self, trade_date: str, source: str) -> int: + started_at = datetime.now().astimezone().isoformat(timespec="seconds") + with self.connect() as connection: + cursor = connection.execute( + """ + INSERT INTO sync_runs (trade_date, source, status, started_at) + VALUES (?, ?, 'running', ?) + """, + (trade_date, source, started_at), + ) + return int(cursor.lastrowid) + + def finish_sync( + self, + sync_id: int, + status: str, + record_count: int = 0, + message: str = "", + source: str | None = None, + ) -> None: + finished_at = datetime.now().astimezone().isoformat(timespec="seconds") + with self.connect() as connection: + connection.execute( + """ + UPDATE sync_runs + SET status = ?, finished_at = ?, record_count = ?, message = ?, + source = COALESCE(?, source) + WHERE id = ? + """, + (status, finished_at, record_count, message[:1000], source, sync_id), + ) + + def status(self) -> dict[str, Any]: + with self.connect() as connection: + last_sync = connection.execute( + """ + SELECT id, trade_date, source, status, started_at, finished_at, + record_count, message + FROM sync_runs ORDER BY id DESC LIMIT 1 + """ + ).fetchone() + snapshot_stats = connection.execute( + """ + SELECT COUNT(*) AS dates, COALESCE(SUM(record_count), 0) AS records, + MAX(updated_at) AS updated_at + FROM dashboard_snapshots + """ + ).fetchone() + watchlist_count = connection.execute("SELECT COUNT(*) FROM watchlist").fetchone()[0] + note_count = connection.execute("SELECT COUNT(*) FROM review_notes").fetchone()[0] + + return { + "database": str(self.path.name), + "snapshot_dates": int(snapshot_stats["dates"]), + "snapshot_records": int(snapshot_stats["records"]), + "updated_at": snapshot_stats["updated_at"], + "last_sync": dict(last_sync) if last_sync else None, + "watchlist_count": int(watchlist_count), + "note_count": int(note_count), + } + diff --git a/backend/features/market/service.py b/backend/features/market/service.py new file mode 100644 index 0000000..3d5dccd --- /dev/null +++ b/backend/features/market/service.py @@ -0,0 +1,949 @@ +from __future__ import annotations + +import copy +import re +from datetime import date, datetime, time as dt_time, timedelta +from typing import Any + +from backend.bootstrap.config import ( + normalize_date, + tushare_code, + validate_stock_code, + validate_text, +) +from backend.data.providers.ifind_client import IfindError +from backend.data.providers.tushare_client import TushareClient, TushareError +from backend.features.market.charts import ChartDataError +from sentiment_engine import SENTIMENT_ENGINE_VERSION + + +SEARCH_INDEXES = ( + {"id": "000001.SH", "code": "000001.SH", "name": "上证指数", "type": "index", "subtitle": "沪市综合指数"}, + {"id": "399001.SZ", "code": "399001.SZ", "name": "深证成指", "type": "index", "subtitle": "深市成份指数"}, + {"id": "399006.SZ", "code": "399006.SZ", "name": "创业板指", "type": "index", "subtitle": "创业板核心指数"}, +) +SEARCH_TYPE_LABELS = { + "stock": "股票", + "sector": "板块", + "theme": "题材", + "index": "指数", +} +THS_SEARCH_TYPES = { + "I": ("sector", "行业板块"), + "R": ("sector", "地域板块"), + "N": ("theme", "概念题材"), +} + + +class MarketServiceMixin: + def _tushare_client(self) -> TushareClient: + gateway = getattr(self, "data_gateway", None) + if gateway is not None: + return gateway.tushare() + # Compatibility for isolated legacy unit-test service stubs. + return TushareClient(self.token) + + def get_dashboard(self, trade_date: str, force: bool = False) -> dict[str, Any]: + normalized_date = normalize_date(trade_date) + now = datetime.now().astimezone() + if ( + normalized_date == now.strftime("%Y%m%d") + and now.time().replace(tzinfo=None) < datetime.strptime("09:15", "%H:%M").time() + ): + previous = self.database.get_latest_real_snapshot(normalized_date, strictly_before=True) + if previous: + carried = self._carry_dashboard(previous, normalized_date, "盘前沿用最近交易日收盘行情") + return self._apply_reason_overrides(self._with_storage(carried, cached=True)) + if not force: + snapshot = self.database.get_snapshot(normalized_date) + if snapshot and str((snapshot.get("meta") or {}).get("source") or "") != "demo": + snapshot = copy.deepcopy(snapshot) + if normalized_date != now.strftime("%Y%m%d"): + snapshot.setdefault("meta", {}).update( + {"realtime": False, "market_status": "closed"} + ) + if not self._dashboard_sentiment_ready(snapshot): + snapshot = self._enrich_dashboard_sentiment(snapshot, normalized_date) + self.database.save_snapshot( + normalized_date, + str((snapshot.get("meta") or {}).get("source") or "tushare"), + snapshot, + ) + snapshot.setdefault("meta", {})["requested_date"] = self._display_compact_date(normalized_date) + return self._apply_reason_overrides(self._with_storage(snapshot, cached=True)) + resolved = self.database.get_data_snapshot( + "dashboard_request_v1", normalized_date + ) + if resolved and str((resolved.get("meta") or {}).get("source") or "") != "demo": + resolved = copy.deepcopy(resolved) + resolved.setdefault("meta", {})["requested_date"] = self._display_compact_date( + normalized_date + ) + return self._apply_reason_overrides( + self._with_storage(resolved, cached=True) + ) + if datetime.strptime(normalized_date, "%Y%m%d").weekday() >= 5: + previous = self.database.get_latest_real_snapshot(normalized_date) + if previous: + carried = self._carry_dashboard( + previous, + normalized_date, + "非交易日沿用最近交易日收盘行情", + ) + self.database.save_data_snapshot( + "dashboard_request_v1", normalized_date, "sqlite", carried + ) + return self._apply_reason_overrides( + self._with_storage(carried, cached=True) + ) + return self.sync_dashboard(normalized_date) + + @staticmethod + def _dashboard_sentiment_ready(dashboard: dict[str, Any]) -> bool: + overview = dashboard.get("overview") or {} + return int(overview.get("sentiment_engine_version") or 0) == SENTIMENT_ENGINE_VERSION and all( + key in overview + for key in ( + "sentiment_score", + "sentiment_label", + "sentiment_phase", + "sentiment_direction", + "sentiment_components", + ) + ) + + @staticmethod + def _display_compact_date(compact: str) -> str: + return f"{compact[:4]}-{compact[4:6]}-{compact[6:8]}" + + def _carry_dashboard( + self, snapshot: dict[str, Any], requested_date: str, reason: str + ) -> dict[str, Any]: + carried = copy.deepcopy(snapshot) + meta = carried.setdefault("meta", {}) + meta.update( + { + "requested_date": self._display_compact_date(requested_date), + "carried_forward": True, + "realtime": False, + "market_status": "closed", + "notice": reason, + } + ) + return carried + + def _realtime_snapshot_due( + self, + normalized_date: str, + snapshot: dict[str, Any], + ) -> bool: + if not self.configured or normalized_date != date.today().strftime("%Y%m%d"): + return False + now = datetime.now().astimezone() + local_time = now.time().replace(tzinfo=None) + realtime_start = datetime.strptime("09:15", "%H:%M").time() + morning_end = datetime.strptime("11:35", "%H:%M").time() + afternoon_start = datetime.strptime("12:55", "%H:%M").time() + realtime_end = datetime.strptime("15:05", "%H:%M").time() + in_session = ( + realtime_start <= local_time < morning_end + or afternoon_start <= local_time < realtime_end + ) + if not in_session: + return False + meta = snapshot.get("meta") or {} + snapshot_trade_date = str(meta.get("trade_date") or "").replace("-", "") + if snapshot_trade_date and snapshot_trade_date != normalized_date: + return False + if not meta.get("realtime"): + return True + try: + updated_at = datetime.fromisoformat(str(meta.get("updated_at") or "")) + if updated_at.tzinfo is None: + updated_at = updated_at.replace(tzinfo=now.tzinfo) + except ValueError: + return True + age_seconds = (now - updated_at.astimezone(now.tzinfo)).total_seconds() + return age_seconds >= 8 + + def sync_dashboard(self, trade_date: str) -> dict[str, Any]: + normalized_date = normalize_date(trade_date) + source = "tushare" + with self.sync_lock: + sync_id = self.database.start_sync(normalized_date, source) + try: + if not self.configured: + raise TushareError("公共行情尚未配置") + dashboard = self._tushare_client().dashboard(normalized_date) + + dashboard["meta"]["source"] = source + dashboard["meta"]["requested_date"] = self._display_compact_date(normalized_date) + dashboard = self._enrich_dashboard_sentiment(dashboard, normalized_date) + record_count = self._record_count(dashboard) + actual_date = normalize_date( + str(dashboard.get("meta", {}).get("trade_date") or normalized_date) + ) + self.database.save_snapshot(actual_date, source, dashboard) + if actual_date != normalized_date: + dashboard.setdefault("meta", {}).update( + { + "carried_forward": True, + "realtime": False, + "market_status": "closed", + } + ) + self.database.save_data_snapshot( + "dashboard_request_v1", normalized_date, source, dashboard + ) + self.database.finish_sync( + sync_id, + "success", + record_count, + dashboard.get("meta", {}).get("notice", ""), + source, + ) + return self._apply_reason_overrides(self._with_storage(dashboard, cached=False)) + except TushareError as exc: + fallback = self.database.get_latest_real_snapshot(normalized_date) + if fallback: + carried = self._carry_dashboard( + fallback, normalized_date, f"最新行情暂不可用，沿用最近收盘快照：{exc}" + ) + self.database.finish_sync( + sync_id, "fallback", self._record_count(carried), str(exc), "tushare" + ) + return self._apply_reason_overrides(self._with_storage(carried, cached=True)) + self.database.finish_sync(sync_id, "failed", message=str(exc)) + raise ValueError("暂无可用的真实行情快照，请等待后台完成首次同步。") from exc + except Exception as exc: + self.database.finish_sync(sync_id, "failed", message=str(exc)) + raise + + def realtime_aggregate_health(self, sector: str = "") -> dict[str, Any]: + sector = validate_text(sector, "板块名称", 50) + return self.realtime_aggregator.health_snapshot(sector) + + def _search_market_directory(self) -> list[dict[str, Any]]: + cached = self.database.get_data_snapshot("search_directory", "ths") or {} + cached_items = list(cached.get("items") or []) + if cached_items and int(cached.get("schema_version") or 0) >= 2: + return cached_items + if not self.configured: + return cached_items + + try: + rows = self._tushare_client().query( + "ths_index", + {}, + "ts_code,name,count,exchange,list_date,type", + ) + except TushareError: + return cached_items + + items = [] + for row in rows: + mapping = THS_SEARCH_TYPES.get(str(row.get("type") or "").upper()) + code = str(row.get("ts_code") or "").strip().upper() + name = str(row.get("name") or "").strip() + if not mapping or not code or not name or str(row.get("exchange") or "").upper() != "A": + continue + entity_type, subtitle = mapping + items.append( + { + "id": code, + "code": code, + "name": name, + "type": entity_type, + "subtitle": subtitle, + "member_count": int(float(row.get("count") or 0)), + } + ) + if items: + self.database.save_data_snapshot( + "search_directory", "ths", "tushare", {"schema_version": 2, "items": items} + ) + return items + + @staticmethod + def _search_match_score(item: dict[str, Any], query: str) -> tuple[int, int, str]: + name = str(item.get("name") or "").casefold() + code = str(item.get("code") or item.get("id") or "").casefold() + needle = query.casefold() + if code == needle: + rank = 0 + elif name == needle: + rank = 1 + elif code.startswith(needle): + rank = 2 + elif name.startswith(needle): + rank = 3 + else: + rank = 4 + return rank, len(name), code + + def search_entities(self, query: str, trade_date: str) -> dict[str, Any]: + needle = str(query or "").strip() + normalized_date = normalize_date(trade_date) + groups: dict[str, list[dict[str, Any]]] = { + "stocks": [], + "sectors": [], + "themes": [], + "indices": [], + } + if not needle: + return {"query": "", "trade_date": normalized_date, "groups": groups} + + stocks = [] + for row in self.database.search_stock_master(needle, 12): + stocks.append( + { + "id": str(row.get("code") or ""), + "code": str(row.get("code") or ""), + "name": str(row.get("name") or "--"), + "type": "stock", + "type_label": SEARCH_TYPE_LABELS["stock"], + "industry": str(row.get("industry") or "其他"), + "market": str(row.get("market") or ""), + "subtitle": " · ".join( + part for part in (str(row.get("industry") or ""), str(row.get("market") or "")) if part + ) or "A股", + } + ) + groups["stocks"] = stocks[:8] + + market_items = list(self._search_market_directory()) + [dict(item) for item in SEARCH_INDEXES] + matched = [ + item for item in market_items + if needle.casefold() in str(item.get("name") or "").casefold() + or needle.casefold() in str(item.get("code") or "").casefold() + ] + matched.sort(key=lambda item: self._search_match_score(item, needle)) + group_keys = {"sector": "sectors", "theme": "themes", "index": "indices"} + for item in matched: + group_key = group_keys.get(str(item.get("type") or "")) + if not group_key or len(groups[group_key]) >= 8: + continue + groups[group_key].append( + { + **item, + "type_label": SEARCH_TYPE_LABELS[str(item["type"])], + } + ) + return {"query": needle, "trade_date": normalized_date, "groups": groups} + + def get_search_detail( + self, entity_type: str, identifier: str, trade_date: str + ) -> dict[str, Any]: + entity_type = str(entity_type or "").strip().lower() + identifier = str(identifier or "").strip().upper() + normalized_date = normalize_date(trade_date) + if entity_type not in {"sector", "theme", "index"}: + raise ValueError("搜索详情类型不支持。") + if not re.fullmatch(r"[A-Z0-9.]{3,24}", identifier): + raise ValueError("搜索详情标识无效。") + if not self.configured: + raise ValueError("行情数据源尚未配置。") + + if entity_type == "index": + index_basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None) + if not index_basic: + raise ValueError("暂不支持该指数详情。") + return self._index_search_detail(index_basic, normalized_date) + + directory = self._search_market_directory() + basic = next( + ( + item for item in directory + if item.get("id") == identifier and item.get("type") == entity_type + ), + None, + ) + if not basic: + raise ValueError("未找到对应的板块或题材。") + return self._ths_search_detail(basic, normalized_date) + + def get_intraday_chart( + self, entity_type: str, identifier: str + ) -> dict[str, Any]: + entity_type = str(entity_type or "").strip().lower() + identifier = str(identifier or "").strip().upper() + if entity_type == "stock": + code = validate_stock_code(identifier) + chart = self.chart_data.stock_intraday(code) + type_label = SEARCH_TYPE_LABELS["stock"] + elif entity_type == "index": + basic = next((item for item in SEARCH_INDEXES if item["id"] == identifier), None) + if not basic: + raise ValueError("暂不支持该指数分时行情。") + chart = self.chart_data.index_intraday(identifier) + type_label = SEARCH_TYPE_LABELS["index"] + elif entity_type in {"sector", "theme"}: + basic = next( + ( + item for item in self._search_market_directory() + if item.get("id") == identifier and item.get("type") == entity_type + ), + None, + ) + if not basic: + raise ValueError("未找到对应的板块或题材。") + chart = self.chart_data.board_intraday(identifier, str(basic.get("name") or "")) + type_label = SEARCH_TYPE_LABELS[entity_type] + else: + raise ValueError("分时行情类型不支持。") + + return { + "meta": { + "trade_date": str(chart.get("trade_date") or ""), + "previous_close": float(chart.get("previous_close") or 0), + }, + "entity": { + "id": identifier, + "code": str(chart.get("code") or identifier), + "name": str(chart.get("name") or ""), + "type": entity_type, + "type_label": type_label, + }, + "points": list(chart.get("points") or []), + } + + def _ths_search_detail( + self, basic: dict[str, Any], trade_date: str + ) -> dict[str, Any]: + client = self._tushare_client() + resolved_date, _ = client.resolve_trade_context(trade_date) + end = datetime.strptime(resolved_date, "%Y%m%d") + start_date = (end - timedelta(days=190)).strftime("%Y%m%d") + identifier = str(basic["id"]) + snapshot = client.sector_snapshot(identifier, resolved_date) + rows = client.query( + "ths_daily", + {"ts_code": identifier, "start_date": start_date, "end_date": resolved_date}, + "ts_code,trade_date,open,high,low,close,pct_change,vol,turnover_rate,total_mv,float_mv", + ) + rows.sort(key=lambda item: str(item.get("trade_date") or "")) + series = [ + { + "trade_date": self._display_compact_date(str(row.get("trade_date") or "")), + "open": float(row.get("open") or 0), + "high": float(row.get("high") or 0), + "low": float(row.get("low") or 0), + "close": float(row.get("close") or 0), + "change": float(row.get("pct_change") or 0), + "volume": float(row.get("vol") or 0), + "turnover_rate": float(row.get("turnover_rate") or 0), + } + for row in rows[-90:] + ] + try: + chart_series = self.chart_data.board_daily(identifier, resolved_date, 90) + if chart_series: + series = chart_series + except (AttributeError, ChartDataError): + pass + latest = series[-1] if series else {} + snapshot_is_current = str(snapshot.get("trade_date") or "").replace("-", "") == resolved_date + change = float( + snapshot.get("change") + if snapshot_is_current and snapshot.get("change") is not None + else latest.get("change") or 0 + ) + if latest.get("realtime"): + change = float(latest.get("change") or 0) + turnover_rate = float( + snapshot.get("turnover_rate") + if snapshot_is_current and snapshot.get("turnover_rate") is not None + else latest.get("turnover_rate") or 0 + ) + metrics = [ + {"label": "涨跌幅", "value": round(change, 2), "unit": "%", "tone": "change"}, + {"label": "换手率", "value": round(turnover_rate, 2), "unit": "%"}, + {"label": "成份数量", "value": int(float(basic.get("member_count") or 0)), "unit": "只"}, + ] + up_count = int(float(snapshot.get("up_count") or 0)) + down_count = int(float(snapshot.get("down_count") or 0)) + if up_count or down_count: + metrics.extend( + [ + {"label": "上涨家数", "value": up_count, "unit": "家"}, + {"label": "下跌家数", "value": down_count, "unit": "家"}, + ] + ) + leader = str(snapshot.get("leader") or "").strip() + if leader and leader != "--": + metrics.extend( + [ + {"label": "领涨标的", "value": leader, "unit": ""}, + {"label": "领涨幅", "value": round(float(snapshot.get("leading_pct") or 0), 2), "unit": "%", "tone": "change"}, + ] + ) + return { + "meta": { + "trade_date": self._display_compact_date(resolved_date), + "realtime": bool(snapshot.get("realtime")), + }, + "entity": { + "id": identifier, + "code": identifier, + "name": str(snapshot.get("name") or basic.get("name") or "--"), + "type": str(basic.get("type") or "sector"), + "type_label": SEARCH_TYPE_LABELS[str(basic.get("type") or "sector")], + "subtitle": str(basic.get("subtitle") or ""), + "value": float(latest.get("close") or 0), + "change": change, + }, + "series": series, + "metrics": metrics, + } + + def _index_search_detail( + self, basic: dict[str, Any], trade_date: str + ) -> dict[str, Any]: + client = self._tushare_client() + resolved_date, _ = client.resolve_trade_context(trade_date) + payload = ( + client.realtime_market_indices(resolved_date) + if client.should_use_realtime(trade_date, resolved_date) + else client.market_indices(resolved_date, 90) + ) + current = next( + (item for item in payload.get("indices") or [] if item.get("ts_code") == basic["id"]), + None, + ) + if not current: + raise ValueError("该指数暂无可用行情。") + end = datetime.strptime(resolved_date, "%Y%m%d") + rows = client.query( + "index_daily", + { + "ts_code": basic["id"], + "start_date": (end - timedelta(days=190)).strftime("%Y%m%d"), + "end_date": resolved_date, + }, + "ts_code,trade_date,open,high,low,close,pct_chg,vol,amount", + ) + rows.sort(key=lambda item: str(item.get("trade_date") or "")) + series = [ + { + "trade_date": self._display_compact_date(str(row.get("trade_date") or "")), + "open": float(row.get("open") or 0), + "high": float(row.get("high") or 0), + "low": float(row.get("low") or 0), + "close": float(row.get("close") or 0), + "change": float(row.get("pct_chg") or 0), + "volume": float(row.get("vol") or 0), + } + for row in rows[-90:] + ] + try: + chart_series = self.chart_data.index_daily(str(basic["id"]), resolved_date, 90) + if chart_series: + series = chart_series + except (AttributeError, ChartDataError): + pass + latest = series[-1] if series else {} + latest_close = float(latest.get("close") or current.get("close") or 0) + latest_change = float(latest.get("change") or current.get("pct_chg") or 0) + + def series_return(days: int) -> float: + if len(series) <= days: + return 0.0 + previous = float(series[-days - 1].get("close") or 0) + return (latest_close / previous - 1) * 100 if previous > 0 else 0.0 + return { + "meta": { + "trade_date": self._display_compact_date(str(current.get("trade_date") or resolved_date)), + "realtime": bool(payload.get("realtime")), + }, + "entity": { + **basic, + "type_label": SEARCH_TYPE_LABELS["index"], + "value": latest_close, + "change": latest_change, + }, + "series": series, + "metrics": [ + {"label": "涨跌幅", "value": round(latest_change, 2), "unit": "%", "tone": "change"}, + {"label": "近5日", "value": round(series_return(5), 2), "unit": "%", "tone": "change"}, + {"label": "近20日", "value": round(series_return(20), 2), "unit": "%", "tone": "change"}, + {"label": "成交额", "value": round(float(current.get("amount_billion") or 0), 2), "unit": "亿"}, + ], + } + + def get_stock_detail( + self, code: str, trade_date: str, force: bool = False + ) -> dict[str, Any]: + code = validate_stock_code(code) + normalized_date = normalize_date(trade_date) + cache_key = f"{code}:{normalized_date}" + if not force: + cached = self.database.get_data_snapshot("stock_detail", cache_key) + if cached and str((cached.get("meta") or {}).get("source") or "") != "demo": + if not self._stock_detail_cache_needs_refresh(cached, normalized_date): + cached["meta"] = {**cached.get("meta", {}), "cached": True} + return self._prepare_stock_detail(cached, code, normalized_date) + + name, sector = self._stock_identity(code, normalized_date) + source = "tushare" + if self.configured: + try: + payload = self._tushare_client().stock_detail( + tushare_code(code), normalized_date + ) + if not payload.get("prices"): + raise TushareError("No price history returned") + except TushareError as exc: + payload = self.database.get_latest_data_snapshot( + "stock_detail", f"{code}:", cache_key, exclude_source="demo" + ) + if not payload: + raise ValueError(f"暂无 {code} 的真实行情数据：{exc}") from exc + payload = copy.deepcopy(payload) + payload["meta"] = { + **payload.get("meta", {}), + "cached": True, + "notice": "最新行情暂不可用，已沿用最近真实收盘数据。", + } + return self._prepare_stock_detail(payload, code, normalized_date) + else: + payload = self.database.get_latest_data_snapshot( + "stock_detail", f"{code}:", cache_key, exclude_source="demo" + ) + if not payload: + raise ValueError(f"暂无 {code} 的真实行情数据，请等待后台完成首次同步。") + payload = copy.deepcopy(payload) + payload["meta"] = { + **payload.get("meta", {}), + "cached": True, + "notice": "公共行情尚未配置，已沿用最近真实收盘数据。", + } + return self._prepare_stock_detail(payload, code, normalized_date) + payload["meta"]["source"] = source + payload["meta"]["cached"] = False + self.database.save_data_snapshot("stock_detail", cache_key, source, payload) + return self._prepare_stock_detail(payload, code, normalized_date) + + @staticmethod + def _stock_detail_bar_date(payload: dict[str, Any]) -> str: + prices = list(payload.get("prices") or []) + return str((prices[-1] if prices else {}).get("trade_date") or "").replace("-", "") + + def _stock_detail_cache_needs_refresh( + self, payload: dict[str, Any], requested_date: str + ) -> bool: + now = datetime.now().astimezone() + return ( + requested_date == now.strftime("%Y%m%d") + and now.time().replace(tzinfo=None) >= dt_time(15, 0) + and self._stock_detail_bar_date(payload) < requested_date + ) + + def _prepare_stock_detail( + self, payload: dict[str, Any], code: str, requested_date: str + ) -> dict[str, Any]: + result = copy.deepcopy(payload) + now = datetime.now().astimezone() + try: + result["prices"] = self.chart_data.stock_daily(code, requested_date, 90) + result["meta"] = {**(result.get("meta") or {}), "chart_source": "market_chart"} + except (AttributeError, ChartDataError): + pass + result = self._sanitize_stock_detail_prices(result, now) + actual_date = self._stock_detail_bar_date(result) + if actual_date: + result["meta"] = { + **(result.get("meta") or {}), + "trade_date": f"{actual_date[:4]}-{actual_date[4:6]}-{actual_date[6:]}", + } + today = now.strftime("%Y%m%d") + should_merge = ( + requested_date == today + and actual_date <= today + and now.weekday() < 5 + and now.time().replace(tzinfo=None) >= dt_time(9, 30) + ) + if should_merge: + quote = self._ifind_realtime_stock_quote(code) + if quote and self._valid_realtime_stock_quote(quote, today): + self._merge_realtime_stock_detail(result, quote, requested_date) + elif self.configured and actual_date < today: + client = self._tushare_client() + try: + resolved_date, _ = client.resolve_trade_context(requested_date) + if resolved_date == today: + quote = client.realtime_stock_quote(tushare_code(code), requested_date) + if self._valid_realtime_stock_quote(quote, today): + self._merge_realtime_stock_detail(result, quote, requested_date) + except TushareError: + pass + return self._enrich_stock_detail(result) + + @staticmethod + def _sanitize_stock_detail_prices( + payload: dict[str, Any], market_now: datetime + ) -> dict[str, Any]: + result = copy.deepcopy(payload) + raw_prices = list(result.get("prices") or []) + raw_latest_date = str( + (raw_prices[-1] if raw_prices else {}).get("trade_date") or "" + ).replace("-", "") + prices = [] + for bar in raw_prices: + open_price = float(bar.get("open") or 0) + high = float(bar.get("high") or 0) + low = float(bar.get("low") or 0) + close = float(bar.get("close") or 0) + if ( + open_price > 0 + and high >= max(open_price, close) + and 0 < low <= min(open_price, close) + and close > 0 + ): + prices.append(bar) + + today = market_now.strftime("%Y%m%d") + market_open = ( + market_now.weekday() < 5 + and market_now.time().replace(tzinfo=None) >= dt_time(9, 30) + ) + if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == today: + current = prices[-1] + has_market_activity = ( + float(current.get("volume") or 0) > 0 + or float(current.get("amount_billion") or 0) > 0 + ) + if not market_open or not has_market_activity: + prices.pop() + + if raw_latest_date == today and ( + not prices + or str(prices[-1].get("trade_date") or "").replace("-", "") != today + ): + result["meta"] = {**(result.get("meta") or {}), "realtime": False} + + result["prices"] = prices + if prices: + latest = prices[-1] + stock = dict(result.get("stock") or {}) + stock.update( + { + "price": float(latest.get("close") or 0), + "change": float(latest.get("change") or 0), + "amount_billion": float(latest.get("amount_billion") or 0), + } + ) + result["stock"] = stock + return result + + @staticmethod + def _valid_realtime_stock_quote(quote: dict[str, Any], trade_date: str) -> bool: + price = float(quote.get("price") or 0) + open_price = float(quote.get("open") or 0) + high = float(quote.get("high") or 0) + low = float(quote.get("low") or 0) + volume = float(quote.get("volume") or 0) + amount = float(quote.get("amount_billion") or 0) + quote_date = str(quote.get("quote_time") or "")[:10].replace("-", "") + return ( + price > 0 + and open_price > 0 + and high >= max(open_price, price) + and 0 < low <= min(open_price, price) + and (volume > 0 or amount > 0) + and (not quote_date or quote_date == trade_date) + ) + + def _ifind_realtime_stock_quote(self, code: str) -> dict[str, Any] | None: + ifind = getattr(self, "ifind", None) + if not ifind or not ifind.configured: + return None + try: + rows = ifind.real_time( + tushare_code(code), + [ + "open", "high", "low", "latest", "preClose", + "volume", "amount", "turnoverRatio", + ], + cache_ttl=10, + ) + except IfindError: + return None + row = rows[0] if rows else {} + price = float(row.get("latest") or 0) + previous_close = float(row.get("preClose") or 0) + if price <= 0: + return None + change = (price / previous_close - 1) * 100 if previous_close > 0 else 0.0 + stock = self._stock_identity(code, date.today().strftime("%Y%m%d")) + return { + "name": stock[0], + "sector": stock[1], + "price": price, + "open": float(row.get("open") or price), + "high": float(row.get("high") or price), + "low": float(row.get("low") or price), + "change": round(change, 4), + "volume": float(row.get("volume") or 0), + "volume_unit": "lots", + "amount_billion": float(row.get("amount") or 0) / 100_000_000, + "turnover_rate": float(row.get("turnoverRatio") or 0), + "quote_time": str(row.get("time") or ""), + } + + @staticmethod + def _merge_realtime_stock_detail( + payload: dict[str, Any], quote: dict[str, Any], trade_date: str + ) -> None: + display_date = f"{trade_date[:4]}-{trade_date[4:6]}-{trade_date[6:]}" + realtime_bar = { + "trade_date": display_date, + "open": quote["open"], + "high": quote["high"], + "low": quote["low"], + "close": quote["price"], + "change": quote["change"], + "volume": quote["volume"] if quote.get("volume_unit") == "lots" else quote["volume"] / 100, + "amount_billion": quote["amount_billion"], + "realtime": True, + } + prices = list(payload.get("prices") or []) + if prices and str(prices[-1].get("trade_date") or "").replace("-", "") == trade_date: + prices[-1] = realtime_bar + else: + prices.append(realtime_bar) + payload["prices"] = prices[-90:] + stock = dict(payload.get("stock") or {}) + stock.update( + { + "name": quote["name"], + "industry": quote["sector"], + "price": quote["price"], + "change": quote["change"], + "amount_billion": quote["amount_billion"], + "turnover_rate": quote["turnover_rate"], + } + ) + payload["stock"] = stock + payload["meta"] = { + **(payload.get("meta") or {}), + "trade_date": display_date, + "realtime": True, + "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), + } + + def get_stock_preview( + self, code: str, trade_date: str, force: bool = False + ) -> dict[str, Any]: + code = validate_stock_code(code) + # Hover previews deliberately follow the latest market day, independent + # from the review date selected by the page. + detail = self.get_stock_detail(code, date.today().strftime("%Y%m%d"), force) + detail_meta = detail.get("meta") or {} + resolved_date = str(detail_meta.get("trade_date") or trade_date) + intraday_points: list[dict[str, Any]] = [] + intraday_status = "unavailable" + intraday_notice = "分时行情暂不可用。" + + intraday_trade_date = "" + intraday_previous_close = 0.0 + try: + intraday = self.chart_data.stock_intraday(code) + intraday_points = list(intraday.get("points") or []) + intraday_trade_date = str(intraday.get("trade_date") or "") + intraday_previous_close = float(intraday.get("previous_close") or 0) + if intraday_points: + intraday_status = "available" + intraday_notice = "" + else: + intraday_status = "empty" + intraday_notice = "最近交易日暂无分时数据。" + except ChartDataError: + intraday_status = "unavailable" + intraday_notice = "分时行情暂不可用，请稍后重试。" + + prices = list(detail.get("prices") or [])[-60:] + stock = dict(detail.get("stock") or {"code": code}) + realtime = bool(detail_meta.get("realtime")) + return { + "meta": { + "trade_date": resolved_date, + "source": detail_meta.get("source") or "unavailable", + "notice": detail_meta.get("notice") or "", + "intraday_status": intraday_status, + "intraday_notice": intraday_notice, + "intraday_trade_date": intraday_trade_date, + "intraday_previous_close": intraday_previous_close, + "realtime": realtime, + "refresh_interval_seconds": 10 if realtime else 0, + }, + "stock": stock, + "prices": prices, + "intraday": intraday_points, + } + + def backfill(self, start_date: str, end_date: str) -> list[dict[str, Any]]: + start = datetime.strptime(normalize_date(start_date), "%Y%m%d").date() + end = datetime.strptime(normalize_date(end_date), "%Y%m%d").date() + if start > end: + raise ValueError("开始日期不能晚于结束日期。") + weekdays = [] + current = start + while current <= end: + if current.weekday() < 5: + weekdays.append(current) + current += timedelta(days=1) + if len(weekdays) > 15: + raise ValueError("单次最多回补 15 个工作日。") + results = [] + for day in weekdays: + dashboard = self.sync_dashboard(day.strftime("%Y%m%d")) + results.append( + { + "requested_date": day.isoformat(), + "trade_date": dashboard["meta"]["trade_date"], + "source": dashboard["meta"]["source"], + "records": self._record_count(dashboard), + } + ) + return results + + def _stock_identity(self, code: str, trade_date: str) -> tuple[str, str]: + snapshot = self.database.get_snapshot(trade_date) or {} + for key in ("limits", "broken", "down_limits"): + for row in snapshot.get(key) or []: + if str(row.get("code")) == code: + return row.get("name") or "--", row.get("sector") or "其他" + for item in self.database.list_watchlist(self.current_user_id): + if item["code"] == code: + return item["name"], item["sector"] or "其他" + return "--", "其他" + + def _enrich_stock_detail(self, payload: dict[str, Any]) -> dict[str, Any]: + result = dict(payload) + stock = dict(payload.get("stock") or {}) + code = str(stock.get("code") or "") + watched = { + item["code"]: item + for item in self.database.list_watchlist(self.current_user_id) + } + stock["watchlist"] = watched.get(code) + result["stock"] = stock + result["notes"] = self.database.list_notes(self.current_user_id, code=code) + return result + + def _with_storage(self, dashboard: dict[str, Any], cached: bool) -> dict[str, Any]: + result = dict(dashboard) + result["meta"] = { + **dashboard.get("meta", {}), + "storage": "sqlite", + "cached": cached, + } + return result + + @staticmethod + def _record_count(dashboard: dict[str, Any]) -> int: + return sum( + len(dashboard.get(key) or []) + for key in ("limits", "broken", "down_limits", "yesterday_limits") + ) + diff --git a/chart_data_provider.py b/chart_data_provider.py index 1259cc5..f07b035 100644 --- a/chart_data_provider.py +++ b/chart_data_provider.py @@ -1,497 +1,7 @@ -from __future__ import annotations +"""Compatibility alias for the canonical market chart clients.""" -import http.client -import json -import re -import time -import urllib.error -import urllib.parse -import urllib.request -from dataclasses import dataclass -from datetime import datetime, time as dt_time, timedelta -from threading import Lock -from typing import Any, ClassVar +import sys -from ifind_client import IfindError, IfindHttpClient +from backend.features.market import charts as _implementation - -class ChartDataError(RuntimeError): - pass - - -TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get" -BOARD_LIST_URL = "https://push2delay.eastmoney.com/api/qt/clist/get" -BROWSER_USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/138.0.0.0 Safari/537.36" -) -INDEX_SECIDS = { - "000001.SH": "1.000001", - "399001.SZ": "0.399001", - "399006.SZ": "0.399006", -} - - -class MarketChartClient: - """Prefer iFinD for display charts and retain Eastmoney as a last resort.""" - - def __init__(self, ifind: IfindHttpClient, fallback: "EastmoneyChartClient") -> None: - self.ifind = ifind - self.fallback = fallback - - def stock_intraday(self, code: str) -> dict[str, Any]: - normalized = str(code or "").strip() - if not re.fullmatch(r"\d{6}", normalized): - raise ChartDataError("Invalid stock code") - ifind_code = _stock_market_code(normalized) - try: - return self._ifind_intraday(ifind_code, "stock", normalized) - except (IfindError, ChartDataError): - return self.fallback.stock_intraday(normalized) - - def stock_daily(self, code: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]: - normalized = str(code or "").strip() - if not re.fullmatch(r"\d{6}", normalized): - raise ChartDataError("Invalid stock code") - return self._ifind_daily(_stock_market_code(normalized), end_date, limit) - - def index_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]: - normalized = str(identifier or "").strip().upper() - if normalized not in INDEX_SECIDS: - raise ChartDataError("Unsupported index") - return self._ifind_daily(normalized, end_date, limit) - - def board_daily(self, identifier: str, end_date: str, limit: int = 90) -> list[dict[str, Any]]: - normalized = str(identifier or "").strip().upper() - if not normalized: - raise ChartDataError("Invalid board code") - return self._ifind_daily(normalized, end_date, limit) - - def index_intraday(self, identifier: str) -> dict[str, Any]: - normalized = str(identifier or "").strip().upper() - if normalized not in INDEX_SECIDS: - raise ChartDataError("Unsupported index") - try: - return self._ifind_intraday(normalized, "index", normalized) - except (IfindError, ChartDataError): - return self.fallback.index_intraday(normalized) - - def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]: - normalized = str(identifier or "").strip().upper() - try: - return self._ifind_intraday(normalized, "board", normalized, name) - except (IfindError, ChartDataError): - return self.fallback.board_intraday(normalized, name) - - def _ifind_intraday( - self, - ifind_code: str, - entity_type: str, - identifier: str, - name: str = "", - ) -> dict[str, Any]: - if not self.ifind.configured: - raise ChartDataError("iFinD is not configured") - now = datetime.now().astimezone() - rows: list[dict[str, Any]] = [] - for offset in range(0, 8): - candidate = now.date() - timedelta(days=offset) - if candidate.weekday() >= 5: - continue - display_date = candidate.isoformat() - rows = self.ifind.intraday( - ifind_code, - f"{display_date} 09:30:00", - f"{display_date} 15:00:00", - cache_ttl=20 if offset == 0 else 6 * 60 * 60, - ) - if rows: - break - points = [point for row in rows if (point := _ifind_point(row))] - if not points: - raise ChartDataError("No iFinD intraday chart data returned") - latest_date = points[-1]["date"] - points = [point for point in points if point["date"] == latest_date] - previous_close = self._previous_close(ifind_code, latest_date, points[0]["open"]) - return { - "entity_type": entity_type, - "identifier": identifier, - "name": name, - "code": identifier, - "trade_date": latest_date, - "previous_close": previous_close, - "points": points, - "source": "ifind", - } - - def _ifind_daily( - self, ifind_code: str, end_date: str, limit: int - ) -> list[dict[str, Any]]: - if not self.ifind.configured: - raise ChartDataError("iFinD is not configured") - compact_end = str(end_date or "").replace("-", "") - if not re.fullmatch(r"\d{8}", compact_end): - raise ChartDataError("Invalid chart end date") - end = datetime.strptime(compact_end, "%Y%m%d") - start = (end - timedelta(days=max(190, limit * 3))).strftime("%Y%m%d") - try: - rows = self.ifind.history( - ifind_code, - ["open", "high", "low", "close", "volume", "amount"], - start, - compact_end, - cache_ttl=300, - ) - except IfindError as exc: - raise ChartDataError("No iFinD daily chart data returned") from exc - normalized = [] - for row in rows: - stamp = str(row.get("time") or "").strip() - trade_date = stamp[:10] - close = _number(row.get("close")) - if not re.fullmatch(r"\d{4}-\d{2}-\d{2}", trade_date) or close <= 0: - continue - normalized.append( - { - "trade_date": trade_date, - "open": _number(row.get("open")), - "high": _number(row.get("high")), - "low": _number(row.get("low")), - "close": close, - "volume": _number(row.get("volume")), - "amount_billion": _number(row.get("amount")) / 100_000_000, - } - ) - normalized.sort(key=lambda row: row["trade_date"]) - for index, row in enumerate(normalized): - previous = normalized[index - 1]["close"] if index > 0 else 0 - row["change"] = round((row["close"] / previous - 1) * 100, 4) if previous else 0.0 - - market_now = datetime.now().astimezone() - today = market_now.strftime("%Y%m%d") - market_open = ( - market_now.weekday() < 5 - and market_now.time().replace(tzinfo=None) >= dt_time(9, 30) - ) - today_display = market_now.date().isoformat() - if normalized and normalized[-1]["trade_date"] == today_display: - current_bar = normalized[-1] - current_bar_is_valid = ( - current_bar["open"] > 0 - and current_bar["high"] >= max(current_bar["open"], current_bar["close"]) - and 0 < current_bar["low"] <= min(current_bar["open"], current_bar["close"]) - and (current_bar["volume"] > 0 or current_bar["amount_billion"] > 0) - ) - if not market_open or not current_bar_is_valid: - normalized.pop() - if compact_end == today and market_open: - try: - quote_rows = self.ifind.real_time( - ifind_code, - ["open", "high", "low", "latest", "preClose", "volume", "amount"], - cache_ttl=10, - ) - quote = quote_rows[0] if quote_rows else {} - latest = _number(quote.get("latest")) - previous = _number(quote.get("preClose")) - open_price = _number(quote.get("open")) - high = _number(quote.get("high")) - low = _number(quote.get("low")) - volume = _number(quote.get("volume")) - amount = _number(quote.get("amount")) - quote_date = str(quote.get("time") or "")[:10].replace("-", "") - quote_is_current = not quote_date or quote_date == today - has_market_activity = volume > 0 or amount > 0 - if ( - latest > 0 - and open_price > 0 - and high >= max(open_price, latest) - and 0 < low <= min(open_price, latest) - and has_market_activity - and quote_is_current - ): - realtime = { - "trade_date": end.strftime("%Y-%m-%d"), - "open": open_price, - "high": high, - "low": low, - "close": latest, - "change": round((latest / previous - 1) * 100, 4) if previous else 0.0, - "volume": volume, - "amount_billion": amount / 100_000_000, - "realtime": True, - } - if normalized and normalized[-1]["trade_date"] == realtime["trade_date"]: - normalized[-1] = realtime - else: - normalized.append(realtime) - except IfindError: - pass - if not normalized: - raise ChartDataError("No iFinD daily chart data returned") - return normalized[-max(20, min(180, int(limit))):] - - def _previous_close(self, code: str, trade_date: str, fallback: float) -> float: - today = datetime.now().astimezone().date().isoformat() - if trade_date == today: - try: - quote = self.ifind.real_time(code, ["preClose"], cache_ttl=20) - value = _number((quote[0] if quote else {}).get("preClose")) - if value > 0: - return value - except IfindError: - pass - end = datetime.strptime(trade_date, "%Y-%m-%d") - try: - rows = self.ifind.history( - code, - ["close"], - (end - timedelta(days=12)).strftime("%Y%m%d"), - end.strftime("%Y%m%d"), - cache_ttl=6 * 60 * 60, - ) - closes = [_number(row.get("close")) for row in rows if _number(row.get("close")) > 0] - if len(closes) >= 2: - return closes[-2] - except IfindError: - pass - return fallback - - -@dataclass -class EastmoneyChartClient: - """Isolated display-only minute chart source. - - The returned data must not be used by market snapshots, scoring, screening, - or divination. Its only consumer is a chart-rendering endpoint. - """ - - timeout: int = 6 - cache_ttl_seconds: int = 20 - retry_attempts: int = 2 - _cache: ClassVar[dict[str, dict[str, Any]]] = {} - _cache_lock: ClassVar[Lock] = Lock() - _board_catalog: ClassVar[dict[str, dict[str, str]]] = {} - _board_catalog_at: ClassVar[float] = 0.0 - _board_catalog_lock: ClassVar[Lock] = Lock() - - def stock_intraday(self, code: str) -> dict[str, Any]: - normalized = str(code or "").strip() - if not re.fullmatch(r"\d{6}", normalized): - raise ChartDataError("Invalid stock code") - market = "1" if normalized.startswith(("5", "6", "9")) else "0" - return self._intraday(f"{market}.{normalized}", "stock", normalized) - - def index_intraday(self, identifier: str) -> dict[str, Any]: - normalized = str(identifier or "").strip().upper() - secid = INDEX_SECIDS.get(normalized) - if not secid: - raise ChartDataError("Unsupported index") - return self._intraday(secid, "index", normalized) - - def board_intraday(self, identifier: str, name: str = "") -> dict[str, Any]: - normalized = str(identifier or "").strip().upper() - if re.fullmatch(r"BK\d{4}", normalized): - board_code = normalized - else: - board_code = self._resolve_board_code(name or identifier) - return self._intraday(f"90.{board_code}", "board", board_code) - - def _intraday(self, secid: str, entity_type: str, identifier: str) -> dict[str, Any]: - cache_key = f"{entity_type}:{identifier}" - cached = self._get_cached(cache_key) - if cached is not None: - return cached - - payload = self._request_json( - TRENDS_URL, - { - "secid": secid, - "fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13", - "fields2": "f51,f52,f53,f54,f55,f56,f57,f58", - "iscr": "0", - "ndays": "1", - }, - "https://quote.eastmoney.com/", - ) - data = payload.get("data") or {} - points = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))] - if not points: - raise ChartDataError("No intraday chart data returned") - - result = { - "entity_type": entity_type, - "identifier": identifier, - "name": str(data.get("name") or ""), - "code": str(data.get("code") or identifier), - "trade_date": points[-1]["date"], - "previous_close": _number(data.get("preClose")), - "points": points, - } - with self._cache_lock: - self._cache[cache_key] = {"created_at": time.time(), "payload": result} - return result - - def _get_cached(self, cache_key: str) -> dict[str, Any] | None: - with self._cache_lock: - cached = self._cache.get(cache_key) - if not cached: - return None - if time.time() - float(cached.get("created_at") or 0) > self.cache_ttl_seconds: - with self._cache_lock: - self._cache.pop(cache_key, None) - return None - return dict(cached["payload"]) - - def _resolve_board_code(self, name: str) -> str: - normalized = _normalize_name(name) - if not normalized: - raise ChartDataError("Board name is required") - catalog = self._load_board_catalog() - item = catalog.get(normalized) - if not item: - raise ChartDataError("No matching chart board") - return item["code"] - - def _load_board_catalog(self) -> dict[str, dict[str, str]]: - now = time.time() - with self._board_catalog_lock: - if self._board_catalog and now - self._board_catalog_at < 6 * 60 * 60: - return dict(self._board_catalog) - - rows: list[dict[str, Any]] = [] - for board_type in ("1", "2", "3"): - for page in range(1, 6): - payload = self._request_json( - BOARD_LIST_URL, - { - "pn": str(page), - "pz": "100", - "po": "1", - "np": "1", - "fltt": "2", - "invt": "2", - "fid": "f3", - "fs": f"m:90+t:{board_type}", - "fields": "f12,f14", - }, - "https://quote.eastmoney.com/center/boardlist.html", - ) - page_rows = (payload.get("data") or {}).get("diff") or [] - rows.extend(page_rows) - if len(page_rows) < 100: - break - - catalog: dict[str, dict[str, str]] = {} - for row in rows: - code = str(row.get("f12") or "").strip().upper() - board_name = str(row.get("f14") or "").strip() - if re.fullmatch(r"BK\d{4}", code) and board_name: - catalog.setdefault(_normalize_name(board_name), {"code": code, "name": board_name}) - if not catalog: - raise ChartDataError("Board chart directory is unavailable") - with self._board_catalog_lock: - type(self)._board_catalog = catalog - type(self)._board_catalog_at = now - return dict(catalog) - - def _request_json( - self, url: str, params: dict[str, str], referer: str - ) -> dict[str, Any]: - request_url = f"{url}?{urllib.parse.urlencode(params)}" - last_error: Exception | None = None - for attempt in range(max(1, int(self.retry_attempts))): - request = urllib.request.Request( - request_url, - headers={ - "Accept": "application/json,text/plain,*/*", - "Connection": "close", - "Referer": referer, - "User-Agent": BROWSER_USER_AGENT, - }, - ) - try: - with urllib.request.urlopen(request, timeout=self.timeout) as response: - payload = json.loads(response.read().decode("utf-8")) - if not isinstance(payload, dict): - raise ChartDataError("Invalid intraday chart response") - return payload - except ( - urllib.error.URLError, - TimeoutError, - ConnectionError, - OSError, - http.client.HTTPException, - json.JSONDecodeError, - ChartDataError, - ) as exc: - last_error = exc - if attempt + 1 < self.retry_attempts: - time.sleep(0.12) - raise ChartDataError("Intraday chart request failed") from last_error - - -def _parse_trend(raw: Any) -> dict[str, Any] | None: - fields = str(raw or "").split(",") - if len(fields) < 8 or " " not in fields[0]: - return None - stamp = fields[0].strip() - trade_date, trade_time = stamp.split(" ", 1) - close = _number(fields[2]) - if close <= 0: - return None - return { - "date": trade_date, - "time": trade_time[:5], - "open": _number(fields[1]), - "close": close, - "high": _number(fields[3]), - "low": _number(fields[4]), - "volume": _number(fields[5]), - "amount": _number(fields[6]), - "average": _number(fields[7]), - } - - -def _ifind_point(row: dict[str, Any]) -> dict[str, Any] | None: - stamp = str(row.get("time") or "").strip() - if " " not in stamp: - return None - trade_date, trade_time = stamp.split(" ", 1) - close = _number(row.get("close")) - if close <= 0: - return None - return { - "date": trade_date, - "time": trade_time[:5], - "open": _number(row.get("open")), - "close": close, - "high": _number(row.get("high")), - "low": _number(row.get("low")), - "volume": _number(row.get("volume")), - "amount": _number(row.get("amount")), - "average": _number(row.get("avgPrice")), - } - - -def _stock_market_code(code: str) -> str: - if code.startswith(("4", "8", "9")): - suffix = "BJ" - elif code.startswith("6"): - suffix = "SH" - else: - suffix = "SZ" - return f"{code}.{suffix}" - - -def _number(value: Any) -> float: - try: - return float(value or 0) - except (TypeError, ValueError): - return 0.0 - - -def _normalize_name(value: Any) -> str: - normalized = re.sub(r"[\s·・（）()\-_/]", "", str(value or "")).casefold() - return re.sub(r"(?:概念|行业|[ⅠⅡⅢ])$", "", normalized) +sys.modules[__name__] = _implementation diff --git a/database.py b/database.py index 9f23746..9a3b60b 100644 --- a/database.py +++ b/database.py @@ -8,6 +8,7 @@ from typing import Any from backend.database import MIGRATIONS, MigrationRunner, SQLiteConnectionFactory from backend.features.accounts.repository import AccountRepositoryMixin +from backend.features.market.repository import MarketRepositoryMixin from backend.features.system.repository import SystemSettingsRepositoryMixin @@ -20,7 +21,11 @@ def _optional_float(value: Any) -> float | None: return None -class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): +class ReviewDatabase( + AccountRepositoryMixin, + MarketRepositoryMixin, + SystemSettingsRepositoryMixin, +): def __init__(self, path: Path) -> None: self.path = path self.path.parent.mkdir(parents=True, exist_ok=True) @@ -690,118 +695,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): ).fetchone() return int(row["total"] if row else 0) - def get_snapshot(self, trade_date: str) -> dict[str, Any] | None: - with self.connect() as connection: - row = connection.execute( - "SELECT payload FROM dashboard_snapshots WHERE trade_date = ?", - (trade_date,), - ).fetchone() - if not row: - return None - try: - return json.loads(row["payload"]) - except json.JSONDecodeError: - return None - - def get_latest_real_snapshot( - self, trade_date: str, strictly_before: bool = False - ) -> dict[str, Any] | None: - operator = "<" if strictly_before else "<=" - with self.connect() as connection: - row = connection.execute( - f""" - SELECT payload FROM dashboard_snapshots - WHERE trade_date {operator} ? AND source != 'demo' - ORDER BY trade_date DESC LIMIT 1 - """, - (trade_date,), - ).fetchone() - if not row: - return None - try: - return json.loads(row["payload"]) - except json.JSONDecodeError: - return None - - def save_snapshot(self, trade_date: str, source: str, payload: dict[str, Any]) -> None: - updated_at = datetime.now().astimezone().isoformat(timespec="seconds") - record_count = sum( - len(payload.get(key) or []) - for key in ("limits", "broken", "down_limits", "yesterday_limits") - ) - content = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) - with self.connect() as connection: - connection.execute( - """ - INSERT INTO dashboard_snapshots - (trade_date, source, payload, record_count, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(trade_date) DO UPDATE SET - source = excluded.source, - payload = excluded.payload, - record_count = excluded.record_count, - updated_at = excluded.updated_at - """, - (trade_date, source, content, record_count, updated_at), - ) - - def get_data_snapshot(self, kind: str, cache_key: str) -> dict[str, Any] | None: - with self.connect() as connection: - row = connection.execute( - "SELECT payload FROM data_snapshots WHERE kind = ? AND cache_key = ?", - (kind, cache_key), - ).fetchone() - if not row: - return None - try: - return json.loads(row["payload"]) - except json.JSONDecodeError: - return None - - def get_latest_data_snapshot( - self, - kind: str, - cache_key_prefix: str, - maximum_cache_key: str, - exclude_source: str = "", - ) -> dict[str, Any] | None: - source_clause = " AND source != ?" if exclude_source else "" - parameters: list[Any] = [kind, f"{cache_key_prefix}%", maximum_cache_key] - if exclude_source: - parameters.append(exclude_source) - with self.connect() as connection: - row = connection.execute( - f""" - SELECT payload FROM data_snapshots - WHERE kind = ? AND cache_key LIKE ? AND cache_key <= ?{source_clause} - ORDER BY cache_key DESC LIMIT 1 - """, - parameters, - ).fetchone() - if not row: - return None - try: - return json.loads(row["payload"]) - except json.JSONDecodeError: - return None - - def save_data_snapshot( - self, kind: str, cache_key: str, source: str, payload: dict[str, Any] - ) -> None: - updated_at = datetime.now().astimezone().isoformat(timespec="seconds") - content = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) - with self.connect() as connection: - connection.execute( - """ - INSERT INTO data_snapshots (kind, cache_key, source, payload, updated_at) - VALUES (?, ?, ?, ?, ?) - ON CONFLICT(kind, cache_key) DO UPDATE SET - source = excluded.source, - payload = excluded.payload, - updated_at = excluded.updated_at - """, - (kind, cache_key, source, content, updated_at), - ) def list_watchlist(self, user_id: int) -> list[dict[str, Any]]: with self.connect() as connection: @@ -869,7 +762,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): (int(user_id), code), ) return cursor.rowcount > 0 - def list_notes( self, user_id: int, @@ -1042,26 +934,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): ) return len(values) - def search_stock_master(self, query: str, limit: int = 12) -> list[dict[str, Any]]: - text = str(query or "").strip() - if not text: - return [] - escaped = text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") - with self.connect() as connection: - rows = connection.execute( - """ - SELECT ts_code, code, name, industry, market, list_date - FROM stock_master - WHERE code = ? OR name = ? OR name LIKE ? ESCAPE '\\' - ORDER BY - CASE WHEN code = ? THEN 0 WHEN name = ? THEN 1 ELSE 2 END, - list_date DESC, - code - LIMIT ? - """, - (text, text, f"%{escaped}%", text, text, max(1, min(30, int(limit)))), - ).fetchall() - return [dict(row) for row in rows] def list_stock_master(self) -> list[dict[str, Any]]: with self.connect() as connection: @@ -1666,24 +1538,6 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): for row in series[-limit:] ] - def list_snapshot_payloads(self, end_date: str, limit: int = 260) -> list[dict[str, Any]]: - with self.connect() as connection: - rows = connection.execute( - """ - SELECT trade_date, payload FROM dashboard_snapshots - WHERE trade_date <= ? ORDER BY trade_date DESC LIMIT ? - """, - (end_date, limit), - ).fetchall() - result: list[dict[str, Any]] = [] - for row in reversed(rows): - try: - payload = json.loads(row["payload"]) - except json.JSONDecodeError: - continue - payload["_snapshot_date"] = row["trade_date"] - result.append(payload) - return result def save_screener_strategy( self, user_id: int | None, name: str, description: str, regimes: list[str], formula: dict[str, Any], @@ -2529,64 +2383,3 @@ class ReviewDatabase(AccountRepositoryMixin, SystemSettingsRepositoryMixin): (int(reading_id), int(user_id)), ) return cursor.rowcount > 0 - - def start_sync(self, trade_date: str, source: str) -> int: - started_at = datetime.now().astimezone().isoformat(timespec="seconds") - with self.connect() as connection: - cursor = connection.execute( - """ - INSERT INTO sync_runs (trade_date, source, status, started_at) - VALUES (?, ?, 'running', ?) - """, - (trade_date, source, started_at), - ) - return int(cursor.lastrowid) - - def finish_sync( - self, - sync_id: int, - status: str, - record_count: int = 0, - message: str = "", - source: str | None = None, - ) -> None: - finished_at = datetime.now().astimezone().isoformat(timespec="seconds") - with self.connect() as connection: - connection.execute( - """ - UPDATE sync_runs - SET status = ?, finished_at = ?, record_count = ?, message = ?, - source = COALESCE(?, source) - WHERE id = ? - """, - (status, finished_at, record_count, message[:1000], source, sync_id), - ) - - def status(self) -> dict[str, Any]: - with self.connect() as connection: - last_sync = connection.execute( - """ - SELECT id, trade_date, source, status, started_at, finished_at, - record_count, message - FROM sync_runs ORDER BY id DESC LIMIT 1 - """ - ).fetchone() - snapshot_stats = connection.execute( - """ - SELECT COUNT(*) AS dates, COALESCE(SUM(record_count), 0) AS records, - MAX(updated_at) AS updated_at - FROM dashboard_snapshots - """ - ).fetchone() - watchlist_count = connection.execute("SELECT COUNT(*) FROM watchlist").fetchone()[0] - note_count = connection.execute("SELECT COUNT(*) FROM review_notes").fetchone()[0] - - return { - "database": str(self.path.name), - "snapshot_dates": int(snapshot_stats["dates"]), - "snapshot_records": int(snapshot_stats["records"]), - "updated_at": snapshot_stats["updated_at"], - "last_sync": dict(last_sync) if last_sync else None, - "watchlist_count": int(watchlist_count), - "note_count": int(note_count), - } diff --git a/ifind_client.py b/ifind_client.py index 2b30fd9..e6cb99e 100644 --- a/ifind_client.py +++ b/ifind_client.py @@ -1,385 +1,7 @@ -from __future__ import annotations +"""Compatibility alias for the canonical iFinD provider implementation.""" -import copy -import json -import threading -import time -import urllib.error -import urllib.request -from datetime import datetime, timedelta -from typing import Any +import sys +from backend.data.providers import ifind_client as _implementation -class IfindError(RuntimeError): - pass - - -class IfindHttpClient: - BASE_URL = "https://quantapi.51ifind.com/api/v1" - AUTH_ENDPOINT = "get_access_token" - AUTH_ERROR_CODES = {-1302, -1303, -1304, -4302, -4303} - - def __init__( - self, - refresh_token: str = "", - access_token: str = "", - timeout: int = 15, - ) -> None: - self.timeout = max(3, int(timeout)) - self._refresh_token = str(refresh_token or "").strip() - self._access_token = str(access_token or "").strip() - self._access_expires_at: datetime | None = None - self._token_lock = threading.Lock() - self._cache_lock = threading.Lock() - self._cache: dict[str, dict[str, Any]] = {} - - @property - def configured(self) -> bool: - return bool(self._refresh_token or self._access_token) - - def set_credentials(self, refresh_token: str, access_token: str = "") -> None: - refresh_token = str(refresh_token or "").strip() - access_token = str(access_token or "").strip() - with self._token_lock: - refresh_changed = refresh_token != self._refresh_token - self._refresh_token = refresh_token - if access_token or refresh_changed: - self._access_token = access_token - self._access_expires_at = None - if refresh_changed: - with self._cache_lock: - self._cache.clear() - - def status(self) -> dict[str, Any]: - return { - "configured": self.configured, - "access_ready": bool(self._access_token), - "access_expires_at": ( - self._access_expires_at.isoformat(timespec="seconds") - if self._access_expires_at - else "" - ), - } - - def test_connection(self) -> dict[str, Any]: - payload = self.real_time( - "000001.SH", - ["open", "high", "low", "latest", "preClose"], - cache_ttl=0, - ) - return { - "ok": bool(payload), - "sample_time": str(payload[0].get("time") or "") if payload else "", - } - - def real_time( - self, - codes: str | list[str], - indicators: list[str], - cache_ttl: int = 10, - ) -> list[dict[str, Any]]: - code_text = self._codes(codes) - payload = self._request( - "real_time_quotation", - {"codes": code_text, "indicators": ",".join(indicators)}, - cache_key=f"rq:{code_text}:{','.join(indicators)}", - cache_ttl=cache_ttl, - ) - return self._table_rows(payload) - - def history( - self, - codes: str | list[str], - indicators: list[str], - start_date: str, - end_date: str, - cache_ttl: int = 300, - ) -> list[dict[str, Any]]: - code_text = self._codes(codes) - payload = self._request( - "cmd_history_quotation", - { - "codes": code_text, - "indicators": ",".join(indicators), - "startdate": self._display_date(start_date), - "enddate": self._display_date(end_date), - "functionpara": {"CPS": "forward1", "Fill": "Omit"}, - }, - cache_key=f"hq:{code_text}:{start_date}:{end_date}:{','.join(indicators)}", - cache_ttl=cache_ttl, - ) - return self._table_rows(payload) - - def intraday( - self, - code: str, - start_time: str, - end_time: str, - cache_ttl: int = 20, - ) -> list[dict[str, Any]]: - indicators = ["open", "high", "low", "close", "volume", "amount", "avgPrice"] - payload = self._request( - "high_frequency", - { - "codes": self._codes(code), - "indicators": ",".join(indicators), - "starttime": start_time, - "endtime": end_time, - "functionpara": { - "CPS": "forward1", - "Fill": "Previous", - "Timeformat": "LocalTime", - "Interval": "1", - "Limitstart": "09:30:00", - "Limitend": "15:00:00", - }, - }, - cache_key=f"hf:{code}:{start_time}:{end_time}", - cache_ttl=cache_ttl, - ) - return self._table_rows(payload) - - def snapshots( - self, - codes: str | list[str], - indicators: list[str], - start_time: str, - end_time: str, - cache_ttl: int = 8, - ) -> list[dict[str, Any]]: - code_text = self._codes(codes) - payload = self._request( - "snap_shot", - { - "codes": code_text, - "indicators": ",".join(indicators), - "starttime": start_time, - "endtime": end_time, - }, - cache_key=f"ss:{code_text}:{start_time}:{end_time}:{','.join(indicators)}", - cache_ttl=cache_ttl, - ) - return self._table_rows(payload) - - def wencai(self, query: str, search_type: str = "stock", cache_ttl: int = 300) -> list[dict[str, Any]]: - normalized = " ".join(str(query or "").split()) - if not normalized: - raise IfindError("问财查询不能为空。") - payload = self._request( - "smart_stock_picking", - {"searchstring": normalized, "searchtype": search_type}, - cache_key=f"wc:{search_type}:{normalized}", - cache_ttl=cache_ttl, - ) - return self._table_rows(payload) - - def report_query( - self, - codes: str | list[str], - begin_date: str, - end_date: str, - cache_ttl: int = 300, - ) -> list[dict[str, Any]]: - code_text = self._codes(codes) - payload = self._request( - "report_query", - { - "codes": code_text, - "beginrDate": self._display_date(begin_date), - "endrDate": self._display_date(end_date), - "outputpara": ( - "reportDate:Y,thscode:Y,secName:Y,ctime:Y," - "reportTitle:Y,pdfURL:Y,seq:Y" - ), - }, - cache_key=f"report:{code_text}:{begin_date}:{end_date}", - cache_ttl=cache_ttl, - ) - return self._table_rows(payload) - - def _request( - self, - endpoint: str, - body: dict[str, Any], - cache_key: str = "", - cache_ttl: int = 0, - ) -> dict[str, Any]: - if not self.configured: - raise IfindError("iFinD 尚未配置。") - if cache_key and cache_ttl > 0: - cached = self._cached(cache_key, cache_ttl) - if cached is not None: - return cached - - payload = self._post(endpoint, body, self._ensure_access_token()) - if self._is_auth_error(payload) and self._refresh_token: - self._invalidate_access_token() - payload = self._post(endpoint, body, self._ensure_access_token(force=True)) - self._validate_payload(payload) - if cache_key and cache_ttl > 0: - with self._cache_lock: - self._cache[cache_key] = { - "created_at": time.time(), - "payload": copy.deepcopy(payload), - } - return payload - - def _ensure_access_token(self, force: bool = False) -> str: - with self._token_lock: - now = datetime.now().astimezone().replace(tzinfo=None) - token_valid = bool(self._access_token) and ( - self._access_expires_at is None - or self._access_expires_at > now + timedelta(minutes=2) - ) - if token_valid and not force: - return self._access_token - if not self._refresh_token: - if self._access_token: - return self._access_token - raise IfindError("iFinD Refresh Token 尚未配置。") - payload = self._post(self.AUTH_ENDPOINT, {}, "", self._refresh_token) - self._validate_payload(payload) - data = payload.get("data") or {} - token = str(data.get("access_token") or "").strip() - if not token: - raise IfindError("iFinD 未返回 Access Token。") - expires_at = self._parse_datetime(data.get("expired_time")) - self._access_token = token - self._access_expires_at = expires_at - return token - - def _post( - self, - endpoint: str, - body: dict[str, Any], - access_token: str, - refresh_token: str = "", - ) -> dict[str, Any]: - headers = { - "Accept": "application/json", - "Content-Type": "application/json", - "User-Agent": "XiaobaiReviewWeb/1.0", - "ifindlang": "cn", - } - if access_token: - headers["access_token"] = access_token - if refresh_token: - headers["refresh_token"] = refresh_token - request = urllib.request.Request( - f"{self.BASE_URL}/{endpoint}", - data=json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), - headers=headers, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=self.timeout) as response: - payload = json.loads(response.read().decode("utf-8")) - except urllib.error.HTTPError as exc: - detail = "" - try: - detail_payload = json.loads(exc.read().decode("utf-8", errors="replace")) - detail = str(detail_payload.get("errmsg") or detail_payload.get("message") or "") - except (json.JSONDecodeError, OSError): - pass - raise IfindError(f"iFinD HTTP {exc.code}{f'：{detail[:160]}' if detail else ''}") from exc - except (urllib.error.URLError, TimeoutError, OSError, json.JSONDecodeError) as exc: - raise IfindError("iFinD 数据请求失败。") from exc - if not isinstance(payload, dict): - raise IfindError("iFinD 返回格式不正确。") - return payload - - def _cached(self, key: str, ttl: int) -> dict[str, Any] | None: - with self._cache_lock: - cached = self._cache.get(key) - if not cached: - return None - if time.time() - float(cached.get("created_at") or 0) > ttl: - self._cache.pop(key, None) - return None - return copy.deepcopy(cached["payload"]) - - def _invalidate_access_token(self) -> None: - with self._token_lock: - self._access_token = "" - self._access_expires_at = None - - @classmethod - def _validate_payload(cls, payload: dict[str, Any]) -> None: - try: - error_code = int(payload.get("errorcode") or 0) - except (TypeError, ValueError): - error_code = -1 - if error_code != 0: - message = str(payload.get("errmsg") or "未知错误") - raise IfindError(f"iFinD 返回错误：{message[:200]}") - - @classmethod - def _is_auth_error(cls, payload: dict[str, Any]) -> bool: - try: - error_code = int(payload.get("errorcode") or 0) - except (TypeError, ValueError): - error_code = 0 - message = str(payload.get("errmsg") or "").casefold() - return error_code in cls.AUTH_ERROR_CODES or "token" in message or "鉴权" in message - - @staticmethod - def _table_rows(payload: dict[str, Any]) -> list[dict[str, Any]]: - tables = payload.get("tables") or [] - if isinstance(tables, dict): - tables = [tables] - rows: list[dict[str, Any]] = [] - for block in tables if isinstance(tables, list) else []: - if not isinstance(block, dict): - continue - table = block.get("table") or {} - if not isinstance(table, dict): - continue - times = block.get("time") or [] - codes = block.get("thscode") or block.get("thscodes") or [] - if isinstance(codes, str): - codes = [codes] - lengths = [len(value) for value in table.values() if isinstance(value, list)] - row_count = max(lengths or [len(times) if isinstance(times, list) else 0, 1 if table else 0]) - for index in range(row_count): - row: dict[str, Any] = {} - if isinstance(times, list) and index < len(times): - row["time"] = times[index] - if codes: - row["thscode"] = codes[index] if index < len(codes) else codes[0] - for field, values in table.items(): - if isinstance(values, list): - row[field] = values[index] if index < len(values) else None - elif index == 0: - row[field] = values - rows.append(row) - return rows - - @staticmethod - def _codes(codes: str | list[str]) -> str: - if isinstance(codes, list): - values = [str(code or "").strip().upper() for code in codes] - else: - values = [part.strip().upper() for part in str(codes or "").split(",")] - values = [value for value in values if value] - if not values: - raise IfindError("iFinD 证券代码不能为空。") - if len(values) > 100: - raise IfindError("iFinD 单次证券代码过多。") - return ",".join(values) - - @staticmethod - def _display_date(value: str) -> str: - compact = str(value or "").replace("-", "") - if len(compact) != 8 or not compact.isdigit(): - raise IfindError("iFinD 日期格式不正确。") - return f"{compact[:4]}-{compact[4:6]}-{compact[6:]}" - - @staticmethod - def _parse_datetime(value: Any) -> datetime | None: - text = str(value or "").strip() - if not text: - return None - try: - return datetime.fromisoformat(text) - except ValueError: - return None +sys.modules[__name__] = _implementation diff --git a/realtime_aggregator.py b/realtime_aggregator.py index d566df2..a0d7ba6 100644 --- a/realtime_aggregator.py +++ b/realtime_aggregator.py @@ -1,426 +1,7 @@ -from __future__ import annotations +"""Compatibility alias for the canonical display-only realtime observer.""" -import copy -import http.client -import json -import time -import urllib.error -import urllib.parse -import urllib.request -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from datetime import datetime -from threading import Lock -from typing import Any, ClassVar +import sys +from backend.data import realtime as _implementation -class RealtimeAggregateError(RuntimeError): - pass - - -EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get" -EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get" -TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006" -THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool" -XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail" -BROWSER_USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/138.0.0.0 Safari/537.36" -) - - -@dataclass -class WebRealtimeAggregator: - timeout: int = 8 - retry_attempts: int = 3 - retry_delay_seconds: float = 0.2 - response_cache_ttl_seconds: int = 90 - _sector_cache: ClassVar[dict[str, Any]] = {} - _sector_cache_lock: ClassVar[Lock] = Lock() - _response_cache: ClassVar[dict[str, dict[str, Any]]] = {} - _response_cache_lock: ClassVar[Lock] = Lock() - - def health_snapshot(self, sector: str = "") -> dict[str, Any]: - started = time.perf_counter() - sources: dict[str, dict[str, Any]] = {} - indices: list[dict[str, Any]] = [] - sector_payload: dict[str, Any] | None = None - - indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices) - if sector.strip(): - sector_payload, sources["eastmoney_sector"] = self._capture( - lambda: self.eastmoney_sector(sector) - ) - ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool) - xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool) - - index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []] - now = datetime.now().astimezone() - max_skew = 120 if now.hour >= 15 else 15 - index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew - ready = ( - bool(indices) - and len(indices) == 3 - and index_consistent - and (not sector.strip() or bool(sector_payload)) - ) - return { - "ready": ready, - "isolated": True, - "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - "elapsed_ms": round((time.perf_counter() - started) * 1000), - "indices": indices or [], - "index_consistent": index_consistent, - "sector": sector_payload, - "sources": sources, - "observations": { - "ths_limit_pool": ths_observation, - "xgb_limit_pool": xgb_observation, - }, - "policy": { - "integration": "heaven_realtime_fallback", - "max_index_time_skew_seconds": max_skew, - "notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。", - }, - } - - def eastmoney_indices(self) -> list[dict[str, Any]]: - try: - payload = self._get_json( - EASTMONEY_INDEX_URL, - { - "secids": "1.000001,0.399001,0.399006", - "fltt": "2", - "invt": "2", - "fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124", - }, - referer="https://quote.eastmoney.com/", - ) - except RealtimeAggregateError: - return self.tencent_indices() - cache_meta = payload.get("_aggregate_cache") or {} - rows = list((payload.get("data") or {}).get("diff") or []) - result = [] - for row in rows: - code = str(row.get("f12") or "") - if code not in {"000001", "399001", "399006"}: - continue - epoch = int(_number(row.get("f124"))) - result.append( - { - "code": code, - "name": row.get("f14") or code, - "price": _number(row.get("f2")), - "change": _number(row.get("f3")), - "change_amount": _number(row.get("f4")), - "open": _number(row.get("f17")), - "high": _number(row.get("f15")), - "low": _number(row.get("f16")), - "previous_close": _number(row.get("f18")), - "amount_billion": round(_number(row.get("f6")) / 100000000, 2), - "quote_time_epoch": epoch, - "quote_time": ( - datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds") - if epoch else "" - ), - "source": ( - "eastmoney_push2_cache" if cache_meta else "eastmoney_push2" - ), - "cache_age_seconds": cache_meta.get("age_seconds", 0), - } - ) - if len(result) != 3: - raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices") - return result - - def tencent_indices(self) -> list[dict[str, Any]]: - raw, cache_age = self._get_text( - TENCENT_INDEX_URL, - referer="https://gu.qq.com/", - encoding="gb18030", - ) - result = [] - for line in raw.splitlines(): - if '="' not in line: - continue - fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~") - if len(fields) < 38: - continue - code = fields[2] - if code not in {"000001", "399001", "399006"}: - continue - try: - quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone() - except ValueError as exc: - raise RealtimeAggregateError( - f"Tencent returned invalid quote time for {code}" - ) from exc - result.append( - { - "code": code, - "name": fields[1] or code, - "price": _number(fields[3]), - "change": _number(fields[32]), - "change_amount": _number(fields[31]), - "open": _number(fields[5]), - "high": _number(fields[33]), - "low": _number(fields[34]), - "previous_close": _number(fields[4]), - "amount_billion": round(_number(fields[37]) / 10000, 2), - "quote_time_epoch": int(quote_time.timestamp()), - "quote_time": quote_time.isoformat(timespec="seconds"), - "source": "tencent_qt_cache" if cache_age else "tencent_qt", - "cache_age_seconds": cache_age, - } - ) - if len(result) != 3: - raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices") - return result - - def eastmoney_sector(self, query: str) -> dict[str, Any]: - target = _normalize_sector(query) - candidates = self._eastmoney_sector_catalog() - matched = _match_sector(candidates, target) - if not matched: - raise RealtimeAggregateError(f"Eastmoney sector not found: {query}") - epoch = int(_number(matched.get("f124"))) - return { - "code": matched.get("f12") or "", - "name": matched.get("f14") or query, - "price": _number(matched.get("f2")), - "change": _number(matched.get("f3")), - "change_amount": _number(matched.get("f4")), - "turnover_rate": _number(matched.get("f8")), - "up_count": int(_number(matched.get("f104"))), - "down_count": int(_number(matched.get("f105"))), - "leader": matched.get("f128") or "--", - "leader_code": matched.get("f140") or "", - "leading_pct": _number(matched.get("f136")), - "quote_time_epoch": epoch, - "quote_time": ( - datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds") - if epoch else "" - ), - "source": "eastmoney_push2", - "match_query": query, - } - - def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]: - now = time.time() - with self._sector_cache_lock: - cached = self._sector_cache.get("eastmoney") - if cached and now - float(cached.get("created_at") or 0) < 600: - return list(cached.get("rows") or []) - - def load_page(page: int) -> list[dict[str, Any]]: - payload = self._get_json( - EASTMONEY_SECTOR_URL, - { - "pn": str(page), - "pz": "100", - "po": "1", - "np": "1", - "fltt": "2", - "invt": "2", - "fid": "f3", - "fs": "m:90+t:2", - "fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124", - }, - referer="https://quote.eastmoney.com/center/boardlist.html", - ) - return list((payload.get("data") or {}).get("diff") or []) - - with ThreadPoolExecutor(max_workers=5) as executor: - pages = list(executor.map(load_page, range(1, 6))) - rows = [row for page in pages for row in page] - if not rows: - raise RealtimeAggregateError("Eastmoney sector catalog is empty") - with self._sector_cache_lock: - self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows} - return rows - - def ths_limit_pool(self) -> dict[str, Any]: - payload = self._get_json( - THS_LIMIT_URL, - {"page": "1", "limit": "3", "field": "199112"}, - referer="https://data.10jqka.com.cn/limit_up/", - ) - data = payload.get("data") or payload - return { - "available": True, - "keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [], - "source": "ths_web_dataapi", - } - - def xgb_limit_pool(self) -> dict[str, Any]: - payload = self._get_json( - XGB_POOL_URL, - {"pool_name": "limit_up"}, - referer="https://xuangubao.cn/", - ) - data = payload.get("data") or {} - rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or [] - return { - "available": True, - "count": len(rows) if isinstance(rows, list) else 0, - "source": "xuangubao_web_api", - } - - def _capture(self, operation): - started = time.perf_counter() - try: - value = operation() - return value, { - "ok": True, - "elapsed_ms": round((time.perf_counter() - started) * 1000), - "error": "", - } - except Exception as exc: - return None, { - "ok": False, - "elapsed_ms": round((time.perf_counter() - started) * 1000), - "error": str(exc)[:500], - } - - def _get_json( - self, - url: str, - params: dict[str, str], - referer: str, - ) -> dict[str, Any]: - request_url = f"{url}?{urllib.parse.urlencode(params)}" - last_error: Exception | None = None - attempts = max(1, int(self.retry_attempts)) - for attempt in range(attempts): - request = urllib.request.Request( - request_url, - headers={ - "Accept": "application/json,text/plain,*/*", - "Connection": "close", - "Referer": referer, - "User-Agent": BROWSER_USER_AGENT, - }, - ) - try: - with urllib.request.urlopen(request, timeout=self.timeout) as response: - content_type = response.headers.get("Content-Type", "") - raw = response.read().decode("utf-8", errors="replace") - if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")): - raise RealtimeAggregateError( - f"non-JSON response: {raw[:120].strip()}" - ) - payload = json.loads(raw) - if not isinstance(payload, dict): - raise RealtimeAggregateError("unexpected response shape") - if payload.get("rc") not in (None, 0): - raise RealtimeAggregateError(f"provider rc={payload.get('rc')}") - with self._response_cache_lock: - self._response_cache[request_url] = { - "created_at": time.time(), - "payload": copy.deepcopy(payload), - } - return payload - except ( - urllib.error.URLError, - TimeoutError, - ConnectionError, - OSError, - http.client.HTTPException, - json.JSONDecodeError, - RealtimeAggregateError, - ) as exc: - last_error = exc - if attempt + 1 < attempts and self.retry_delay_seconds > 0: - time.sleep(self.retry_delay_seconds * (attempt + 1)) - - now = time.time() - with self._response_cache_lock: - cached = self._response_cache.get(request_url) - cache_age = now - float((cached or {}).get("created_at") or 0) - if cached and cache_age <= self.response_cache_ttl_seconds: - payload = copy.deepcopy(cached.get("payload") or {}) - payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)} - return payload - raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error - - def _get_text( - self, - request_url: str, - referer: str, - encoding: str = "utf-8", - ) -> tuple[str, float]: - cache_key = f"text:{request_url}" - last_error: Exception | None = None - attempts = max(1, int(self.retry_attempts)) - for attempt in range(attempts): - request = urllib.request.Request( - request_url, - headers={ - "Accept": "text/plain,*/*", - "Connection": "close", - "Referer": referer, - "User-Agent": BROWSER_USER_AGENT, - }, - ) - try: - with urllib.request.urlopen(request, timeout=self.timeout) as response: - raw = response.read().decode(encoding, errors="replace") - if not raw.strip(): - raise RealtimeAggregateError("empty text response") - with self._response_cache_lock: - self._response_cache[cache_key] = { - "created_at": time.time(), - "payload": raw, - } - return raw, 0 - except ( - urllib.error.URLError, - TimeoutError, - ConnectionError, - OSError, - http.client.HTTPException, - RealtimeAggregateError, - ) as exc: - last_error = exc - if attempt + 1 < attempts and self.retry_delay_seconds > 0: - time.sleep(self.retry_delay_seconds * (attempt + 1)) - - now = time.time() - with self._response_cache_lock: - cached = self._response_cache.get(cache_key) - cache_age = now - float((cached or {}).get("created_at") or 0) - if cached and cache_age <= self.response_cache_ttl_seconds: - return str(cached.get("payload") or ""), round(cache_age, 1) - raise RealtimeAggregateError( - f"text request failed after {attempts} attempts: {last_error}" - ) from last_error - - -def _normalize_sector(value: Any) -> str: - text = str(value or "").strip().replace(" ", "") - for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"): - text = text.replace(suffix, "") - aliases = {"元器件": "元件", "电子元器件": "元件"} - return aliases.get(text, text) - - -def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None: - exact = [row for row in rows if _normalize_sector(row.get("f14")) == target] - if exact: - return min(exact, key=lambda row: len(str(row.get("f14") or ""))) - fuzzy = [ - row for row in rows - if target and ( - target in _normalize_sector(row.get("f14")) - or _normalize_sector(row.get("f14")) in target - ) - ] - return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None - - -def _number(value: Any, default: float = 0.0) -> float: - try: - return float(value) - except (TypeError, ValueError): - return default +sys.modules[__name__] = _implementation diff --git a/tests/test_data_gateway.py b/tests/test_data_gateway.py index a3058f4..32cfe04 100644 --- a/tests/test_data_gateway.py +++ b/tests/test_data_gateway.py @@ -48,7 +48,11 @@ class DataGatewayTests(unittest.TestCase): from pathlib import Path source = ( - Path(__file__).resolve().parents[1] / "backend" / "application.py" + Path(__file__).resolve().parents[1] + / "backend" + / "features" + / "market" + / "service.py" ).read_text(encoding="utf-8") self.assertEqual(source.count("TushareClient(self.token)"), 1) self.assertIn("return gateway.tushare()", source) diff --git a/tests/test_preservation_slice_market.py b/tests/test_preservation_slice_market.py new file mode 100644 index 0000000..e0bbb29 --- /dev/null +++ b/tests/test_preservation_slice_market.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import ast +import hashlib +import unittest +from pathlib import Path + +import chart_data_provider +import ifind_client +import realtime_aggregator +import tushare_client +from backend.data import realtime +from backend.data.providers import ifind_client as canonical_ifind +from backend.data.providers import tushare_client as canonical_tushare +from backend.features.market import charts + + +APP_ROOT = Path(__file__).resolve().parents[1] +ORIGINAL_ROOT = APP_ROOT.parent + +MARKET_METHODS = { + "_tushare_client", + "get_dashboard", + "_dashboard_sentiment_ready", + "_display_compact_date", + "_carry_dashboard", + "_realtime_snapshot_due", + "sync_dashboard", + "realtime_aggregate_health", + "_search_market_directory", + "_search_match_score", + "search_entities", + "get_search_detail", + "get_intraday_chart", + "_ths_search_detail", + "_index_search_detail", + "get_stock_detail", + "_stock_detail_bar_date", + "_stock_detail_cache_needs_refresh", + "_prepare_stock_detail", + "_sanitize_stock_detail_prices", + "_valid_realtime_stock_quote", + "_ifind_realtime_stock_quote", + "_merge_realtime_stock_detail", + "get_stock_preview", + "backfill", + "_stock_identity", + "_enrich_stock_detail", + "_with_storage", + "_record_count", +} + +MARKET_REPOSITORY_METHODS = { + "get_snapshot", + "get_latest_real_snapshot", + "save_snapshot", + "get_data_snapshot", + "get_latest_data_snapshot", + "save_data_snapshot", + "search_stock_master", + "list_snapshot_payloads", + "start_sync", + "finish_sync", + "status", +} + + +def class_methods(path: Path, class_name: str) -> dict[str, str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + owner = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ) + return { + node.name: ast.dump(node, include_attributes=False) + for node in owner.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def top_level_definitions(path: Path) -> dict[str, str]: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + return { + node.name: ast.dump(node, include_attributes=False) + for node in tree.body + if isinstance(node, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +class MarketSliceSourceEquivalenceTests(unittest.TestCase): + def test_market_service_methods_are_exact_original_ast(self) -> None: + original = class_methods(ORIGINAL_ROOT / "server.py", "DashboardService") + migrated = class_methods( + APP_ROOT / "backend" / "features" / "market" / "service.py", + "MarketServiceMixin", + ) + self.assertEqual(set(migrated), MARKET_METHODS) + for name in sorted(MARKET_METHODS): + self.assertEqual(migrated[name], original[name], name) + + def test_market_repository_methods_are_exact_original_ast(self) -> None: + original = class_methods(ORIGINAL_ROOT / "database.py", "ReviewDatabase") + migrated = class_methods( + APP_ROOT / "backend" / "features" / "market" / "repository.py", + "MarketRepositoryMixin", + ) + self.assertEqual(set(migrated), MARKET_REPOSITORY_METHODS) + for name in sorted(MARKET_REPOSITORY_METHODS): + self.assertEqual(migrated[name], original[name], name) + + def test_original_classes_no_longer_duplicate_moved_methods(self) -> None: + remaining_service = class_methods(APP_ROOT / "backend" / "application.py", "DashboardService") + remaining_database = class_methods(APP_ROOT / "database.py", "ReviewDatabase") + self.assertTrue(MARKET_METHODS.isdisjoint(remaining_service)) + self.assertTrue(MARKET_REPOSITORY_METHODS.isdisjoint(remaining_database)) + + def test_provider_compatibility_modules_are_canonical_aliases(self) -> None: + self.assertIs(tushare_client.TushareClient, canonical_tushare.TushareClient) + self.assertIs(ifind_client.IfindHttpClient, canonical_ifind.IfindHttpClient) + self.assertIs(realtime_aggregator.WebRealtimeAggregator, realtime.WebRealtimeAggregator) + self.assertIs(chart_data_provider.MarketChartClient, charts.MarketChartClient) + + def test_provider_logic_is_the_original_implementation(self) -> None: + exact_moves = ( + ("tushare_client.py", "backend/data/providers/tushare_client.py"), + ("ifind_client.py", "backend/data/providers/ifind_client.py"), + ("realtime_aggregator.py", "backend/data/realtime.py"), + ) + for original, migrated in exact_moves: + self.assertEqual(sha256(ORIGINAL_ROOT / original), sha256(APP_ROOT / migrated)) + self.assertEqual( + top_level_definitions(ORIGINAL_ROOT / "chart_data_provider.py"), + top_level_definitions(APP_ROOT / "backend/features/market/charts.py"), + ) + + def test_unchanged_frontend_assets_match_the_original(self) -> None: + for relative in ( + "index.html", + "app.js", + "styles.css", + "renovation.css", + "redesign-v2.css", + "theme.css", + "wentian-v2.css", + ): + self.assertEqual( + sha256(APP_ROOT / "static" / relative), + sha256(ORIGINAL_ROOT / "static" / relative), + relative, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_stock_detail_realtime.py b/tests/test_stock_detail_realtime.py index c6760b6..b33743a 100644 --- a/tests/test_stock_detail_realtime.py +++ b/tests/test_stock_detail_realtime.py @@ -90,8 +90,8 @@ class StockDetailRealtimeTests(unittest.TestCase): "moneyflow": {}, } - with patch("backend.application.datetime", FixedMarketDatetime), patch( - "backend.application.TushareClient", RealtimeClientStub + with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch( + "backend.features.market.service.TushareClient", RealtimeClientStub ): result = self.service._prepare_stock_detail(cached, "002141", today) @@ -112,8 +112,8 @@ class StockDetailRealtimeTests(unittest.TestCase): "stock": {"code": "002141", "price": 10, "change": 1.2}, "prices": [{"trade_date": historical, "close": 10, "change": 1.2}], } - with patch("backend.application.datetime", FixedMarketDatetime), patch( - "backend.application.TushareClient", RealtimeClientStub + with patch("backend.features.market.service.datetime", FixedMarketDatetime), patch( + "backend.features.market.service.TushareClient", RealtimeClientStub ): result = self.service._prepare_stock_detail(payload, "002141", historical) @@ -151,8 +151,8 @@ class StockDetailRealtimeTests(unittest.TestCase): }, ], } - with patch("backend.application.datetime", FixedPreopenDatetime), patch( - "backend.application.TushareClient", RealtimeClientStub + with patch("backend.features.market.service.datetime", FixedPreopenDatetime), patch( + "backend.features.market.service.TushareClient", RealtimeClientStub ): result = self.service._prepare_stock_detail(payload, "002141", today) diff --git a/tools/move_class_methods.py b/tools/move_class_methods.py new file mode 100644 index 0000000..12e50bb --- /dev/null +++ b/tools/move_class_methods.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import argparse +import ast +from pathlib import Path + + +MARKER = " # PRESERVATION_METHODS\n" + + +def method_span(node: ast.FunctionDef | ast.AsyncFunctionDef) -> tuple[int, int]: + start = min((decorator.lineno for decorator in node.decorator_list), default=node.lineno) + if node.end_lineno is None: + raise ValueError(f"Missing end position for {node.name}") + return start - 1, node.end_lineno + + +def move_methods( + source_path: Path, + class_name: str, + target_path: Path, + method_names: list[str], +) -> None: + source = source_path.read_text(encoding="utf-8") + tree = ast.parse(source, filename=str(source_path)) + owner = next( + ( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == class_name + ), + None, + ) + if owner is None: + raise ValueError(f"Class not found: {class_name}") + + methods = { + node.name: node + for node in owner.body + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + missing = [name for name in method_names if name not in methods] + if missing: + raise ValueError(f"Methods not found in {class_name}: {', '.join(missing)}") + + lines = source.splitlines(keepends=True) + ordered = sorted((methods[name] for name in method_names), key=lambda node: node.lineno) + blocks = ["".join(lines[start:end]).rstrip() for start, end in map(method_span, ordered)] + + for start, end in sorted(map(method_span, ordered), reverse=True): + del lines[start:end] + while start < len(lines) - 1 and lines[start] == "\n" and lines[start + 1] == "\n": + del lines[start] + + target = target_path.read_text(encoding="utf-8") + if target.count(MARKER) != 1: + raise ValueError(f"Target must contain exactly one method marker: {target_path}") + target = target.replace(MARKER, "\n\n".join(blocks) + "\n") + + source_path.write_text("".join(lines), encoding="utf-8") + target_path.write_text(target, encoding="utf-8") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Mechanically move class methods between modules") + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--class-name", required=True) + parser.add_argument("--target", type=Path, required=True) + parser.add_argument("methods", nargs="+") + args = parser.parse_args() + move_methods(args.source, args.class_name, args.target, args.methods) + + +if __name__ == "__main__": + main() + diff --git a/tushare_client.py b/tushare_client.py index a2f65b0..793a84c 100644 --- a/tushare_client.py +++ b/tushare_client.py @@ -1,2175 +1,7 @@ -from __future__ import annotations +"""Compatibility alias for the canonical Tushare provider implementation.""" -import json -import math -import re -import urllib.error -import urllib.request -from collections import Counter -from dataclasses import dataclass -from datetime import datetime, time as dt_time, timedelta -from threading import Lock -from typing import Any, ClassVar +import sys -from sentiment_engine import apply_sentiment_to_dashboard +from backend.data.providers import tushare_client as _implementation - -TUSHARE_URL = "http://api.tushare.pro" - - -class TushareError(RuntimeError): - pass - - -@dataclass -class TushareClient: - token: str - timeout: int = 30 - _realtime_reference_cache: ClassVar[dict[str, dict[str, Any]]] = {} - _realtime_reference_lock: ClassVar[Lock] = Lock() - _capital_cache: ClassVar[dict[str, dict[str, Any]]] = {} - _latest_realtime_market: ClassVar[dict[str, dict[str, Any]]] = {} - _stock_activity_cache: ClassVar[dict[str, dict[str, Any]]] = {} - _stock_listing_cache: ClassVar[dict[str, Any]] = {} - _stock_listing_lock: ClassVar[Lock] = Lock() - _suspension_cache: ClassVar[dict[str, dict[str, str] | None]] = {} - _suspension_lock: ClassVar[Lock] = Lock() - - def query( - self, - api_name: str, - params: dict[str, Any] | None = None, - fields: str = "", - ) -> list[dict[str, Any]]: - payload = json.dumps( - { - "api_name": api_name, - "token": self.token, - "params": params or {}, - "fields": fields, - } - ).encode("utf-8") - request = urllib.request.Request( - TUSHARE_URL, - data=payload, - headers={"Content-Type": "application/json", "User-Agent": "XiaobaiReviewWeb/0.2"}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=self.timeout) as response: - result = json.loads(response.read().decode("utf-8")) - except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc: - raise TushareError(f"Tushare request failed: {exc}") from exc - - if result.get("code") != 0: - raise TushareError(result.get("msg") or "Tushare returned an unknown error") - - data = result.get("data") or {} - columns = data.get("fields") or [] - return [dict(zip(columns, item)) for item in data.get("items") or []] - - def dashboard(self, requested_date: str) -> dict[str, Any]: - trade_date, previous_trade_date = self.resolve_trade_context(requested_date) - if self.should_use_realtime(requested_date, trade_date): - return self._realtime_dashboard( - requested_date, - trade_date, - previous_trade_date, - ) - - daily = self._load_daily(trade_date) - if ( - not daily - and requested_date == datetime.now().astimezone().strftime("%Y%m%d") - and trade_date == requested_date - and datetime.now().astimezone().time().replace(tzinfo=None) >= dt_time(9, 15) - ): - return self._realtime_dashboard( - requested_date, - trade_date, - previous_trade_date, - ) - if not daily: - raise TushareError(f"No daily data returned for {trade_date}") - - notices: list[str] = [] - try: - limit_rows = self._load_limit_lists(trade_date) - previous_limit_rows = self._load_limit_type(previous_trade_date, "U") - if not limit_rows: - notices.append("涨跌停高级接口当日数据尚未更新，已使用日线数据推算。") - limit_rows = self._derive_limits(trade_date, daily) - except TushareError as exc: - notices.append(f"涨跌停高级接口不可用，已使用日线数据推算：{exc}") - limit_rows = self._derive_limits(trade_date, daily) - previous_daily = self._load_daily(previous_trade_date) - previous_limit_rows = [ - row for row in self._derive_limits(previous_trade_date, previous_daily) - if row.get("limit_type") == "U" - ] - - up_rows = [row for row in limit_rows if row.get("limit_type") == "U"] - down_rows = [row for row in limit_rows if row.get("limit_type") == "D"] - broken_rows = [row for row in limit_rows if row.get("limit_type") == "Z"] - limits = [self._normalize_limit(row, "涨停") for row in up_rows] - broken = [self._normalize_limit(row, "炸板") for row in broken_rows] - down_limits = [self._normalize_limit(row, "跌停") for row in down_rows] - previous_limits = [self._normalize_limit(row, "涨停") for row in previous_limit_rows] - yesterday_limits = _build_yesterday_performance( - previous_limits, - daily, - limits, - broken, - down_limits, - ) - sectors = _build_sectors(limits) - previous_sectors = _build_sectors(previous_limits) - - dashboard = { - "meta": { - "requested_date": _display_date(requested_date), - "trade_date": _display_date(trade_date), - "previous_trade_date": _display_date(previous_trade_date), - "source": "tushare", - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - "notice": "；".join(notices), - }, - "overview": _build_overview(daily, up_rows, down_rows, broken_rows), - "limits": limits, - "broken": broken, - "down_limits": down_limits, - "yesterday_limits": yesterday_limits, - "limit_performance": _build_limit_performance(yesterday_limits), - "ladders": _build_ladders(limits), - "sectors": sectors, - "sector_rotation": _build_sector_rotation(sectors, previous_sectors), - } - return apply_sentiment_to_dashboard(dashboard) - - @staticmethod - def should_use_realtime(requested_date: str, trade_date: str) -> bool: - """Use rt_k for today's open market until end-of-day datasets settle.""" - now = datetime.now().astimezone() - today = now.strftime("%Y%m%d") - return ( - requested_date == today - and trade_date == today - and dt_time(9, 15) <= now.time().replace(tzinfo=None) < dt_time(16, 30) - ) - - def _realtime_dashboard( - self, - requested_date: str, - trade_date: str, - previous_trade_date: str, - ) -> dict[str, Any]: - reference = self._load_realtime_reference(trade_date, previous_trade_date) - basic_rows = list(reference["basic_rows"]) - codes = ",".join( - str(row.get("ts_code") or "") for row in basic_rows if row.get("ts_code") - ) - if not codes: - raise TushareError("No active stock codes available for rt_k") - quotes = self.query("rt_k", {"ts_code": codes}) - if not quotes: - raise TushareError(f"No realtime data returned for {trade_date}") - - basic_map = {str(row.get("ts_code") or ""): row for row in basic_rows} - daily: list[dict[str, Any]] = [] - for quote in quotes: - close = _number(quote.get("close")) - previous_close = _number(quote.get("pre_close")) - if close <= 0 or previous_close <= 0: - continue - basic = basic_map.get(str(quote.get("ts_code") or ""), {}) - daily.append( - { - **quote, - "trade_date": trade_date, - "name": str(quote.get("name") or basic.get("name") or "--").strip(), - "industry": basic.get("industry") or "其他", - "pct_chg": round((close / previous_close - 1) * 100, 4), - "amount_unit": "yuan", - } - ) - with self._realtime_reference_lock: - self._latest_realtime_market[trade_date] = { - "rows": daily, - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - } - if len(self._latest_realtime_market) > 3: - oldest = next(iter(self._latest_realtime_market)) - self._latest_realtime_market.pop(oldest, None) - - limit_rows = self._derive_limits( - trade_date, - daily, - price_limits=list(reference["price_limits"]), - basic_rows=basic_rows, - previous_limit_rows=list(reference["previous_limit_rows"]), - capital_rows=list(reference["capital_rows"]), - ) - previous_limit_rows = list(reference["previous_limit_rows"]) - up_rows = [row for row in limit_rows if row.get("limit_type") == "U"] - down_rows = [row for row in limit_rows if row.get("limit_type") == "D"] - broken_rows = [row for row in limit_rows if row.get("limit_type") == "Z"] - limits = [self._normalize_limit(row, "涨停") for row in up_rows] - broken = [self._normalize_limit(row, "炸板") for row in broken_rows] - down_limits = [self._normalize_limit(row, "跌停") for row in down_rows] - previous_limits = [self._normalize_limit(row, "涨停") for row in previous_limit_rows] - yesterday_limits = _build_yesterday_performance( - previous_limits, - daily, - limits, - broken, - down_limits, - ) - sectors = _build_sectors(limits) - previous_sectors = _build_sectors(previous_limits) - now = datetime.now().astimezone() - market_status = _realtime_market_status(now.time().replace(tzinfo=None)) - dashboard = { - "meta": { - "requested_date": _display_date(requested_date), - "trade_date": _display_date(trade_date), - "previous_trade_date": _display_date(previous_trade_date), - "source": "tushare", - "mode": "realtime", - "realtime": True, - "market_status": market_status, - "refresh_mode": "manual", - "auto_refresh": False, - "quote_count": len(daily), - "updated_at": now.isoformat(timespec="seconds"), - "notice": "盘中行情由 Tushare rt_k 实时计算；涨停原因、封板时间和开板次数以盘后榜单校正为准。", - }, - "overview": _build_overview(daily, up_rows, down_rows, broken_rows), - "limits": limits, - "broken": broken, - "down_limits": down_limits, - "yesterday_limits": yesterday_limits, - "limit_performance": _build_limit_performance(yesterday_limits), - "ladders": _build_ladders(limits), - "sectors": sectors, - "sector_rotation": _build_sector_rotation(sectors, previous_sectors), - } - return apply_sentiment_to_dashboard(dashboard) - - def _load_realtime_reference( - self, - trade_date: str, - previous_trade_date: str, - ) -> dict[str, Any]: - cache_key = f"{trade_date}:{previous_trade_date}" - with self._realtime_reference_lock: - cached = self._realtime_reference_cache.get(cache_key) - if cached: - return cached - - basic_rows = self.query( - "stock_basic", - {"exchange": "", "list_status": "L"}, - "ts_code,name,industry,market,list_date", - ) - price_limits = self.query( - "stk_limit", - {"trade_date": trade_date}, - "ts_code,trade_date,up_limit,down_limit", - ) - previous_limit_rows = self._load_limit_type(previous_trade_date, "U") - capital_rows = self.query( - "daily_basic", - {"trade_date": previous_trade_date}, - "ts_code,trade_date,total_share,float_share,free_share,total_mv,circ_mv", - ) - if not basic_rows or not price_limits: - raise TushareError(f"Realtime reference data is incomplete for {trade_date}") - result = { - "basic_rows": basic_rows, - "price_limits": price_limits, - "previous_limit_rows": previous_limit_rows, - "capital_rows": capital_rows, - } - with self._realtime_reference_lock: - self._realtime_reference_cache[cache_key] = result - if len(self._realtime_reference_cache) > 3: - oldest = next(iter(self._realtime_reference_cache)) - self._realtime_reference_cache.pop(oldest, None) - return result - - def realtime_stock_quote( - self, - ts_code: str, - reference_date: str = "", - ) -> dict[str, Any]: - rows = self.query("rt_k", {"ts_code": ts_code}) - if not rows: - raise TushareError(f"No realtime quote returned for {ts_code}") - row = rows[0] - close = _number(row.get("close")) - previous_close = _number(row.get("pre_close")) - if close <= 0 or previous_close <= 0: - raise TushareError(f"Realtime quote is unavailable for {ts_code}") - - basic: dict[str, Any] = {} - with self._realtime_reference_lock: - references = list(self._realtime_reference_cache.values()) - for reference in reversed(references): - basic = next( - ( - item for item in reference.get("basic_rows") or [] - if str(item.get("ts_code") or "") == ts_code - ), - {}, - ) - if basic: - break - if not basic: - basics = self.query( - "stock_basic", - {"ts_code": ts_code}, - "ts_code,name,industry,market,list_date", - ) - basic = basics[0] if basics else {} - capital = self._latest_capital(ts_code, reference_date) - float_share = _number(capital.get("float_share")) - # rt_k volume is shares; daily_basic float_share is reported in 10k shares. - turnover_rate = _number(row.get("vol")) / float_share / 100 if float_share else 0 - market_date = reference_date or datetime.now().astimezone().strftime("%Y%m%d") - self._ensure_realtime_market_cache(market_date) - with self._realtime_reference_lock: - market_rows = list((self._latest_realtime_market.get(market_date) or {}).get("rows") or []) - references = list(self._realtime_reference_cache.values()) - capital_map: dict[str, dict[str, Any]] = {} - for reference in reversed(references): - capital_map = { - str(item.get("ts_code") or ""): item - for item in reference.get("capital_rows") or [] - } - if capital_map: - break - market_amounts = [_number(item.get("amount")) for item in market_rows if _number(item.get("amount")) > 0] - amount_percentile = _value_percentile(_number(row.get("amount")), market_amounts) - market_turnovers = [] - for item in market_rows: - item_capital = capital_map.get(str(item.get("ts_code") or ""), {}) - item_float_share = _number(item_capital.get("float_share")) - if item_float_share: - market_turnovers.append(_number(item.get("vol")) / item_float_share / 100) - market_turnover = ( - sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0 - ) - turnover_relative = turnover_rate / market_turnover if market_turnover else 0 - activity = self._stock_activity_metrics( - ts_code, - market_date, - _number(row.get("vol")) / 100, - ) - return { - "code": ts_code.split(".")[0], - "ts_code": ts_code, - "name": str(row.get("name") or basic.get("name") or "--").strip(), - "sector": basic.get("industry") or "其他", - "price": round(close, 3), - "change": round((close / previous_close - 1) * 100, 4), - "open": round(_number(row.get("open")), 3), - "high": round(_number(row.get("high")), 3), - "low": round(_number(row.get("low")), 3), - "previous_close": round(previous_close, 3), - "amount_billion": round(_number(row.get("amount")) / 100000000, 3), - "volume": _number(row.get("vol")), - "trade_count": int(_number(row.get("num"))), - "turnover_rate": round(turnover_rate, 4), - "market_turnover_rate": round(market_turnover, 4), - "turnover_relative": round(turnover_relative, 4), - "amount_percentile": round(amount_percentile * 100, 2), - "volume_activity_ratio": activity.get("volume_activity_ratio", 0), - "activity_history_date": activity.get("history_trade_date", ""), - "activity_source": activity.get("source", "unavailable"), - "float_share_10k": float_share, - "capital_trade_date": str(capital.get("trade_date") or ""), - "turnover_source": "rt_volume/latest_float_share" if float_share else "unavailable", - "data_source": "tushare", - "realtime": True, - } - - def _stock_activity_metrics( - self, - ts_code: str, - reference_date: str, - current_volume_lots: float, - ) -> dict[str, Any]: - cache_key = f"{ts_code}:{reference_date}" - with self._realtime_reference_lock: - history = self._stock_activity_cache.get(cache_key) - if history is None: - try: - end = datetime.strptime(reference_date, "%Y%m%d") - except ValueError: - end = datetime.now().astimezone().replace(tzinfo=None) - rows = self.query( - "daily", - { - "ts_code": ts_code, - "start_date": (end - timedelta(days=30)).strftime("%Y%m%d"), - "end_date": reference_date, - }, - "ts_code,trade_date,vol,amount", - ) - completed = [ - item for item in rows - if str(item.get("trade_date") or "") < reference_date and _number(item.get("vol")) > 0 - ] - completed.sort(key=lambda item: str(item.get("trade_date") or "")) - recent = completed[-5:] - history = { - "average_volume_lots": ( - sum(_number(item.get("vol")) for item in recent) / len(recent) - if recent else 0 - ), - "history_trade_date": str(recent[-1].get("trade_date") or "") if recent else "", - } - with self._realtime_reference_lock: - self._stock_activity_cache[cache_key] = history - if len(self._stock_activity_cache) > 256: - oldest = next(iter(self._stock_activity_cache)) - self._stock_activity_cache.pop(oldest, None) - average_volume = _number(history.get("average_volume_lots")) - progress = _trading_session_progress(datetime.now().astimezone().time().replace(tzinfo=None)) - expected_volume = average_volume * progress - ratio = current_volume_lots / expected_volume if expected_volume else 0 - return { - **history, - "volume_activity_ratio": round(ratio, 4), - "session_progress": round(progress, 4), - "source": "rt_volume/5d_average_at_same_progress" if expected_volume else "unavailable", - } - - def realtime_factor_snapshot(self, requested_date: str) -> dict[str, Any]: - trade_date, previous_trade_date = self.resolve_trade_context(requested_date) - reference = self._load_realtime_reference(trade_date, previous_trade_date) - codes = [ - str(row.get("ts_code") or "") - for row in reference.get("basic_rows") or [] - if row.get("ts_code") - ] - quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") - capital_map = { - str(row.get("ts_code") or ""): row - for row in reference.get("capital_rows") or [] - } - rows = [] - for quote in quotes: - ts_code = str(quote.get("ts_code") or "") - close = _number(quote.get("close")) - previous_close = _number(quote.get("pre_close")) - if not ts_code or close <= 0 or previous_close <= 0: - continue - capital = capital_map.get(ts_code, {}) - float_share = _number(capital.get("float_share")) - rows.append( - { - "ts_code": ts_code, - "trade_date": trade_date, - "open": _number(quote.get("open")), - "high": _number(quote.get("high")), - "low": _number(quote.get("low")), - "close": close, - "pct_chg": (close / previous_close - 1) * 100, - "vol": _number(quote.get("vol")) / 100, - "amount": _number(quote.get("amount")), - "turnover_rate": ( - _number(quote.get("vol")) / float_share / 100 if float_share else 0 - ), - "capital_trade_date": str(capital.get("trade_date") or ""), - } - ) - if not rows: - raise TushareError(f"No realtime factor snapshot returned for {trade_date}") - return { - "trade_date": trade_date, - "previous_trade_date": previous_trade_date, - "source": "tushare_rt_k", - "realtime": True, - "rows": rows, - } - - def _ensure_realtime_market_cache(self, requested_date: str) -> list[dict[str, Any]]: - with self._realtime_reference_lock: - cached = list( - (self._latest_realtime_market.get(requested_date) or {}).get("rows") or [] - ) - if cached: - return cached - trade_date, previous_trade_date = self.resolve_trade_context(requested_date) - if trade_date != requested_date: - return [] - reference = self._load_realtime_reference(trade_date, previous_trade_date) - codes = [ - str(row.get("ts_code") or "") - for row in reference.get("basic_rows") or [] - if row.get("ts_code") - ] - quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") - rows = [ - row for row in quotes - if _number(row.get("close")) > 0 and _number(row.get("pre_close")) > 0 - ] - with self._realtime_reference_lock: - self._latest_realtime_market[trade_date] = { - "rows": rows, - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - } - return rows - - def _latest_capital(self, ts_code: str, reference_date: str = "") -> dict[str, Any]: - end_date = reference_date or datetime.now().astimezone().strftime("%Y%m%d") - cache_key = f"{ts_code}:{end_date}" - with self._realtime_reference_lock: - cached = self._capital_cache.get(cache_key) - if cached: - return cached - try: - end = datetime.strptime(end_date, "%Y%m%d") - except ValueError: - end = datetime.now().astimezone().replace(tzinfo=None) - end_date = end.strftime("%Y%m%d") - start_date = (end - timedelta(days=20)).strftime("%Y%m%d") - rows = self.query( - "daily_basic", - {"ts_code": ts_code, "start_date": start_date, "end_date": end_date}, - "ts_code,trade_date,turnover_rate,volume_ratio,total_share,float_share," - "free_share,total_mv,circ_mv", - ) - rows.sort(key=lambda item: str(item.get("trade_date") or "")) - result = rows[-1] if rows else {} - with self._realtime_reference_lock: - self._capital_cache[cache_key] = result - if len(self._capital_cache) > 256: - oldest = next(iter(self._capital_cache)) - self._capital_cache.pop(oldest, None) - return result - - def market_indices(self, requested_date: str, lookback_days: int = 45) -> dict[str, Any]: - trade_date, _ = self.resolve_trade_context(requested_date) - end = datetime.strptime(trade_date, "%Y%m%d") - start_date = (end - timedelta(days=max(30, lookback_days * 2))).strftime("%Y%m%d") - index_names = { - "000001.SH": "上证指数", - "399001.SZ": "深证成指", - "399006.SZ": "创业板指", - } - indices = [] - for ts_code, name in index_names.items(): - rows = self.query( - "index_daily", - {"ts_code": ts_code, "start_date": start_date, "end_date": trade_date}, - "ts_code,trade_date,close,pct_chg,vol,amount", - ) - rows.sort(key=lambda item: str(item.get("trade_date") or "")) - if not rows: - continue - latest = rows[-1] - close = _number(latest.get("close")) - close_5d = _number(rows[-6].get("close")) if len(rows) >= 6 else _number(rows[0].get("close")) - close_20d = _number(rows[-21].get("close")) if len(rows) >= 21 else _number(rows[0].get("close")) - indices.append( - { - "ts_code": ts_code, - "name": name, - "trade_date": str(latest.get("trade_date") or trade_date), - "close": close, - "pct_chg": round(_number(latest.get("pct_chg")), 3), - "return_5d": round((close / close_5d - 1) * 100, 3) if close_5d else 0, - "return_20d": round((close / close_20d - 1) * 100, 3) if close_20d else 0, - "amount_billion": round(_number(latest.get("amount")) / 100000, 2), - } - ) - if not indices: - raise TushareError(f"No index data returned for {trade_date}") - return { - "trade_date": trade_date, - "source": "tushare", - "realtime": False, - "precise": all(item["trade_date"] == trade_date for item in indices), - "indices": indices, - "aggregate": { - "average_pct_chg": round(sum(item["pct_chg"] for item in indices) / len(indices), 3), - "average_return_5d": round(sum(item["return_5d"] for item in indices) / len(indices), 3), - "average_return_20d": round(sum(item["return_20d"] for item in indices) / len(indices), 3), - }, - } - - def realtime_market_indices(self, requested_date: str) -> dict[str, Any]: - trade_date, _ = self.resolve_trade_context(requested_date) - index_names = { - "000001.SH": "上证指数", - "399001.SZ": "深证成指", - "399006.SZ": "创业板指", - } - rows = self.query("rt_idx_k", {"ts_code": ",".join(index_names)}, "") - row_map = {str(row.get("ts_code") or ""): row for row in rows} - indices = [] - for ts_code, name in index_names.items(): - row = row_map.get(ts_code) - if not row: - continue - close = _number(row.get("close")) - previous_close = _number(row.get("pre_close")) - if close <= 0 or previous_close <= 0: - continue - history = self.query( - "index_daily", - { - "ts_code": ts_code, - "start_date": (datetime.strptime(trade_date, "%Y%m%d") - timedelta(days=20)).strftime("%Y%m%d"), - "end_date": trade_date, - }, - "ts_code,trade_date,close,pct_chg", - ) - history.sort(key=lambda item: str(item.get("trade_date") or "")) - previous_closes = [ - _number(item.get("close")) for item in history - if str(item.get("trade_date") or "") < trade_date and _number(item.get("close")) > 0 - ] - close_5d = previous_closes[-5] if len(previous_closes) >= 5 else previous_closes[0] if previous_closes else previous_close - indices.append( - { - "ts_code": ts_code, - "name": str(row.get("name") or name).strip(), - "trade_date": trade_date, - "close": close, - "pct_chg": round((close / previous_close - 1) * 100, 3), - "return_5d": round((close / close_5d - 1) * 100, 3) if close_5d else 0, - "amount_billion": round(_number(row.get("amount")) / 100000000, 2), - } - ) - if len(indices) != len(index_names): - raise TushareError("Realtime index quotes are incomplete") - return { - "trade_date": trade_date, - "source": "tushare_rt_idx_k", - "realtime": True, - "precise": True, - "indices": indices, - "aggregate": { - "average_pct_chg": round(sum(item["pct_chg"] for item in indices) / len(indices), 3), - "average_return_5d": round(sum(item["return_5d"] for item in indices) / len(indices), 3), - "average_return_20d": 0, - }, - } - - def sw_stock_industry(self, ts_code: str, trade_date: str) -> dict[str, Any]: - """Return the Shenwan industry active for a stock on trade_date.""" - rows = [] - for is_new in ("Y", "N"): - rows.extend( - self.query( - "index_member_all", - {"ts_code": ts_code, "is_new": is_new}, - "l1_code,l1_name,l2_code,l2_name,l3_code,l3_name," - "ts_code,name,in_date,out_date,is_new", - ) - ) - rows = _reconcile_membership_rows(rows) - matched = [row for row in rows if _membership_active_on(row, trade_date)] - if not matched: - matched = [ - row for row in rows - if row.get("is_new") == "Y" - and str(row.get("in_date") or "") <= trade_date - ] - if not matched: - raise TushareError(f"No Shenwan industry returned for {ts_code}") - row = max( - matched, - key=lambda item: ( - str(item.get("in_date") or ""), - 1 if item.get("is_new") == "Y" else 0, - str(item.get("l3_code") or item.get("l2_code") or ""), - ), - ) - return { - "l1_code": str(row.get("l1_code") or ""), - "l1_name": str(row.get("l1_name") or ""), - "l2_code": str(row.get("l2_code") or ""), - "l2_name": str(row.get("l2_name") or ""), - "l3_code": str(row.get("l3_code") or ""), - "l3_name": str(row.get("l3_name") or ""), - "in_date": str(row.get("in_date") or ""), - "out_date": str(row.get("out_date") or ""), - "is_new": str(row.get("is_new") or ""), - } - - def sw_sector_snapshot( - self, - ts_code: str, - requested_date: str, - realtime_expected: bool = False, - allow_realtime_close: bool = False, - ) -> dict[str, Any]: - """Build the single Shenwan L2 sector context used by heaven trend.""" - trade_date, previous_trade_date = self.resolve_trade_context(requested_date) - industry = self.sw_stock_industry(ts_code, trade_date) - sector_code = str(industry.get("l2_code") or "") - if not sector_code: - raise TushareError(f"Shenwan L2 code is unavailable for {ts_code}") - members = self._sw_sector_members(sector_code, trade_date) - if not members: - raise TushareError(f"No Shenwan members returned for {sector_code}") - raw_member_count = len(members) - members, excluded_members = _filter_members_by_listing( - members, - self._stock_listing_reference(), - trade_date, - ) - if not members: - raise TushareError(f"No listed Shenwan members returned for {sector_code}") - - if realtime_expected: - snapshot = self._sw_realtime_sector_snapshot( - industry, - members, - trade_date, - previous_trade_date, - finalized=False, - ) - snapshot.update({ - "raw_member_count": raw_member_count, - "excluded_member_count": len(excluded_members), - "excluded_members": excluded_members, - }) - return snapshot - - member_set = {str(item.get("ts_code") or "") for item in members} - member_names = { - str(item.get("ts_code") or ""): str(item.get("name") or "") - for item in members - } - member_rows = [ - row for row in self._load_daily(trade_date) - if str(row.get("ts_code") or "") in member_set - ] - quoted_codes = {str(row.get("ts_code") or "") for row in member_rows} - suspended_members = self._confirmed_suspended_members( - members, quoted_codes, trade_date - ) - up_count = sum(_number(row.get("pct_chg")) > 0 for row in member_rows) - down_count = sum(_number(row.get("pct_chg")) < 0 for row in member_rows) - leader = max(member_rows, key=lambda row: _number(row.get("pct_chg")), default={}) - leader_code = str(leader.get("ts_code") or "") - equal_change = ( - sum(_number(row.get("pct_chg")) for row in member_rows) / len(member_rows) - if member_rows else 0 - ) - coverage = len(member_rows) / max(len(members), 1) * 100 - explained_count = len(member_rows) + len(suspended_members) - explained_coverage = explained_count / max(len(members), 1) * 100 - coverage_issue = _sector_coverage_issue( - len(members), - len(member_rows), - explained_coverage, - explained_count, - ) - inner_precise = not coverage_issue - inner_error = coverage_issue - amount_billion = sum(_number(row.get("amount")) for row in member_rows) / 100000 - rows = self.query( - "sw_daily", - {"ts_code": sector_code, "trade_date": trade_date}, - "ts_code,trade_date,name,close,pct_change,vol,amount,pe,pb,float_mv,total_mv", - ) - daily = rows[0] if rows else {} - actual_trade_date = str(daily.get("trade_date") or "") - outer_precise = actual_trade_date == trade_date - outer_error = "" if outer_precise else ( - f"No Shenwan daily returned for {sector_code} on {trade_date}" - ) - if not outer_precise and allow_realtime_close: - try: - return self._sw_realtime_sector_snapshot( - industry, - members, - trade_date, - previous_trade_date, - finalized=True, - ) - except TushareError as exc: - outer_error = f"{outer_error}; realtime close fallback failed: {exc}" - - official_change = _number(daily.get("pct_change")) if outer_precise else None - return { - "code": sector_code, - "name": industry.get("l2_name") or daily.get("name") or sector_code, - "leader": str(leader.get("name") or member_names.get(leader_code) or "--"), - "leader_code": leader_code, - "leading_pct": round(_number(leader.get("pct_chg")), 3), - "change": round(official_change, 3) if official_change is not None else None, - "member_equal_change": round(equal_change, 3), - "turnover_rate": 0, - "up_count": up_count, - "down_count": down_count, - "flat_count": len(member_rows) - up_count - down_count, - "member_count": len(members), - "raw_member_count": raw_member_count, - "excluded_member_count": len(excluded_members), - "excluded_members": excluded_members, - "quote_count": len(member_rows), - "coverage": round(coverage, 1), - "explained_count": explained_count, - "explained_coverage": round(explained_coverage, 1), - "suspended_count": len(suspended_members), - "suspended_members": suspended_members, - "strength": round(max(0, min(100, 50 + (official_change if official_change is not None else equal_change) * 5)), 1), - "amount_billion": round(amount_billion, 2), - "count": 0, - "max_streak": 0, - "source": "tushare_sw_daily+member_daily" if outer_precise else "tushare_member_daily", - "inner_source": "tushare_member_daily", - "outer_source": "tushare_sw_daily" if outer_precise else "unavailable", - "taxonomy": "sw_l2", - "industry": industry, - "trade_date": trade_date, - "inner_trade_date": trade_date if member_rows else "", - "outer_trade_date": actual_trade_date, - "realtime": False, - "finalized": True, - "inner_precise": inner_precise, - "outer_precise": outer_precise, - "precise": inner_precise and outer_precise, - "inner_error": inner_error, - "outer_error": outer_error, - "schema_version": 6, - "methodology": "外显使用申万二级行业官方日线；内核独立使用当日成分日线宽度与等权涨跌聚合", - } - - def _sw_sector_members( - self, - sector_code: str, - trade_date: str, - ) -> list[dict[str, Any]]: - rows = [] - for is_new in ("Y", "N"): - rows.extend( - self.query( - "index_member_all", - {"l2_code": sector_code, "is_new": is_new}, - "l2_code,l2_name,ts_code,name,in_date,out_date,is_new", - ) - ) - deduped: dict[str, dict[str, Any]] = {} - for row in _reconcile_membership_rows(rows): - code = str(row.get("ts_code") or "") - if code and _membership_active_on(row, trade_date): - current = deduped.get(code) - if current is None or str(row.get("in_date") or "") > str(current.get("in_date") or ""): - deduped[code] = row - return list(deduped.values()) - - def sw_sector_members(self, sector_code: str, trade_date: str) -> list[dict[str, Any]]: - """Return constituents active in a Shenwan L2 industry on the target date.""" - return self._sw_sector_members(sector_code, trade_date) - - def _stock_listing_reference(self) -> dict[str, dict[str, Any]]: - now = datetime.now().astimezone() - with self._stock_listing_lock: - loaded_at = self._stock_listing_cache.get("loaded_at") - cached = self._stock_listing_cache.get("rows") - if ( - isinstance(loaded_at, datetime) - and isinstance(cached, dict) - and now - loaded_at < timedelta(hours=6) - ): - return cached - - rows: list[dict[str, Any]] = [] - try: - for status in ("L", "D", "P"): - rows.extend(self.query( - "stock_basic", - {"list_status": status}, - "ts_code,name,list_status,list_date,delist_date", - )) - except TushareError: - # Unknown status must remain in the denominator so a reference-data - # failure cannot silently improve coverage. - return {} - reference = { - str(row.get("ts_code") or ""): dict(row) - for row in rows - if row.get("ts_code") - } - with self._stock_listing_lock: - type(self)._stock_listing_cache = {"loaded_at": now, "rows": reference} - return reference - - def _confirmed_suspended_members( - self, - members: list[dict[str, Any]], - quoted_codes: set[str], - trade_date: str, - ) -> list[dict[str, str]]: - suspended: list[dict[str, str]] = [] - for member in members: - code = str(member.get("ts_code") or "") - if not code or code in quoted_codes: - continue - cache_key = f"{trade_date}:{code}" - with self._suspension_lock: - cached = self._suspension_cache.get(cache_key, "missing") - if cached == "missing": - try: - rows = self.query( - "suspend_d", - {"ts_code": code}, - "ts_code,suspend_date,resume_date,ann_date,suspend_reason,reason_type", - ) - except TushareError: - rows = [] - active = [ - row for row in rows - if str(row.get("suspend_date") or "") - and str(row.get("suspend_date") or "") <= trade_date - and ( - not str(row.get("resume_date") or "") - or trade_date < str(row.get("resume_date") or "") - ) - ] - row = max( - active, - key=lambda item: str(item.get("suspend_date") or ""), - default=None, - ) - cached = ({ - "ts_code": code, - "name": str(member.get("name") or code), - "suspend_date": str(row.get("suspend_date") or ""), - "resume_date": str(row.get("resume_date") or ""), - "reason": str(row.get("suspend_reason") or row.get("reason_type") or "已确认停牌"), - } if row else None) - with self._suspension_lock: - type(self)._suspension_cache[cache_key] = cached - if isinstance(cached, dict): - suspended.append(cached) - return suspended - - def _sw_realtime_sector_snapshot( - self, - industry: dict[str, Any], - members: list[dict[str, Any]], - trade_date: str, - previous_trade_date: str, - finalized: bool = False, - ) -> dict[str, Any]: - sector_code = str(industry.get("l2_code") or "") - sw_rows = self.query( - "rt_sw_k", - {"ts_code": sector_code}, - "ts_code,name,trade_time,close,pre_close,high,open,low,vol,amount,pct_change", - ) - sw_row = sw_rows[0] if sw_rows else {} - trade_time = str(sw_row.get("trade_time") or "") - quote_date = trade_time[:10].replace("-", "") - quote_clock = trade_time[11:19] if len(trade_time) >= 19 else "" - outer_precise = bool(sw_row and quote_date == trade_date) - if finalized and (not quote_clock or quote_clock < "15:00:00"): - outer_precise = False - official_change = _number(sw_row.get("pct_change")) - if not official_change: - close = _number(sw_row.get("close")) - pre_close = _number(sw_row.get("pre_close")) - official_change = (close / pre_close - 1) * 100 if close and pre_close else 0 - if not outer_precise: - official_change = None - outer_error = "" - if not sw_row: - outer_error = f"No Shenwan realtime index returned for {sector_code}" - elif quote_date != trade_date: - outer_error = f"Shenwan realtime index date is {quote_date or 'unknown'}, expected {trade_date}" - elif finalized and (not quote_clock or quote_clock < "15:00:00"): - outer_error = f"Shenwan realtime index is not a close snapshot ({trade_time})" - - valid: list[dict[str, Any]] = [] - codes: list[str] = [] - reference: dict[str, Any] = {} - inner_error = "" - try: - reference = self._load_realtime_reference(trade_date, previous_trade_date) - active_codes = { - str(row.get("ts_code") or "") - for row in reference.get("basic_rows") or [] - if row.get("ts_code") - } - codes = [ - str(row.get("ts_code") or "") - for row in members - if str(row.get("ts_code") or "") in active_codes - ] - if codes: - quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") - for row in quotes: - close = _number(row.get("close")) - previous_close = _number(row.get("pre_close")) - if close <= 0 or previous_close <= 0: - continue - valid.append({**row, "change": (close / previous_close - 1) * 100}) - else: - inner_error = f"No active Shenwan members returned for {sector_code}" - except TushareError as exc: - inner_error = str(exc) - - coverage = len(valid) / max(len(codes), 1) * 100 - valid_codes = {str(item.get("ts_code") or "") for item in valid} - suspended_members = self._confirmed_suspended_members( - members, valid_codes, trade_date - ) - explained_count = len(valid) + len(suspended_members) - explained_coverage = explained_count / max(len(codes), 1) * 100 - coverage_issue = _sector_coverage_issue( - len(codes), len(valid), explained_coverage, explained_count - ) - inner_precise = bool(codes) and not coverage_issue - if not inner_precise and not inner_error: - inner_error = coverage_issue or "申万实时有效成分为空" - up_count = sum(item["change"] > 0 for item in valid) - down_count = sum(item["change"] < 0 for item in valid) - leader = max(valid, key=lambda item: item["change"], default={}) - leader_code = str(leader.get("ts_code") or "") - member_names = { - str(item.get("ts_code") or ""): str(item.get("name") or "") - for item in members - } - equal_change = sum(item["change"] for item in valid) / len(valid) if valid else 0 - amount_billion = sum(_number(item.get("amount")) for item in valid) / 100000000 - try: - self._ensure_realtime_market_cache(trade_date) - with self._realtime_reference_lock: - market_rows = list( - (self._latest_realtime_market.get(trade_date) or {}).get("rows") or [] - ) - except TushareError as exc: - market_rows = [] - inner_precise = False - inner_error = inner_error or str(exc) - capital_map = { - str(item.get("ts_code") or ""): item - for item in reference.get("capital_rows") or [] - } - sector_turnovers = [] - for item in valid: - capital = capital_map.get(str(item.get("ts_code") or ""), {}) - float_share = _number(capital.get("float_share")) - if float_share: - sector_turnovers.append(_number(item.get("vol")) / float_share / 100) - market_turnovers = [] - for item in market_rows: - capital = capital_map.get(str(item.get("ts_code") or ""), {}) - float_share = _number(capital.get("float_share")) - if float_share: - market_turnovers.append(_number(item.get("vol")) / float_share / 100) - average_turnover = sum(sector_turnovers) / len(sector_turnovers) if sector_turnovers else 0 - market_turnover = sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0 - relative_turnover = average_turnover / market_turnover if market_turnover else 0 - if not relative_turnover: - inner_precise = False - inner_error = inner_error or "Shenwan member relative turnover is unavailable" - return { - "code": sector_code, - "name": str(industry.get("l2_name") or sw_row.get("name") or ""), - "leader": str(leader.get("name") or member_names.get(leader_code) or "--").strip(), - "leader_code": leader_code, - "leading_pct": round(_number(leader.get("change")), 3), - "change": round(official_change, 3) if official_change is not None else None, - "member_equal_change": round(equal_change, 3), - "turnover_rate": round(average_turnover, 4), - "market_turnover_rate": round(market_turnover, 4), - "relative_turnover": round(relative_turnover, 4), - "up_count": up_count, - "down_count": down_count, - "flat_count": len(valid) - up_count - down_count, - "member_count": len(codes), - "quote_count": len(valid), - "coverage": round(coverage, 1), - "explained_count": explained_count, - "explained_coverage": round(explained_coverage, 1), - "suspended_count": len(suspended_members), - "suspended_members": suspended_members, - "strength": round(max(0, min(100, 50 + (official_change if official_change is not None else equal_change) * 5)), 1), - "amount_billion": round(amount_billion, 2), - "count": sum(item["change"] >= 9.5 for item in valid), - "max_streak": 0, - "source": "tushare_rt_sw_k+sw_members_rt_k", - "inner_source": "tushare_sw_members+rt_k", - "outer_source": "tushare_rt_sw_k", - "taxonomy": "sw_l2", - "industry": industry, - "trade_date": trade_date, - "inner_trade_date": trade_date if valid else "", - "outer_trade_date": quote_date, - "trade_time": trade_time, - "realtime": True, - "finalized": finalized, - "inner_precise": inner_precise, - "outer_precise": outer_precise, - "precise": inner_precise and outer_precise, - "inner_error": inner_error, - "outer_error": outer_error, - "schema_version": 6, - "methodology": "外显使用申万官方 rt_sw_k；内核独立使用申万成分 rt_k 宽度与相对换手聚合", - } - - def sector_snapshot( - self, - identifier: str, - requested_date: str, - realtime_expected: bool | None = None, - ) -> dict[str, Any]: - trade_date, _ = self.resolve_trade_context(requested_date) - raw_identifier = identifier.strip() - if not raw_identifier: - raise TushareError("Sector identifier is empty") - errors = [] - now = datetime.now().astimezone() - if realtime_expected is None: - realtime_expected = ( - trade_date == now.strftime("%Y%m%d") - and dt_time(9, 15) <= now.time().replace(tzinfo=None) <= dt_time(15, 5) - ) - try: - dc_params = {"trade_date": trade_date} - if re.fullmatch(r"[A-Z0-9.]+", raw_identifier.upper()) and "." in raw_identifier: - dc_params["ts_code"] = raw_identifier.upper() - else: - dc_params["name"] = raw_identifier - dc_rows = self.query( - "dc_index", - dc_params, - "ts_code,trade_date,name,leading,leading_code,pct_change,leading_pct," - "total_mv,turnover_rate,up_num,down_num", - ) - if not dc_rows and "name" in dc_params: - dc_rows = self.query( - "dc_index", - {"trade_date": trade_date}, - "ts_code,trade_date,name,leading,leading_code,pct_change,leading_pct," - "total_mv,turnover_rate,up_num,down_num", - ) - dc_row = _match_sector_row(dc_rows, raw_identifier) - if dc_row and not realtime_expected: - change = _number(dc_row.get("pct_change")) - actual_trade_date = str(dc_row.get("trade_date") or "") - return { - "code": dc_row.get("ts_code") or "", - "name": dc_row.get("name") or raw_identifier, - "leader": dc_row.get("leading") or "--", - "leader_code": dc_row.get("leading_code") or "", - "leading_pct": _number(dc_row.get("leading_pct")), - "change": change, - "turnover_rate": _number(dc_row.get("turnover_rate")), - "up_count": int(_number(dc_row.get("up_num"))), - "down_count": int(_number(dc_row.get("down_num"))), - "total_mv": _number(dc_row.get("total_mv")), - "strength": round(max(0, min(100, 50 + change * 5)), 1), - "amount_billion": 0, - "count": 0, - "max_streak": 0, - "source": "tushare_dc", - "trade_date": actual_trade_date, - "realtime": False, - "precise": actual_trade_date == trade_date, - } - except TushareError as exc: - errors.append(f"DC: {exc}") - - ts_code = raw_identifier.upper() - if re.fullmatch(r"\d{6}", ts_code): - ts_code = f"{ts_code}.TI" - try: - if re.fullmatch(r"\d{6}\.TI", ts_code): - index_rows = self.query( - "ths_index", - {"ts_code": ts_code}, - "ts_code,name,count,exchange,list_date,type", - ) - else: - index_rows = self.query( - "ths_index", - {}, - "ts_code,name,count,exchange,list_date,type", - ) - basic = _match_sector_row(index_rows, raw_identifier) - if not basic: - raise TushareError(f"No THS sector returned for {raw_identifier}") - except TushareError as exc: - errors.append(f"THS: {exc}") - raise TushareError("; ".join(errors)) from exc - actual_code = str(basic.get("ts_code") or ts_code) - if realtime_expected: - try: - realtime_sector = self._realtime_sector_snapshot(actual_code, basic, trade_date) - if realtime_sector: - return realtime_sector - except TushareError as exc: - errors.append(f"THS realtime members: {exc}") - daily_rows = self.query( - "ths_daily", - {"ts_code": actual_code, "trade_date": trade_date}, - "ts_code,trade_date,close,pct_change,vol,turnover_rate,total_mv,float_mv", - ) - daily = daily_rows[0] if daily_rows else {} - actual_trade_date = str(daily.get("trade_date") or "") - change = _number(daily.get("pct_change")) - return { - "code": actual_code, - "name": basic.get("name") or raw_identifier, - "leader": "--", - "change": change, - "leading_pct": change, - "turnover_rate": _number(daily.get("turnover_rate")), - "up_count": 0, - "down_count": 0, - "strength": round(max(0, min(100, 50 + change * 5)), 1), - "amount_billion": 0, - "count": 0, - "max_streak": 0, - "source": "tushare_ths", - "trade_date": actual_trade_date, - "realtime": False, - "precise": actual_trade_date == trade_date, - } - - def _realtime_sector_snapshot( - self, - sector_code: str, - basic: dict[str, Any], - trade_date: str, - ) -> dict[str, Any] | None: - members = self.query( - "ths_member", - {"ts_code": sector_code, "is_new": "Y"}, - "ts_code,con_code,con_name,is_new", - ) - codes = [str(row.get("con_code") or "") for row in members if row.get("con_code")] - if not codes: - return None - quotes = self.query("rt_k", {"ts_code": ",".join(codes)}, "") - valid = [] - for row in quotes: - close = _number(row.get("close")) - previous_close = _number(row.get("pre_close")) - if close <= 0 or previous_close <= 0: - continue - valid.append( - { - **row, - "change": (close / previous_close - 1) * 100, - } - ) - minimum = max(1, math.ceil(len(codes) * 0.9)) - if len(valid) < minimum: - raise TushareError( - f"Realtime sector coverage is insufficient ({len(valid)}/{len(codes)})" - ) - up_count = sum(item["change"] > 0 for item in valid) - down_count = sum(item["change"] < 0 for item in valid) - flat_count = len(valid) - up_count - down_count - leader = max(valid, key=lambda item: item["change"]) - change = sum(item["change"] for item in valid) / len(valid) - amount_billion = sum(_number(item.get("amount")) for item in valid) / 100000000 - self._ensure_realtime_market_cache(trade_date) - with self._realtime_reference_lock: - references = list(self._realtime_reference_cache.values()) - market_rows = list((self._latest_realtime_market.get(trade_date) or {}).get("rows") or []) - capital_map: dict[str, dict[str, Any]] = {} - for reference in reversed(references): - capital_map = { - str(item.get("ts_code") or ""): item - for item in reference.get("capital_rows") or [] - } - if capital_map: - break - sector_turnovers = [] - for item in valid: - capital = capital_map.get(str(item.get("ts_code") or ""), {}) - float_share = _number(capital.get("float_share")) - if float_share: - sector_turnovers.append(_number(item.get("vol")) / float_share / 100) - market_turnovers = [] - for item in market_rows: - capital = capital_map.get(str(item.get("ts_code") or ""), {}) - float_share = _number(capital.get("float_share")) - if float_share: - market_turnovers.append(_number(item.get("vol")) / float_share / 100) - average_turnover = sum(sector_turnovers) / len(sector_turnovers) if sector_turnovers else 0 - market_turnover = sum(market_turnovers) / len(market_turnovers) if market_turnovers else 0 - relative_turnover = average_turnover / market_turnover if market_turnover else 0 - return { - "code": sector_code, - "name": basic.get("name") or sector_code, - "leader": str(leader.get("name") or "--").strip(), - "leader_code": leader.get("ts_code") or "", - "leading_pct": round(leader["change"], 3), - "change": round(change, 3), - "turnover_rate": round(average_turnover, 4), - "market_turnover_rate": round(market_turnover, 4), - "relative_turnover": round(relative_turnover, 4), - "up_count": up_count, - "down_count": down_count, - "flat_count": flat_count, - "member_count": len(codes), - "quote_count": len(valid), - "coverage": round(len(valid) / len(codes) * 100, 1), - "strength": round(max(0, min(100, 50 + change * 5)), 1), - "amount_billion": round(amount_billion, 2), - "count": sum(item["change"] >= 9.5 for item in valid), - "max_streak": 0, - "source": "tushare_rt_ths_members", - "trade_date": trade_date, - "realtime": True, - "precise": True, - "methodology": "同花顺行业最新成分股的 rt_k 等权涨跌、宽度与成交额聚合", - } - - def hot_money_profiles(self) -> dict[str, Any]: - rows = self.query("hm_list", {}, "name,desc,orgs") - profiles: list[dict[str, Any]] = [] - seen_names: set[str] = set() - for row in rows: - name = str(row.get("name") or "").strip() - if not name or name in seen_names: - continue - seen_names.add(name) - description = _text(row.get("desc")) - organization_text = _text(row.get("orgs")) - parsed_organizations: Any = None - if organization_text.startswith("["): - try: - parsed_organizations = json.loads(organization_text) - except json.JSONDecodeError: - parsed_organizations = None - organization_parts = ( - parsed_organizations - if isinstance(parsed_organizations, list) - else re.split(r"[,，;；\n]+", organization_text) - ) - organizations = list(dict.fromkeys( - _text(part) - for part in organization_parts - if _text(part) - )) - profiles.append( - { - "id": f"hot-money-profile-{len(profiles) + 1}", - "name": name, - "description": description, - "organizations": organizations, - "organization_count": len(organizations), - } - ) - return { - "meta": { - "source": "tushare", - "status": "success" if profiles else "empty", - "schema_version": 1, - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - "notice": "", - }, - "summary": { - "profile_count": len(profiles), - "described_count": sum(bool(item["description"]) for item in profiles), - "organization_count": sum(item["organization_count"] for item in profiles), - }, - "profiles": profiles, - } - - def dragon_tiger(self, requested_date: str) -> dict[str, Any]: - trade_date, _ = self.resolve_trade_context(requested_date) - detail_rows = self.query( - "hm_detail", - {"trade_date": trade_date}, - "trade_date,ts_code,ts_name,buy_amount,sell_amount,net_amount," - "hm_name,hm_orgs,tag", - ) - - notices: list[str] = [] - try: - directory_rows = self.query("hm_list", {}, "name,desc,orgs") - except TushareError as exc: - directory_rows = [] - notices.append(f"游资名录暂不可用：{exc}") - directory = { - str(row.get("name") or "").strip(): { - "description": _text(row.get("desc")), - "orgs": _text(row.get("orgs")), - } - for row in directory_rows - if str(row.get("name") or "").strip() - } - - # 个股龙虎榜仅用于补充涨幅和上榜原因，不参与游资身份识别。 - try: - top_rows = self.query( - "top_list", - {"trade_date": trade_date}, - "trade_date,ts_code,name,pct_change,reason", - ) - except TushareError as exc: - top_rows = [] - notices.append(f"个股龙虎榜辅助信息暂不可用：{exc}") - stock_context: dict[str, dict[str, Any]] = {} - for row in top_rows: - ts_code = str(row.get("ts_code") or "") - if ts_code and ts_code not in stock_context: - stock_context[ts_code] = row - - groups: dict[str, dict[str, Any]] = {} - for row in detail_rows: - trader_name = str(row.get("hm_name") or "未命名游资").strip() - ts_code = str(row.get("ts_code") or "").strip() - stock = stock_context.get(ts_code, {}) - directory_item = directory.get(trader_name, {}) - seat_name = _text(row.get("hm_orgs")) or directory_item.get("orgs") or "--" - buy = round(_number(row.get("buy_amount")) / 1000000, 2) - sell = round(_number(row.get("sell_amount")) / 1000000, 2) - net_buy = round(_number(row.get("net_amount")) / 1000000, 2) - group = groups.setdefault( - trader_name, - { - "name": trader_name, - "description": directory_item.get("description") or "", - "directory_orgs": directory_item.get("orgs") or "", - "identity_type": "trader", - "identity_source": "tushare_hm", - "recognized": True, - "buy_million": 0.0, - "sell_million": 0.0, - "net_buy_million": 0.0, - "seat_names": set(), - "stock_codes": set(), - "operations": [], - }, - ) - group["buy_million"] += buy - group["sell_million"] += sell - group["net_buy_million"] += net_buy - if seat_name != "--": - group["seat_names"].add(seat_name) - code = ts_code.split(".")[0] - if code: - group["stock_codes"].add(code) - group["operations"].append( - { - "code": code, - "ts_code": ts_code, - "name": row.get("ts_name") or stock.get("name") or "--", - "change": ( - _number(stock.get("pct_change")) - if stock.get("pct_change") is not None - else None - ), - "direction": "买入" if net_buy > 0 else "卖出" if net_buy < 0 else "持平", - "buy_million": buy, - "sell_million": sell, - "net_buy_million": net_buy, - "seat_name": seat_name, - "seat_alias": trader_name, - "tag": _text(row.get("tag")) or "--", - "reason": _text(stock.get("reason")) or "--", - } - ) - - traders = list(groups.values()) - traders.sort(key=lambda item: abs(item["net_buy_million"]), reverse=True) - for index, group in enumerate(traders, start=1): - group["id"] = f"hot-money-{index}" - group["buy_million"] = round(group["buy_million"], 2) - group["sell_million"] = round(group["sell_million"], 2) - group["net_buy_million"] = round(group["net_buy_million"], 2) - group["seat_count"] = len(group.pop("seat_names")) - group["stock_count"] = len(group.pop("stock_codes")) - group["operation_count"] = len(group["operations"]) - group["operations"].sort( - key=lambda item: abs(float(item.get("net_buy_million") or 0)), reverse=True - ) - - operation_count = sum(item["operation_count"] for item in traders) - active_stocks = { - operation["code"] for item in traders for operation in item["operations"] - if operation["code"] - } - net_buy_total = round(sum(item["net_buy_million"] for item in traders), 2) - status = "success" if detail_rows else "partial" if top_rows else "empty" - if not detail_rows: - notices.insert( - 0, - f"当日有 {len(stock_context)} 只股票上榜，但未返回可识别的游资每日明细。" - if top_rows - else "该交易日未返回龙虎榜或游资每日明细。", - ) - return { - "meta": { - "requested_date": _display_date(requested_date), - "trade_date": _display_date(trade_date), - "source": "tushare", - "status": status, - "schema_version": 3, - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - "notice": "；".join(notices), - }, - "summary": { - "trader_count": len(traders), - "identity_count": len(traders), - "operation_count": operation_count, - "active_stock_count": len(active_stocks), - "seat_net_buy_million": net_buy_total, - "unclassified_count": 0, - "directory_count": len(directory), - "official_stock_count": len(stock_context), - }, - "traders": traders, - "unclassified_seats": [], - "rows": [], - } - - def stock_detail(self, ts_code: str, requested_date: str) -> dict[str, Any]: - trade_date, _ = self.resolve_trade_context(requested_date) - end = datetime.strptime(trade_date, "%Y%m%d") - start_date = (end - timedelta(days=190)).strftime("%Y%m%d") - daily = self.query( - "daily", - {"ts_code": ts_code, "start_date": start_date, "end_date": trade_date}, - "ts_code,trade_date,open,high,low,close,pct_chg,vol,amount", - ) - factors = self.query( - "adj_factor", - {"ts_code": ts_code, "start_date": start_date, "end_date": trade_date}, - "ts_code,trade_date,adj_factor", - ) - basics = self.query( - "stock_basic", - {"ts_code": ts_code}, - "ts_code,symbol,name,area,industry,market,list_date", - ) - daily_basics = self.query( - "daily_basic", - {"ts_code": ts_code, "trade_date": trade_date}, - "ts_code,trade_date,turnover_rate,volume_ratio,total_mv,circ_mv", - ) - moneyflow = self.query( - "moneyflow", - {"ts_code": ts_code, "trade_date": trade_date}, - "ts_code,trade_date,buy_sm_amount,sell_sm_amount,buy_md_amount,sell_md_amount," - "buy_lg_amount,sell_lg_amount,buy_elg_amount,sell_elg_amount,net_mf_amount", - ) - factor_map = {row["trade_date"]: _number(row.get("adj_factor"), 1) for row in factors} - latest_factor = max(factor_map.values(), default=1) or 1 - prices = [] - for row in sorted(daily, key=lambda item: item.get("trade_date", ""))[-90:]: - factor = factor_map.get(row.get("trade_date"), latest_factor) - ratio = factor / latest_factor - prices.append( - { - "trade_date": _display_date(str(row.get("trade_date", ""))), - "open": round(_number(row.get("open")) * ratio, 3), - "high": round(_number(row.get("high")) * ratio, 3), - "low": round(_number(row.get("low")) * ratio, 3), - "close": round(_number(row.get("close")) * ratio, 3), - "change": _number(row.get("pct_chg")), - "volume": _number(row.get("vol")), - "amount_billion": round(_number(row.get("amount")) / 100000, 2), - } - ) - flow = moneyflow[0] if moneyflow else {} - basic = basics[0] if basics else {} - daily_basic = daily_basics[0] if daily_basics else {} - latest = prices[-1] if prices else {} - actual_trade_date = max( - (str(row.get("trade_date") or "") for row in daily), - default=trade_date, - ) or trade_date - return { - "meta": { - "requested_date": _display_date(requested_date), - "trade_date": _display_date(actual_trade_date), - "source": "tushare", - "updated_at": datetime.now().astimezone().isoformat(timespec="seconds"), - "notice": "", - }, - "stock": { - "code": ts_code.split(".")[0], - "ts_code": ts_code, - "name": basic.get("name") or "--", - "industry": basic.get("industry") or "其他", - "area": basic.get("area") or "--", - "market": basic.get("market") or "--", - "list_date": _display_date(str(basic.get("list_date") or "")), - "price": latest.get("close", 0), - "change": latest.get("change", 0), - "turnover_rate": _number(daily_basic.get("turnover_rate")), - "volume_ratio": _number(daily_basic.get("volume_ratio")), - "amount_billion": latest.get("amount_billion", 0), - }, - "prices": prices, - "moneyflow": { - "net_million": round(_number(flow.get("net_mf_amount")) / 100, 2), - "large_million": round( - (_number(flow.get("buy_lg_amount")) + _number(flow.get("buy_elg_amount")) - - _number(flow.get("sell_lg_amount")) - _number(flow.get("sell_elg_amount"))) / 100, - 2, - ), - "medium_million": round( - (_number(flow.get("buy_md_amount")) - _number(flow.get("sell_md_amount"))) / 100, - 2, - ), - "small_million": round( - (_number(flow.get("buy_sm_amount")) - _number(flow.get("sell_sm_amount"))) / 100, - 2, - ), - }, - } - - def stock_intraday(self, ts_code: str, requested_date: str) -> dict[str, Any]: - trade_date, _ = self.resolve_trade_context(requested_date) - display_date = _display_date(trade_date) - rows = self.query( - "stk_mins", - { - "ts_code": ts_code, - "freq": "1min", - "start_date": f"{display_date} 09:00:00", - "end_date": f"{display_date} 15:30:00", - }, - "ts_code,trade_time,open,close,high,low,vol,amount", - ) - points = [] - for row in sorted(rows, key=lambda item: str(item.get("trade_time") or "")): - trade_time = str(row.get("trade_time") or "") - if not trade_time: - continue - points.append( - { - "time": trade_time[-8:-3] if len(trade_time) >= 8 else trade_time, - "open": round(_number(row.get("open")), 3), - "high": round(_number(row.get("high")), 3), - "low": round(_number(row.get("low")), 3), - "close": round(_number(row.get("close")), 3), - "volume": _number(row.get("vol")), - "amount": _number(row.get("amount")), - } - ) - return {"trade_date": display_date, "points": points} - - def resolve_trade_context(self, requested: str) -> tuple[str, str]: - requested_rows = self.query( - "trade_cal", - {"exchange": "SSE", "start_date": requested, "end_date": requested}, - "cal_date,is_open,pretrade_date", - ) - if not requested_rows: - trade_date = requested - else: - row = requested_rows[0] - trade_date = row["cal_date"] if row.get("is_open") == 1 else row.get("pretrade_date", requested) - - resolved_rows = self.query( - "trade_cal", - {"exchange": "SSE", "start_date": trade_date, "end_date": trade_date}, - "cal_date,is_open,pretrade_date", - ) - previous = resolved_rows[0].get("pretrade_date") if resolved_rows else "" - return trade_date, previous or trade_date - - def _load_daily(self, trade_date: str) -> list[dict[str, Any]]: - return self.query( - "daily", - {"trade_date": trade_date}, - "ts_code,trade_date,open,high,low,close,pct_chg,amount", - ) - - def _load_limit_type(self, trade_date: str, limit_type: str) -> list[dict[str, Any]]: - fields = ( - "trade_date,ts_code,industry,name,close,pct_chg,amount,limit_amount," - "float_mv,total_mv,turnover_ratio,fd_amount,first_time,last_time," - "open_times,up_stat,limit_times" - ) - rows = self.query( - "limit_list_d", - {"trade_date": trade_date, "limit_type": limit_type}, - fields, - ) - for row in rows: - row["limit_type"] = limit_type - row["amount_unit"] = "yuan" - return rows - - def _load_limit_lists(self, trade_date: str) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for limit_type in ("U", "D", "Z"): - rows.extend(self._load_limit_type(trade_date, limit_type)) - return rows - - def _derive_limits( - self, - trade_date: str, - daily: list[dict[str, Any]], - price_limits: list[dict[str, Any]] | None = None, - basic_rows: list[dict[str, Any]] | None = None, - previous_limit_rows: list[dict[str, Any]] | None = None, - capital_rows: list[dict[str, Any]] | None = None, - ) -> list[dict[str, Any]]: - if price_limits is None: - price_limits = self.query( - "stk_limit", - {"trade_date": trade_date}, - "ts_code,trade_date,up_limit,down_limit", - ) - limit_map = {row["ts_code"]: row for row in price_limits} - if basic_rows is None: - basic_rows = self.query( - "stock_basic", - {"list_status": "L"}, - "ts_code,name,industry", - ) - basic_map = {row["ts_code"]: row for row in basic_rows} - previous_limit_map = { - str(row.get("ts_code") or ""): row for row in (previous_limit_rows or []) - } - capital_map = { - str(row.get("ts_code") or ""): row for row in (capital_rows or []) - } - - result: list[dict[str, Any]] = [] - for row in daily: - bounds = limit_map.get(row.get("ts_code")) - if not bounds or row.get("close") is None: - continue - limit_type = "" - if _prices_equal(row["close"], bounds.get("up_limit")): - limit_type = "U" - elif _prices_equal(row["close"], bounds.get("down_limit")): - limit_type = "D" - elif _prices_equal(row.get("high"), bounds.get("up_limit")): - limit_type = "Z" - if not limit_type: - continue - basic = basic_map.get(row["ts_code"], {}) - previous_limit = previous_limit_map.get(str(row.get("ts_code") or ""), {}) - streak = ( - max(1, int(_number(previous_limit.get("limit_times"), 1)) + 1) - if limit_type == "U" and previous_limit - else 1 - ) - item = { - **row, - "name": basic.get("name", "--"), - "industry": basic.get("industry") or "其他", - "limit_type": limit_type, - "limit_times": streak, - "open_times": 1 if limit_type == "Z" else 0, - "amount_unit": row.get("amount_unit") or "thousand_yuan", - } - if row.get("amount_unit") == "yuan": - capital = capital_map.get(str(row.get("ts_code") or ""), {}) - if not capital and capital_rows is None: - capital = self._latest_capital(str(row.get("ts_code") or ""), trade_date) - float_share = _number(capital.get("float_share")) - item["turnover_ratio"] = ( - _number(row.get("vol")) / float_share / 100 if float_share else 0 - ) - item["turnover_source"] = ( - "rt_volume/latest_float_share" if float_share else "unavailable" - ) - item["capital_trade_date"] = str(capital.get("trade_date") or "") - result.append(item) - return result - - @staticmethod - def _normalize_limit(row: dict[str, Any], status: str) -> dict[str, Any]: - amount = _number(row.get("amount")) - if row.get("amount_unit") == "thousand_yuan": - amount_billion = amount / 100000 - else: - amount_billion = amount / 100000000 - return { - "code": str(row.get("ts_code", "")).split(".")[0], - "ts_code": row.get("ts_code", ""), - "name": row.get("name") or "--", - "price": _number(row.get("close")), - "change": _number(row.get("pct_chg")), - "sector": row.get("industry") or "其他", - "reason": row.get("industry") or "待补充", - "first_time": _display_time(row.get("first_time")), - "last_time": _display_time(row.get("last_time")), - "open_times": int(_number(row.get("open_times"))), - "streak": max(1, int(_number(row.get("limit_times"), 1))), - "turnover_rate": _number(row.get("turnover_ratio")), - "turnover_source": row.get("turnover_source") or "provider", - "capital_trade_date": row.get("capital_trade_date") or "", - "amount_billion": round(amount_billion, 2), - "seal_amount_million": round(_number(row.get("fd_amount")) / 10000, 0), - "float_mv_billion": round(_number(row.get("float_mv")) / 100000000, 1), - "status": status, - } - - -def _number(value: Any, default: float = 0.0) -> float: - try: - number = float(value) - return number if math.isfinite(number) else default - except (TypeError, ValueError): - return default - - -def _text(value: Any) -> str: - if isinstance(value, (list, tuple, set)): - return "、".join(str(item).strip() for item in value if str(item).strip()) - return str(value or "").strip() - - -def _filter_members_by_listing( - members: list[dict[str, Any]], - listing_reference: dict[str, dict[str, Any]], - trade_date: str, -) -> tuple[list[dict[str, Any]], list[dict[str, str]]]: - eligible: list[dict[str, Any]] = [] - excluded: list[dict[str, str]] = [] - for member in members: - code = str(member.get("ts_code") or "") - listing = listing_reference.get(code) - if not listing: - eligible.append(member) - continue - list_date = str(listing.get("list_date") or "") - delist_date = str(listing.get("delist_date") or "") - reason = "" - effective_date = "" - if delist_date and delist_date <= trade_date: - reason = "目标日期前已退市" - effective_date = delist_date - elif list_date and list_date > trade_date: - reason = "目标日期尚未上市" - effective_date = list_date - if not reason: - eligible.append(member) - continue - excluded.append({ - "ts_code": code, - "name": str(member.get("name") or listing.get("name") or code), - "reason": reason, - "effective_date": effective_date, - }) - return eligible, excluded - - -def _sector_coverage_issue( - member_count: int, - quote_count: int, - coverage: float | None = None, - explained_count: int | None = None, -) -> str: - members = max(0, int(member_count or 0)) - quotes = max(0, min(int(quote_count or 0), members)) - if members <= 0: - if coverage is not None and float(coverage) >= 90: - return "" - if coverage is not None: - return "行业成分行情覆盖率低于90%" - return "申万有效成分为空" - explained = quotes if explained_count is None else max( - quotes, min(int(explained_count or 0), members) - ) - actual_coverage = ( - float(coverage) - if coverage is not None - else explained / members * 100 - ) - missing = members - explained - if members <= 7 and missing: - return f"小型行业有效成分状态仅确认 {explained}/{members}，要求全部可解释" - if members <= 20 and (actual_coverage < 90 or missing > 1): - return f"中型行业有效成分状态仅确认 {explained}/{members}，要求覆盖率至少90%且最多缺1只" - if members > 20 and actual_coverage < 90: - return f"行业有效成分状态仅确认 {explained}/{members}，覆盖率低于90%" - return "" - - -def _membership_active_on(row: dict[str, Any], trade_date: str) -> bool: - start = str(row.get("in_date") or "") - end = str(row.get("out_date") or "") - return (not start or start <= trade_date) and (not end or end > trade_date) - - -def _reconcile_membership_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Merge duplicate Y/N membership rows before evaluating their date interval.""" - reconciled: dict[tuple[str, str, str, str, str], dict[str, Any]] = {} - for raw in rows: - row = dict(raw) - key = ( - str(row.get("ts_code") or ""), - str(row.get("l1_code") or ""), - str(row.get("l2_code") or ""), - str(row.get("l3_code") or ""), - str(row.get("in_date") or ""), - ) - current = reconciled.get(key) - if current is None: - reconciled[key] = row - continue - current_end = str(current.get("out_date") or "") - candidate_end = str(row.get("out_date") or "") - if candidate_end and not current_end: - current["out_date"] = candidate_end - current["is_new"] = row.get("is_new") or current.get("is_new") - for field, value in row.items(): - if not current.get(field) and value not in (None, ""): - current[field] = value - return list(reconciled.values()) - - -def _match_sector_row(rows: list[dict[str, Any]], identifier: str) -> dict[str, Any] | None: - if not rows: - return None - target = identifier.strip().upper() - code_match = next( - (row for row in rows if str(row.get("ts_code") or "").strip().upper() == target), - None, - ) - if code_match: - return code_match - - def normalized(value: Any) -> str: - text = str(value or "").strip().replace(" ", "") - for suffix in ("板块", "概念", "行业"): - text = text.removesuffix(suffix) - aliases = { - "元器件": "元件", - "电子元器件": "元件", - } - return aliases.get(text, text) - - target_name = normalized(identifier) - exact = [row for row in rows if normalized(row.get("name")) == target_name] - if exact: - return min(exact, key=_sector_match_priority) - fuzzy = [ - row for row in rows - if target_name and ( - target_name in normalized(row.get("name")) - or normalized(row.get("name")) in target_name - ) - ] - return min( - fuzzy, - key=lambda row: (len(normalized(row.get("name"))), *_sector_match_priority(row)), - ) if fuzzy else None - - -def _sector_match_priority(row: dict[str, Any]) -> tuple[int, int, int]: - code = str(row.get("ts_code") or "") - exchange = str(row.get("exchange") or "").upper() - return ( - 0 if exchange == "A" else 1, - 0 if code.startswith("881") else 1, - 0 if _number(row.get("count")) > 0 else 1, - ) - - -def _prices_equal(left: Any, right: Any) -> bool: - if left is None or right is None: - return False - return abs(_number(left) - _number(right)) < 0.005 - - -def _value_percentile(value: float, population: list[float]) -> float: - valid = sorted(item for item in population if item >= 0) - if not valid: - return 0.0 - below = sum(item < value for item in valid) - equal = sum(item == value for item in valid) - return (below + equal * 0.5) / len(valid) - - -def _trading_session_progress(current_time: dt_time) -> float: - morning_start = dt_time(9, 30) - morning_end = dt_time(11, 30) - afternoon_start = dt_time(13, 0) - afternoon_end = dt_time(15, 0) - if current_time <= morning_start: - return 0.05 - if current_time <= morning_end: - minutes = (current_time.hour * 60 + current_time.minute) - (9 * 60 + 30) - return max(0.05, min(0.5, minutes / 240)) - if current_time < afternoon_start: - return 0.5 - if current_time <= afternoon_end: - minutes = (current_time.hour * 60 + current_time.minute) - 13 * 60 - return max(0.5, min(1.0, 0.5 + minutes / 240)) - return 1.0 - - -def _display_time(value: Any) -> str: - raw = str(value or "").replace(":", "").zfill(6) - if not raw.strip("0"): - return "--" - return f"{raw[:2]}:{raw[2:4]}:{raw[4:6]}" - - -def _display_date(value: str) -> str: - return f"{value[:4]}-{value[4:6]}-{value[6:8]}" if len(value) == 8 else value - - -def _realtime_market_status(current_time: dt_time) -> str: - if current_time < dt_time(9, 25): - return "pre_open" - if current_time < dt_time(9, 30): - return "auction" - if current_time <= dt_time(11, 30) or dt_time(13, 0) <= current_time <= dt_time(15, 0): - return "trading" - if current_time < dt_time(13, 0): - return "lunch_break" - return "closed" - - -def _build_overview( - daily: list[dict[str, Any]], - up_rows: list[dict[str, Any]], - down_rows: list[dict[str, Any]], - broken_rows: list[dict[str, Any]], -) -> dict[str, Any]: - up_count = sum(1 for row in daily if _number(row.get("pct_chg")) > 0) - down_count = sum(1 for row in daily if _number(row.get("pct_chg")) < 0) - flat_count = len(daily) - up_count - down_count - amount_billion = sum( - _number(row.get("amount")) - / (100000000 if row.get("amount_unit") == "yuan" else 100000) - for row in daily - ) - limit_count = len(up_rows) - broken_count = len(broken_rows) - seal_rate = round(limit_count / max(limit_count + broken_count, 1) * 100, 1) - return { - "up_count": up_count, - "down_count": down_count, - "flat_count": flat_count, - "limit_up_count": limit_count, - "limit_down_count": len(down_rows), - "broken_count": broken_count, - "amount_billion": round(amount_billion, 1), - "seal_rate": seal_rate, - } - - -def _build_ladders(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - groups: dict[int, list[dict[str, Any]]] = {} - for row in rows: - groups.setdefault(int(row.get("streak") or 1), []).append(row) - return [ - { - "level": level, - "label": "首板" if level == 1 else f"{level}板", - "count": len(stocks), - "stocks": sorted(stocks, key=lambda item: item.get("first_time") or "99:99:99"), - } - for level, stocks in sorted(groups.items(), reverse=True) - ] - - -def _build_sectors(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - counts = Counter(row.get("sector") or "其他" for row in rows) - result: list[dict[str, Any]] = [] - for name, count in counts.most_common(20): - stocks = [row for row in rows if (row.get("sector") or "其他") == name] - max_streak = max(item.get("streak", 1) for item in stocks) - leader = max(stocks, key=lambda item: (item.get("streak", 1), item.get("amount_billion", 0))) - result.append( - { - "name": name, - "count": count, - "strength": min(100, 44 + count * 8 + max_streak * 5), - "amount_billion": round(sum(item.get("amount_billion", 0) for item in stocks), 1), - "leader": leader.get("name", "--"), - "change": round(sum(item.get("change", 0) for item in stocks) / count, 2), - "max_streak": max_streak, - } - ) - return result - - -def _build_yesterday_performance( - previous_limits: list[dict[str, Any]], - daily: list[dict[str, Any]], - current_limits: list[dict[str, Any]], - current_broken: list[dict[str, Any]], - current_down: list[dict[str, Any]], -) -> list[dict[str, Any]]: - daily_map = {str(row.get("ts_code", "")).split(".")[0]: row for row in daily} - limit_map = {row["code"]: row for row in current_limits} - broken_codes = {row["code"] for row in current_broken} - down_codes = {row["code"] for row in current_down} - result = [] - for previous in previous_limits: - code = previous["code"] - daily_row = daily_map.get(code, {}) - current = limit_map.get(code) - if current: - outcome = "晋级" - elif code in broken_codes: - outcome = "炸板" - elif code in down_codes: - outcome = "跌停" - else: - outcome = "断板" - result.append( - { - "code": code, - "name": previous["name"], - "prior_streak": previous.get("streak", 1), - "current_streak": current.get("streak", 0) if current else 0, - "current_change": _number(daily_row.get("pct_chg")), - "current_price": _number(daily_row.get("close")), - "sector": previous.get("sector", "其他"), - "reason": previous.get("reason", "待补充"), - "outcome": outcome, - } - ) - return result - - -def _build_limit_performance(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - result = [] - for level in sorted({int(row.get("prior_streak") or 1) for row in rows}, reverse=True): - group = [row for row in rows if int(row.get("prior_streak") or 1) == level] - advanced = sum(row.get("outcome") == "晋级" for row in group) - positive = sum(_number(row.get("current_change")) > 0 for row in group) - result.append( - { - "level": level, - "label": "昨日首板" if level == 1 else f"昨日{level}板", - "count": len(group), - "advanced": advanced, - "advance_rate": round(advanced / len(group) * 100, 1), - "positive_rate": round(positive / len(group) * 100, 1), - "average_change": round(sum(_number(row.get("current_change")) for row in group) / len(group), 2), - } - ) - return result - - -def _build_sector_rotation( - current: list[dict[str, Any]], previous: list[dict[str, Any]] -) -> list[dict[str, Any]]: - previous_map = {row["name"]: row for row in previous} - result = [] - for index, sector in enumerate(current, start=1): - previous_count = int(previous_map.get(sector["name"], {}).get("count", 0)) - delta = int(sector["count"]) - previous_count - result.append( - { - **sector, - "rank": index, - "previous_count": previous_count, - "delta": delta, - "trend": "升温" if delta > 0 else "降温" if delta < 0 else "持平", - } - ) - return result +sys.modules[__name__] = _implementation