from __future__ import annotations import copy import http.client import json import time import urllib.error import urllib.parse import urllib.request from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass from datetime import datetime from threading import Lock from typing import Any, ClassVar class RealtimeAggregateError(RuntimeError): pass EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get" EASTMONEY_STOCK_URL = "https://push2.eastmoney.com/api/qt/stock/get" EASTMONEY_STOCK_FIELDS = "f43,f44,f45,f46,f47,f48,f57,f58,f60,f86,f168" EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get" EASTMONEY_A_SHARE_BOARDS = ( "m:0+t:6", "m:0+t:80", "m:1+t:2", "m:1+t:23", "m:0+t:81", ) EASTMONEY_QUOTE_FIELDS = "f12,f13,f14,f2,f3,f4,f5,f6,f15,f16,f17,f18,f8,f124" EASTMONEY_MARKET_PAGE_SIZE = 100 TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006" TENCENT_QUOTE_URL = "https://qt.gtimg.cn/q=" THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool" XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail" BROWSER_USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/138.0.0.0 Safari/537.36" ) @dataclass class WebRealtimeAggregator: timeout: int = 8 retry_attempts: int = 3 retry_delay_seconds: float = 0.2 response_cache_ttl_seconds: int = 90 _sector_cache: ClassVar[dict[str, Any]] = {} _sector_cache_lock: ClassVar[Lock] = Lock() _response_cache: ClassVar[dict[str, dict[str, Any]]] = {} _response_cache_lock: ClassVar[Lock] = Lock() def health_snapshot(self, sector: str = "") -> dict[str, Any]: started = time.perf_counter() sources: dict[str, dict[str, Any]] = {} indices: list[dict[str, Any]] = [] sector_payload: dict[str, Any] | None = None indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices) if sector.strip(): sector_payload, sources["eastmoney_sector"] = self._capture( lambda: self.eastmoney_sector(sector) ) ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool) xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool) index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []] now = datetime.now().astimezone() max_skew = 120 if now.hour >= 15 else 15 index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew ready = ( bool(indices) and len(indices) == 3 and index_consistent and (not sector.strip() or bool(sector_payload)) ) return { "ready": ready, "isolated": True, "generated_at": datetime.now().astimezone().isoformat(timespec="seconds"), "elapsed_ms": round((time.perf_counter() - started) * 1000), "indices": indices or [], "index_consistent": index_consistent, "sector": sector_payload, "sources": sources, "observations": { "ths_limit_pool": ths_observation, "xgb_limit_pool": xgb_observation, }, "policy": { "integration": "heaven_realtime_fallback", "max_index_time_skew_seconds": max_skew, "notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。", }, } def eastmoney_indices(self) -> list[dict[str, Any]]: try: payload = self._get_json( EASTMONEY_INDEX_URL, { "secids": "1.000001,0.399001,0.399006", "fltt": "2", "invt": "2", "fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124", }, referer="https://quote.eastmoney.com/", ) except RealtimeAggregateError: return self.tencent_indices() cache_meta = payload.get("_aggregate_cache") or {} rows = list((payload.get("data") or {}).get("diff") or []) result = [] for row in rows: code = str(row.get("f12") or "") if code not in {"000001", "399001", "399006"}: continue epoch = int(_number(row.get("f124"))) result.append( { "code": code, "name": row.get("f14") or code, "price": _number(row.get("f2")), "change": _number(row.get("f3")), "change_amount": _number(row.get("f4")), "open": _number(row.get("f17")), "high": _number(row.get("f15")), "low": _number(row.get("f16")), "previous_close": _number(row.get("f18")), "amount_billion": round(_number(row.get("f6")) / 100000000, 2), "quote_time_epoch": epoch, "quote_time": ( datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds") if epoch else "" ), "source": ( "eastmoney_push2_cache" if cache_meta else "eastmoney_push2" ), "cache_age_seconds": cache_meta.get("age_seconds", 0), } ) if len(result) != 3: raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices") return result def eastmoney_market_quotes(self, expected_date: str = "") -> list[dict[str, Any]]: """Full A-share snapshot via Eastmoney clist, used when Tushare rt_k is unavailable.""" now = time.time() cache_key = "assembled:eastmoney_market" with self._response_cache_lock: cached = self._response_cache.get(cache_key) cache_age = now - float((cached or {}).get("created_at") or 0) if cached and cache_age <= min(20, self.response_cache_ttl_seconds): quotes = list(cached.get("payload") or []) return self._filter_quotes_by_date(quotes, expected_date) rows: list[dict[str, Any]] = [] board_errors: list[str] = [] for board in EASTMONEY_A_SHARE_BOARDS: try: rows.extend(self._eastmoney_board_quotes(board)) except Exception as exc: board_errors.append(f"{board}:{exc}") quotes = [] seen: set[str] = set() for row in rows: quote = _normalize_eastmoney_quote(row) ts_code = str((quote or {}).get("ts_code") or "") if not quote or ts_code in seen: continue seen.add(ts_code) quotes.append(quote) if len(quotes) < 200: detail = f"；{'; '.join(board_errors)}" if board_errors else "" raise RealtimeAggregateError( f"Eastmoney market snapshot too small: {len(quotes)}{detail}" ) quotes = self._filter_quotes_by_date(quotes, expected_date) with self._response_cache_lock: self._response_cache[cache_key] = {"created_at": now, "payload": quotes} return quotes def _eastmoney_board_quotes(self, board: str) -> list[dict[str, Any]]: first = self._eastmoney_market_page(board, 1) data = first.get("data") or {} rows = _diff_rows(data) total = int(_number(data.get("total"))) page_count = 1 if total > 0: page_count = max(1, (total + EASTMONEY_MARKET_PAGE_SIZE - 1) // EASTMONEY_MARKET_PAGE_SIZE) for page in range(2, min(page_count, 40) + 1): payload = self._eastmoney_market_page(board, page) rows.extend(_diff_rows(payload.get("data") or {})) return rows def _eastmoney_market_page(self, board: str, page: int) -> dict[str, Any]: return self._get_json( EASTMONEY_SECTOR_URL, { "pn": str(page), "pz": str(EASTMONEY_MARKET_PAGE_SIZE), "po": "1", "np": "1", "fltt": "2", "invt": "2", "fid": "f12", "fs": board, "fields": EASTMONEY_QUOTE_FIELDS, }, referer="https://quote.eastmoney.com/center/gridlist.html", ) def _filter_quotes_by_date( self, quotes: list[dict[str, Any]], expected_date: str, ) -> list[dict[str, Any]]: want = str(expected_date or "").replace("-", "") if not want or not quotes: return quotes dated = [item for item in quotes if str(item.get("quote_date") or "") == want] if dated and len(dated) >= max(100, int(len(quotes) * 0.2)): return dated if dated: return dated if all(not item.get("quote_date") for item in quotes): return quotes raise RealtimeAggregateError(f"Eastmoney quotes are not for {want}") def tencent_market_quotes( self, codes: list[str], expected_date: str = "", ) -> list[dict[str, Any]]: symbols: list[str] = [] seen: set[str] = set() for raw in codes: ts = str(raw or "").strip().upper() if not ts: continue symbol = ts.split(".")[0] if not symbol.isdigit() or len(symbol) != 6 or symbol in seen: continue seen.add(symbol) if ts.endswith(".SH") or symbol.startswith(("5", "6", "9")): symbols.append(f"sh{symbol}") elif ts.endswith(".BJ") or symbol.startswith(("4", "8")): symbols.append(f"bj{symbol}") else: symbols.append(f"sz{symbol}") if not symbols: raise RealtimeAggregateError("No stock codes available for Tencent quotes") quotes: list[dict[str, Any]] = [] batch_size = 80 def load_batch(batch: list[str]) -> list[dict[str, Any]]: raw, _cache_age = self._get_text( f"{TENCENT_QUOTE_URL}{','.join(batch)}", referer="https://gu.qq.com/", encoding="gb18030", ) return [ quote for line in raw.splitlines() if (quote := _parse_tencent_stock_quote(line)) ] batches = [symbols[index:index + batch_size] for index in range(0, len(symbols), batch_size)] errors: list[str] = [] with ThreadPoolExecutor(max_workers=4) as executor: for result in executor.map(self._capture, [lambda batch=batch: load_batch(batch) for batch in batches]): rows, status = result if status.get("ok") and rows: quotes.extend(rows) elif not status.get("ok"): errors.append(str(status.get("error") or "batch failed")) if len(quotes) < 200: detail = f"；{'; '.join(errors[:3])}" if errors else "" raise RealtimeAggregateError( f"Tencent market snapshot too small: {len(quotes)}{detail}" ) return self._filter_quotes_by_date(quotes, expected_date) def tencent_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]: symbol, _secid, ts_code = _a_share_identity(code) raw, _cache_age = self._get_text( f"{TENCENT_QUOTE_URL}{symbol}", referer="https://gu.qq.com/", encoding="gb18030", ) quote = next( ( item for line in raw.splitlines() if (item := _parse_tencent_stock_quote(line)) ), None, ) if not quote: raise RealtimeAggregateError(f"Tencent stock quote unavailable for {ts_code}") return _require_quote_date(quote, expected_date) def eastmoney_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]: _symbol, secid, ts_code = _a_share_identity(code) payload = self._get_json( EASTMONEY_STOCK_URL, { "secid": secid, "invt": "2", "fltt": "2", "fields": EASTMONEY_STOCK_FIELDS, }, referer="https://quote.eastmoney.com/", ) quote = _normalize_eastmoney_stock_quote(payload.get("data") or {}, ts_code) if not quote: raise RealtimeAggregateError(f"Eastmoney stock quote unavailable for {ts_code}") return _require_quote_date(quote, expected_date) def tencent_indices(self) -> list[dict[str, Any]]: raw, cache_age = self._get_text( TENCENT_INDEX_URL, referer="https://gu.qq.com/", encoding="gb18030", ) result = [] for line in raw.splitlines(): if '="' not in line: continue fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~") if len(fields) < 38: continue code = fields[2] if code not in {"000001", "399001", "399006"}: continue try: quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone() except ValueError as exc: raise RealtimeAggregateError( f"Tencent returned invalid quote time for {code}" ) from exc result.append( { "code": code, "name": fields[1] or code, "price": _number(fields[3]), "change": _number(fields[32]), "change_amount": _number(fields[31]), "open": _number(fields[5]), "high": _number(fields[33]), "low": _number(fields[34]), "previous_close": _number(fields[4]), "amount_billion": round(_number(fields[37]) / 10000, 2), "quote_time_epoch": int(quote_time.timestamp()), "quote_time": quote_time.isoformat(timespec="seconds"), "source": "tencent_qt_cache" if cache_age else "tencent_qt", "cache_age_seconds": cache_age, } ) if len(result) != 3: raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices") return result def eastmoney_sector(self, query: str) -> dict[str, Any]: target = _normalize_sector(query) candidates = self._eastmoney_sector_catalog() matched = _match_sector(candidates, target) if not matched: raise RealtimeAggregateError(f"Eastmoney sector not found: {query}") epoch = int(_number(matched.get("f124"))) return { "code": matched.get("f12") or "", "name": matched.get("f14") or query, "price": _number(matched.get("f2")), "change": _number(matched.get("f3")), "change_amount": _number(matched.get("f4")), "turnover_rate": _number(matched.get("f8")), "up_count": int(_number(matched.get("f104"))), "down_count": int(_number(matched.get("f105"))), "leader": matched.get("f128") or "--", "leader_code": matched.get("f140") or "", "leading_pct": _number(matched.get("f136")), "quote_time_epoch": epoch, "quote_time": ( datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds") if epoch else "" ), "source": "eastmoney_push2", "match_query": query, } def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]: now = time.time() with self._sector_cache_lock: cached = self._sector_cache.get("eastmoney") if cached and now - float(cached.get("created_at") or 0) < 600: return list(cached.get("rows") or []) def load_page(page: int) -> list[dict[str, Any]]: payload = self._get_json( EASTMONEY_SECTOR_URL, { "pn": str(page), "pz": "100", "po": "1", "np": "1", "fltt": "2", "invt": "2", "fid": "f3", "fs": "m:90+t:2", "fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124", }, referer="https://quote.eastmoney.com/center/boardlist.html", ) return list((payload.get("data") or {}).get("diff") or []) with ThreadPoolExecutor(max_workers=5) as executor: pages = list(executor.map(load_page, range(1, 6))) rows = [row for page in pages for row in page] if not rows: raise RealtimeAggregateError("Eastmoney sector catalog is empty") with self._sector_cache_lock: self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows} return rows def ths_limit_pool(self) -> dict[str, Any]: payload = self._get_json( THS_LIMIT_URL, {"page": "1", "limit": "3", "field": "199112"}, referer="https://data.10jqka.com.cn/limit_up/", ) data = payload.get("data") or payload return { "available": True, "keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [], "source": "ths_web_dataapi", } def xgb_limit_pool(self) -> dict[str, Any]: payload = self._get_json( XGB_POOL_URL, {"pool_name": "limit_up"}, referer="https://xuangubao.cn/", ) data = payload.get("data") or {} rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or [] return { "available": True, "count": len(rows) if isinstance(rows, list) else 0, "source": "xuangubao_web_api", } def _capture(self, operation): started = time.perf_counter() try: value = operation() return value, { "ok": True, "elapsed_ms": round((time.perf_counter() - started) * 1000), "error": "", } except Exception as exc: return None, { "ok": False, "elapsed_ms": round((time.perf_counter() - started) * 1000), "error": str(exc)[:500], } def _get_json( self, url: str, params: dict[str, str], referer: str, ) -> dict[str, Any]: request_url = f"{url}?{urllib.parse.urlencode(params)}" last_error: Exception | None = None attempts = max(1, int(self.retry_attempts)) for attempt in range(attempts): request = urllib.request.Request( request_url, headers={ "Accept": "application/json,text/plain,*/*", "Connection": "close", "Referer": referer, "User-Agent": BROWSER_USER_AGENT, }, ) try: with urllib.request.urlopen(request, timeout=self.timeout) as response: content_type = response.headers.get("Content-Type", "") raw = response.read().decode("utf-8", errors="replace") if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")): raise RealtimeAggregateError( f"non-JSON response: {raw[:120].strip()}" ) payload = json.loads(raw) if not isinstance(payload, dict): raise RealtimeAggregateError("unexpected response shape") if payload.get("rc") not in (None, 0): raise RealtimeAggregateError(f"provider rc={payload.get('rc')}") with self._response_cache_lock: self._response_cache[request_url] = { "created_at": time.time(), "payload": copy.deepcopy(payload), } return payload except ( urllib.error.URLError, TimeoutError, ConnectionError, OSError, http.client.HTTPException, json.JSONDecodeError, RealtimeAggregateError, ) as exc: last_error = exc if attempt + 1 < attempts and self.retry_delay_seconds > 0: time.sleep(self.retry_delay_seconds * (attempt + 1)) now = time.time() with self._response_cache_lock: cached = self._response_cache.get(request_url) cache_age = now - float((cached or {}).get("created_at") or 0) if cached and cache_age <= self.response_cache_ttl_seconds: payload = copy.deepcopy(cached.get("payload") or {}) payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)} return payload raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error def _get_text( self, request_url: str, referer: str, encoding: str = "utf-8", ) -> tuple[str, float]: cache_key = f"text:{request_url}" last_error: Exception | None = None attempts = max(1, int(self.retry_attempts)) for attempt in range(attempts): request = urllib.request.Request( request_url, headers={ "Accept": "text/plain,*/*", "Connection": "close", "Referer": referer, "User-Agent": BROWSER_USER_AGENT, }, ) try: with urllib.request.urlopen(request, timeout=self.timeout) as response: raw = response.read().decode(encoding, errors="replace") if not raw.strip(): raise RealtimeAggregateError("empty text response") with self._response_cache_lock: self._response_cache[cache_key] = { "created_at": time.time(), "payload": raw, } return raw, 0 except ( urllib.error.URLError, TimeoutError, ConnectionError, OSError, http.client.HTTPException, RealtimeAggregateError, ) as exc: last_error = exc if attempt + 1 < attempts and self.retry_delay_seconds > 0: time.sleep(self.retry_delay_seconds * (attempt + 1)) now = time.time() with self._response_cache_lock: cached = self._response_cache.get(cache_key) cache_age = now - float((cached or {}).get("created_at") or 0) if cached and cache_age <= self.response_cache_ttl_seconds: return str(cached.get("payload") or ""), round(cache_age, 1) raise RealtimeAggregateError( f"text request failed after {attempts} attempts: {last_error}" ) from last_error def _diff_rows(data: dict[str, Any]) -> list[dict[str, Any]]: diff = data.get("diff") or [] if isinstance(diff, dict): return [row for row in diff.values() if isinstance(row, dict)] return [row for row in diff if isinstance(row, dict)] def _a_share_identity(code: str) -> tuple[str, str, str]: raw = str(code or "").strip().upper() symbol = raw.split(".")[0] if not symbol.isdigit() or len(symbol) != 6: raise RealtimeAggregateError("Invalid stock code") if raw.endswith(".SH") or symbol.startswith(("5", "6", "9")): return f"sh{symbol}", f"1.{symbol}", f"{symbol}.SH" if raw.endswith(".BJ") or symbol.startswith(("4", "8")): return f"bj{symbol}", f"0.{symbol}", f"{symbol}.BJ" return f"sz{symbol}", f"0.{symbol}", f"{symbol}.SZ" def _require_quote_date(quote: dict[str, Any], expected_date: str) -> dict[str, Any]: want = str(expected_date or "").replace("-", "") got = str(quote.get("quote_date") or "") if want and got != want: raise RealtimeAggregateError(f"quote date {got or 'empty'} is not {want}") return quote def _normalize_eastmoney_stock_quote( row: dict[str, Any], ts_code: str ) -> dict[str, Any] | None: close = _number(row.get("f43")) previous_close = _number(row.get("f60")) if close <= 0 or previous_close <= 0: return None epoch = int(_number(row.get("f86"))) quote_date = "" if epoch > 0: quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") return { "ts_code": ts_code, "name": row.get("f58") or ts_code.split(".")[0], "pre_close": previous_close, "open": _number(row.get("f46")), "high": _number(row.get("f44")), "low": _number(row.get("f45")), "close": close, "vol": _number(row.get("f47")) * 100, "amount": _number(row.get("f48")), "num": 0, "quote_date": quote_date, "quote_time_epoch": epoch, "turnover_rate": _number(row.get("f168")), "source": "eastmoney_stock", } def _parse_tencent_stock_quote(line: str) -> dict[str, Any] | None: if '="' not in line: return None prefix, payload = line.split('="', 1) fields = payload.rsplit('";', 1)[0].split("~") if len(fields) < 38: return None symbol = fields[2] if not symbol.isdigit() or len(symbol) != 6: return None close = _number(fields[3]) previous_close = _number(fields[4]) if close <= 0 or previous_close <= 0: return None marker = prefix.lower() if "sh" in marker: ts_code = f"{symbol}.SH" elif "bj" in marker: ts_code = f"{symbol}.BJ" else: ts_code = f"{symbol}.SZ" try: quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S") quote_date = quote_time.strftime("%Y%m%d") epoch = int(quote_time.timestamp()) except ValueError: quote_date = "" epoch = 0 return { "ts_code": ts_code, "name": fields[1] or symbol, "pre_close": previous_close, "open": _number(fields[5]), "high": _number(fields[33]), "low": _number(fields[34]), "close": close, "vol": _number(fields[6]) * 100, "amount": _number(fields[37]) * 10000, "num": 0, "quote_date": quote_date, "quote_time_epoch": epoch, "source": "tencent_qt", } def _normalize_eastmoney_quote(row: dict[str, Any]) -> dict[str, Any] | None: symbol = str(row.get("f12") or "").strip() if not symbol.isdigit() or len(symbol) != 6: return None close = _number(row.get("f2")) previous_close = _number(row.get("f18")) if close <= 0 or previous_close <= 0: return None market = int(_number(row.get("f13"))) if market == 1 or symbol.startswith(("5", "6", "9")): ts_code = f"{symbol}.SH" elif symbol.startswith(("4", "8")): ts_code = f"{symbol}.BJ" else: ts_code = f"{symbol}.SZ" epoch = int(_number(row.get("f124"))) quote_date = "" if epoch > 0: quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") return { "ts_code": ts_code, "name": row.get("f14") or symbol, "pre_close": previous_close, "open": _number(row.get("f17")), "high": _number(row.get("f15")), "low": _number(row.get("f16")), "close": close, "vol": _number(row.get("f5")) * 100, "amount": _number(row.get("f6")), "num": 0, "quote_date": quote_date, "quote_time_epoch": epoch, "source": "eastmoney_clist", } def _normalize_sector(value: Any) -> str: text = str(value or "").strip().replace(" ", "") for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"): text = text.replace(suffix, "") aliases = {"元器件": "元件", "电子元器件": "元件"} return aliases.get(text, text) def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None: exact = [row for row in rows if _normalize_sector(row.get("f14")) == target] if exact: return min(exact, key=lambda row: len(str(row.get("f14") or ""))) fuzzy = [ row for row in rows if target and ( target in _normalize_sector(row.get("f14")) or _normalize_sector(row.get("f14")) in target ) ] return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None def _number(value: Any, default: float = 0.0) -> float: try: return float(value) except (TypeError, ValueError): return default