3e828b346c
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: multica-agent <github@multica.ai>
934 lines
36 KiB
Python
934 lines
36 KiB
Python
from __future__ import annotations
|
||
|
||
import copy
|
||
import http.client
|
||
import json
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from threading import Lock
|
||
from typing import Any, ClassVar
|
||
|
||
|
||
class RealtimeAggregateError(RuntimeError):
|
||
pass
|
||
|
||
|
||
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
|
||
EASTMONEY_STOCK_URL = "https://push2.eastmoney.com/api/qt/stock/get"
|
||
EASTMONEY_STOCK_FIELDS = "f43,f44,f45,f46,f47,f48,f57,f58,f60,f86,f168,f62,f66,f72,f78,f84"
|
||
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
|
||
EASTMONEY_ZT_POOL_URL = "https://push2ex.eastmoney.com/getTopicZTPool"
|
||
EASTMONEY_ZB_POOL_URL = "https://push2ex.eastmoney.com/getTopicZBPool"
|
||
EASTMONEY_A_SHARE_BOARDS = (
|
||
"m:0+t:6",
|
||
"m:0+t:80",
|
||
"m:1+t:2",
|
||
"m:1+t:23",
|
||
"m:0+t:81",
|
||
)
|
||
EASTMONEY_QUOTE_FIELDS = "f12,f13,f14,f2,f3,f4,f5,f6,f15,f16,f17,f18,f8,f124"
|
||
EASTMONEY_MARKET_PAGE_SIZE = 100
|
||
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
|
||
TENCENT_QUOTE_URL = "https://qt.gtimg.cn/q="
|
||
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
|
||
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
|
||
BROWSER_USER_AGENT = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/138.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class WebRealtimeAggregator:
|
||
timeout: int = 8
|
||
retry_attempts: int = 3
|
||
retry_delay_seconds: float = 0.2
|
||
response_cache_ttl_seconds: int = 90
|
||
_sector_cache: ClassVar[dict[str, Any]] = {}
|
||
_sector_cache_lock: ClassVar[Lock] = Lock()
|
||
_response_cache: ClassVar[dict[str, dict[str, Any]]] = {}
|
||
_response_cache_lock: ClassVar[Lock] = Lock()
|
||
|
||
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
|
||
started = time.perf_counter()
|
||
sources: dict[str, dict[str, Any]] = {}
|
||
indices: list[dict[str, Any]] = []
|
||
sector_payload: dict[str, Any] | None = None
|
||
|
||
indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices)
|
||
if sector.strip():
|
||
sector_payload, sources["eastmoney_sector"] = self._capture(
|
||
lambda: self.eastmoney_sector(sector)
|
||
)
|
||
ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool)
|
||
xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool)
|
||
|
||
index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []]
|
||
now = datetime.now().astimezone()
|
||
max_skew = 120 if now.hour >= 15 else 15
|
||
index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew
|
||
ready = (
|
||
bool(indices)
|
||
and len(indices) == 3
|
||
and index_consistent
|
||
and (not sector.strip() or bool(sector_payload))
|
||
)
|
||
return {
|
||
"ready": ready,
|
||
"isolated": True,
|
||
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||
"elapsed_ms": round((time.perf_counter() - started) * 1000),
|
||
"indices": indices or [],
|
||
"index_consistent": index_consistent,
|
||
"sector": sector_payload,
|
||
"sources": sources,
|
||
"observations": {
|
||
"ths_limit_pool": ths_observation,
|
||
"xgb_limit_pool": xgb_observation,
|
||
},
|
||
"policy": {
|
||
"integration": "heaven_realtime_fallback",
|
||
"max_index_time_skew_seconds": max_skew,
|
||
"notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。",
|
||
},
|
||
}
|
||
|
||
def eastmoney_indices(self) -> list[dict[str, Any]]:
|
||
try:
|
||
payload = self._get_json(
|
||
EASTMONEY_INDEX_URL,
|
||
{
|
||
"secids": "1.000001,0.399001,0.399006",
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
except RealtimeAggregateError:
|
||
return self.tencent_indices()
|
||
cache_meta = payload.get("_aggregate_cache") or {}
|
||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||
result = []
|
||
for row in rows:
|
||
code = str(row.get("f12") or "")
|
||
if code not in {"000001", "399001", "399006"}:
|
||
continue
|
||
epoch = int(_number(row.get("f124")))
|
||
result.append(
|
||
{
|
||
"code": code,
|
||
"name": row.get("f14") or code,
|
||
"price": _number(row.get("f2")),
|
||
"change": _number(row.get("f3")),
|
||
"change_amount": _number(row.get("f4")),
|
||
"open": _number(row.get("f17")),
|
||
"high": _number(row.get("f15")),
|
||
"low": _number(row.get("f16")),
|
||
"previous_close": _number(row.get("f18")),
|
||
"amount_billion": round(_number(row.get("f6")) / 100000000, 2),
|
||
"quote_time_epoch": epoch,
|
||
"quote_time": (
|
||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||
if epoch else ""
|
||
),
|
||
"source": (
|
||
"eastmoney_push2_cache" if cache_meta else "eastmoney_push2"
|
||
),
|
||
"cache_age_seconds": cache_meta.get("age_seconds", 0),
|
||
}
|
||
)
|
||
if len(result) != 3:
|
||
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
|
||
return result
|
||
|
||
def eastmoney_market_quotes(self, expected_date: str = "") -> list[dict[str, Any]]:
|
||
"""Full A-share snapshot via Eastmoney clist, used when Tushare rt_k is unavailable."""
|
||
now = time.time()
|
||
cache_key = "assembled:eastmoney_market"
|
||
with self._response_cache_lock:
|
||
cached = self._response_cache.get(cache_key)
|
||
cache_age = now - float((cached or {}).get("created_at") or 0)
|
||
if cached and cache_age <= min(20, self.response_cache_ttl_seconds):
|
||
quotes = list(cached.get("payload") or [])
|
||
return self._filter_quotes_by_date(quotes, expected_date)
|
||
|
||
rows: list[dict[str, Any]] = []
|
||
board_errors: list[str] = []
|
||
for board in EASTMONEY_A_SHARE_BOARDS:
|
||
try:
|
||
rows.extend(self._eastmoney_board_quotes(board))
|
||
except Exception as exc:
|
||
board_errors.append(f"{board}:{exc}")
|
||
quotes = []
|
||
seen: set[str] = set()
|
||
for row in rows:
|
||
quote = _normalize_eastmoney_quote(row)
|
||
ts_code = str((quote or {}).get("ts_code") or "")
|
||
if not quote or ts_code in seen:
|
||
continue
|
||
seen.add(ts_code)
|
||
quotes.append(quote)
|
||
if len(quotes) < 200:
|
||
detail = f"；{'; '.join(board_errors)}" if board_errors else ""
|
||
raise RealtimeAggregateError(
|
||
f"Eastmoney market snapshot too small: {len(quotes)}{detail}"
|
||
)
|
||
quotes = self._filter_quotes_by_date(quotes, expected_date)
|
||
with self._response_cache_lock:
|
||
self._response_cache[cache_key] = {"created_at": now, "payload": quotes}
|
||
return quotes
|
||
|
||
def _eastmoney_board_quotes(self, board: str) -> list[dict[str, Any]]:
|
||
first = self._eastmoney_market_page(board, 1)
|
||
data = first.get("data") or {}
|
||
rows = _diff_rows(data)
|
||
total = int(_number(data.get("total")))
|
||
page_count = 1
|
||
if total > 0:
|
||
page_count = max(1, (total + EASTMONEY_MARKET_PAGE_SIZE - 1) // EASTMONEY_MARKET_PAGE_SIZE)
|
||
for page in range(2, min(page_count, 40) + 1):
|
||
payload = self._eastmoney_market_page(board, page)
|
||
rows.extend(_diff_rows(payload.get("data") or {}))
|
||
return rows
|
||
|
||
def _eastmoney_market_page(self, board: str, page: int) -> dict[str, Any]:
|
||
return self._get_json(
|
||
EASTMONEY_SECTOR_URL,
|
||
{
|
||
"pn": str(page),
|
||
"pz": str(EASTMONEY_MARKET_PAGE_SIZE),
|
||
"po": "1",
|
||
"np": "1",
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fid": "f12",
|
||
"fs": board,
|
||
"fields": EASTMONEY_QUOTE_FIELDS,
|
||
},
|
||
referer="https://quote.eastmoney.com/center/gridlist.html",
|
||
)
|
||
|
||
def _filter_quotes_by_date(
|
||
self,
|
||
quotes: list[dict[str, Any]],
|
||
expected_date: str,
|
||
) -> list[dict[str, Any]]:
|
||
want = str(expected_date or "").replace("-", "")
|
||
if not want or not quotes:
|
||
return quotes
|
||
dated = [item for item in quotes if str(item.get("quote_date") or "") == want]
|
||
if dated and len(dated) >= max(100, int(len(quotes) * 0.2)):
|
||
return dated
|
||
if dated:
|
||
return dated
|
||
if all(not item.get("quote_date") for item in quotes):
|
||
return quotes
|
||
raise RealtimeAggregateError(f"Eastmoney quotes are not for {want}")
|
||
|
||
def tencent_market_quotes(
|
||
self,
|
||
codes: list[str],
|
||
expected_date: str = "",
|
||
) -> list[dict[str, Any]]:
|
||
symbols: list[str] = []
|
||
seen: set[str] = set()
|
||
for raw in codes:
|
||
ts = str(raw or "").strip().upper()
|
||
if not ts:
|
||
continue
|
||
symbol = ts.split(".")[0]
|
||
if not symbol.isdigit() or len(symbol) != 6 or symbol in seen:
|
||
continue
|
||
seen.add(symbol)
|
||
if ts.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||
symbols.append(f"sh{symbol}")
|
||
elif ts.endswith(".BJ") or symbol.startswith(("4", "8")):
|
||
symbols.append(f"bj{symbol}")
|
||
else:
|
||
symbols.append(f"sz{symbol}")
|
||
if not symbols:
|
||
raise RealtimeAggregateError("No stock codes available for Tencent quotes")
|
||
|
||
quotes: list[dict[str, Any]] = []
|
||
batch_size = 80
|
||
|
||
def load_batch(batch: list[str]) -> list[dict[str, Any]]:
|
||
raw, _cache_age = self._get_text(
|
||
f"{TENCENT_QUOTE_URL}{','.join(batch)}",
|
||
referer="https://gu.qq.com/",
|
||
encoding="gb18030",
|
||
)
|
||
return [
|
||
quote
|
||
for line in raw.splitlines()
|
||
if (quote := _parse_tencent_stock_quote(line))
|
||
]
|
||
|
||
batches = [symbols[index:index + batch_size] for index in range(0, len(symbols), batch_size)]
|
||
errors: list[str] = []
|
||
with ThreadPoolExecutor(max_workers=4) as executor:
|
||
for result in executor.map(self._capture, [lambda batch=batch: load_batch(batch) for batch in batches]):
|
||
rows, status = result
|
||
if status.get("ok") and rows:
|
||
quotes.extend(rows)
|
||
elif not status.get("ok"):
|
||
errors.append(str(status.get("error") or "batch failed"))
|
||
if len(quotes) < 200:
|
||
detail = f"；{'; '.join(errors[:3])}" if errors else ""
|
||
raise RealtimeAggregateError(
|
||
f"Tencent market snapshot too small: {len(quotes)}{detail}"
|
||
)
|
||
return self._filter_quotes_by_date(quotes, expected_date)
|
||
|
||
def tencent_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]:
|
||
symbol, _secid, ts_code = _a_share_identity(code)
|
||
raw, _cache_age = self._get_text(
|
||
f"{TENCENT_QUOTE_URL}{symbol}",
|
||
referer="https://gu.qq.com/",
|
||
encoding="gb18030",
|
||
)
|
||
quote = next(
|
||
(
|
||
item
|
||
for line in raw.splitlines()
|
||
if (item := _parse_tencent_stock_quote(line))
|
||
),
|
||
None,
|
||
)
|
||
if not quote:
|
||
raise RealtimeAggregateError(f"Tencent stock quote unavailable for {ts_code}")
|
||
return _require_quote_date(quote, expected_date)
|
||
|
||
def eastmoney_stock_quote(self, code: str, expected_date: str = "") -> dict[str, Any]:
|
||
_symbol, secid, ts_code = _a_share_identity(code)
|
||
payload = self._get_json(
|
||
EASTMONEY_STOCK_URL,
|
||
{
|
||
"secid": secid,
|
||
"invt": "2",
|
||
"fltt": "2",
|
||
"fields": EASTMONEY_STOCK_FIELDS,
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
quote = _normalize_eastmoney_stock_quote(payload.get("data") or {}, ts_code)
|
||
if not quote:
|
||
raise RealtimeAggregateError(f"Eastmoney stock quote unavailable for {ts_code}")
|
||
return _require_quote_date(quote, expected_date)
|
||
|
||
def eastmoney_stock_quotes(
|
||
self,
|
||
codes: list[str],
|
||
expected_date: str = "",
|
||
) -> list[dict[str, Any]]:
|
||
secids = []
|
||
for code in codes:
|
||
try:
|
||
_symbol, secid, _ts = _a_share_identity(code)
|
||
except RealtimeAggregateError:
|
||
continue
|
||
secids.append(secid)
|
||
quotes: list[dict[str, Any]] = []
|
||
for index in range(0, len(secids), 60):
|
||
payload = self._get_json(
|
||
EASTMONEY_INDEX_URL,
|
||
{
|
||
"secids": ",".join(secids[index:index + 60]),
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fields": EASTMONEY_QUOTE_FIELDS,
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
for row in _diff_rows(payload.get("data") or {}):
|
||
quote = _normalize_eastmoney_quote(row)
|
||
if quote:
|
||
quotes.append(quote)
|
||
return self._filter_quotes_by_date(quotes, expected_date)
|
||
|
||
def eastmoney_shenwan_quote(
|
||
self,
|
||
ts_code: str,
|
||
expected_date: str = "",
|
||
) -> dict[str, Any]:
|
||
code = str(ts_code or "").split(".")[0]
|
||
if not code:
|
||
raise RealtimeAggregateError("Invalid Shenwan code")
|
||
payload = self._get_json(
|
||
EASTMONEY_INDEX_URL,
|
||
{
|
||
"secids": f"90.{code}",
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f8,f104,f105,f128,f136,f140,f124",
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
row = next((item for item in _diff_rows(payload.get("data") or {}) if item), None)
|
||
if not row:
|
||
raise RealtimeAggregateError(f"Eastmoney Shenwan quote missing for {code}")
|
||
epoch = int(_number(row.get("f124")))
|
||
quote_time = (
|
||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||
if epoch
|
||
else ""
|
||
)
|
||
close = _number(row.get("f2"))
|
||
previous = _number(row.get("f18"))
|
||
if close <= 0 or previous <= 0:
|
||
raise RealtimeAggregateError(f"Eastmoney Shenwan quote empty for {code}")
|
||
result = {
|
||
"ts_code": f"{code}.SI",
|
||
"code": f"{code}.SI",
|
||
"name": row.get("f14") or code,
|
||
"price": close,
|
||
"close": close,
|
||
"pre_close": previous,
|
||
"previous_close": previous,
|
||
"open": _number(row.get("f17")),
|
||
"high": _number(row.get("f15")),
|
||
"low": _number(row.get("f16")),
|
||
"change": _number(row.get("f3")),
|
||
"pct_change": _number(row.get("f3")),
|
||
"amount": _number(row.get("f6")),
|
||
"leader": row.get("f128") or "--",
|
||
"leader_code": row.get("f140") or "",
|
||
"leading_pct": _number(row.get("f136")),
|
||
"up_count": int(_number(row.get("f104"))),
|
||
"down_count": int(_number(row.get("f105"))),
|
||
"quote_time": quote_time,
|
||
"trade_time": quote_time,
|
||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||
"quote_time_epoch": epoch,
|
||
"source": "eastmoney_sw",
|
||
}
|
||
return _require_quote_date(result, expected_date) if expected_date else result
|
||
|
||
def eastmoney_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]]:
|
||
day = str(trade_date or "").replace("-", "")
|
||
rows: list[dict[str, Any]] = []
|
||
for url, limit_type in (
|
||
(EASTMONEY_ZT_POOL_URL, "U"),
|
||
(EASTMONEY_ZB_POOL_URL, "Z"),
|
||
):
|
||
try:
|
||
payload = self._get_json(
|
||
url,
|
||
{
|
||
"ut": "7eea3edcaed734bea9cbfc24409ed989",
|
||
"dpt": "wz.ztzt",
|
||
"PageIndex": "0",
|
||
"PageSize": "200",
|
||
"sort": "fbt:asc",
|
||
"date": day,
|
||
},
|
||
referer="https://quote.eastmoney.com/ztb/detail",
|
||
)
|
||
except RealtimeAggregateError:
|
||
continue
|
||
pool = (payload.get("data") or {}).get("pool") or []
|
||
if isinstance(pool, dict):
|
||
pool = list(pool.values())
|
||
for item in pool:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
parsed = _normalize_eastmoney_limit_row(item, limit_type)
|
||
if parsed:
|
||
rows.append(parsed)
|
||
return rows
|
||
|
||
def tencent_indices(self) -> list[dict[str, Any]]:
|
||
raw, cache_age = self._get_text(
|
||
TENCENT_INDEX_URL,
|
||
referer="https://gu.qq.com/",
|
||
encoding="gb18030",
|
||
)
|
||
result = []
|
||
for line in raw.splitlines():
|
||
if '="' not in line:
|
||
continue
|
||
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
|
||
if len(fields) < 38:
|
||
continue
|
||
code = fields[2]
|
||
if code not in {"000001", "399001", "399006"}:
|
||
continue
|
||
try:
|
||
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
|
||
except ValueError as exc:
|
||
raise RealtimeAggregateError(
|
||
f"Tencent returned invalid quote time for {code}"
|
||
) from exc
|
||
result.append(
|
||
{
|
||
"code": code,
|
||
"name": fields[1] or code,
|
||
"price": _number(fields[3]),
|
||
"change": _number(fields[32]),
|
||
"change_amount": _number(fields[31]),
|
||
"open": _number(fields[5]),
|
||
"high": _number(fields[33]),
|
||
"low": _number(fields[34]),
|
||
"previous_close": _number(fields[4]),
|
||
"amount_billion": round(_number(fields[37]) / 10000, 2),
|
||
"quote_time_epoch": int(quote_time.timestamp()),
|
||
"quote_time": quote_time.isoformat(timespec="seconds"),
|
||
"source": "tencent_qt_cache" if cache_age else "tencent_qt",
|
||
"cache_age_seconds": cache_age,
|
||
}
|
||
)
|
||
if len(result) != 3:
|
||
raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices")
|
||
return result
|
||
|
||
def eastmoney_sector(self, query: str) -> dict[str, Any]:
|
||
target = _normalize_sector(query)
|
||
candidates = self._eastmoney_sector_catalog()
|
||
matched = _match_sector(candidates, target)
|
||
if not matched:
|
||
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
|
||
epoch = int(_number(matched.get("f124")))
|
||
quote_time = (
|
||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||
if epoch else ""
|
||
)
|
||
return {
|
||
"code": matched.get("f12") or "",
|
||
"name": matched.get("f14") or query,
|
||
"price": _number(matched.get("f2")),
|
||
"close": _number(matched.get("f2")),
|
||
"change": _number(matched.get("f3")),
|
||
"pct_change": _number(matched.get("f3")),
|
||
"change_amount": _number(matched.get("f4")),
|
||
"turnover_rate": _number(matched.get("f8")),
|
||
"up_count": int(_number(matched.get("f104"))),
|
||
"down_count": int(_number(matched.get("f105"))),
|
||
"leader": matched.get("f128") or "--",
|
||
"leader_code": matched.get("f140") or "",
|
||
"leading_pct": _number(matched.get("f136")),
|
||
"quote_time_epoch": epoch,
|
||
"quote_time": quote_time,
|
||
"trade_time": quote_time,
|
||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||
"source": "eastmoney_push2",
|
||
"match_query": query,
|
||
}
|
||
|
||
def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]:
|
||
now = time.time()
|
||
with self._sector_cache_lock:
|
||
cached = self._sector_cache.get("eastmoney")
|
||
if cached and now - float(cached.get("created_at") or 0) < 600:
|
||
return list(cached.get("rows") or [])
|
||
|
||
def load_page(page: int) -> list[dict[str, Any]]:
|
||
payload = self._get_json(
|
||
EASTMONEY_SECTOR_URL,
|
||
{
|
||
"pn": str(page),
|
||
"pz": "100",
|
||
"po": "1",
|
||
"np": "1",
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fid": "f3",
|
||
"fs": "m:90+t:2",
|
||
"fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124",
|
||
},
|
||
referer="https://quote.eastmoney.com/center/boardlist.html",
|
||
)
|
||
return list((payload.get("data") or {}).get("diff") or [])
|
||
|
||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||
pages = list(executor.map(load_page, range(1, 6)))
|
||
rows = [row for page in pages for row in page]
|
||
if not rows:
|
||
raise RealtimeAggregateError("Eastmoney sector catalog is empty")
|
||
with self._sector_cache_lock:
|
||
self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows}
|
||
return rows
|
||
|
||
def ths_limit_pool(self) -> dict[str, Any]:
|
||
payload = self._get_json(
|
||
THS_LIMIT_URL,
|
||
{"page": "1", "limit": "3", "field": "199112"},
|
||
referer="https://data.10jqka.com.cn/limit_up/",
|
||
)
|
||
data = payload.get("data") or payload
|
||
return {
|
||
"available": True,
|
||
"keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [],
|
||
"source": "ths_web_dataapi",
|
||
}
|
||
|
||
def xgb_limit_pool(self) -> dict[str, Any]:
|
||
payload = self._get_json(
|
||
XGB_POOL_URL,
|
||
{"pool_name": "limit_up"},
|
||
referer="https://xuangubao.cn/",
|
||
)
|
||
data = payload.get("data") or {}
|
||
rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or []
|
||
return {
|
||
"available": True,
|
||
"count": len(rows) if isinstance(rows, list) else 0,
|
||
"source": "xuangubao_web_api",
|
||
}
|
||
|
||
def _capture(self, operation):
|
||
started = time.perf_counter()
|
||
try:
|
||
value = operation()
|
||
return value, {
|
||
"ok": True,
|
||
"elapsed_ms": round((time.perf_counter() - started) * 1000),
|
||
"error": "",
|
||
}
|
||
except Exception as exc:
|
||
return None, {
|
||
"ok": False,
|
||
"elapsed_ms": round((time.perf_counter() - started) * 1000),
|
||
"error": str(exc)[:500],
|
||
}
|
||
|
||
def _get_json(
|
||
self,
|
||
url: str,
|
||
params: dict[str, str],
|
||
referer: str,
|
||
) -> dict[str, Any]:
|
||
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
||
last_error: Exception | None = None
|
||
attempts = max(1, int(self.retry_attempts))
|
||
for attempt in range(attempts):
|
||
request = urllib.request.Request(
|
||
request_url,
|
||
headers={
|
||
"Accept": "application/json,text/plain,*/*",
|
||
"Connection": "close",
|
||
"Referer": referer,
|
||
"User-Agent": BROWSER_USER_AGENT,
|
||
},
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||
content_type = response.headers.get("Content-Type", "")
|
||
raw = response.read().decode("utf-8", errors="replace")
|
||
if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")):
|
||
raise RealtimeAggregateError(
|
||
f"non-JSON response: {raw[:120].strip()}"
|
||
)
|
||
payload = json.loads(raw)
|
||
if not isinstance(payload, dict):
|
||
raise RealtimeAggregateError("unexpected response shape")
|
||
if payload.get("rc") not in (None, 0):
|
||
raise RealtimeAggregateError(f"provider rc={payload.get('rc')}")
|
||
with self._response_cache_lock:
|
||
self._response_cache[request_url] = {
|
||
"created_at": time.time(),
|
||
"payload": copy.deepcopy(payload),
|
||
}
|
||
return payload
|
||
except (
|
||
urllib.error.URLError,
|
||
TimeoutError,
|
||
ConnectionError,
|
||
OSError,
|
||
http.client.HTTPException,
|
||
json.JSONDecodeError,
|
||
RealtimeAggregateError,
|
||
) as exc:
|
||
last_error = exc
|
||
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
|
||
time.sleep(self.retry_delay_seconds * (attempt + 1))
|
||
|
||
now = time.time()
|
||
with self._response_cache_lock:
|
||
cached = self._response_cache.get(request_url)
|
||
cache_age = now - float((cached or {}).get("created_at") or 0)
|
||
if cached and cache_age <= self.response_cache_ttl_seconds:
|
||
payload = copy.deepcopy(cached.get("payload") or {})
|
||
payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)}
|
||
return payload
|
||
raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error
|
||
|
||
def _get_text(
|
||
self,
|
||
request_url: str,
|
||
referer: str,
|
||
encoding: str = "utf-8",
|
||
) -> tuple[str, float]:
|
||
cache_key = f"text:{request_url}"
|
||
last_error: Exception | None = None
|
||
attempts = max(1, int(self.retry_attempts))
|
||
for attempt in range(attempts):
|
||
request = urllib.request.Request(
|
||
request_url,
|
||
headers={
|
||
"Accept": "text/plain,*/*",
|
||
"Connection": "close",
|
||
"Referer": referer,
|
||
"User-Agent": BROWSER_USER_AGENT,
|
||
},
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||
raw = response.read().decode(encoding, errors="replace")
|
||
if not raw.strip():
|
||
raise RealtimeAggregateError("empty text response")
|
||
with self._response_cache_lock:
|
||
self._response_cache[cache_key] = {
|
||
"created_at": time.time(),
|
||
"payload": raw,
|
||
}
|
||
return raw, 0
|
||
except (
|
||
urllib.error.URLError,
|
||
TimeoutError,
|
||
ConnectionError,
|
||
OSError,
|
||
http.client.HTTPException,
|
||
RealtimeAggregateError,
|
||
) as exc:
|
||
last_error = exc
|
||
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
|
||
time.sleep(self.retry_delay_seconds * (attempt + 1))
|
||
|
||
now = time.time()
|
||
with self._response_cache_lock:
|
||
cached = self._response_cache.get(cache_key)
|
||
cache_age = now - float((cached or {}).get("created_at") or 0)
|
||
if cached and cache_age <= self.response_cache_ttl_seconds:
|
||
return str(cached.get("payload") or ""), round(cache_age, 1)
|
||
raise RealtimeAggregateError(
|
||
f"text request failed after {attempts} attempts: {last_error}"
|
||
) from last_error
|
||
|
||
|
||
def _diff_rows(data: dict[str, Any]) -> list[dict[str, Any]]:
|
||
diff = data.get("diff") or []
|
||
if isinstance(diff, dict):
|
||
return [row for row in diff.values() if isinstance(row, dict)]
|
||
return [row for row in diff if isinstance(row, dict)]
|
||
|
||
|
||
def _a_share_identity(code: str) -> tuple[str, str, str]:
|
||
raw = str(code or "").strip().upper()
|
||
symbol = raw.split(".")[0]
|
||
if not symbol.isdigit() or len(symbol) != 6:
|
||
raise RealtimeAggregateError("Invalid stock code")
|
||
if raw.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||
return f"sh{symbol}", f"1.{symbol}", f"{symbol}.SH"
|
||
if raw.endswith(".BJ") or symbol.startswith(("4", "8")):
|
||
return f"bj{symbol}", f"0.{symbol}", f"{symbol}.BJ"
|
||
return f"sz{symbol}", f"0.{symbol}", f"{symbol}.SZ"
|
||
|
||
|
||
def _require_quote_date(quote: dict[str, Any], expected_date: str) -> dict[str, Any]:
|
||
want = str(expected_date or "").replace("-", "")
|
||
got = str(quote.get("quote_date") or "")
|
||
if want and got != want:
|
||
raise RealtimeAggregateError(f"quote date {got or 'empty'} is not {want}")
|
||
return quote
|
||
|
||
|
||
def _normalize_eastmoney_stock_quote(
|
||
row: dict[str, Any], ts_code: str
|
||
) -> dict[str, Any] | None:
|
||
close = _number(row.get("f43"))
|
||
previous_close = _number(row.get("f60"))
|
||
if close <= 0 or previous_close <= 0:
|
||
return None
|
||
epoch = int(_number(row.get("f86")))
|
||
quote_date = ""
|
||
if epoch > 0:
|
||
quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||
return {
|
||
"ts_code": ts_code,
|
||
"name": row.get("f58") or ts_code.split(".")[0],
|
||
"pre_close": previous_close,
|
||
"open": _number(row.get("f46")),
|
||
"high": _number(row.get("f44")),
|
||
"low": _number(row.get("f45")),
|
||
"close": close,
|
||
"vol": _number(row.get("f47")) * 100,
|
||
"amount": _number(row.get("f48")),
|
||
"num": 0,
|
||
"quote_date": quote_date,
|
||
"quote_time_epoch": epoch,
|
||
"turnover_rate": _number(row.get("f168")),
|
||
"net_mf_amount": _eastmoney_flow_wan(row.get("f62")),
|
||
"large_amount": _eastmoney_flow_wan(row.get("f62")),
|
||
"medium_amount": _eastmoney_flow_wan(row.get("f78")),
|
||
"small_amount": _eastmoney_flow_wan(row.get("f84")),
|
||
"source": "eastmoney_stock",
|
||
}
|
||
|
||
|
||
def _parse_tencent_stock_quote(line: str) -> dict[str, Any] | None:
|
||
if '="' not in line:
|
||
return None
|
||
prefix, payload = line.split('="', 1)
|
||
fields = payload.rsplit('";', 1)[0].split("~")
|
||
if len(fields) < 38:
|
||
return None
|
||
symbol = fields[2]
|
||
if not symbol.isdigit() or len(symbol) != 6:
|
||
return None
|
||
close = _number(fields[3])
|
||
previous_close = _number(fields[4])
|
||
if close <= 0 or previous_close <= 0:
|
||
return None
|
||
marker = prefix.lower()
|
||
if "sh" in marker:
|
||
ts_code = f"{symbol}.SH"
|
||
elif "bj" in marker:
|
||
ts_code = f"{symbol}.BJ"
|
||
else:
|
||
ts_code = f"{symbol}.SZ"
|
||
try:
|
||
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S")
|
||
quote_date = quote_time.strftime("%Y%m%d")
|
||
epoch = int(quote_time.timestamp())
|
||
except ValueError:
|
||
quote_date = ""
|
||
epoch = 0
|
||
return {
|
||
"ts_code": ts_code,
|
||
"name": fields[1] or symbol,
|
||
"pre_close": previous_close,
|
||
"open": _number(fields[5]),
|
||
"high": _number(fields[33]),
|
||
"low": _number(fields[34]),
|
||
"close": close,
|
||
"vol": _number(fields[6]) * 100,
|
||
"amount": _number(fields[37]) * 10000,
|
||
"num": 0,
|
||
"quote_date": quote_date,
|
||
"quote_time_epoch": epoch,
|
||
"source": "tencent_qt",
|
||
}
|
||
|
||
|
||
def _normalize_eastmoney_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||
symbol = str(row.get("f12") or "").strip()
|
||
if not symbol.isdigit() or len(symbol) != 6:
|
||
return None
|
||
close = _number(row.get("f2"))
|
||
previous_close = _number(row.get("f18"))
|
||
if close <= 0 or previous_close <= 0:
|
||
return None
|
||
market = int(_number(row.get("f13")))
|
||
if market == 1 or symbol.startswith(("5", "6", "9")):
|
||
ts_code = f"{symbol}.SH"
|
||
elif symbol.startswith(("4", "8")):
|
||
ts_code = f"{symbol}.BJ"
|
||
else:
|
||
ts_code = f"{symbol}.SZ"
|
||
epoch = int(_number(row.get("f124")))
|
||
quote_date = ""
|
||
if epoch > 0:
|
||
quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||
return {
|
||
"ts_code": ts_code,
|
||
"name": row.get("f14") or symbol,
|
||
"pre_close": previous_close,
|
||
"open": _number(row.get("f17")),
|
||
"high": _number(row.get("f15")),
|
||
"low": _number(row.get("f16")),
|
||
"close": close,
|
||
"vol": _number(row.get("f5")) * 100,
|
||
"amount": _number(row.get("f6")),
|
||
"num": 0,
|
||
"quote_date": quote_date,
|
||
"quote_time_epoch": epoch,
|
||
"source": "eastmoney_clist",
|
||
}
|
||
|
||
|
||
def _eastmoney_flow_wan(value: Any) -> float | None:
|
||
if value in (None, "", "-"):
|
||
return None
|
||
amount = _number(value, default=float("nan"))
|
||
if amount != amount:
|
||
return None
|
||
return amount / 10000
|
||
|
||
|
||
def _board_clock(value: Any) -> str:
|
||
digits = "".join(character for character in str(value or "") if character.isdigit())
|
||
if len(digits) >= 6:
|
||
return f"{digits[:2]}:{digits[2:4]}:{digits[4:6]}"
|
||
if len(digits) == 5:
|
||
digits = digits.zfill(6)
|
||
return f"{digits[:2]}:{digits[2:4]}:{digits[4:6]}"
|
||
if len(digits) == 4:
|
||
return f"{digits[:2]}:{digits[2:]}:00"
|
||
return ""
|
||
|
||
|
||
def _normalize_eastmoney_limit_row(row: dict[str, Any], limit_type: str) -> dict[str, Any] | None:
|
||
symbol = str(row.get("c") or row.get("code") or "").strip()
|
||
if not symbol.isdigit() or len(symbol) != 6:
|
||
return None
|
||
market = int(_number(row.get("m") if row.get("m") not in (None, "") else row.get("market")))
|
||
if market == 1 or symbol.startswith(("5", "6", "9")):
|
||
ts_code = f"{symbol}.SH"
|
||
elif symbol.startswith(("4", "8")):
|
||
ts_code = f"{symbol}.BJ"
|
||
else:
|
||
ts_code = f"{symbol}.SZ"
|
||
first_time = _board_clock(row.get("fbt") if row.get("fbt") not in (None, "") else row.get("first_time"))
|
||
last_time = _board_clock(row.get("lbt") if row.get("lbt") not in (None, "") else row.get("last_time"))
|
||
fund = row.get("fund")
|
||
if fund in (None, ""):
|
||
fund = row.get("fd_amount")
|
||
return {
|
||
"ts_code": ts_code,
|
||
"name": row.get("n") or row.get("name") or symbol,
|
||
"limit_type": limit_type,
|
||
"first_time": first_time or None,
|
||
"last_time": last_time or None,
|
||
"open_times": int(_number(row.get("zbc") if row.get("zbc") not in (None, "") else row.get("open_times"))),
|
||
"limit_times": max(1, int(_number(row.get("lbc") if row.get("lbc") not in (None, "") else 1))),
|
||
"turnover_ratio": _number(row.get("hs") if row.get("hs") not in (None, "") else row.get("turnover_ratio")),
|
||
"fd_amount": _number(fund) if fund not in (None, "", "-") else None,
|
||
"source": "eastmoney_zt_pool",
|
||
}
|
||
|
||
|
||
def _normalize_sector(value: Any) -> str:
|
||
text = str(value or "").strip().replace(" ", "")
|
||
for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"):
|
||
text = text.replace(suffix, "")
|
||
aliases = {"元器件": "元件", "电子元器件": "元件"}
|
||
return aliases.get(text, text)
|
||
|
||
|
||
def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None:
|
||
exact = [row for row in rows if _normalize_sector(row.get("f14")) == target]
|
||
if exact:
|
||
return min(exact, key=lambda row: len(str(row.get("f14") or "")))
|
||
fuzzy = [
|
||
row for row in rows
|
||
if target and (
|
||
target in _normalize_sector(row.get("f14"))
|
||
or _normalize_sector(row.get("f14")) in target
|
||
)
|
||
]
|
||
return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None
|
||
|
||
|
||
def _number(value: Any, default: float = 0.0) -> float:
|
||
try:
|
||
return float(value)
|
||
except (TypeError, ValueError):
|
||
return default
|