migration: preserve market data and search slice
This commit is contained in:
leefer
@@ -0,0 +1,426 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import http.client
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from threading import Lock
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class RealtimeAggregateError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
|
||||
EASTMONEY_SECTOR_URL = "https://push2.eastmoney.com/api/qt/clist/get"
|
||||
TENCENT_INDEX_URL = "https://qt.gtimg.cn/q=sh000001,sz399001,sz399006"
|
||||
THS_LIMIT_URL = "https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool"
|
||||
XGB_POOL_URL = "https://flash-api.xuangubao.cn/api/pool/detail"
|
||||
BROWSER_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/138.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WebRealtimeAggregator:
|
||||
timeout: int = 8
|
||||
retry_attempts: int = 3
|
||||
retry_delay_seconds: float = 0.2
|
||||
response_cache_ttl_seconds: int = 90
|
||||
_sector_cache: ClassVar[dict[str, Any]] = {}
|
||||
_sector_cache_lock: ClassVar[Lock] = Lock()
|
||||
_response_cache: ClassVar[dict[str, dict[str, Any]]] = {}
|
||||
_response_cache_lock: ClassVar[Lock] = Lock()
|
||||
|
||||
def health_snapshot(self, sector: str = "") -> dict[str, Any]:
|
||||
started = time.perf_counter()
|
||||
sources: dict[str, dict[str, Any]] = {}
|
||||
indices: list[dict[str, Any]] = []
|
||||
sector_payload: dict[str, Any] | None = None
|
||||
|
||||
indices, sources["eastmoney_indices"] = self._capture(self.eastmoney_indices)
|
||||
if sector.strip():
|
||||
sector_payload, sources["eastmoney_sector"] = self._capture(
|
||||
lambda: self.eastmoney_sector(sector)
|
||||
)
|
||||
ths_observation, sources["ths_limit_pool"] = self._capture(self.ths_limit_pool)
|
||||
xgb_observation, sources["xgb_limit_pool"] = self._capture(self.xgb_limit_pool)
|
||||
|
||||
index_times = [int(item.get("quote_time_epoch") or 0) for item in indices or []]
|
||||
now = datetime.now().astimezone()
|
||||
max_skew = 120 if now.hour >= 15 else 15
|
||||
index_consistent = bool(index_times) and max(index_times) - min(index_times) <= max_skew
|
||||
ready = (
|
||||
bool(indices)
|
||||
and len(indices) == 3
|
||||
and index_consistent
|
||||
and (not sector.strip() or bool(sector_payload))
|
||||
)
|
||||
return {
|
||||
"ready": ready,
|
||||
"isolated": True,
|
||||
"generated_at": datetime.now().astimezone().isoformat(timespec="seconds"),
|
||||
"elapsed_ms": round((time.perf_counter() - started) * 1000),
|
||||
"indices": indices or [],
|
||||
"index_consistent": index_consistent,
|
||||
"sector": sector_payload,
|
||||
"sources": sources,
|
||||
"observations": {
|
||||
"ths_limit_pool": ths_observation,
|
||||
"xgb_limit_pool": xgb_observation,
|
||||
},
|
||||
"policy": {
|
||||
"integration": "heaven_realtime_fallback",
|
||||
"max_index_time_skew_seconds": max_skew,
|
||||
"notice": "聚合源仅作为盘中观势的实时指数与板块外显，主行情快照仍由Tushare维护。",
|
||||
},
|
||||
}
|
||||
|
||||
def eastmoney_indices(self) -> list[dict[str, Any]]:
|
||||
try:
|
||||
payload = self._get_json(
|
||||
EASTMONEY_INDEX_URL,
|
||||
{
|
||||
"secids": "1.000001,0.399001,0.399006",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/",
|
||||
)
|
||||
except RealtimeAggregateError:
|
||||
return self.tencent_indices()
|
||||
cache_meta = payload.get("_aggregate_cache") or {}
|
||||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||||
result = []
|
||||
for row in rows:
|
||||
code = str(row.get("f12") or "")
|
||||
if code not in {"000001", "399001", "399006"}:
|
||||
continue
|
||||
epoch = int(_number(row.get("f124")))
|
||||
result.append(
|
||||
{
|
||||
"code": code,
|
||||
"name": row.get("f14") or code,
|
||||
"price": _number(row.get("f2")),
|
||||
"change": _number(row.get("f3")),
|
||||
"change_amount": _number(row.get("f4")),
|
||||
"open": _number(row.get("f17")),
|
||||
"high": _number(row.get("f15")),
|
||||
"low": _number(row.get("f16")),
|
||||
"previous_close": _number(row.get("f18")),
|
||||
"amount_billion": round(_number(row.get("f6")) / 100000000, 2),
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch else ""
|
||||
),
|
||||
"source": (
|
||||
"eastmoney_push2_cache" if cache_meta else "eastmoney_push2"
|
||||
),
|
||||
"cache_age_seconds": cache_meta.get("age_seconds", 0),
|
||||
}
|
||||
)
|
||||
if len(result) != 3:
|
||||
raise RealtimeAggregateError(f"Eastmoney returned {len(result)}/3 indices")
|
||||
return result
|
||||
|
||||
def tencent_indices(self) -> list[dict[str, Any]]:
|
||||
raw, cache_age = self._get_text(
|
||||
TENCENT_INDEX_URL,
|
||||
referer="https://gu.qq.com/",
|
||||
encoding="gb18030",
|
||||
)
|
||||
result = []
|
||||
for line in raw.splitlines():
|
||||
if '="' not in line:
|
||||
continue
|
||||
fields = line.split('="', 1)[1].rsplit('";', 1)[0].split("~")
|
||||
if len(fields) < 38:
|
||||
continue
|
||||
code = fields[2]
|
||||
if code not in {"000001", "399001", "399006"}:
|
||||
continue
|
||||
try:
|
||||
quote_time = datetime.strptime(fields[30], "%Y%m%d%H%M%S").astimezone()
|
||||
except ValueError as exc:
|
||||
raise RealtimeAggregateError(
|
||||
f"Tencent returned invalid quote time for {code}"
|
||||
) from exc
|
||||
result.append(
|
||||
{
|
||||
"code": code,
|
||||
"name": fields[1] or code,
|
||||
"price": _number(fields[3]),
|
||||
"change": _number(fields[32]),
|
||||
"change_amount": _number(fields[31]),
|
||||
"open": _number(fields[5]),
|
||||
"high": _number(fields[33]),
|
||||
"low": _number(fields[34]),
|
||||
"previous_close": _number(fields[4]),
|
||||
"amount_billion": round(_number(fields[37]) / 10000, 2),
|
||||
"quote_time_epoch": int(quote_time.timestamp()),
|
||||
"quote_time": quote_time.isoformat(timespec="seconds"),
|
||||
"source": "tencent_qt_cache" if cache_age else "tencent_qt",
|
||||
"cache_age_seconds": cache_age,
|
||||
}
|
||||
)
|
||||
if len(result) != 3:
|
||||
raise RealtimeAggregateError(f"Tencent returned {len(result)}/3 indices")
|
||||
return result
|
||||
|
||||
def eastmoney_sector(self, query: str) -> dict[str, Any]:
|
||||
target = _normalize_sector(query)
|
||||
candidates = self._eastmoney_sector_catalog()
|
||||
matched = _match_sector(candidates, target)
|
||||
if not matched:
|
||||
raise RealtimeAggregateError(f"Eastmoney sector not found: {query}")
|
||||
epoch = int(_number(matched.get("f124")))
|
||||
return {
|
||||
"code": matched.get("f12") or "",
|
||||
"name": matched.get("f14") or query,
|
||||
"price": _number(matched.get("f2")),
|
||||
"change": _number(matched.get("f3")),
|
||||
"change_amount": _number(matched.get("f4")),
|
||||
"turnover_rate": _number(matched.get("f8")),
|
||||
"up_count": int(_number(matched.get("f104"))),
|
||||
"down_count": int(_number(matched.get("f105"))),
|
||||
"leader": matched.get("f128") or "--",
|
||||
"leader_code": matched.get("f140") or "",
|
||||
"leading_pct": _number(matched.get("f136")),
|
||||
"quote_time_epoch": epoch,
|
||||
"quote_time": (
|
||||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||||
if epoch else ""
|
||||
),
|
||||
"source": "eastmoney_push2",
|
||||
"match_query": query,
|
||||
}
|
||||
|
||||
def _eastmoney_sector_catalog(self) -> list[dict[str, Any]]:
|
||||
now = time.time()
|
||||
with self._sector_cache_lock:
|
||||
cached = self._sector_cache.get("eastmoney")
|
||||
if cached and now - float(cached.get("created_at") or 0) < 600:
|
||||
return list(cached.get("rows") or [])
|
||||
|
||||
def load_page(page: int) -> list[dict[str, Any]]:
|
||||
payload = self._get_json(
|
||||
EASTMONEY_SECTOR_URL,
|
||||
{
|
||||
"pn": str(page),
|
||||
"pz": "100",
|
||||
"po": "1",
|
||||
"np": "1",
|
||||
"fltt": "2",
|
||||
"invt": "2",
|
||||
"fid": "f3",
|
||||
"fs": "m:90+t:2",
|
||||
"fields": "f12,f14,f2,f3,f4,f8,f104,f105,f128,f136,f140,f124",
|
||||
},
|
||||
referer="https://quote.eastmoney.com/center/boardlist.html",
|
||||
)
|
||||
return list((payload.get("data") or {}).get("diff") or [])
|
||||
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
pages = list(executor.map(load_page, range(1, 6)))
|
||||
rows = [row for page in pages for row in page]
|
||||
if not rows:
|
||||
raise RealtimeAggregateError("Eastmoney sector catalog is empty")
|
||||
with self._sector_cache_lock:
|
||||
self._sector_cache["eastmoney"] = {"created_at": now, "rows": rows}
|
||||
return rows
|
||||
|
||||
def ths_limit_pool(self) -> dict[str, Any]:
|
||||
payload = self._get_json(
|
||||
THS_LIMIT_URL,
|
||||
{"page": "1", "limit": "3", "field": "199112"},
|
||||
referer="https://data.10jqka.com.cn/limit_up/",
|
||||
)
|
||||
data = payload.get("data") or payload
|
||||
return {
|
||||
"available": True,
|
||||
"keys": sorted(str(key) for key in data.keys()) if isinstance(data, dict) else [],
|
||||
"source": "ths_web_dataapi",
|
||||
}
|
||||
|
||||
def xgb_limit_pool(self) -> dict[str, Any]:
|
||||
payload = self._get_json(
|
||||
XGB_POOL_URL,
|
||||
{"pool_name": "limit_up"},
|
||||
referer="https://xuangubao.cn/",
|
||||
)
|
||||
data = payload.get("data") or {}
|
||||
rows = data if isinstance(data, list) else data.get("pool") or data.get("list") or []
|
||||
return {
|
||||
"available": True,
|
||||
"count": len(rows) if isinstance(rows, list) else 0,
|
||||
"source": "xuangubao_web_api",
|
||||
}
|
||||
|
||||
def _capture(self, operation):
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
value = operation()
|
||||
return value, {
|
||||
"ok": True,
|
||||
"elapsed_ms": round((time.perf_counter() - started) * 1000),
|
||||
"error": "",
|
||||
}
|
||||
except Exception as exc:
|
||||
return None, {
|
||||
"ok": False,
|
||||
"elapsed_ms": round((time.perf_counter() - started) * 1000),
|
||||
"error": str(exc)[:500],
|
||||
}
|
||||
|
||||
def _get_json(
|
||||
self,
|
||||
url: str,
|
||||
params: dict[str, str],
|
||||
referer: str,
|
||||
) -> dict[str, Any]:
|
||||
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
||||
last_error: Exception | None = None
|
||||
attempts = max(1, int(self.retry_attempts))
|
||||
for attempt in range(attempts):
|
||||
request = urllib.request.Request(
|
||||
request_url,
|
||||
headers={
|
||||
"Accept": "application/json,text/plain,*/*",
|
||||
"Connection": "close",
|
||||
"Referer": referer,
|
||||
"User-Agent": BROWSER_USER_AGENT,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
content_type = response.headers.get("Content-Type", "")
|
||||
raw = response.read().decode("utf-8", errors="replace")
|
||||
if "json" not in content_type.lower() and not raw.lstrip().startswith(("{", "[")):
|
||||
raise RealtimeAggregateError(
|
||||
f"non-JSON response: {raw[:120].strip()}"
|
||||
)
|
||||
payload = json.loads(raw)
|
||||
if not isinstance(payload, dict):
|
||||
raise RealtimeAggregateError("unexpected response shape")
|
||||
if payload.get("rc") not in (None, 0):
|
||||
raise RealtimeAggregateError(f"provider rc={payload.get('rc')}")
|
||||
with self._response_cache_lock:
|
||||
self._response_cache[request_url] = {
|
||||
"created_at": time.time(),
|
||||
"payload": copy.deepcopy(payload),
|
||||
}
|
||||
return payload
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
http.client.HTTPException,
|
||||
json.JSONDecodeError,
|
||||
RealtimeAggregateError,
|
||||
) as exc:
|
||||
last_error = exc
|
||||
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
|
||||
time.sleep(self.retry_delay_seconds * (attempt + 1))
|
||||
|
||||
now = time.time()
|
||||
with self._response_cache_lock:
|
||||
cached = self._response_cache.get(request_url)
|
||||
cache_age = now - float((cached or {}).get("created_at") or 0)
|
||||
if cached and cache_age <= self.response_cache_ttl_seconds:
|
||||
payload = copy.deepcopy(cached.get("payload") or {})
|
||||
payload["_aggregate_cache"] = {"age_seconds": round(cache_age, 1)}
|
||||
return payload
|
||||
raise RealtimeAggregateError(f"request failed after {attempts} attempts: {last_error}") from last_error
|
||||
|
||||
def _get_text(
|
||||
self,
|
||||
request_url: str,
|
||||
referer: str,
|
||||
encoding: str = "utf-8",
|
||||
) -> tuple[str, float]:
|
||||
cache_key = f"text:{request_url}"
|
||||
last_error: Exception | None = None
|
||||
attempts = max(1, int(self.retry_attempts))
|
||||
for attempt in range(attempts):
|
||||
request = urllib.request.Request(
|
||||
request_url,
|
||||
headers={
|
||||
"Accept": "text/plain,*/*",
|
||||
"Connection": "close",
|
||||
"Referer": referer,
|
||||
"User-Agent": BROWSER_USER_AGENT,
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||||
raw = response.read().decode(encoding, errors="replace")
|
||||
if not raw.strip():
|
||||
raise RealtimeAggregateError("empty text response")
|
||||
with self._response_cache_lock:
|
||||
self._response_cache[cache_key] = {
|
||||
"created_at": time.time(),
|
||||
"payload": raw,
|
||||
}
|
||||
return raw, 0
|
||||
except (
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
http.client.HTTPException,
|
||||
RealtimeAggregateError,
|
||||
) as exc:
|
||||
last_error = exc
|
||||
if attempt + 1 < attempts and self.retry_delay_seconds > 0:
|
||||
time.sleep(self.retry_delay_seconds * (attempt + 1))
|
||||
|
||||
now = time.time()
|
||||
with self._response_cache_lock:
|
||||
cached = self._response_cache.get(cache_key)
|
||||
cache_age = now - float((cached or {}).get("created_at") or 0)
|
||||
if cached and cache_age <= self.response_cache_ttl_seconds:
|
||||
return str(cached.get("payload") or ""), round(cache_age, 1)
|
||||
raise RealtimeAggregateError(
|
||||
f"text request failed after {attempts} attempts: {last_error}"
|
||||
) from last_error
|
||||
|
||||
|
||||
def _normalize_sector(value: Any) -> str:
|
||||
text = str(value or "").strip().replace(" ", "")
|
||||
for suffix in ("板块", "概念", "行业", "Ⅱ", "Ⅲ", "(A股)", "（A股）"):
|
||||
text = text.replace(suffix, "")
|
||||
aliases = {"元器件": "元件", "电子元器件": "元件"}
|
||||
return aliases.get(text, text)
|
||||
|
||||
|
||||
def _match_sector(rows: list[dict[str, Any]], target: str) -> dict[str, Any] | None:
|
||||
exact = [row for row in rows if _normalize_sector(row.get("f14")) == target]
|
||||
if exact:
|
||||
return min(exact, key=lambda row: len(str(row.get("f14") or "")))
|
||||
fuzzy = [
|
||||
row for row in rows
|
||||
if target and (
|
||||
target in _normalize_sector(row.get("f14"))
|
||||
or _normalize_sector(row.get("f14")) in target
|
||||
)
|
||||
]
|
||||
return min(fuzzy, key=lambda row: len(_normalize_sector(row.get("f14")))) if fuzzy else None
|
||||
|
||||
|
||||
def _number(value: Any, default: float = 0.0) -> float:
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
Reference in New Issue
Block a user