525 lines
21 KiB
Python
525 lines
21 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
import time
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from datetime import datetime
|
||
from typing import Any
|
||
|
||
from datahub.adapters.base import AdapterError, MarketAdapter
|
||
from datahub.numbers import finite_number, round4
|
||
|
||
EASTMONEY_INDEX_URL = "https://push2.eastmoney.com/api/qt/ulist.np/get"
|
||
EASTMONEY_CLIST_URL = "https://push2.eastmoney.com/api/qt/clist/get"
|
||
EASTMONEY_A_SHARE_BOARDS = (
|
||
"m:0+t:6",
|
||
"m:0+t:80",
|
||
"m:1+t:2",
|
||
"m:1+t:23",
|
||
"m:0+t:81",
|
||
)
|
||
EASTMONEY_QUOTE_FIELDS = "f12,f13,f14,f2,f3,f4,f5,f6,f15,f16,f17,f18,f8,f124"
|
||
EASTMONEY_MARKET_PAGE_SIZE = 100
|
||
TRENDS_URL = "https://push2delay.eastmoney.com/api/qt/stock/trends2/get"
|
||
HIS_TRENDS_URL = "https://push2his.eastmoney.com/api/qt/stock/trends2/get"
|
||
BROWSER_UA = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
|
||
)
|
||
INDEX_SECIDS = {
|
||
"000001.SH": "1.000001",
|
||
"399001.SZ": "0.399001",
|
||
"399006.SZ": "0.399006",
|
||
}
|
||
|
||
|
||
class EastmoneyAdapter(MarketAdapter):
|
||
name = "eastmoney"
|
||
|
||
def __init__(self, timeout: int = 8) -> None:
|
||
self.timeout = timeout
|
||
|
||
def probe(self) -> dict[str, Any]:
|
||
started = time.perf_counter()
|
||
try:
|
||
rows = self.fetch_indices()
|
||
state = "ok" if len(rows) == 3 else "empty"
|
||
except AdapterError as exc:
|
||
return {
|
||
"provider": self.name,
|
||
"configured": True,
|
||
"state": "error",
|
||
"message": str(exc),
|
||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||
}
|
||
return {
|
||
"provider": self.name,
|
||
"configured": True,
|
||
"state": state,
|
||
"latency_ms": round((time.perf_counter() - started) * 1000),
|
||
}
|
||
|
||
def fetch(self, dataset: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
||
if dataset in {"indexes_quotes", "index_quotes"}:
|
||
return self.fetch_indices()
|
||
if dataset in {"quotes", "quotes_latest"}:
|
||
codes = params.get("codes") or []
|
||
if isinstance(codes, str):
|
||
codes = [item.strip() for item in codes.split(",") if item.strip()]
|
||
if codes:
|
||
return self.fetch_quotes(list(codes))
|
||
return self.fetch_market_quotes()
|
||
if dataset in {"quotes_market", "market_quotes"}:
|
||
return self.fetch_market_quotes()
|
||
raise AdapterError(f"{self.name} unsupported dataset: {dataset}")
|
||
|
||
def normalize(self, dataset: str, rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
return list(rows)
|
||
|
||
def fetch_indices(self) -> list[dict[str, Any]]:
|
||
payload = self._get_json(
|
||
EASTMONEY_INDEX_URL,
|
||
{
|
||
"secids": "1.000001,0.399001,0.399006",
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f124",
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||
result = []
|
||
for row in rows:
|
||
code = str(row.get("f12") or "")
|
||
if code not in {"000001", "399001", "399006"}:
|
||
continue
|
||
epoch = int(finite_number(row.get("f124")) or 0)
|
||
ts_code = f"{code}.SH" if code.startswith("0") and code == "000001" else f"{code}.SZ"
|
||
if code == "000001":
|
||
ts_code = "000001.SH"
|
||
result.append(
|
||
{
|
||
"ts_code": ts_code,
|
||
"code": code,
|
||
"name": row.get("f14") or code,
|
||
"price": round4(finite_number(row.get("f2"))),
|
||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||
"change_amount": round4(finite_number(row.get("f4"))),
|
||
"open": round4(finite_number(row.get("f17"))),
|
||
"high": round4(finite_number(row.get("f15"))),
|
||
"low": round4(finite_number(row.get("f16"))),
|
||
"previous_close": round4(finite_number(row.get("f18"))),
|
||
"amount": round4(finite_number(row.get("f6"))),
|
||
"quote_time_epoch": epoch,
|
||
"quote_time": (
|
||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||
if epoch
|
||
else ""
|
||
),
|
||
"source": "eastmoney_push2",
|
||
}
|
||
)
|
||
if len(result) != 3:
|
||
raise AdapterError(f"Eastmoney returned {len(result)}/3 indices")
|
||
return result
|
||
|
||
def fetch_quotes(self, codes: list[str]) -> list[dict[str, Any]]:
|
||
# Eastmoney ulist.np accepts ~60 secids per request; page remaining codes.
|
||
secids = []
|
||
suffixes: dict[str, str] = {}
|
||
for code in codes:
|
||
ts = str(code or "").upper()
|
||
symbol = ts.split(".")[0]
|
||
if ts.endswith(".BJ") or symbol.startswith(("4", "8", "92")):
|
||
secids.append(f"0.{symbol}")
|
||
suffixes[symbol] = "BJ"
|
||
elif ts.endswith(".SH") or symbol.startswith(("5", "6", "9")):
|
||
secids.append(f"1.{symbol}")
|
||
suffixes[symbol] = "SH"
|
||
else:
|
||
secids.append(f"0.{symbol}")
|
||
suffixes[symbol] = "SZ"
|
||
if not secids:
|
||
return []
|
||
result: list[dict[str, Any]] = []
|
||
for index in range(0, len(secids), 60):
|
||
payload = self._get_json(
|
||
EASTMONEY_INDEX_URL,
|
||
{
|
||
"secids": ",".join(secids[index:index + 60]),
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f5,f6,f8,f124",
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||
for row in rows:
|
||
symbol = str(row.get("f12") or "")
|
||
if not symbol:
|
||
continue
|
||
ts_code = f"{symbol}.{suffixes.get(symbol, 'SZ')}"
|
||
epoch = int(finite_number(row.get("f124")) or 0)
|
||
close = round4(finite_number(row.get("f2")))
|
||
previous = round4(finite_number(row.get("f18")))
|
||
quote_date = (
|
||
datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||
if epoch
|
||
else ""
|
||
)
|
||
result.append(
|
||
{
|
||
"ts_code": ts_code,
|
||
"name": row.get("f14") or symbol,
|
||
"price": close,
|
||
"close": close,
|
||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||
"change_amount": round4(finite_number(row.get("f4"))),
|
||
"open": round4(finite_number(row.get("f17"))),
|
||
"high": round4(finite_number(row.get("f15"))),
|
||
"low": round4(finite_number(row.get("f16"))),
|
||
"pre_close": previous,
|
||
"previous_close": previous,
|
||
"volume": round4(finite_number(row.get("f5"))),
|
||
"vol": round4(finite_number(row.get("f5")) * 100),
|
||
"amount": round4(finite_number(row.get("f6"))),
|
||
"turnover_rate": round4(finite_number(row.get("f8"))),
|
||
"quote_date": quote_date,
|
||
"quote_time_epoch": epoch,
|
||
"quote_time": (
|
||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||
if epoch
|
||
else ""
|
||
),
|
||
"source": "eastmoney_push2",
|
||
}
|
||
)
|
||
return result
|
||
|
||
def fetch_market_quotes(self) -> list[dict[str, Any]]:
|
||
rows: list[dict[str, Any]] = []
|
||
board_errors: list[str] = []
|
||
for board in EASTMONEY_A_SHARE_BOARDS:
|
||
try:
|
||
rows.extend(self._board_quotes(board))
|
||
except Exception as exc:
|
||
board_errors.append(f"{board}:{exc}")
|
||
quotes: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for row in rows:
|
||
quote = _normalize_market_quote(row)
|
||
ts_code = str((quote or {}).get("ts_code") or "")
|
||
if not quote or ts_code in seen:
|
||
continue
|
||
seen.add(ts_code)
|
||
quotes.append(quote)
|
||
if len(quotes) < 200:
|
||
detail = f"；{'; '.join(board_errors)}" if board_errors else ""
|
||
raise AdapterError(f"Eastmoney market snapshot too small: {len(quotes)}{detail}")
|
||
return quotes
|
||
|
||
def _board_quotes(self, board: str) -> list[dict[str, Any]]:
|
||
first = self._market_page(board, 1)
|
||
data = first.get("data") or {}
|
||
rows = list(data.get("diff") or [])
|
||
total = int(finite_number(data.get("total")) or 0)
|
||
page_count = 1
|
||
if total > 0:
|
||
page_count = max(1, (total + EASTMONEY_MARKET_PAGE_SIZE - 1) // EASTMONEY_MARKET_PAGE_SIZE)
|
||
for page in range(2, min(page_count, 40) + 1):
|
||
payload = self._market_page(board, page)
|
||
rows.extend(list((payload.get("data") or {}).get("diff") or []))
|
||
return rows
|
||
|
||
def _market_page(self, board: str, page: int) -> dict[str, Any]:
|
||
return self._get_json(
|
||
EASTMONEY_CLIST_URL,
|
||
{
|
||
"pn": str(page),
|
||
"pz": str(EASTMONEY_MARKET_PAGE_SIZE),
|
||
"po": "1",
|
||
"np": "1",
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fid": "f12",
|
||
"fs": board,
|
||
"fields": EASTMONEY_QUOTE_FIELDS,
|
||
},
|
||
referer="https://quote.eastmoney.com/center/gridlist.html",
|
||
)
|
||
|
||
def fetch_intraday(self, ts_code: str, date: str = "") -> dict[str, Any]:
|
||
code = str(ts_code or "").upper()
|
||
if code in INDEX_SECIDS:
|
||
secid = INDEX_SECIDS[code]
|
||
entity = "index"
|
||
identifier = code
|
||
elif code.startswith("BK") or code.endswith((".TI", ".SI")):
|
||
symbol = code.split(".")[0]
|
||
secid = f"90.{symbol}"
|
||
entity = "board"
|
||
identifier = symbol
|
||
else:
|
||
symbol = code.split(".")[0]
|
||
market = "1" if symbol.startswith(("5", "6", "9")) else "0"
|
||
secid = f"{market}.{symbol}"
|
||
entity = "stock"
|
||
identifier = symbol
|
||
params = {
|
||
"secid": secid,
|
||
"fields1": "f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f11,f12,f13",
|
||
"fields2": "f51,f52,f53,f54,f55,f56,f57,f58",
|
||
"iscr": "0",
|
||
}
|
||
data: dict[str, Any] = {}
|
||
points: list[dict[str, Any]] = []
|
||
last_error: Exception | None = None
|
||
for url, ndays in ((TRENDS_URL, "1"), (TRENDS_URL, "5"), (HIS_TRENDS_URL, "5")):
|
||
try:
|
||
payload = self._get_json(
|
||
url,
|
||
{**params, "ndays": ndays},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
except AdapterError as exc:
|
||
last_error = exc
|
||
continue
|
||
data = payload.get("data") or {}
|
||
parsed = [point for raw in data.get("trends") or [] if (point := _parse_trend(raw))]
|
||
points = _preferred_session(parsed, date)
|
||
if points:
|
||
break
|
||
if not points:
|
||
raise AdapterError("No intraday chart data returned") from last_error
|
||
return {
|
||
"entity_type": entity,
|
||
"identifier": identifier,
|
||
"ts_code": code if "." in code else f"{identifier}.{'SH' if identifier.startswith(('5','6','9')) else 'SZ'}",
|
||
"name": str(data.get("name") or ""),
|
||
"code": str(data.get("code") or identifier),
|
||
"trade_date": points[-1]["date"],
|
||
"previous_close": round4(finite_number(data.get("preClose"))),
|
||
"points": points,
|
||
"source": "eastmoney_trends2",
|
||
}
|
||
|
||
def fetch_shenwan_quote(self, ts_code: str) -> dict[str, Any]:
|
||
code = str(ts_code or "").split(".")[0]
|
||
if not code:
|
||
raise AdapterError("Invalid Shenwan code")
|
||
rows = self.fetch_shenwan_quotes([code])
|
||
if not rows:
|
||
raise AdapterError(f"Eastmoney Shenwan quote missing for {code}")
|
||
return rows[0]
|
||
|
||
def fetch_shenwan_quotes(self, ts_codes: list[str]) -> list[dict[str, Any]]:
|
||
codes = [str(item or "").split(".")[0] for item in ts_codes]
|
||
codes = list(dict.fromkeys(code for code in codes if code))
|
||
if not codes:
|
||
return []
|
||
payload = self._get_json(
|
||
EASTMONEY_INDEX_URL,
|
||
{
|
||
"secids": ",".join(f"90.{code}" for code in codes),
|
||
"fltt": "2",
|
||
"invt": "2",
|
||
"fields": "f12,f14,f2,f3,f4,f15,f16,f17,f18,f6,f8,f104,f105,f128,f136,f140,f124",
|
||
},
|
||
referer="https://quote.eastmoney.com/",
|
||
)
|
||
rows = list((payload.get("data") or {}).get("diff") or [])
|
||
result = [quote for row in rows if row and (quote := self._normalize_shenwan_quote(row))]
|
||
if not result:
|
||
raise AdapterError(f"Eastmoney Shenwan quotes missing for {','.join(codes)}")
|
||
return result
|
||
|
||
@staticmethod
|
||
def _normalize_shenwan_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||
code = str(row.get("f12") or "")
|
||
if not code:
|
||
return None
|
||
epoch = int(finite_number(row.get("f124")) or 0)
|
||
close = round4(finite_number(row.get("f2")))
|
||
previous = round4(finite_number(row.get("f18")))
|
||
if close <= 0 or previous <= 0:
|
||
return None
|
||
quote_time = (
|
||
datetime.fromtimestamp(epoch).astimezone().isoformat(timespec="seconds")
|
||
if epoch
|
||
else ""
|
||
)
|
||
return {
|
||
"ts_code": f"{code}.SI",
|
||
"code": f"{code}.SI",
|
||
"name": row.get("f14") or code,
|
||
"price": close,
|
||
"close": close,
|
||
"pre_close": previous,
|
||
"previous_close": previous,
|
||
"open": round4(finite_number(row.get("f17"))),
|
||
"high": round4(finite_number(row.get("f15"))),
|
||
"low": round4(finite_number(row.get("f16"))),
|
||
"change": round4(finite_number(row.get("f3"))),
|
||
"pct_change": round4(finite_number(row.get("f3"))),
|
||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||
"amount": round4(finite_number(row.get("f6"))),
|
||
"leader": row.get("f128") or "--",
|
||
"leader_code": row.get("f140") or "",
|
||
"leading_pct": round4(finite_number(row.get("f136"))),
|
||
"up_count": int(finite_number(row.get("f104")) or 0),
|
||
"down_count": int(finite_number(row.get("f105")) or 0),
|
||
"quote_time": quote_time,
|
||
"trade_time": quote_time,
|
||
"quote_date": datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d") if epoch else "",
|
||
"quote_time_epoch": epoch,
|
||
"source": "eastmoney_sw",
|
||
}
|
||
|
||
def fetch_limit_pool(self, trade_date: str = "") -> list[dict[str, Any]]:
|
||
day = str(trade_date or "").replace("-", "")
|
||
rows: list[dict[str, Any]] = []
|
||
for url, limit_type in (
|
||
("https://push2ex.eastmoney.com/getTopicZTPool", "U"),
|
||
("https://push2ex.eastmoney.com/getTopicZBPool", "Z"),
|
||
):
|
||
params = {
|
||
"ut": "7eea3edcaed734bea9cbfc24409ed989",
|
||
"dpt": "wz.ztzt",
|
||
"PageIndex": "0",
|
||
"PageSize": "200",
|
||
"sort": "fbt:asc",
|
||
"stat": "1",
|
||
}
|
||
if day:
|
||
params["date"] = day
|
||
try:
|
||
payload = self._get_json(url, params, referer="https://quote.eastmoney.com/")
|
||
except AdapterError:
|
||
continue
|
||
pool = ((payload.get("data") or {}).get("pool") or []) if isinstance(payload.get("data"), dict) else []
|
||
for item in pool:
|
||
code = str(item.get("c") or item.get("code") or "")
|
||
if not code:
|
||
continue
|
||
market = str(item.get("m") or item.get("market") or "")
|
||
suffix = "SH" if market in {"1", "SH"} or code.startswith(("5", "6", "9")) else "SZ"
|
||
first = str(item.get("fbt") or item.get("first_time") or "")
|
||
last = str(item.get("lbt") or item.get("last_time") or "")
|
||
rows.append(
|
||
{
|
||
"ts_code": f"{code}.{suffix}",
|
||
"limit_type": limit_type,
|
||
"first_time": first,
|
||
"last_time": last,
|
||
"fd_amount": item.get("fund") or item.get("fd_amount"),
|
||
"open_times": item.get("zbc") or item.get("open_times"),
|
||
"limit_times": item.get("lbc") or item.get("limit_times"),
|
||
"turnover_ratio": item.get("hs") or item.get("turnover_ratio"),
|
||
"source": "eastmoney_zt_pool",
|
||
}
|
||
)
|
||
if not rows:
|
||
raise AdapterError("Eastmoney limit pool empty")
|
||
return rows
|
||
|
||
def _get_json(self, url: str, params: dict[str, str], referer: str) -> dict[str, Any]:
|
||
request_url = f"{url}?{urllib.parse.urlencode(params)}"
|
||
request = urllib.request.Request(
|
||
request_url,
|
||
headers={
|
||
"Accept": "application/json,text/plain,*/*",
|
||
"User-Agent": BROWSER_UA,
|
||
"Referer": referer,
|
||
},
|
||
method="GET",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=self.timeout) as response:
|
||
return json.loads(response.read().decode("utf-8"))
|
||
except Exception as exc:
|
||
raise AdapterError(f"eastmoney request failed: {exc}") from exc
|
||
|
||
|
||
def _preferred_session(points: list[dict[str, Any]], preferred_date: str = "") -> list[dict[str, Any]]:
|
||
if not points:
|
||
return []
|
||
want = ""
|
||
digits = str(preferred_date or "").replace("-", "")[:8]
|
||
if len(digits) == 8 and digits.isdigit():
|
||
want = f"{digits[:4]}-{digits[4:6]}-{digits[6:8]}"
|
||
if want:
|
||
matched = [point for point in points if str(point.get("date") or "") == want]
|
||
if matched:
|
||
return matched
|
||
latest = max(str(point.get("date") or "") for point in points)
|
||
if not latest:
|
||
return points
|
||
return [point for point in points if str(point.get("date") or "") == latest]
|
||
|
||
|
||
def _normalize_market_quote(row: dict[str, Any]) -> dict[str, Any] | None:
|
||
symbol = str(row.get("f12") or "").strip()
|
||
if not symbol.isdigit() or len(symbol) != 6:
|
||
return None
|
||
close = round4(finite_number(row.get("f2")))
|
||
previous_close = round4(finite_number(row.get("f18")))
|
||
if close <= 0 or previous_close <= 0:
|
||
return None
|
||
market = int(finite_number(row.get("f13")) or 0)
|
||
if market == 0 and symbol.startswith(("4", "8", "92")):
|
||
ts_code = f"{symbol}.BJ"
|
||
elif market == 1 or symbol.startswith(("5", "6", "9")):
|
||
ts_code = f"{symbol}.SH"
|
||
else:
|
||
ts_code = f"{symbol}.SZ"
|
||
epoch = int(finite_number(row.get("f124")) or 0)
|
||
quote_date = ""
|
||
if epoch > 0:
|
||
quote_date = datetime.fromtimestamp(epoch).astimezone().strftime("%Y%m%d")
|
||
return {
|
||
"ts_code": ts_code,
|
||
"name": row.get("f14") or symbol,
|
||
"pre_close": previous_close,
|
||
"previous_close": previous_close,
|
||
"open": round4(finite_number(row.get("f17"))),
|
||
"high": round4(finite_number(row.get("f15"))),
|
||
"low": round4(finite_number(row.get("f16"))),
|
||
"close": close,
|
||
"price": close,
|
||
"pct_chg": round4(finite_number(row.get("f3"))),
|
||
"vol": round4(finite_number(row.get("f5")) * 100),
|
||
"volume": round4(finite_number(row.get("f5")) * 100),
|
||
"amount": round4(finite_number(row.get("f6"))),
|
||
"quote_date": quote_date,
|
||
"quote_time_epoch": epoch,
|
||
"source": "eastmoney_clist",
|
||
}
|
||
|
||
|
||
def _parse_trend(raw: Any) -> dict[str, Any] | None:
|
||
text = str(raw or "")
|
||
parts = text.split(",")
|
||
if len(parts) < 8:
|
||
return None
|
||
stamp = parts[0]
|
||
try:
|
||
when = datetime.strptime(stamp, "%Y-%m-%d %H:%M")
|
||
except ValueError:
|
||
return None
|
||
close = round4(finite_number(parts[2]))
|
||
if close <= 0:
|
||
return None
|
||
return {
|
||
"time": when.strftime("%H:%M"),
|
||
"date": when.strftime("%Y-%m-%d"),
|
||
"open": round4(finite_number(parts[1])),
|
||
"close": close,
|
||
"high": round4(finite_number(parts[3])),
|
||
"low": round4(finite_number(parts[4])),
|
||
"avg_price": round4(finite_number(parts[7] if len(parts) > 7 else parts[2])),
|
||
"volume": round4(finite_number(parts[5])),
|
||
"amount": round4(finite_number(parts[6])),
|
||
}
|